allow passing multiple arguments in test_args (#118)

Co-authored-by: Markus Hauru <markus@mhauru.org>
Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 99
+    labels:
+      - dependencies
+      - github-actions
+

.github/workflows/backup.yml

@@ -1,33 +0,0 @@
-name: Backup
-
-on:
-  schedule:
-    - cron: '5 4 * * 0'
-
-  workflow_dispatch:
-
-jobs:
-  backup:
-    runs-on: ubuntu-20.04
-
-    steps:
-      - name: Configure cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ env.GITHUB_WORKSPACE }}
-            ~/.cache/restic
-          key: ${{ runner.os }}
-
-      - name: Install the correct Python version
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
-
-      - name: Run backup action
-        uses: julia-actions/restic-action@main
-        env: # Options: https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables
-          RESTIC_REPOSITORY: b2:${{ secrets.B2_BUCKET }}:${{ github.repository }}
-          RESTIC_PASSWORD: ${{ secrets.RESTIC_PASSWORD }}
-          B2_ACCOUNT_ID: ${{ secrets.B2_ACCOUNT_ID }}
-          B2_ACCOUNT_KEY: ${{ secrets.B2_ACCOUNT_KEY }}

.github/workflows/ci.yml

@@ -5,13 +5,7 @@ on:
     branches:
       - "main"
       - "master"
-    paths-ignore:
-      - "README.md"
-      - "LICENSE"
   pull_request:
-    paths-ignore:
-      - "README.md"
-      - "LICENSE"
 
 jobs:
   test:
@@ -38,7 +32,7 @@ jobs:
 
     steps:
       - name: Checkout Example.jl
-        uses: actions/checkout@v2
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           repository: julia-actions/Example.jl
 
@@ -47,16 +41,16 @@ jobs:
         shell: bash
 
       - name: Checkout julia-runtest
-        uses: actions/checkout@v2
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           path: ./.github/actions/julia-runtest
 
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@512e2ab44f5848ea62bddd21d1ee906e989b0360 # v2.4.0
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
 
-      - uses: actions/cache@v1
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
         env:
           cache-name: cache-artifacts
         with:
@@ -67,15 +61,18 @@ jobs:
             ${{ runner.os }}-test-
             ${{ runner.os }}-
 
-      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-buildpkg@90dd6f23eb49626e4e6612cb9d64d456f86e6a1c
 
       - uses: ./.github/actions/julia-runtest
 
-      - uses: julia-actions/julia-processcoverage@v1
+      - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38
 
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
         with:
           file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail-ci-if-error: true
+          
 
       - name: Verify that test output file exists
         run: test -f "$HOME/julia-runtest"

.github/workflows/test_logger_ci.yml

@@ -0,0 +1,53 @@
+name: TestLogger CI
+
+on:
+  push:
+    branches:
+      - "main"
+      - "master"
+  pull_request:
+
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "1.6"
+          - "1" # automatically expands to the latest stable 1.x release of Julia
+          - nightly
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x64
+          - x86
+        # 32-bit Julia binaries are not available on macOS
+        exclude:
+          - os: macOS-latest
+            arch: x86
+
+    steps:
+      - name: Checkout julia-runtest
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+
+      - uses: julia-actions/setup-julia@512e2ab44f5848ea62bddd21d1ee906e989b0360 # v2.4.0
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+
+      - run: julia --color=yes --check-bounds=yes test_logger_tests.jl

README.md

@@ -6,19 +6,29 @@ This action runs the tests in a Julia package.
 
 Julia needs to be installed before this action can run. This can easily be achieved with the [setup-julia](https://github.com/marketplace/actions/setup-julia-environment) action.
 
-And example workflow that uses this action might look like this:
+An example workflow that uses this action might look like this:
 
 ```yaml
 name: Run tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        julia-version: ['1.0', '1', 'nightly']
+        julia-version: ['lts', '1', 'pre']
         julia-arch: [x64, x86]
         os: [ubuntu-latest, windows-latest, macOS-latest]
         exclude:
@@ -26,17 +36,24 @@ jobs:
             julia-arch: x86
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.julia-arch }}
+      - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+        # with:
+        #   annotate: true
 ```
 
 You can add this workflow to your repository by placing it in a file called `test.yml` in the folder `.github/workflows/`. [More info here](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions).
 
+Here, setting `annotate: true` causes GitHub "annotations" to appear when reviewing the PR, pointing to failing tests, if any.
+This functionality is only enabled on Julia 1.8 (even if `annotate` is set to `true`), since currently it does not work on other Julia versions (see [#76](https://github.com/julia-actions/julia-runtest/issues/76)).
+
+By default, `annotate` is set to false, but that may change in future releases of this action.
 ### Prefixing the Julia command
 
 In some packages, you may want to prefix the `julia` command with another command, e.g. for running tests of certain graphical libraries with `xvfb-run`.
@@ -55,7 +72,7 @@ If you only want to add this prefix on certain builds, you can [include addition
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        version: ['1.0', '1', 'nightly']
+        version: ['lts', '1', 'pre']
         arch: [x64]
         include:
           - os: ubuntu-latest
@@ -69,3 +86,51 @@ If you only want to add this prefix on certain builds, you can [include addition
 ```
 
 This will add the prefix `xvfb-run` to all builds where the `os` is `ubuntu-latest`.
+
+### Pass Arguments to Test Suite
+
+You can pass arguments from the workflow specification to the test script via the `test_args` parameter.
+
+This is useful, for example, to specify separate workflows for fast and slow tests, or conditionally enabling quality assurance tests.
+
+The functionality can be incorporated as follows:
+
+```yaml
+    # ...
+    steps:
+    # ...
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          test_args: 'slow_tests "quality assurance"'
+    # ...
+```
+
+The value of `test_args` can be accessed in `runtest.jl` via the `ARGS` variable. An example for `runtest.jl` is given below.
+
+```julia
+using Test
+# ...
+
+# run fast tests by default
+include("fast_tests.jl")
+
+if "slow_tests" in ARGS
+    # run slow tests
+    include("slow_tests.jl")
+end
+
+if "quality assurance" in ARGS
+    # run quality assurance tests
+    include("qa.jl")
+end
+```
+
+
+### Registry flavor preference
+
+This actions defines (and exports for subsequent steps of the workflow) the
+environmental variable `JULIA_PKG_SERVER_REGISTRY_PREFERENCE=eager` unless it
+is already set. If you want another registry flavor (i.e. `conservative`) this
+should be defined in the `env:` section of the relevant workflow or step. See
+[Registry flavors](https://pkgdocs.julialang.org/dev/registries/#Registry-flavors)
+for more information.

action.yml

@@ -29,34 +29,51 @@ inputs:
   project:
     description: 'Value passed to the --project flag. The default value is the repository root: "@."'
     default: '@.'
+  annotate:
+    description: 'Whether or not to attempt to create GitHub annotations to show test failures inline. Only effective on Julia 1.8+.'
+    default: 'false'
+  compiled_modules:
+    description: 'Whether to run tests with `compiled-modules`. For possible values, refer to https://docs.julialang.org/en/v1/manual/command-line-interface/#command-line-interface'
+    default: 'yes'
+  allow_reresolve:
+    description: 'Whether to allow re-resolving of package versions in the test environment. Only effective on Julia 1.9+. Options: true | false. Default value: true'
+    default: 'true'
+  test_args:
+    description: 'Arguments string that is passed on to test.'
+    default: ''
 
 runs:
   using: 'composite'
   steps:
-    # Occasionally, there are rather large delays (> a few hours)
-    # between the time a package is registered in General and
-    # propagated to pkg.julialang.org.  We can avoid this by manually
-    # cloning ~/.julia/registries/General/ in Julia 1.5 and later.
-    # See:
-    # * https://github.com/JuliaLang/Pkg.jl/issues/2011
-    # * https://github.com/JuliaRegistries/General/issues/16777
-    # * https://github.com/JuliaPackaging/PkgServer.jl/issues/60
-    - run: julia --color=yes "$GITHUB_ACTION_PATH"/add_general_registry.jl
+    - name: Set and export registry flavor preference
+      run: echo "JULIA_PKG_SERVER_REGISTRY_PREFERENCE=${JULIA_PKG_SERVER_REGISTRY_PREFERENCE:-eager}" >> ${GITHUB_ENV}
       shell: bash
-      env:
-        # We set `JULIA_PKG_SERVER` only for this step to enforce
-        # `Pkg.Registry.add` to use Git.  This way, Pkg.jl can send
-        # the request metadata to pkg.julialang.org when installing
-        # packages via `Pkg.test`.
-        JULIA_PKG_SERVER: ""
+    - name: Install dependencies in their own (shared) environment
+      run: |
+        # Functionality only currently works on a narrow range of Julia versions... see #76
+        if v"1.8pre" < VERSION < v"1.9.0-beta3"
+          using Pkg
+          Pkg.activate("tests-logger-env"; shared=true)
+          Pkg.add(Pkg.PackageSpec(name="GitHubActions", version="0.1"))
+        end
+      shell: julia --color=yes {0}
+      if: inputs.annotate == 'true'
     - run: |
         # The Julia command that will be executed
-        julia_cmd=( julia --color=yes --depwarn=${{ inputs.depwarn }} --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'import Pkg;include(joinpath(ENV["GITHUB_ACTION_PATH"], "kwargs.jl"));kwargs = Kwargs.kwargs(;coverage = :(${{ inputs.coverage }}),force_latest_compatible_version = :(${{ inputs.force_latest_compatible_version }}), julia_args = ["--check-bounds=${{ inputs.check_bounds }}"]);Pkg.test(; kwargs...)' )
+        julia_cmd=( julia --color=yes --depwarn=${{ inputs.depwarn }} --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'include(joinpath(ENV["GITHUB_ACTION_PATH"], "test_harness.jl"))' -- ${{inputs.test_args}} )
 
         # Add the prefix in front of the command if there is one
-        prefix="${{ inputs.prefix }}"
-        [[ -n $prefix ]] && julia_cmd=( "$prefix" "${julia_cmd[@]}" )
+        prefix=( ${{ inputs.prefix }} )
+        [[ -n ${prefix[*]} ]] && julia_cmd=( "${prefix[@]}" "${julia_cmd[@]}" )
 
         # Run the Julia command
+        echo "::debug::Executing Julia: ${julia_cmd[*]}"
         "${julia_cmd[@]}"
       shell: bash
+      env:
+        ANNOTATE: ${{ inputs.annotate }}
+        COVERAGE: ${{ inputs.coverage }}
+        FORCE_LATEST_COMPATIBLE_VERSION: ${{ inputs.force_latest_compatible_version }}
+        CHECK_BOUNDS: ${{ inputs.check_bounds }}
+        COMPILED_MODULES: ${{ inputs.compiled_modules }}
+        ALLOW_RERESOLVE: ${{ inputs.allow_reresolve }}

add_general_registry.jl

@@ -1,49 +0,0 @@
-using Pkg
-
-function general_registry_location()
-    general_registry_dir = joinpath(DEPOT_PATH[1], "registries", "General")
-    registry_toml_file = joinpath(general_registry_dir, "Registry.toml")
-    return general_registry_dir, registry_toml_file
-end
-
-function general_registry_exists()
-    general_registry_dir, registry_toml_file = general_registry_location()
-    if !isdir(general_registry_dir)
-        return false
-    elseif !isfile(registry_toml_file)
-        return false
-    else
-        return true
-    end
-end
-
-function add_general_registry()
-    @info("Attempting to clone the General registry")
-    general_registry_dir, registry_toml_file = general_registry_location()
-    rm(general_registry_dir; force = true, recursive = true)
-    Pkg.Registry.add("General")
-    isfile(registry_toml_file) || throw(ErrorException("the Registry.toml file does not exist"))
-    return nothing
-end
-
-function main(; n = 10, max_delay = 120)
-    VERSION >= v"1.5-" || return
-
-    if general_registry_exists()
-        @info("The General registry already exists locally")
-        return
-    end
-
-    delays = ExponentialBackOff(; n = n, max_delay = max_delay)
-    try
-        retry(add_general_registry; delays = delays)()
-        @info("Successfully added the General registry")
-    catch ex
-        msg = "I was unable to add the General registry. However, the build will continue."
-        @error(msg, exception=(ex,catch_backtrace()))
-    end
-
-    return
-end
-
-main()

kwargs.jl

@@ -4,9 +4,28 @@ import Pkg
 
 include(joinpath(@__DIR__, "autodetect-dependabot.jl"))
 
-function kwargs(; coverage::Bool,
-                  force_latest_compatible_version::Union{Bool, Symbol},
-                  julia_args::AbstractVector{<:AbstractString}=String[])
+function kwargs(; coverage,
+                  force_latest_compatible_version,
+                  allow_reresolve,
+                  julia_args::AbstractVector{<:AbstractString}=String[],
+                  test_args::AbstractVector{<:AbstractString}=String[],
+                  )
+    if coverage isa AbstractString
+        coverage = parse(Bool, coverage)
+    end
+    coverage isa Bool || error("Unexpected type of `coverage`: $(typeof(coverage))")
+
+    if force_latest_compatible_version isa AbstractString
+        res = tryparse(Bool, force_latest_compatible_version)
+        if res === nothing
+            res = Symbol(force_latest_compatible_version)
+        end
+        force_latest_compatible_version = res
+    end
+    if !(force_latest_compatible_version isa Union{Bool, Symbol})
+        error("Unexpected type of `force_latest_compatible_version`: $(typeof(force_latest_compatible_version))")
+    end
+
     if !(force_latest_compatible_version isa Bool) && (force_latest_compatible_version != :auto)
         throw(ArgumentError("Invalid value for force_latest_compatible_version: $(force_latest_compatible_version)"))
     end
@@ -35,6 +54,12 @@ function kwargs(; coverage::Bool,
         kwargs_dict[:force_latest_compatible_version] = force_latest_compatible_version::Bool
     end
 
+    if VERSION >= v"1.9"
+        kwargs_dict[:allow_reresolve] = parse(Bool, allow_reresolve)
+    end
+    
+    kwargs_dict[:test_args] = test_args
+
     return kwargs_dict
 end
 

test_harness.jl

@@ -0,0 +1,51 @@
+import Pkg
+include("kwargs.jl")
+kwargs = Kwargs.kwargs(; coverage=ENV["COVERAGE"],
+                         force_latest_compatible_version=ENV["FORCE_LATEST_COMPATIBLE_VERSION"],
+                         allow_reresolve=ENV["ALLOW_RERESOLVE"],
+                         julia_args=[string("--check-bounds=", ENV["CHECK_BOUNDS"]),
+                                     string("--compiled-modules=", ENV["COMPILED_MODULES"])],
+                         test_args=ARGS,
+                         )
+
+kwargs_reprs = map(kv -> string(kv[1], "=", repr(kv[2])), collect(kwargs))
+kwargs_repr = join(kwargs_reprs, ", ")
+
+print("""
+│
+│ To reproduce this CI run locally run the following from the same repository state on julia version $VERSION:
+│
+│ `import Pkg; Pkg.test(;$kwargs_repr)`
+│
+""")
+
+if parse(Bool, ENV["ANNOTATE"]) && v"1.8pre" < VERSION < v"1.9.0-beta3"
+    push!(LOAD_PATH, "@tests-logger-env") # access dependencies
+    using GitHubActions, Logging
+    global_logger(GitHubActionsLogger())
+    include("test_logger.jl")
+    pop!(LOAD_PATH)
+    try
+        TestLogger.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
+else
+    try
+        Pkg.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
+end

test_logger.jl

@@ -0,0 +1,97 @@
+module TestLogger
+using Pkg
+
+function parse_file_line(failed_line)
+    # The bits like `\e[91m\e[1m` are color codes that get printed by `Pkg.test`. We
+    # match with or without them.
+    r = r"(\e\[91m\e\[1m)?Test Failed(\e\[22m\e\[39m)? at (\e\[39m\e\[1m)?(?<path>[^\s\e]+)(\e\[22m)?"
+    m = match(r, failed_line)
+    m === nothing && return (nothing, nothing)
+
+    if m[:path] === nothing
+        return (nothing, nothing)
+    else
+        path_split_results = rsplit(m[:path], ":", limit=2)
+        if length(path_split_results) == 1
+            return (m[:path], nothing)
+        else
+            path, line_no = path_split_results
+
+            # Try to make sure line number is parseable to avoid false positives
+            line_no = tryparse(Int, line_no) === nothing ? nothing : line_no
+            return (path, line_no)
+        end
+    end
+    return (nothing, nothing)
+end
+
+function readlines_until(f, stream; keep_lines=true, io)
+    lines = String[]
+    while true
+        line = readline(stream; keep=true)
+        print(io, line)
+
+        # with `keep=true`, this should only happen when we're done?
+        # I think so...
+        if line == ""
+            return line, lines
+        end
+        if f(line)
+            return line, lines
+        else
+            keep_lines && push!(lines, line)
+        end
+    end
+end
+
+function has_test_failure(line)
+    contains(line, "Test Failed") || return false
+    file, line_no = parse_file_line(line)
+    return !isnothing(file) && !isnothing(line_no)
+end
+
+function build_stream(io)
+    stream = Base.BufferStream()
+    t = @async begin
+        while !eof(stream)
+            # Iterate through and print until we get to "Test Failed" and can parse it
+            failed_line, _ = readlines_until(has_test_failure, stream; keep_lines=false, io)
+            @label found_failed_line
+            # Parse file and line out
+            file, line_no = parse_file_line(failed_line)
+
+            # Grab everything until the stacktrace, OR we hit another `Test Failed`
+            stopped_line, msg_lines = readlines_until(stream; io) do line
+                contains(line, "Stacktrace:") || has_test_failure(line)
+            end
+
+            # If we stopped because we hit a 2nd test failure,
+            # let's assume somehow the stacktrace didn't show up for the first one.
+            # Let's continue by trying to find the info for this one, by jumping back.
+            if has_test_failure(stopped_line)
+                failed_line = stopped_line
+                @goto found_failed_line
+            end
+
+            if !isempty(msg_lines)
+                msg = string("Test Failed\n", chomp(join(msg_lines)))
+                # Now log it out
+                @error msg _file=file _line=line_no
+            end
+        end
+    end
+    return stream, t
+end
+
+
+function test(args...; kwargs...)
+    stream, t = build_stream(stdout)
+    Base.errormonitor(t)
+    return try
+        Pkg.test(args...; kwargs..., io=stream)
+    finally
+        close(stream)
+    end
+end
+
+end # module

test_logger_tests.jl

@@ -0,0 +1,71 @@
+include("test_logger.jl")
+using Logging, Test
+
+function simulate(text)
+    logger = Test.TestLogger()
+    output = IOBuffer()
+    with_logger(logger) do
+        stream, t = TestLogger.build_stream(output)
+        for line in eachline(IOBuffer(text); keep=true)
+            write(stream, line)
+        end
+        close(stream)
+        wait(t)
+    end
+    return String(take!(output)), logger.logs
+end
+
+@testset "TestLogger" begin
+
+    for input in (
+            """
+            Test Failed at file.txt:1
+            1
+            2
+            3
+            4
+            5
+            6
+            Stacktrace:
+            Hi
+            """,
+            # Let us mess with the stacktrace line
+            """
+            Test Failed at file.txt:1
+            1
+            2
+            3
+            4
+            5
+            6
+               Stacktrace:   extra stuff
+            Hi
+            """)
+
+        output, logs = simulate(input)
+        @test output == input
+        log = only(logs)
+        @test log.message == "Test Failed\n1\n2\n3\n4\n5\n6"
+        @test log.file == "file.txt"
+        @test log.line == "1"
+    end
+
+    # Next, check that if we hit a Test Failed, and then hit another one before we get a stacktrace,
+    # we just move on to handling the new one.
+    input = """
+    Test Failed at file.txt:1
+    Nah
+    Test Failed at file.txt:1
+    Correct
+    Stacktrace:
+    Hi
+    """
+
+    output, logs = simulate(input)
+    @test output == input
+
+    log = only(logs)
+    @test log.message == "Test Failed\nCorrect"
+    @test log.file == "file.txt"
+    @test log.line == "1"
+end