warn if merge commit content differs to branch head (#152)

Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.5.0 to 5.5.1.
- [Release notes](https://github.com/codecov/codecov-action/releases)
- [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md)
- [Commits](fdcc847654...5a1091511a)

---
updated-dependencies:
- dependency-name: codecov/codecov-action
  dependency-version: 5.5.1
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/CODEOWNERS

@@ -0,0 +1 @@
+*       @julia-actions/reviewers

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 99
+    labels:
+      - dependencies
+      - github-actions
+

.github/workflows/ci.yml

@@ -0,0 +1,70 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - "main"
+      - "master"
+  pull_request:
+
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "1.0"
+          - "1" # automatically expands to the latest stable 1.x release of Julia
+          - nightly
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x64
+          - x86
+        # 32-bit Julia binaries are not available on macOS
+        exclude:
+          - os: macOS-latest
+            arch: x86
+
+    steps:
+      - name: Checkout Example.jl
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          repository: julia-actions/Example.jl
+
+      - name: Verify that test output file does not exist yet
+        run: test ! -f "$HOME/julia-runtest"
+        shell: bash
+
+      - name: Checkout julia-runtest
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          path: ./.github/actions/julia-runtest
+
+      - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+
+      - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0
+
+      - uses: julia-actions/julia-buildpkg@e3eb439fad4f9aba7da2667e7510e4a46ebc46e1 # v1.7.0
+
+      - uses: ./.github/actions/julia-runtest
+
+      - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38
+
+      - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
+        with:
+          files: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail-ci-if-error: true
+          
+
+      - name: Verify that test output file exists
+        run: test -f "$HOME/julia-runtest"
+        shell: bash

.github/workflows/test_logger_ci.yml

@@ -0,0 +1,44 @@
+name: TestLogger CI
+
+on:
+  push:
+    branches:
+      - "main"
+      - "master"
+  pull_request:
+
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - "1.6"
+          - "1" # automatically expands to the latest stable 1.x release of Julia
+          - nightly
+        os:
+          - ubuntu-latest
+          - macOS-latest
+          - windows-latest
+        arch:
+          - x64
+          - x86
+        # 32-bit Julia binaries are not available on macOS
+        exclude:
+          - os: macOS-latest
+            arch: x86
+
+    steps:
+      - name: Checkout julia-runtest
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+
+      - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0
+
+      - run: julia --color=yes --check-bounds=yes test_logger_tests.jl

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018-2020 GitHub, Inc., David Anthoff and contributors
+Copyright (c) 2018-2021 GitHub, Inc., David Anthoff and contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -6,19 +6,29 @@ This action runs the tests in a Julia package.
 
 Julia needs to be installed before this action can run. This can easily be achieved with the [setup-julia](https://github.com/marketplace/actions/setup-julia-environment) action.
 
-And example workflow that uses this action might look like this:
+An example workflow that uses this action might look like this:
 
-```
+```yaml
 name: Run tests
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+      - main
+  pull_request:
+
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
 
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        julia-version: ['1.0', '1', 'nightly']
+        julia-version: ['lts', '1', 'pre']
         julia-arch: [x64, x86]
         os: [ubuntu-latest, windows-latest, macOS-latest]
         exclude:
@@ -26,10 +36,101 @@ jobs:
             julia-arch: x86
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@latest
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.julia-version }}
-      - uses: julia-actions/julia-buildpkg@latest
+          arch: ${{ matrix.julia-arch }}
-      - uses: julia-actions/julia-runtest@latest
+      - uses: julia-actions/cache@v2
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+        # with:
+        #   annotate: true
 ```
+
+You can add this workflow to your repository by placing it in a file called `test.yml` in the folder `.github/workflows/`. [More info here](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions).
+
+Here, setting `annotate: true` causes GitHub "annotations" to appear when reviewing the PR, pointing to failing tests, if any.
+This functionality is only enabled on Julia 1.8 (even if `annotate` is set to `true`), since currently it does not work on other Julia versions (see [#76](https://github.com/julia-actions/julia-runtest/issues/76)).
+
+By default, `annotate` is set to false, but that may change in future releases of this action.
+### Prefixing the Julia command
+
+In some packages, you may want to prefix the `julia` command with another command, e.g. for running tests of certain graphical libraries with `xvfb-run`.
+In that case, you can add an input called `prefix` containing the command that will be inserted to your workflow:
+
+```yaml
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          prefix: xvfb-run
+```
+
+If you only want to add this prefix on certain builds, you can [include additional values into a combination](https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#example-including-additional-values-into-combinations) of your build matrix, e.g.:
+
+```yaml
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        version: ['lts', '1', 'pre']
+        arch: [x64]
+        include:
+          - os: ubuntu-latest
+            prefix: xvfb-run
+    steps:
+    # ...
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          prefix: ${{ matrix.prefix }}
+    # ...
+```
+
+This will add the prefix `xvfb-run` to all builds where the `os` is `ubuntu-latest`.
+
+### Pass Arguments to Test Suite
+
+You can pass arguments from the workflow specification to the test script via the `test_args` parameter.
+
+This is useful, for example, to specify separate workflows for fast and slow tests, or conditionally enabling quality assurance tests.
+
+The functionality can be incorporated as follows:
+
+```yaml
+    # ...
+    steps:
+    # ...
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          test_args: 'slow_tests "quality assurance"'
+    # ...
+```
+
+The value of `test_args` can be accessed in `runtest.jl` via the `ARGS` variable. An example for `runtest.jl` is given below.
+
+```julia
+using Test
+# ...
+
+# run fast tests by default
+include("fast_tests.jl")
+
+if "slow_tests" in ARGS
+    # run slow tests
+    include("slow_tests.jl")
+end
+
+if "quality assurance" in ARGS
+    # run quality assurance tests
+    include("qa.jl")
+end
+```
+
+
+### Registry flavor preference
+
+This actions defines (and exports for subsequent steps of the workflow) the
+environmental variable `JULIA_PKG_SERVER_REGISTRY_PREFERENCE=eager` unless it
+is already set. If you want another registry flavor (i.e. `conservative`) this
+should be defined in the `env:` section of the relevant workflow or step. See
+[Registry flavors](https://pkgdocs.julialang.org/dev/registries/#Registry-flavors)
+for more information.

action.yml

@@ -7,8 +7,8 @@ branding:
   color: 'gray-dark'
 
 inputs:
-  inline:
+  check_bounds:
-    description: 'Value passed to the --inline flag. Options: yes | no. Default value: yes.'
+    description: 'Value determining which bounds checking setting to use. Options: yes | no | auto. Default value: yes.'
     default: 'yes'
   coverage:
     description: 'Value determining whether to test with coverage or not. Options: true | false. Default value: true.'
@@ -16,25 +16,65 @@ inputs:
   depwarn:
     description: 'Value passed to the --depwarn flag. Options: yes | no | error. Default value: yes.'
     default: 'yes'
+  force_latest_compatible_version:
+    description: 'If true, then, for each [compat] entry in the active project, only allow the latest compatible version. If the value is auto and the pull request has been opened by Dependabot or CompatHelper, then force_latest_compatible_version will be set to true, otherwise it will be set to false. Options: true | false | auto. Default value: auto.'
+    default: 'auto'
+  inline:
+    description: 'Value passed to the --inline flag. Options: yes | no. Default value: yes.'
+    default: 'yes'
+  prefix:
+    description: 'Value inserted in front of the julia command, e.g. for running xvfb-run julia [...]'
+    default: ''
+    required: false
+  project:
+    description: 'Value passed to the --project flag. The default value is the repository root: "@."'
+    default: '@.'
+  annotate:
+    description: 'Whether or not to attempt to create GitHub annotations to show test failures inline. Only effective on Julia 1.8+.'
+    default: 'false'
+  compiled_modules:
+    description: 'Whether to run tests with `compiled-modules`. For possible values, refer to https://docs.julialang.org/en/v1/manual/command-line-interface/#command-line-interface'
+    default: 'yes'
+  allow_reresolve:
+    description: 'Whether to allow re-resolving of package versions in the test environment. Only effective on Julia 1.9+. Options: true | false. Default value: true'
+    default: 'true'
+  test_args:
+    description: 'Arguments string that is passed on to test.'
+    default: ''
 
 runs:
   using: 'composite'
   steps:
-    # Occasionally, there are rather large delays (> a few hours)
+    - name: Set and export registry flavor preference
-    # between the time a package is registered in General and
+      run: echo "JULIA_PKG_SERVER_REGISTRY_PREFERENCE=${JULIA_PKG_SERVER_REGISTRY_PREFERENCE:-eager}" >> ${GITHUB_ENV}
-    # propagated to pkg.julialang.org.  We can avoid this by manually
+      shell: bash
-    # cloning ~/.julia/registries/General/ in Julia 1.5 and later.
+    - name: Install dependencies in their own (shared) environment
-    # See:
+      run: |
-    # * https://github.com/JuliaLang/Pkg.jl/issues/2011
+        # Functionality only currently works on a narrow range of Julia versions... see #76
-    # * https://github.com/JuliaRegistries/General/issues/16777
+        if v"1.8pre" < VERSION < v"1.9.0-beta3"
-    # * https://github.com/JuliaPackaging/PkgServer.jl/issues/60
+          using Pkg
-    - run: julia --color=yes -e 'using Pkg; VERSION >= v"1.5-" && !isdir(joinpath(DEPOT_PATH[1], "registries", "General")) && Pkg.Registry.add("General")'
+          Pkg.activate("tests-logger-env"; shared=true)
+          Pkg.add(Pkg.PackageSpec(name="GitHubActions", version="0.1"))
+        end
+      shell: julia --color=yes {0}
+      if: inputs.annotate == 'true'
+    - run: |
+        # The Julia command that will be executed
+        julia_cmd=( julia --color=yes --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'include(joinpath(ENV["GITHUB_ACTION_PATH"], "test_harness.jl"))' -- ${{inputs.test_args}} )
+
+        # Add the prefix in front of the command if there is one
+        prefix=( ${{ inputs.prefix }} )
+        [[ -n ${prefix[*]} ]] && julia_cmd=( "${prefix[@]}" "${julia_cmd[@]}" )
+
+        # Run the Julia command
+        echo "::debug::Executing Julia: ${julia_cmd[*]}"
+        "${julia_cmd[@]}"
       shell: bash
       env:
-        # We set `JULIA_PKG_SERVER` only for this step to enforce
+        ANNOTATE: ${{ inputs.annotate }}
-        # `Pkg.Registry.add` to use Git.  This way, Pkg.jl can send
+        COVERAGE: ${{ inputs.coverage }}
-        # the request metadata to pkg.julialang.org when installing
+        FORCE_LATEST_COMPATIBLE_VERSION: ${{ inputs.force_latest_compatible_version }}
-        # packages via `Pkg.test`.
+        CHECK_BOUNDS: ${{ inputs.check_bounds }}
-        JULIA_PKG_SERVER: ""
+        COMPILED_MODULES: ${{ inputs.compiled_modules }}
-    - run: julia --color=yes --check-bounds=yes --inline=${{ inputs.inline }} --depwarn=${{ inputs.depwarn }} --project -e 'using Pkg; Pkg.test(coverage=${{ inputs.coverage }})'
+        ALLOW_RERESOLVE: ${{ inputs.allow_reresolve }}
-      shell: bash
+        DEPWARN: ${{ inputs.depwarn }}

autodetect-dependabot.jl

@@ -0,0 +1,25 @@
+module AutodetectDependabot
+
+function _get_possible_branch_names()
+    possible_branch_names = [
+        get(ENV, "GITHUB_BASE_REF", ""),
+        get(ENV, "GITHUB_HEAD_REF", ""),
+        get(ENV, "GITHUB_REF", ""),
+    ]
+    return possible_branch_names
+end
+
+function _chop_refs_head(branch_name::AbstractString)
+    replace(branch_name, r"^(refs\/heads\/)" => "")
+end
+
+function _is_dependabot_branch(branch_name::AbstractString)
+    return startswith(branch_name, "dependabot/julia") || startswith(branch_name, "compathelper/")
+end
+
+function is_dependabot_job()
+    possible_branch_names = _get_possible_branch_names()
+    return any(_is_dependabot_branch.(_chop_refs_head.(possible_branch_names)))
+end
+
+end # module

kwargs.jl

@@ -0,0 +1,68 @@
+module Kwargs
+
+import Pkg
+
+include(joinpath(@__DIR__, "autodetect-dependabot.jl"))
+
+function kwargs(; coverage,
+                  force_latest_compatible_version,
+                  allow_reresolve,
+                  julia_args::AbstractVector{<:AbstractString}=String[],
+                  test_args::AbstractVector{<:AbstractString}=String[],
+                  )
+    if coverage isa AbstractString
+        coverage = parse(Bool, coverage)
+    end
+    coverage isa Bool || error("Unexpected type of `coverage`: $(typeof(coverage))")
+
+    if force_latest_compatible_version isa AbstractString
+        res = tryparse(Bool, force_latest_compatible_version)
+        if res === nothing
+            res = Symbol(force_latest_compatible_version)
+        end
+        force_latest_compatible_version = res
+    end
+    if !(force_latest_compatible_version isa Union{Bool, Symbol})
+        error("Unexpected type of `force_latest_compatible_version`: $(typeof(force_latest_compatible_version))")
+    end
+
+    if !(force_latest_compatible_version isa Bool) && (force_latest_compatible_version != :auto)
+        throw(ArgumentError("Invalid value for force_latest_compatible_version: $(force_latest_compatible_version)"))
+    end
+
+    kwargs_dict = Dict{Symbol, Any}()
+    kwargs_dict[:coverage] = coverage
+
+    if VERSION >= v"1.6.0"
+        kwargs_dict[:julia_args] = julia_args
+    elseif julia_args == ["--check-bounds=yes"]
+        # silently don't add this default julia_args value as < 1.6 doesn't support julia_args, but it's the default state
+    else
+        println("::warning::The Pkg.test bounds checking behavior cannot be changed before Julia 1.6. VERSION=$VERSION, julia_args=$julia_args")
+    end
+
+    if VERSION < v"1.7.0-" || !hasmethod(Pkg.Operations.test, Tuple{Pkg.Types.Context, Vector{Pkg.Types.PackageSpec}}, (:force_latest_compatible_version,))
+        (force_latest_compatible_version != :auto) && @warn("The `force_latest_compatible_version` option requires at least Julia 1.7", VERSION, force_latest_compatible_version)
+        return kwargs_dict
+    end
+
+    if force_latest_compatible_version == :auto
+        is_dependabot_job = AutodetectDependabot.is_dependabot_job()
+        is_dependabot_job && @info("This is a Dependabot/CompatHelper job, so `force_latest_compatible_version` has been set to `true`")
+        kwargs_dict[:force_latest_compatible_version] = is_dependabot_job
+    else
+        kwargs_dict[:force_latest_compatible_version] = force_latest_compatible_version::Bool
+    end
+
+    if VERSION >= v"1.9"
+        kwargs_dict[:allow_reresolve] = parse(Bool, allow_reresolve)
+    end
+
+    if !isempty(test_args)
+        kwargs_dict[:test_args] = test_args
+    end
+
+    return kwargs_dict
+end
+
+end # module

test_harness.jl

@@ -0,0 +1,83 @@
+import Pkg
+include("kwargs.jl")
+kwargs = Kwargs.kwargs(; coverage=ENV["COVERAGE"],
+                         force_latest_compatible_version=ENV["FORCE_LATEST_COMPATIBLE_VERSION"],
+                         allow_reresolve=ENV["ALLOW_RERESOLVE"],
+                         julia_args=[string("--check-bounds=", ENV["CHECK_BOUNDS"]),
+                                     string("--compiled-modules=", ENV["COMPILED_MODULES"]),
+                                     # Needs to be done via `julia_args` to ensure `depwarn: no` is respected:
+                                     # https://github.com/JuliaLang/Pkg.jl/pull/1763#discussion_r406819660
+                                     string("--depwarn=", ENV["DEPWARN"]),],
+                         test_args=ARGS,
+                         )
+
+kwargs_reprs = map(kv -> string(kv[1], "=", repr(kv[2])), collect(kwargs))
+kwargs_repr = join(kwargs_reprs, ", ")
+
+# Warn if running on a merge commit (different from branch HEAD)
+git_note = ""
+if haskey(ENV, "GITHUB_SHA") && get(ENV, "GITHUB_EVENT_NAME", "") == "pull_request" && haskey(ENV, "GITHUB_HEAD_REF")
+    # For pull_request events, GITHUB_SHA is the merge commit, not the PR head commit
+    try
+        merge_commit = ENV["GITHUB_SHA"]
+        pr_branch = ENV["GITHUB_HEAD_REF"]
+        base_branch_name = get(ENV, "GITHUB_BASE_REF", "")
+
+        # Check if there's any difference between the merge commit and the PR head
+        # In GitHub Actions, HEAD^2 is the PR head (second parent of merge commit)
+        # success() returns true if the command exits with 0 (no differences)
+        has_diff = !success(`git diff --quiet --exit-code HEAD^2 HEAD`)
+
+        if has_diff
+            base_branch = isempty(base_branch_name) ? "the base branch" : "'$base_branch_name'"
+            global git_note = """
+            │ Note: This is being run on merge commit $merge_commit (merge of PR branch '$pr_branch' into $base_branch).
+            │ The content differs from the actual commit on your PR branch.
+            │ To reproduce locally, update your branch with $base_branch first.
+            │
+            """
+        end
+    catch e
+        @warn "Error while checking git diff" exception=(e, catch_backtrace())
+    end
+end
+
+print("""
+│
+│ To reproduce this CI run locally run the following from the same repository state on julia version $VERSION:
+│
+│ `import Pkg; Pkg.test(;$kwargs_repr)`
+│
+""")
+print(git_note)
+
+if parse(Bool, ENV["ANNOTATE"]) && v"1.8pre" < VERSION < v"1.9.0-beta3"
+    push!(LOAD_PATH, "@tests-logger-env") # access dependencies
+    using GitHubActions, Logging
+    global_logger(GitHubActionsLogger())
+    include("test_logger.jl")
+    pop!(LOAD_PATH)
+    try
+        TestLogger.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
+else
+    try
+        Pkg.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
+end

test_logger.jl

@@ -0,0 +1,97 @@
+module TestLogger
+using Pkg
+
+function parse_file_line(failed_line)
+    # The bits like `\e[91m\e[1m` are color codes that get printed by `Pkg.test`. We
+    # match with or without them.
+    r = r"(\e\[91m\e\[1m)?Test Failed(\e\[22m\e\[39m)? at (\e\[39m\e\[1m)?(?<path>[^\s\e]+)(\e\[22m)?"
+    m = match(r, failed_line)
+    m === nothing && return (nothing, nothing)
+
+    if m[:path] === nothing
+        return (nothing, nothing)
+    else
+        path_split_results = rsplit(m[:path], ":", limit=2)
+        if length(path_split_results) == 1
+            return (m[:path], nothing)
+        else
+            path, line_no = path_split_results
+
+            # Try to make sure line number is parseable to avoid false positives
+            line_no = tryparse(Int, line_no) === nothing ? nothing : line_no
+            return (path, line_no)
+        end
+    end
+    return (nothing, nothing)
+end
+
+function readlines_until(f, stream; keep_lines=true, io)
+    lines = String[]
+    while true
+        line = readline(stream; keep=true)
+        print(io, line)
+
+        # with `keep=true`, this should only happen when we're done?
+        # I think so...
+        if line == ""
+            return line, lines
+        end
+        if f(line)
+            return line, lines
+        else
+            keep_lines && push!(lines, line)
+        end
+    end
+end
+
+function has_test_failure(line)
+    contains(line, "Test Failed") || return false
+    file, line_no = parse_file_line(line)
+    return !isnothing(file) && !isnothing(line_no)
+end
+
+function build_stream(io)
+    stream = Base.BufferStream()
+    t = @async begin
+        while !eof(stream)
+            # Iterate through and print until we get to "Test Failed" and can parse it
+            failed_line, _ = readlines_until(has_test_failure, stream; keep_lines=false, io)
+            @label found_failed_line
+            # Parse file and line out
+            file, line_no = parse_file_line(failed_line)
+
+            # Grab everything until the stacktrace, OR we hit another `Test Failed`
+            stopped_line, msg_lines = readlines_until(stream; io) do line
+                contains(line, "Stacktrace:") || has_test_failure(line)
+            end
+
+            # If we stopped because we hit a 2nd test failure,
+            # let's assume somehow the stacktrace didn't show up for the first one.
+            # Let's continue by trying to find the info for this one, by jumping back.
+            if has_test_failure(stopped_line)
+                failed_line = stopped_line
+                @goto found_failed_line
+            end
+
+            if !isempty(msg_lines)
+                msg = string("Test Failed\n", chomp(join(msg_lines)))
+                # Now log it out
+                @error msg _file=file _line=line_no
+            end
+        end
+    end
+    return stream, t
+end
+
+
+function test(args...; kwargs...)
+    stream, t = build_stream(stdout)
+    Base.errormonitor(t)
+    return try
+        Pkg.test(args...; kwargs..., io=stream)
+    finally
+        close(stream)
+    end
+end
+
+end # module

test_logger_tests.jl

@@ -0,0 +1,71 @@
+include("test_logger.jl")
+using Logging, Test
+
+function simulate(text)
+    logger = Test.TestLogger()
+    output = IOBuffer()
+    with_logger(logger) do
+        stream, t = TestLogger.build_stream(output)
+        for line in eachline(IOBuffer(text); keep=true)
+            write(stream, line)
+        end
+        close(stream)
+        wait(t)
+    end
+    return String(take!(output)), logger.logs
+end
+
+@testset "TestLogger" begin
+
+    for input in (
+            """
+            Test Failed at file.txt:1
+            1
+            2
+            3
+            4
+            5
+            6
+            Stacktrace:
+            Hi
+            """,
+            # Let us mess with the stacktrace line
+            """
+            Test Failed at file.txt:1
+            1
+            2
+            3
+            4
+            5
+            6
+               Stacktrace:   extra stuff
+            Hi
+            """)
+
+        output, logs = simulate(input)
+        @test output == input
+        log = only(logs)
+        @test log.message == "Test Failed\n1\n2\n3\n4\n5\n6"
+        @test log.file == "file.txt"
+        @test log.line == "1"
+    end
+
+    # Next, check that if we hit a Test Failed, and then hit another one before we get a stacktrace,
+    # we just move on to handling the new one.
+    input = """
+    Test Failed at file.txt:1
+    Nah
+    Test Failed at file.txt:1
+    Correct
+    Stacktrace:
+    Hi
+    """
+
+    output, logs = simulate(input)
+    @test output == input
+
+    log = only(logs)
+    @test log.message == "Test Failed\nCorrect"
+    @test log.file == "file.txt"
+    @test log.line == "1"
+end