Skip to content

Add maxfailures option to limit test failures before stopping #61560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VanitasCodes wants to merge 2 commits into
base: master
Choose a base branch
from
+136 −2
Open

Add maxfailures option to limit test failures before stopping #61560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fe95410
Add maxfailures option to limit test failures before stopping
Apr 11, 2026
db67957
Addressed review feedback
Apr 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 23 additions & 2 deletions stdlib/Test/src/Test.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@ const global_fail_fast = OncePerProcess{Bool}() do
return Base.get_bool_env("JULIA_TEST_FAILFAST", false)
end

# Global state for maxfailures tracking (0 = disabled)
global_failure_count = Ref(0)
global_failure_limit = Ref(0)

#-----------------------------------------------------------------------

# Backtrace utility functions
Expand Down Expand Up @@ -1547,6 +1551,11 @@ extract_file(::Nothing) = nothing

struct FailFastError <: Exception end

struct MaxFailuresError <: Exception
limit::Int
count::Int
end

# For a broken result, simply store the result
record(ts::DefaultTestSet, t::Broken) = ((@lock ts.results_lock push!(ts.results, t)); t)
# For a passed result, do not store the result since it uses a lot of memory, unless
Expand Down Expand Up @@ -1580,6 +1589,12 @@ function record(ts::DefaultTestSet, t::Union{Fail, Error}; print_result::Bool=TE
end
@lock ts.results_lock push!(ts.results, t)
ts.failfast && throw(FailFastError())
limit = global_failure_limit[]
if limit > 0
global_failure_count[] += 1
count = global_failure_count[]
count >= limit && throw(MaxFailuresError(limit, count))
end
return t
end

Expand Down Expand Up @@ -2144,6 +2159,7 @@ trigger_test_failure_break(@nospecialize(err)) =
ccall(:jl_test_failure_breakpoint, Cvoid, (Any,), err)

is_failfast_error(err::FailFastError) = true
is_failfast_error(err::MaxFailuresError) = true
is_failfast_error(err::LoadError) = is_failfast_error(err.error) # handle `include` barrier
is_failfast_error(err) = false

Expand Down Expand Up @@ -2252,7 +2268,7 @@ function testset_beginend_call(args, tests, source)
# error in this test set
trigger_test_failure_break(err)
if is_failfast_error(err)
get_testset_depth() > 0 ? rethrow() : failfast_print()
get_testset_depth() > 0 ? rethrow() : (err isa MaxFailuresError ? maxfailures_print(err) : failfast_print())
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing))
end
Expand All @@ -2275,6 +2291,11 @@ function failfast_print()
printstyled(" Fail or Error occurred\n\n"; color = Base.error_color())
end

function maxfailures_print(err::MaxFailuresError)
printstyled("\nMax failures reached:"; color = Base.error_color(), bold=true)
printstyled(" $(err.count) failures (limit=$(err.limit))\n\n"; color = Base.error_color())
end

"""
Generate the code for a `@testset` with a `for` loop argument
"""
Expand Down Expand Up @@ -2332,7 +2353,7 @@ function testset_forloop(args, testloop, source)
# error in this test set
trigger_test_failure_break(err)
if is_failfast_error(err)
get_testset_depth() > 0 ? rethrow() : failfast_print()
get_testset_depth() > 0 ? rethrow() : (err isa MaxFailuresError ? maxfailures_print(err) : failfast_print())
else
record(ts, Error(:nontest_error, Expr(:tuple), err, Base.current_exceptions(), $(QuoteNode(source)), nothing))
end
Expand Down
113 changes: 113 additions & 0 deletions stdlib/Test/test/runtests.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1952,6 +1952,119 @@ end
end
end

@testset "maxfailures option" begin
@testset "default (no limit)" begin
mktemp() do f, _
write(f,
"""
using Test
Test.global_failure_count[] = 0
@testset "outer" begin
@testset "First" begin
@test false
@test false
end
@testset "Second" begin
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin("Test Summary:", result)
@test occursin("First", result)
@test occursin("Second", result)
end
end
@testset "stop after 1 failure" begin
mktemp() do f, _
write(f,
"""
using Test
Test.global_failure_count[] = 0
Test.global_failure_limit[] = 1
@testset "outer" begin
@testset "First" begin
@test false
end
@testset "Second" begin
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin("Max failures reached: 1", result)
@test occursin("First", result)
@test !occursin(r"Test Summary:.*\n.*Second", result)
Copy link

Copilot AI Apr 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These regexes assume \n line endings. To make the tests more robust across platforms/environments, consider matching \r?\n (or otherwise avoiding hard-coding newline style) so the assertion doesn’t become Windows-sensitive.

Copilot uses AI. Check for mistakes.
end
end
@testset "stop after N failures" begin
mktemp() do f, _
write(f,
"""
using Test
Test.global_failure_count[] = 0
Test.global_failure_limit[] = 2
@testset "outer" begin
@testset "First" begin
@test false
@test false
end
@testset "Second" begin
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin("Max failures reached: 2", result)
@test occursin("First", result)
@test !occursin(r"Test Summary:.*\n.*Second", result)
Copy link

Copilot AI Apr 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These regexes assume \n line endings. To make the tests more robust across platforms/environments, consider matching \r?\n (or otherwise avoiding hard-coding newline style) so the assertion doesn’t become Windows-sensitive.

Copilot uses AI. Check for mistakes.
end
end
@testset "errors count toward limit" begin
mktemp() do f, _
write(f,
"""
using Test
Test.global_failure_count[] = 0
Test.global_failure_limit[] = 1
@testset "outer" begin
@testset "First" begin
@test error("oops")
end
@testset "Second" begin
@test true
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
result = read(pipeline(ignorestatus(cmd), stderr=devnull), String)
@test occursin("Max failures reached: 1", result)
@test occursin("First", result)
@test !occursin(r"Test Summary:.*\n.*Second", result)
Copy link

Copilot AI Apr 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These regexes assume \n line endings. To make the tests more robust across platforms/environments, consider matching \r?\n (or otherwise avoiding hard-coding newline style) so the assertion doesn’t become Windows-sensitive.

Copilot uses AI. Check for mistakes.
end
end
@testset "process exits with failure" begin
mktemp() do f, _
write(f,
"""
using Test
Test.global_failure_count[] = 0
Test.global_failure_limit[] = 1
@testset "outer" begin
@testset "First" begin
@test false
end
end
""")
cmd = `$(Base.julia_cmd()) --startup-file=no --color=no $f`
@test !success(ignorestatus(cmd))
end
end
end

# Non-booleans in @test (#35888)
struct T35888 end
Base.isequal(::T35888, ::T35888) = T35888()
Expand Down
Loading