From ac4ec804ce1347dceda3210076b2c843d32082e3 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Sat, 14 Aug 2021 07:52:52 -0400 Subject: [PATCH 1/3] Support `@test_throws` just checking the error message With this PR, @test_throws "reducing over an empty collection" reduce(+, ()) allows you to check the displayed error message without caring about the details of which specific Exception subtype is used. The pattern-matching options are the same as for `@test_warn`. --- NEWS.md | 3 ++ stdlib/Test/src/Test.jl | 76 +++++++++++++++++++++++++----------- stdlib/Test/test/runtests.jl | 36 ++++++++++++++++- 3 files changed, 91 insertions(+), 24 deletions(-) diff --git a/NEWS.md b/NEWS.md index cb298402ea88d..ec05c7ae9e16c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -37,6 +37,9 @@ New library functions New library features -------------------- +* `@test_throws "some message" triggers_error()` can now be used to check whether the displayed error text + contains "some message" regardless of the specific exception type. + Regular expressions are also supported. ([#41888) Standard library changes ------------------------ diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index b4f08b8e00d5d..8afbdc2bc78c5 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -86,8 +86,9 @@ struct Pass <: Result data value source::Union{Nothing,LineNumberNode} - function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing) - return new(test_type, orig_expr, data, thrown isa String ? "String" : thrown, source) + message_only::Bool + function Pass(test_type::Symbol, orig_expr, data, thrown, source=nothing, message_only=false) + return new(test_type, orig_expr, data, thrown, source, message_only) end end @@ -98,7 +99,11 @@ function Base.show(io::IO, t::Pass) end if t.test_type === :test_throws # The correct type of exception was thrown - print(io, "\n Thrown: ", t.value isa String ? t.value : typeof(t.value)) + if t.message_only + print(io, "\n Message: ", t.value) + else + print(io, "\n Thrown: ", typeof(t.value)) + end elseif t.test_type === :test && t.data !== nothing # The test was an expression, so display the term-by-term # evaluated version as well @@ -118,12 +123,14 @@ struct Fail <: Result data::Union{Nothing, String} value::String source::LineNumberNode - function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode) + message_only::Bool + function Fail(test_type::Symbol, orig_expr, data, value, source::LineNumberNode, message_only::Bool=false) return new(test_type, string(orig_expr), data === nothing ? nothing : string(data), string(isa(data, Type) ? typeof(value) : value), - source) + source, + message_only) end end @@ -132,18 +139,24 @@ function Base.show(io::IO, t::Fail) print(io, " at ") printstyled(io, something(t.source.file, :none), ":", t.source.line, "\n"; bold=true, color=:default) print(io, " Expression: ", t.orig_expr) + value, data = t.value, t.data if t.test_type === :test_throws_wrong # An exception was thrown, but it was of the wrong type - print(io, "\n Expected: ", t.data) - print(io, "\n Thrown: ", t.value) + if t.message_only + print(io, "\n Expected: ", data) + print(io, "\n Message: ", value) + else + print(io, "\n Expected: ", data) + print(io, "\n Thrown: ", value) + end elseif t.test_type === :test_throws_nothing # An exception was expected, but no exception was thrown - print(io, "\n Expected: ", t.data) + print(io, "\n Expected: ", data) print(io, "\n No exception thrown") - elseif t.test_type === :test && t.data !== nothing + elseif t.test_type === :test && data !== nothing # The test was an expression, so display the term-by-term # evaluated version as well - print(io, "\n Evaluated: ", t.data) + print(io, "\n Evaluated: ", data) end end @@ -238,6 +251,7 @@ function Serialization.serialize(s::Serialization.AbstractSerializer, t::Pass) Serialization.serialize(s, t.data === nothing ? nothing : string(t.data)) Serialization.serialize(s, string(t.value)) Serialization.serialize(s, t.source === nothing ? nothing : t.source) + Serialization.serialize(s, t.message_only) nothing end @@ -657,6 +671,7 @@ end Tests that the expression `expr` throws `exception`. The exception may specify either a type, +a string or regular expression occurring in the displayed error message, or a value (which will be tested for equality by comparing fields). Note that `@test_throws` does not support a trailing keyword form. @@ -671,6 +686,11 @@ julia> @test_throws DimensionMismatch [1, 2, 3] + [1, 2] Test Passed Expression: [1, 2, 3] + [1, 2] Thrown: DimensionMismatch + +julia> @test_throws "Try sqrt(Complex" sqrt(-1) +Test Passed + Expression: sqrt(-1) + Message: "DomainError with -1.0:\\nsqrt will only return a complex result if called with a complex argument. Try sqrt(Complex(x))." ``` """ macro test_throws(extype, ex) @@ -697,6 +717,7 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) if isa(result, Threw) # Check that the right type of exception was thrown success = false + message_only = false exc = result.exception # NB: Throwing LoadError from macroexpands is deprecated, but in order to limit # the breakage in package tests we add extra logic here. @@ -704,6 +725,9 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) orig_expr isa Expr && orig_expr.head in (:call, :macrocall) && orig_expr.args[1] in MACROEXPAND_LIKE + if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) + extype = extype.error # deprecated + end if isa(extype, Type) success = if from_macroexpand && extype == LoadError && exc isa Exception @@ -712,24 +736,30 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) else isa(exc, extype) end - else - if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) - extype = extype.error # deprecated - end - if isa(exc, typeof(extype)) - success = true - for fld in 1:nfields(extype) - if !isequal(getfield(extype, fld), getfield(exc, fld)) - success = false - break - end + elseif isa(exc, typeof(extype)) + success = true + for fld in 1:nfields(extype) + if !isequal(getfield(extype, fld), getfield(exc, fld)) + success = false + break end end + elseif isa(extype, Exception) + else + message_only = true + exc = sprint(showerror, exc) + success = contains_warn(exc, extype) + exc = '"' * escape_string(exc) * '"' + if isa(extype, AbstractString) + extype = '"' * escape_string(extype) * '"' + elseif isa(extype, Function) + extype = "< match function >" + end end if success - testres = Pass(:test_throws, orig_expr, extype, exc, result.source) + testres = Pass(:test_throws, orig_expr, extype, exc, result.source, message_only) else - testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source) + testres = Fail(:test_throws_wrong, orig_expr, extype, exc, result.source, message_only) end else testres = Fail(:test_throws_nothing, orig_expr, extype, nothing, result.source) diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 3c56ad8feba51..36d7c75d2fe0a 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -96,6 +96,14 @@ end "Thrown: ErrorException") @test endswith(sprint(show, @test_throws ErrorException("test") error("test")), "Thrown: ErrorException") + @test endswith(sprint(show, @test_throws "a test" error("a test")), + "Message: \"a test\"") + @test occursin("Message: \"DomainError", + sprint(show, @test_throws r"sqrt\([Cc]omplex" sqrt(-1))) + @test endswith(sprint(show, @test_throws str->occursin("a t", str) error("a test")), + "Message: \"a test\"") + @test endswith(sprint(show, @test_throws ["BoundsError", "access", "1-element", "at index [2]"] [1][2]), + "Message: \"BoundsError: attempt to access 1-element Vector{$Int} at index [2]\"") end # Test printing of Fail results include("nothrow_testset.jl") @@ -148,6 +156,11 @@ let fails = @testset NoThrowTestSet begin @test contains(str1, str2) # 22 - Fail - Type Comparison @test typeof(1) <: typeof("julia") + # 23 - 26 - Fail - wrong message + @test_throws "A test" error("a test") + @test_throws r"sqrt\([Cc]omplx" sqrt(-1) + @test_throws str->occursin("a T", str) error("a test") + @test_throws ["BoundsError", "acess", "1-element", "at index [2]"] [1][2] end for fail in fails @test fail isa Test.Fail @@ -262,6 +275,27 @@ let fails = @testset NoThrowTestSet begin @test occursin("Expression: typeof(1) <: typeof(\"julia\")", str) @test occursin("Evaluated: $(typeof(1)) <: $(typeof("julia"))", str) end + + let str = sprint(show, fails[23]) + @test occursin("Expected: \"A test\"", str) + @test occursin("Message: \"a test\"", str) + end + + let str = sprint(show, fails[24]) + @test occursin("Expected: r\"sqrt\\([Cc]omplx\"", str) + @test occursin(r"Message: .*Try sqrt\(Complex", str) + end + + let str = sprint(show, fails[25]) + @test occursin("Expected: < match function >", str) + @test occursin("Message: \"a test\"", str) + end + + let str = sprint(show, fails[26]) + @test occursin("Expected: [\"BoundsError\", \"acess\", \"1-element\", \"at index [2]\"]", str) + @test occursin(r"Message: \"BoundsError.* 1-element.*at index \[2\]", str) + end + end let errors = @testset NoThrowTestSet begin @@ -1202,4 +1236,4 @@ Test.finish(ts::PassInformationTestSet) = ts @test ts.results[2].data == ErrorException @test ts.results[2].value == ErrorException("Msg") @test ts.results[2].source == LineNumberNode(test_throws_line_number, @__FILE__) -end \ No newline at end of file +end From 016ddf9a804971ccedec164e8ab40855618f3b81 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 17 Aug 2021 11:05:46 -0400 Subject: [PATCH 2/3] Address review comments Co-authored by: Jameson Nash Co-authored by: Takafumi Arakaki --- NEWS.md | 2 +- stdlib/Test/src/Test.jl | 25 +++++++++++++------------ stdlib/Test/test/runtests.jl | 2 ++ 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/NEWS.md b/NEWS.md index ec05c7ae9e16c..45536c83e6833 100644 --- a/NEWS.md +++ b/NEWS.md @@ -39,7 +39,7 @@ New library features * `@test_throws "some message" triggers_error()` can now be used to check whether the displayed error text contains "some message" regardless of the specific exception type. - Regular expressions are also supported. ([#41888) + Regular expressions, lists of strings, and matching functions are also supported. ([#41888) Standard library changes ------------------------ diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 8afbdc2bc78c5..5ca6b792a9724 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -725,9 +725,6 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) orig_expr isa Expr && orig_expr.head in (:call, :macrocall) && orig_expr.args[1] in MACROEXPAND_LIKE - if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) - extype = extype.error # deprecated - end if isa(extype, Type) success = if from_macroexpand && extype == LoadError && exc isa Exception @@ -736,22 +733,26 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) else isa(exc, extype) end - elseif isa(exc, typeof(extype)) - success = true - for fld in 1:nfields(extype) - if !isequal(getfield(extype, fld), getfield(exc, fld)) - success = false - break + elseif isa(extype, Exception) || !isa(exc, Exception) + if extype isa LoadError && !(exc isa LoadError) && typeof(extype.error) == typeof(exc) + extype = extype.error # deprecated + end + if isa(exc, typeof(extype)) + success = true + for fld in 1:nfields(extype) + if !isequal(getfield(extype, fld), getfield(exc, fld)) + success = false + break + end end end - elseif isa(extype, Exception) else message_only = true exc = sprint(showerror, exc) success = contains_warn(exc, extype) - exc = '"' * escape_string(exc) * '"' + exc = repr(exc) if isa(extype, AbstractString) - extype = '"' * escape_string(extype) * '"' + extype = repr(extype) elseif isa(extype, Function) extype = "< match function >" end diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index 36d7c75d2fe0a..d260c5840d626 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -104,6 +104,8 @@ end "Message: \"a test\"") @test endswith(sprint(show, @test_throws ["BoundsError", "access", "1-element", "at index [2]"] [1][2]), "Message: \"BoundsError: attempt to access 1-element Vector{$Int} at index [2]\"") + @test_throws "\"" throw("\"") + @test_throws Returns(false) throw(Returns(false)) end # Test printing of Fail results include("nothrow_testset.jl") From 5147b46ed1a88d4b2231eded337a8b20300740ee Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 17 Aug 2021 11:47:32 -0400 Subject: [PATCH 3/3] Better NEWS and docstring --- stdlib/Test/src/Test.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/stdlib/Test/src/Test.jl b/stdlib/Test/src/Test.jl index 5ca6b792a9724..0c9cfd5ccafd1 100644 --- a/stdlib/Test/src/Test.jl +++ b/stdlib/Test/src/Test.jl @@ -671,7 +671,8 @@ end Tests that the expression `expr` throws `exception`. The exception may specify either a type, -a string or regular expression occurring in the displayed error message, +a string, regular expression, or list of strings occurring in the displayed error message, +a matching function, or a value (which will be tested for equality by comparing fields). Note that `@test_throws` does not support a trailing keyword form. @@ -692,6 +693,12 @@ Test Passed Expression: sqrt(-1) Message: "DomainError with -1.0:\\nsqrt will only return a complex result if called with a complex argument. Try sqrt(Complex(x))." ``` + +In the final example, instead of matching a single string it could alternatively have been performed with: + +- `["Try", "Complex"]` (a list of strings) +- `r"Try sqrt\\([Cc]omplex"` (a regular expression) +- `str -> occursin("complex", str)` (a matching function) """ macro test_throws(extype, ex) orig_ex = Expr(:inert, ex)