diff --git a/NEWS.md b/NEWS.md index d13c3c130da66..70c31ddbf3d53 100644 --- a/NEWS.md +++ b/NEWS.md @@ -149,6 +149,9 @@ Library improvements implemented; the semantics are as if the `Nullable` were a container with zero or one elements ([#16961]). + * New `@test_warn` and `@test_nowarn` macros in the `Base.Test` module to + test for the presence or absence of warning messages ([#19903]). + * `logging` can be used to redirect `info`, `warn`, and `error` messages either universally or on a per-module/function basis ([#16213]). diff --git a/base/test.jl b/base/test.jl index 69d6567eb646f..7ba45e4028fd5 100644 --- a/base/test.jl +++ b/base/test.jl @@ -13,7 +13,7 @@ and summarize them at the end of the test set with `@testset`. """ module Test -export @test, @test_throws, @test_broken, @test_skip +export @test, @test_throws, @test_broken, @test_skip, @test_warn, @test_nowarn export @testset # Legacy approximate testing functions, yet to be included export @test_approx_eq_eps, @inferred @@ -356,6 +356,57 @@ function do_test_throws(result::ExecutionResult, orig_expr, extype) record(get_testset(), testres) end +#----------------------------------------------------------------------- +# Test for warning messages + +ismatch_warn(s::AbstractString, output) = contains(output, s) +ismatch_warn(s::Regex, output) = ismatch(s, output) +ismatch_warn(s::Function, output) = s(output) +ismatch_warn(S::Union{AbstractArray,Tuple}, output) = all(s -> ismatch_warn(s, output), S) + +""" + @test_warn msg expr + +Test whether evaluating `expr` results in [`STDERR`](@ref) output that contains +the `msg` string or matches the `msg` regular expression. If `msg` is +a boolean function, tests whether `msg(output)` returns `true`. If `msg` is a +tuple or array, checks that the error output contains/matches each item in `msg`. +Returns the result of evaluating `expr`. + +See also [`@test_nowarn`](@ref) to check for the absence of error output. +""" +macro test_warn(msg, expr) + quote + let fname = tempname(), have_color = Base.have_color + try + eval(Base, :(have_color = false)) + ret = open(fname, "w") do f + redirect_stderr(f) do + $(esc(expr)) + end + end + @test ismatch_warn($(esc(msg)), readstring(fname)) + ret + finally + eval(Base, Expr(:(=), :have_color, have_color)) + rm(fname, force=true) + end + end + end +end + +""" + @test_nowarn expr + +Test whether evaluating `expr` results in empty [`STDERR`](@ref) output +(no warnings or other messages). Returns the result of evaluating `expr`. +""" +macro test_nowarn(expr) + quote + @test_warn r"^(?!.)"s $expr + end +end + #----------------------------------------------------------------------- # The AbstractTestSet interface is defined by two methods: diff --git a/doc/src/stdlib/test.md b/doc/src/stdlib/test.md index 8c4a719b79da9..6b58fbd065753 100644 --- a/doc/src/stdlib/test.md +++ b/doc/src/stdlib/test.md @@ -171,6 +171,8 @@ checks using either `@test a ≈ b` (where `≈`, typed via tab completion of `\ ```@docs Base.Test.@inferred +Base.Test.@test_warn +Base.Test.@test_nowarn ``` ## Broken Tests diff --git a/test/compile.jl b/test/compile.jl index fd7be47f004b0..044bd21281e18 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -2,20 +2,6 @@ using Base.Test -function redirected_stderr(expected) - rd, wr = redirect_stderr() - t = @async begin - read = readstring(rd) # also makes sure the kernel isn't being forced to buffer the output - if !contains(read, expected) - @show expected - @show read - @test false - end - nothing - end - return t -end - Foo_module = :Foo4b3a94a1a081a8cb FooBase_module = :FooBase4b3a94a1a081a8cb @eval module ConflictingBindings @@ -29,7 +15,6 @@ using .ConflictingBindings # so we disable it for the tests below withenv( "JULIA_DEBUG_LOADING" => nothing ) do -olderr = STDERR dir = mktempdir() dir2 = mktempdir() insert!(LOAD_PATH, 1, dir) @@ -136,14 +121,9 @@ try # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: - t = redirected_stderr("WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition ") - try + @test_warn "WARNING: replacing module Foo4b3a94a1a081a8cb.\nWARNING: Method definition " begin @test isa(Base._require_from_serialized(myid(), Foo_module, cachefile, #=broadcast-load=#false), Array{Any,1}) - finally - close(STDERR) - redirect_stderr(olderr) end - wait(t) let Foo = getfield(Main, Foo_module) @test_throws MethodError Foo.foo(17) # world shouldn't be visible yet @@ -217,17 +197,13 @@ try end """) - t = redirected_stderr("ERROR: LoadError: Declaring __precompile__(false) is not allowed in files that are being precompiled.\nStacktrace:\n [1] __precompile__") - try + @test_warn "ERROR: LoadError: Declaring __precompile__(false) is not allowed in files that are being precompiled.\nStacktrace:\n [1] __precompile__" try Base.compilecache("Baz") # from __precompile__(false) error("__precompile__ disabled test failed") catch exc - close(STDERR) - redirect_stderr(olderr) isa(exc, ErrorException) || rethrow(exc) !isempty(search(exc.msg, "__precompile__(false)")) && rethrow(exc) end - wait(t) # Issue #12720 FooBar1_file = joinpath(dir, "FooBar1.jl") @@ -273,14 +249,7 @@ try fb_uuid1 = Base.module_uuid(Main.FooBar1) @test fb_uuid != fb_uuid1 - t = redirected_stderr("WARNING: replacing module FooBar.") - try - reload("FooBar") - finally - close(STDERR) - redirect_stderr(olderr) - end - wait(t) + @test_warn "WARNING: replacing module FooBar." reload("FooBar") @test fb_uuid != Base.module_uuid(Main.FooBar) @test fb_uuid1 == Base.module_uuid(Main.FooBar1) fb_uuid = Base.module_uuid(Main.FooBar) @@ -289,14 +258,7 @@ try @test !Base.stale_cachefile(FooBar1_file, joinpath(dir2, "FooBar1.ji")) @test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji")) - t = redirected_stderr("WARNING: replacing module FooBar1.") - try - reload("FooBar1") - finally - close(STDERR) - redirect_stderr(olderr) - end - wait(t) + @test_warn "WARNING: replacing module FooBar1." reload("FooBar1") @test fb_uuid == Base.module_uuid(Main.FooBar) @test fb_uuid1 != Base.module_uuid(Main.FooBar1) @@ -314,22 +276,14 @@ try error("break me") end """) - t = redirected_stderr("ERROR: LoadError: break me\nStacktrace:\n [1] error") - try + @test_warn "ERROR: LoadError: break me\nStacktrace:\n [1] error" try Base.require(:FooBar) error("\"LoadError: break me\" test failed") catch exc - close(STDERR) - redirect_stderr(olderr) isa(exc, ErrorException) || rethrow(exc) !isempty(search(exc.msg, "ERROR: LoadError: break me")) && rethrow(exc) end - wait(t) finally - if STDERR != olderr - close(STDERR) - redirect_stderr(olderr) - end splice!(Base.LOAD_CACHE_PATH, 1:2) splice!(LOAD_PATH, 1) rm(dir, recursive=true) diff --git a/test/core.jl b/test/core.jl index d0a43d40effbd..5a4b892060809 100644 --- a/test/core.jl +++ b/test/core.jl @@ -4822,6 +4822,17 @@ end @test f14893() == 14893 @test M14893.f14893() == 14893 +# issue #18725 +@test_nowarn eval(Main, :(begin + f18725(x) = 1 + f18725(x) = 2 +end)) +@test Main.f18725(0) == 2 +@test_warn "WARNING: Method definition f18725(Any) in module Module18725" eval(Main, :(module Module18725 + f18725(x) = 1 + f18725(x) = 2 +end)) + # issue #19599 f19599{T}(x::((S)->Vector{S})(T)...) = 1 @test f19599([1],[1]) == 1 diff --git a/test/pkg.jl b/test/pkg.jl index 92530cd9cc1d8..5c5ea9e9813fa 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -2,6 +2,22 @@ import Base.Pkg.PkgError +function capture_stdout(f::Function) + let fname = tempname() + try + open(fname, "w") do fout + redirect_stdout(fout) do + f() + end + end + return readstring(fname) + finally + rm(fname, force=true) + end + end +end + + function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) # Used in tests below to set up and tear down a sandboxed package directory const tmpdir = joinpath(tempdir(),randstring()) @@ -18,45 +34,6 @@ function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) end end -macro grab_outputs(ex) - quote - local err::String, out::String - local ferrname = "" - local foutname = "" - local ret - try - OLD_STDERR = STDERR - ferrname = tempname() - ferr = open(ferrname, "w") - try - OLD_STDOUT = STDOUT - foutname = tempname() - fout = open(foutname, "w") - try - redirect_stderr(ferr) - try - redirect_stdout(fout) - ret = $(esc(ex)) - finally - redirect_stdout(OLD_STDOUT) - close(fout) - end - finally - redirect_stderr(OLD_STDERR) - close(ferr) - end - out = readstring(foutname) - err = readstring(ferrname) - finally - isfile(foutname) && rm(foutname) - end - finally - isfile(ferrname) && rm(ferrname) - end - ret, out, err - end -end - # Test basic operations: adding or removing a package, status, free # Also test for the existence of REQUIRE and META_BRANCH temp_pkg_dir() do @@ -287,72 +264,46 @@ temp_pkg_dir() do end # Various pin/free/re-pin/change-pin patterns (issue #17176) - begin - ret, out, err = @grab_outputs Pkg.free("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Freeing Example") - - ret, out, err = @grab_outputs Pkg.pin("Example") - @test ret === nothing && out == "" - @test ismatch(r"INFO: Creating Example branch pinned\.[0-9a-f]{8}\.tmp", err) - @test !contains(err, "INFO: No packages to install, update or remove") - branchid = replace(err, r".*pinned\.([0-9a-f]{8})\.tmp.*"s, s"\1") + @test "" == capture_stdout() do + @test_warn "INFO: Freeing Example" Pkg.free("Example") + + @test_warn r"^INFO: Creating Example branch pinned\.[0-9a-f]{8}\.tmp$" Pkg.pin("Example") vers = Pkg.installed("Example") + branch = LibGit2.with(LibGit2.GitRepo, Pkg.dir("Example")) do repo + LibGit2.branch(repo) + end - ret, out, err = @grab_outputs Pkg.free("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Freeing Example") + @test_warn "INFO: Freeing Example" Pkg.free("Example") - ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") - @test ret === nothing && out == "" - @test contains(err, "INFO: Creating Example branch pinned.b1990792.tmp") - @test contains(err, "INFO: No packages to install, update or remove") + @test_warn ("INFO: Creating Example branch pinned.b1990792.tmp", + "INFO: No packages to install, update or remove") Pkg.pin("Example", v"0.4.0") @test Pkg.installed("Example") == v"0.4.0" - ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") - @test ret === nothing && out == "" - @test contains(err, "INFO: Package Example is already pinned to the selected commit") - @test !contains(err, "INFO: No packages to install, update or remove") + @test_warn r"^INFO: Package Example is already pinned to the selected commit$" Pkg.pin("Example", v"0.4.0") @test Pkg.installed("Example") == v"0.4.0" - ret, out, err = @grab_outputs Pkg.pin("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Package Example is already pinned") - @test !contains(err, "INFO: No packages to install, update or remove") + @test_warn r"^INFO: Package Example is already pinned$" Pkg.pin("Example") @test Pkg.installed("Example") == v"0.4.0" - ret, out, err = @grab_outputs Pkg.update() - @test ret === nothing && out == "" - @test contains(err, "INFO: Package Example: skipping update (pinned)...") + @test_warn "INFO: Package Example: skipping update (pinned)..." Pkg.update() @test Pkg.installed("Example") == v"0.4.0" - ret, out, err = @grab_outputs Pkg.pin("Example", v"0.3.1") - @test ret === nothing && out == "" - @test contains(err, "INFO: Creating Example branch pinned.d1ef7b00.tmp") - @test contains(err, "INFO: No packages to install, update or remove") + @test_warn ("INFO: Creating Example branch pinned.d1ef7b00.tmp", + "INFO: No packages to install, update or remove") Pkg.pin("Example", v"0.3.1") @test Pkg.installed("Example") == v"0.3.1" - ret, out, err = @grab_outputs Pkg.pin("Example", v"0.4.0") - @test ret === nothing && out == "" - @test contains(err, "INFO: Package Example: checking out existing branch pinned.b1990792.tmp") - @test contains(err, "INFO: No packages to install, update or remove") + @test_warn ("INFO: Package Example: checking out existing branch pinned.b1990792.tmp", + "INFO: No packages to install, update or remove") Pkg.pin("Example", v"0.4.0") @test Pkg.installed("Example") == v"0.4.0" - ret, out, err = @grab_outputs Pkg.free("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Freeing Example") - @test contains(err, "INFO: No packages to install, update or remove") + @test_warn ("INFO: Freeing Example", + "INFO: No packages to install, update or remove") Pkg.free("Example") @test Pkg.installed("Example") == vers - ret, out, err = @grab_outputs Pkg.pin("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Package Example: checking out existing branch pinned.$branchid.tmp") - @test !contains(err, "INFO: No packages to install, update or remove") + @test_warn Regex("^INFO: Package Example: checking out existing branch $branch\$") Pkg.pin("Example") @test Pkg.installed("Example") == vers - ret, out, err = @grab_outputs Pkg.free("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Freeing Example") + @test_warn "INFO: Freeing Example" Pkg.free("Example") @test Pkg.installed("Example") == vers end @@ -470,47 +421,35 @@ temp_pkg_dir() do end # partial Pkg.update - begin - nothingtodomsg = "INFO: No packages to install, update or remove\n" + @test "" == capture_stdout() do + nothingtodomsg = "INFO: No packages to install, update or remove" - ret, out, err = @grab_outputs begin + @test_warn "INFO: Installing Example v" begin Pkg.rm("Example") Pkg.add("Example") end - @test ret === nothing && out == "" - @test contains(err, "INFO: Installing Example v") - ret, out, err = @grab_outputs Pkg.update("Example") - @test ret === nothing && out == "" - @test contains(err, nothingtodomsg) + @test_warn nothingtodomsg Pkg.update("Example") - ret, out, err = @grab_outputs begin + @test_warn "INFO: Installing Example v0.4.0" begin Pkg.rm("Example") Pkg.add("Example", v"0", v"0.4.1-") # force version to be < 0.4.1 end - @test ret === nothing && out == "" - @test contains(err, "INFO: Installing Example v0.4.0") - ret, out, err = @grab_outputs Pkg.update("Example") - @test ret === nothing && out == "" - @test ismatch(r"INFO: Package Example was set to version 0\.4\.0, but a higher version \d+\.\d+\.\d+\S* exists.\n", err) - @test contains(err, "The update is prevented by explicit requirements constraints. Edit your REQUIRE file to change this.\n") - @test contains(err, nothingtodomsg) + @test_warn (r"INFO: Package Example was set to version 0\.4\.0, but a higher version \d+\.\d+\.\d+\S* exists.", + "The update is prevented by explicit requirements constraints. Edit your REQUIRE file to change this.", + nothingtodomsg) Pkg.update("Example") - ret, out, err = @grab_outputs begin + @test_warn "INFO: Installing Example" begin Pkg.rm("Example") Pkg.add("Example") Pkg.pin("Example", v"0.4.0") end - @test ret === nothing && out == "" - @test contains(err, "INFO: Installing Example") - ret, out, err = @grab_outputs Pkg.update("Example") - @test ret === nothing && out == "" - @test contains(err, "INFO: Package Example: skipping update (pinned)...\n") - @test ismatch(r"INFO: Package Example was set to version 0\.4\.0, but a higher version \d+\.\d+\.\d+\S* exists.\n", err) - @test contains(err, "The package is fixed. You can try using `Pkg.free(\"Example\")` to update it.") - @test contains(err, nothingtodomsg) + @test_warn ("INFO: Package Example: skipping update (pinned)...", + r"INFO: Package Example was set to version 0\.4\.0, but a higher version \d+\.\d+\.\d+\S* exists.", + "The package is fixed. You can try using `Pkg.free(\"Example\")` to update it.", + nothingtodomsg) Pkg.update("Example") metadata_dir = Pkg.dir("METADATA") const old_commit = "313bfaafa301e82d40574a778720e893c559a7e2" @@ -523,23 +462,18 @@ temp_pkg_dir() do LibGit2.reset!(repo, LibGit2.Oid(old_commit), LibGit2.Consts.RESET_HARD) end - ret, out, err = @grab_outputs Pkg.add("Colors") - @test ret === nothing && out == "" - @test contains(err, "INFO: Installing Colors v0.6.4") - @test contains(err, "INFO: Installing ColorTypes v0.2.2") - @test contains(err, "INFO: Installing FixedPointNumbers v0.1.3") - @test contains(err, "INFO: Installing Compat v0.7.18") - @test contains(err, "INFO: Installing Reexport v0.0.3") - - ret, out, err = @grab_outputs Pkg.update("ColorTypes") - @test ismatch(r"INFO: Upgrading ColorTypes: v0\.2\.2 => v\d+\.\d+\.\d+", err) - @test ismatch(r"INFO: Upgrading Compat: v0\.7\.18 => v\d+\.\d+\.\d+", err) - @test !contains(err, "INFO: Upgrading Colors: ") + @test_warn ("INFO: Installing Colors v0.6.4", + "INFO: Installing ColorTypes v0.2.2", + "INFO: Installing FixedPointNumbers v0.1.3", + "INFO: Installing Compat v0.7.18", + "INFO: Installing Reexport v0.0.3") Pkg.add("Colors") + + @test_warn (r"INFO: Upgrading ColorTypes: v0\.2\.2 => v\d+\.\d+\.\d+", + r"INFO: Upgrading Compat: v0\.7\.18 => v\d+\.\d+\.\d+", + s -> !contains(s, "INFO: Upgrading Colors: ")) Pkg.update("ColorTypes") @test Pkg.installed("Colors") == v"0.6.4" - ret, out, err = @grab_outputs Pkg.update("FixedPointNumbers") - @test ret === nothing && out == "" - @test contains(err, nothingtodomsg) + @test_warn nothingtodomsg Pkg.update("FixedPointNumbers") end # issue #18239 diff --git a/test/test.jl b/test/test.jl index c6542f9b273c5..b2ad932183e43 100644 --- a/test/test.jl +++ b/test/test.jl @@ -23,6 +23,10 @@ @test_skip false @test_skip gobbeldygook +# Test @test_warn +@test 1234 === @test_nowarn(1234) +@test 5678 === @test_warn("WARNING: foo", begin warn("foo"); 5678; end) + a = Array(Float64, 2, 2, 2, 2, 2) a[1,1,1,1,1] = 10 @test a[1,1,1,1,1] == 10