From cabe156ae32f948e20c9beb14b51a969520a75ab Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Mon, 5 Apr 2021 22:41:18 +0900 Subject: [PATCH] package analysis, enter analysis from method signature (#101) * wip: package analysis, enter analysis from method signature * fix report uniqify logic when printing * configurable * test --- .JET.toml | 1 + benchmark/benchmarks.jl | 10 +- docs/src/internals.md | 2 +- docs/src/usages.md | 6 +- src/JET.jl | 27 +++-- src/abstractinterpretation.jl | 2 +- src/abstractinterpreterinterface.jl | 12 ++- src/legacy/abstractinterpretation | 2 +- src/print.jl | 23 +++- src/reports.jl | 10 -- src/typeinfer.jl | 12 ++- src/virtualprocess.jl | 162 ++++++++++++++++++++-------- test/interactive_utils.jl | 4 +- test/test_virtualprocess.jl | 88 ++++++++++++++- 14 files changed, 278 insertions(+), 83 deletions(-) diff --git a/.JET.toml b/.JET.toml index a843fddb5..6adcea77c 100644 --- a/.JET.toml +++ b/.JET.toml @@ -1 +1,2 @@ +analyze_from_definitions = true concretization_patterns = ["EGAL_TYPES = x_", "_JET_CONFIGURATIONS = x_"] diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 002b8073c..07a1832ed 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -128,9 +128,13 @@ SUITE["invalidation"] = @jetbenchmarkable (@analyze_call println(QuoteNode(nothi end end SUITE["self profiling"] = @jetbenchmarkable( - analyze_call(JET.virtual_process!, - (AbstractString, AbstractString, Module, JET.JETInterpreter, JET.ToplevelConfig,), - ), + analyze_call(JET.virtual_process, (AbstractString, + AbstractString, + Module, + JET.JETInterpreter, + JET.ToplevelConfig, + Module, + )), setup = begin using JET @analyze_call identity(nothing) diff --git a/docs/src/internals.md b/docs/src/internals.md index 1fabaf65e..02df8b11e 100644 --- a/docs/src/internals.md +++ b/docs/src/internals.md @@ -23,7 +23,7 @@ JET.AbstractGlobal ## Top-level Analysis ```@docs -JET.virtual_process! +JET.virtual_process JET.ConcreteInterpreter JET.partially_interpret! ``` diff --git a/docs/src/usages.md b/docs/src/usages.md index d249f6d45..62d7da41d 100644 --- a/docs/src/usages.md +++ b/docs/src/usages.md @@ -11,12 +11,12 @@ JET can analyze your "top-level" code. This means your can just give your Julia file or code to JET and get error reports. -[`report_and_watch_file`](@ref), [`report_file`](@ref) and [`report_text`](@ref) are the main entry points for that. +[`report_file`](@ref), [`report_and_watch_file`](@ref) and [`report_text`](@ref) are the main entry points for that. JET will analyze your code "half-statically" – JET will selectively interpret "top-level definitions" (like a function definition) and try to simulate Julia's top-level code execution, while it tries to avoid executing any other parts of code like function calls, but analyze them using [abstract interpretation](https://en.wikipedia.org/wiki/Abstract_interpretation) (this is a part where JET "statically" analyzes your code). -If you're interested in how JET selects "top-level definitions", please see [`JET.virtual_process!`](@ref). +If you're interested in how JET selects "top-level definitions", please see [`JET.virtual_process`](@ref). !!! warning Because JET will actually interpret "top-level definitions" in your code, it certainly _runs_ your code. @@ -24,9 +24,9 @@ If you're interested in how JET selects "top-level definitions", please see [`JE macros used in your code, and so the side effects involved with macro expansions will also happen in JET's analysis process. ```@docs -report_text report_file report_and_watch_file +report_text ``` diff --git a/src/JET.jl b/src/JET.jl index 3a5139874..117873853 100644 --- a/src/JET.jl +++ b/src/JET.jl @@ -134,7 +134,8 @@ import JuliaInterpreter: maybe_evaluate_builtin, collect_args, is_return, - is_quotenode_egal + is_quotenode_egal, + @lookup import MacroTools: @capture @@ -449,6 +450,16 @@ This function will look for `$CONFIG_FILE_NAME` configuration file in the direct When found, the configurations specified in the file will overwrite the given `jetconfigs`. See [Configuration File](@ref) for more details. +!!! tip + When you want to analyze your package, but any file using it isn't available, the + `analyze_from_definitions` option can be useful (see [`ToplevelConfig`](@ref)'s `analyze_from_definitions` option). \\ + For example, JET can analyze JET itself like below: + ```julia + # from the root directory of JET.jl + julia> report_file("src/JET"; + analyze_from_definitions = true) + ``` + !!! note This function will enable the toplevel logger by default with the default logging level (see [Logging Configurations](@ref) for more details). @@ -585,13 +596,13 @@ function analyze_text(text::AbstractString, jetconfigs...) interp = JETInterpreter(; jetconfigs...) config = ToplevelConfig(; jetconfigs...) - return virtual_process!(text, - filename, - actualmod, - interp, - config, - virtualmod, - ) + return virtual_process(text, + filename, + actualmod, + interp, + config, + virtualmod, + ) end function analyze_toplevel!(interp::JETInterpreter, src::CodeInfo) diff --git a/src/abstractinterpretation.jl b/src/abstractinterpretation.jl index d5ad9ff1b..30ba708fe 100644 --- a/src/abstractinterpretation.jl +++ b/src/abstractinterpretation.jl @@ -77,7 +77,7 @@ end An overload for `abstract_call_gf_by_type(interp::JETInterpreter, ...)`, which keeps inference on non-concrete call sites in a toplevel frame created by - [`virtual_process!`](@ref). + [`virtual_process`](@ref). """ function CC.bail_out_toplevel_call(interp::JETInterpreter, @nospecialize(sig), sv) return isa(sv.linfo.def, Module) && !isdispatchtuple(sig) && !istoplevel(interp, sv) diff --git a/src/abstractinterpreterinterface.jl b/src/abstractinterpreterinterface.jl index 2d5166d39..8a2eba4e4 100644 --- a/src/abstractinterpreterinterface.jl +++ b/src/abstractinterpreterinterface.jl @@ -298,10 +298,10 @@ end analysis_params = nothing, inf_params = nothing, opt_params = nothing, - concretized = BitVector(), - toplevelmod = __toplevelmod__, - toplevelmods = Set{Module}(), - global_slots = Dict{Int,Symbol}(), + concretized = _CONCRETIZED, + toplevelmod = _TOPLEVELMOD, + toplevelmods = _TOPLEVELMODS, + global_slots = _GLOBAL_SLOTS, logger = nothing, depth = 0, jetconfigs...) @@ -328,6 +328,10 @@ end # dummies for non-toplevel analysis module __toplevelmod__ end +const _CONCRETIZED = BitVector() +const _TOPLEVELMOD = __toplevelmod__ +const _TOPLEVELMODS = Set{Module}() +const _GLOBAL_SLOTS = Dict{Int,Symbol}() # constructor for sequential toplevel JET analysis function JETInterpreter(interp::JETInterpreter, concretized, toplevelmod) diff --git a/src/legacy/abstractinterpretation b/src/legacy/abstractinterpretation index bd57d35ab..fb4ea45e2 100644 --- a/src/legacy/abstractinterpretation +++ b/src/legacy/abstractinterpretation @@ -6,7 +6,7 @@ the aims of this overload are: 1. report `NoMethodErrorReport` on empty method signature matching -2. keep inference on non-concrete call sites in a toplevel frame created by [`virtual_process!`](@ref) +2. keep inference on non-concrete call sites in a toplevel frame created by [`virtual_process`](@ref) 3. don't bail out even after the current return type grows up to `Any` and collects as much error points as possible; of course it slows down inference performance, but hopefully it stays to be "practical" speed (because the number of matching methods is limited beforehand) diff --git a/src/print.jl b/src/print.jl index fd2b9ce90..f659d1229 100644 --- a/src/print.jl +++ b/src/print.jl @@ -253,8 +253,9 @@ function print_reports(io::IO, jetconfigs...) config = PrintConfig(; jetconfigs...) - # XXX the same hack is already imposed in `_typeinf`, so we may not need this - reports = unique(get_identity_key, reports) + # here we more aggressively uniqify reports, ignoring the difference between different `MethodInstance`s + # as far as the report location and its signature are the same + reports = unique(print_identity_key, reports) if isempty(reports) if config.print_inference_success @@ -283,6 +284,24 @@ function print_reports(io::IO, return true end +@withmixedhash struct VirtualFrameNoLinfo + file::Symbol + line::Int + sig::Vector{Any} + # linfo::MethodInstance +end +VirtualFrameNoLinfo(vf::VirtualFrame) = VirtualFrameNoLinfo(vf.file, vf.line, vf.sig) + +@withmixedhash struct PrintIdentityKey + T::Type{<:InferenceErrorReport} + sig::Vector{Any} + # entry_frame::VirtualFrame + error_frame::VirtualFrameNoLinfo +end + +print_identity_key(report::T) where {T<:InferenceErrorReport} = + PrintIdentityKey(T, report.sig, #=VirtualFrameNoLinfo(first(report.st)),=# VirtualFrameNoLinfo(last(report.st))) + # traverse abstract call stack, print frames function print_report(io, report::InferenceErrorReport, config, wrote_linfos, depth = 1) if length(report.st) == depth # error here diff --git a/src/reports.jl b/src/reports.jl index c71870062..49cfe373b 100644 --- a/src/reports.jl +++ b/src/reports.jl @@ -130,16 +130,6 @@ function restore_cached_report(cache::InferenceErrorReportCache) return T(st, cache.msg, cache.sig, cache.spec_args)::InferenceErrorReport end -@withmixedhash struct IdentityKey - T::Type{<:InferenceErrorReport} - sig::Vector{Any} - # entry_frame::VirtualFrame - error_frame::VirtualFrame -end - -get_identity_key(report::T) where {T<:InferenceErrorReport} = - IdentityKey(T, report.sig, #=first(report.st),=# last(report.st)) - macro reportdef(ex, kwargs...) T = esc(first(ex.args)) args = map(ex.args) do x diff --git a/src/typeinfer.jl b/src/typeinfer.jl index f9d12961d..84e876cfb 100644 --- a/src/typeinfer.jl +++ b/src/typeinfer.jl @@ -122,7 +122,7 @@ function CC._typeinf(interp::JETInterpreter, frame::InferenceState) # XXX this is a dirty fix for performance problem, we need more "proper" fix # https://github.com/aviatesk/JET.jl/issues/75 - unique!(get_identity_key, reports) + unique!(report_identity_key, reports) reports_after = Set(reports) @@ -250,6 +250,16 @@ end is_unreachable(@nospecialize(x)) = isa(x, ReturnNode) && !isdefined(x, :val) +@withmixedhash struct ReportIdentityKey + T::Type{<:InferenceErrorReport} + sig::Vector{Any} + # entry_frame::VirtualFrame + error_frame::VirtualFrame +end + +report_identity_key(report::T) where {T<:InferenceErrorReport} = + ReportIdentityKey(T, report.sig, #=first(report.st),=# last(report.st)) + # basically same as `is_throw_call`, but also toplevel module handling added function is_throw_call_expr(interp::JETInterpreter, frame::InferenceState, @nospecialize(e)) if isa(e, Expr) diff --git a/src/virtualprocess.jl b/src/virtualprocess.jl index 65996be1a..635c2b3ec 100644 --- a/src/virtualprocess.jl +++ b/src/virtualprocess.jl @@ -2,6 +2,25 @@ Configurations for top-level analysis. These configurations will be active for all the top-level entries explained in [Analysis entry points](@ref). +--- +- `analyze_from_definitions::Bool = false` \\ + If `true`, JET will start analysis using signatures of top-level definitions (e.g. method signatures), + after the top-level interpretation has been done (unless no serious top-level error has + happened, like errors involved within a macro expansion). + + This is useful when you want to analyze a package, which usually contains only definitions + but not top-level callsites. + With this option, JET can enter analysis just with method or type definitions, and we don't + need to pass a file that uses the target package. + + !!! warning + This feature is very experimental at this point, and you may face lots of false positive + errors, especially when trying to analyze a big package with lots of dependencies. + If a file that contains top-level callsites (e.g. `test/runtests.jl`) is available, + JET analysis entered from there will produce more accurate analysis results than + using with this configuration. + + Also see: [`report_file`](@ref), [`report_and_watch_file`](@ref) --- - `concretization_patterns::Vector{<:Any} = Expr[]` \\ Specifies a customized top-level code concretization strategy. @@ -92,14 +111,17 @@ These configurations will be active for all the top-level entries explained in [ [toplevel-debug] exited from test/fixtures/concretization_patterns.jl (took 0.018 sec) ``` - Also see: [Logging Configurations](@ref), [`virtual_process!`](@ref). + Also see: [Logging Configurations](@ref), [`virtual_process`](@ref). --- """ struct ToplevelConfig + analyze_from_definitions::Bool concretization_patterns::Vector{<:Any} - @jetconfigurable ToplevelConfig(; concretization_patterns = Expr[], + @jetconfigurable ToplevelConfig(; analyze_from_definitions::Bool = false, + concretization_patterns::Vector{<:Any} = Expr[], ) = - return new(concretization_patterns, + return new(analyze_from_definitions, + concretization_patterns, ) end @@ -109,6 +131,7 @@ const VirtualProcessResult = @NamedTuple begin included_files::Set{String} toplevel_error_reports::Vector{ToplevelErrorReport} inference_error_reports::Vector{InferenceErrorReport} + toplevel_signatures::Vector{Type} actual2virtual::Actual2Virtual end @@ -116,6 +139,7 @@ function gen_virtual_process_result(actualmod, virtualmod) return (; included_files = Set{String}(), toplevel_error_reports = ToplevelErrorReport[], inference_error_reports = InferenceErrorReport[], + toplevel_signatures = Type[], actual2virtual = Actual2Virtual(actualmod, virtualmod), )::VirtualProcessResult end @@ -124,22 +148,13 @@ gen_virtual_module(actualmod = Main) = Core.eval(actualmod, :(module $(gensym(:JETVirtualModule)) end))::Module """ - virtual_process!(s::AbstractString, - filename::AbstractString, - actualmod::Module, - interp::JETInterpreter, - config::ToplevelConfig, - virtualmod::Module = gen_virtual_module(actualmod) - res::VirtualProcessResult = gen_virtual_process_result(actualmod, virtualmod), - ) -> VirtualProcessResult - virtual_process!(toplevelex::Expr, - filename::AbstractString, - actualmod::Module, - interp::JETInterpreter, - config::ToplevelConfig, - virtualmod::Module, - res::VirtualProcessResult, - ) -> VirtualProcessResult + virtual_process(s::AbstractString, + filename::AbstractString, + actualmod::Module, + interp::JETInterpreter, + config::ToplevelConfig, + virtualmod::Module, + ) -> VirtualProcessResult Simulates Julia's toplevel execution and collects error points, and finally returns `res::VirtualProcessResult`, which keeps the following information: @@ -149,11 +164,12 @@ Simulates Julia's toplevel execution and collects error points, and finally retu have precedence over `inference_error_reports` - `res.inference_error_reports::Vector{InferenceErrorReport}`: possible error reports found by `JETInterpreter` +- `res.toplevel_signatures`: signatures of methods defined within the analyzed files - `res.actual2virtual::$Actual2Virtual`: keeps actual and virtual module This function first parses `s::AbstractString` into `toplevelex::Expr` and then iterate the following steps on each code block (`blk`) of `toplevelex`: -1. if `blk` is a `:module` expression, recusively call `virtual_process!` with an newly defined +1. if `blk` is a `:module` expression, recusively enters analysis into an newly defined virtual module 2. `lower`s `blk` into `:thunk` expression `lwr` (macros are also expanded in this step) 3. replaces self-references of the original root module (i.e. `actualmod`) @@ -163,7 +179,7 @@ This function first parses `s::AbstractString` into `toplevelex::Expr` and then 4. finally, `JETInterpreter` analyzes the remaining statements by abstract interpretation !!! warning - In order to process the toplevel code sequentially as Julia runtime does, `virtual_process!` + In order to process the toplevel code sequentially as Julia runtime does, `virtual_process` splits the entire code, and then iterate a simulation process on each code block. With this approach, we can't track the inter-code-block level dependencies, and so a partial interpretation of toplevle definitions will fail if it needs an access to global @@ -172,14 +188,51 @@ This function first parses `s::AbstractString` into `toplevelex::Expr` and then allows us to customize JET's concretization strategy. See [`ToplevelConfig`](@ref) for more details. """ -function virtual_process!(s::AbstractString, - filename::AbstractString, - actualmod::Module, - interp::JETInterpreter, - config::ToplevelConfig, - virtualmod::Module = gen_virtual_module(actualmod), - res::VirtualProcessResult = gen_virtual_process_result(actualmod, virtualmod), - )::VirtualProcessResult +function virtual_process(s::AbstractString, + filename::AbstractString, + actualmod::Module, + interp::JETInterpreter, + config::ToplevelConfig, + virtualmod::Module, + ) + res = _virtual_process!(s, + filename, + actualmod, + interp, + config, + virtualmod, + gen_virtual_process_result(actualmod, virtualmod), + ) + + # analyze collected signatures unless critical error happened + if config.analyze_from_definitions && isempty(res.toplevel_error_reports) + for tt in res.toplevel_signatures + mms = _methods_by_ftype(tt, -1, get_world_counter()) + isa(mms, Bool) && continue + filter!(mm::MethodMatch->mm.spec_types===tt, mms) + if length(mms) == 1 + interp = JETInterpreter(interp, _CONCRETIZED, _TOPLEVELMOD) + mm = first(mms) + analyze_method_signature!(interp, mm.method, mm.spec_types, mm.sparams) + append!(res.inference_error_reports, interp.reports) + else + # @info "skipped" tt length(mms) + continue + end + end + end + + return res +end + +function _virtual_process!(s::AbstractString, + filename::AbstractString, + actualmod::Module, + interp::JETInterpreter, + config::ToplevelConfig, + virtualmod::Module, + res::VirtualProcessResult, + )::VirtualProcessResult start = time() with_toplevel_logger(interp) do io @@ -196,7 +249,7 @@ function virtual_process!(s::AbstractString, elseif isnothing(toplevelex) # just return if there is nothing to analyze else - res = virtual_process!(toplevelex, filename, actualmod, interp, config, virtualmod, res) + res = _virtual_process!(toplevelex, filename, actualmod, interp, config, virtualmod, res) end with_toplevel_logger(interp) do io @@ -207,14 +260,14 @@ function virtual_process!(s::AbstractString, return res end -function virtual_process!(toplevelex::Expr, - filename::AbstractString, - actualmod::Module, - interp::JETInterpreter, - config::ToplevelConfig, - virtualmod::Module, - res::VirtualProcessResult, - )::VirtualProcessResult +function _virtual_process!(toplevelex::Expr, + filename::AbstractString, + actualmod::Module, + interp::JETInterpreter, + config::ToplevelConfig, + virtualmod::Module, + res::VirtualProcessResult, + )::VirtualProcessResult @assert @isexpr(toplevelex, :toplevel) local lnn::LineNumberNode = LineNumberNode(0, filename) @@ -311,7 +364,7 @@ function virtual_process!(toplevelex::Expr, isnothing(newvirtualmod) && continue # error happened, e.g. duplicated naming - virtual_process!(newtoplevelex, filename, actualmod, interp, config, newvirtualmod::Module, res) + _virtual_process!(newtoplevelex, filename, actualmod, interp, config, newvirtualmod::Module, res) continue end @@ -438,8 +491,7 @@ Partially interprets statements in `src` using JuliaInterpreter.jl: - concretizes user-specified toplevel code (see [`ToplevelConfig`](@ref)) - directly evaluates module usage expressions and report error of invalid module usages (TODO: enter into the loaded module and keep JET analysis) -- special-cases `include` calls so that [`virtual_process!`](@ref) recursively runs on the - included file +- special-cases `include` calls so that top-level analysis recursively enters the included file """ function partially_interpret!(interp::ConcreteInterpreter, mod::Module, src::CodeInfo) concretize = select_statements(src, interp.config) @@ -538,7 +590,25 @@ function JuliaInterpreter.step_expr!(interp::ConcreteInterpreter, frame::Frame, return nothing end - return @invoke step_expr!(interp, frame, node, istoplevel::Bool) + res = @invoke step_expr!(interp, frame, node, istoplevel::Bool) + + interp.config.analyze_from_definitions && collect_toplevel_signature!(interp, frame, node) + + return res +end + +function collect_toplevel_signature!(interp::ConcreteInterpreter, frame::Frame, @nospecialize(node)) + if @isexpr(node, :method, 3) + sigs = node.args[2] + atype_params, sparams, _ = @lookup(frame, sigs)::SimpleVector + # t = atype_params[1] + # if isdefined(t, :name) + # # XXX ignore constructor methods, just because it can lead to false positives ... + # t.name === CC._TYPE_NAME && return + # end + atype = Tuple{(atype_params::SimpleVector)...} + push!(interp.res.toplevel_signatures, atype) + end end ismoduleusage(@nospecialize(x)) = @isexpr(x, (:import, :using, :export)) @@ -584,8 +654,8 @@ function JuliaInterpreter.evaluate_call_recurse!(interp::ConcreteInterpreter, fr if isinclude(f) return handle_include(interp, fargs) else - # `virtual_process!` iteratively interpret toplevel expressions but it doesn't hit toplevel - # we may want to make `virtual_process!` hit the toplevel on each interation rather than + # `_virtual_process!` iteratively interpret toplevel expressions but it doesn't hit toplevel + # we may want to make `_virtual_process!` hit the toplevel on each interation rather than # using `invokelatest` here, but assuming concretized calls are supposed only to be # used for other toplevel definitions and as such not so computational heavy, # I'd like to go with this simplest way @@ -618,9 +688,9 @@ function handle_include(interp, fargs) isnothing(include_text) && return nothing # typically no file error - virtual_process!(include_text::String, include_file, interp.actualmod, interp.interp, interp.config, interp.virtualmod, interp.res) + _virtual_process!(include_text::String, include_file, interp.actualmod, interp.interp, interp.config, interp.virtualmod, interp.res) - # TODO: actually, here we need to try to get the lastly analyzed result of the `virtual_process!` call above + # TODO: actually, here we need to try to get the lastly analyzed result of the `_virtual_process!` call above return nothing end diff --git a/test/interactive_utils.jl b/test/interactive_utils.jl index dbf114f6d..5d5889e2e 100644 --- a/test/interactive_utils.jl +++ b/test/interactive_utils.jl @@ -12,7 +12,7 @@ import JET: analyze_text, get_result, ToplevelConfig, - virtual_process!, + _virtual_process!, gen_virtual_module, ToplevelErrorReport, InferenceErrorReport, @@ -78,7 +78,7 @@ function analyze_toplevel(ex, lnn, actualmod, virtualmod, jetconfigs) interp = JETInterpreter(; $(map(esc, jetconfigs)...)) config = ToplevelConfig(; $(map(esc, jetconfigs)...)) res = $(JET.gen_virtual_process_result)(actualmod, virtualmod) - $virtual_process!($toplevelex, $(string(lnn.file)), actualmod, interp, config, virtualmod, res) + $_virtual_process!($toplevelex, $(string(lnn.file)), actualmod, interp, config, virtualmod, res) end end end diff --git a/test/test_virtualprocess.jl b/test/test_virtualprocess.jl index acfe5cb32..87bd10abe 100644 --- a/test/test_virtualprocess.jl +++ b/test/test_virtualprocess.jl @@ -1166,7 +1166,7 @@ end @test isempty(res.toplevel_error_reports) end -@testset "avoid too much bail out from `virtual_process!`" begin +@testset "avoid too much bail out from `_virtual_process!`" begin let res = @analyze_toplevel begin sin′ @@ -1261,3 +1261,89 @@ end cd(back) end end + +@testset "`collect_toplevel_signature!`" begin + res = @analyze_toplevel begin + foo() = return + bar(a) = return a + baz(a) = return a + baz(a::Int) = return a + qux(a::T) where T<:Integer = return a + end + @test isempty(res.toplevel_signatures) + + vmod = gen_virtual_module() + res = @analyze_toplevel analyze_from_definitions=true vmod begin + foo() = return + bar(a) = return a + baz(a) = return a + baz(a::Int) = return a + qux(a::T) where T<:Integer = return a + end + @test !isempty(res.toplevel_signatures) + @test any(res.toplevel_signatures) do sig + sig === Tuple{typeof(vmod.foo)} + end + @test any(res.toplevel_signatures) do sig + sig === Tuple{typeof(vmod.bar),Any} + end + @test any(res.toplevel_signatures) do sig + sig === Tuple{typeof(vmod.baz),Any} + end + @test any(res.toplevel_signatures) do sig + sig === Tuple{typeof(vmod.baz),Int} + end + @test any(res.toplevel_signatures) do sig + sig <: Tuple{typeof(vmod.qux),Integer} # `Tuple{typeof(vmod.qux),TypeVar}` + end +end + +@testset "analyze from definitions" begin + let + s = quote + foo() = return undefvar + end |> string + + res = analyze_text(s; analyze_from_definitions = false) + @test isempty(res.inference_error_reports) + + res = analyze_text(s; analyze_from_definitions = true) + @test !isempty(res.inference_error_reports) + @test any(res.inference_error_reports) do err + isa(err, GlobalUndefVarErrorReport) && + err.name === :undefvar + end + end + + let + s = quote + foo(a) = b # typo + bar() = foo("julia") + end |> string + + res = analyze_text(s; analyze_from_definitions = true) + @test length(res.inference_error_reports) == 2 + # report analyzed from `foo` + @test any(res.inference_error_reports) do err + isa(err, GlobalUndefVarErrorReport) && + err.name === :b && + length(err.st) == 1 + end + # report analyzed from `bar` + @test any(res.inference_error_reports) do err + isa(err, GlobalUndefVarErrorReport) && + err.name === :b && + length(err.st) == 2 + end + end + + let + s = quote + foo(a) = sum(a) + bar() = foo("julia") + end |> string + + res = analyze_text(s; analyze_from_definitions = true) + test_sum_over_string(res) + end +end