From ab02f4d21ac22e6d424e7e13498b0283038fb36e Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sat, 2 Nov 2019 03:40:06 +0900 Subject: [PATCH 01/13] allow escape submodules in `toplevelitems` --- src/static/static.jl | 3 +++ src/static/toplevel.jl | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/static/static.jl b/src/static/static.jl index c927536d..46c8defb 100644 --- a/src/static/static.jl +++ b/src/static/static.jl @@ -50,6 +50,9 @@ function isinclude(expr::CSTParser.EXPR) endswith(expr.args[3].val, ".jl") end +ismodule(expr::CSTParser.EXPR) = + expr.typ === CSTParser.ModuleH || expr.typ === CSTParser.BareModule + function isdoc(expr::CSTParser.EXPR) expr.typ === CSTParser.MacroCall && length(expr.args) >= 1 && diff --git a/src/static/toplevel.jl b/src/static/toplevel.jl index 849531de..c59f2db6 100644 --- a/src/static/toplevel.jl +++ b/src/static/toplevel.jl @@ -2,7 +2,6 @@ Find toplevel items (bind / call) - downstreams: modules.jl, outline.jl, goto.jl -- TODO: crate `ToplevelScope` and allow to escape modules =# @@ -25,7 +24,11 @@ struct ToplevelTupleH <: ToplevelItem lines::UnitRange{Int} end -function toplevelitems(expr, text, items::Vector{ToplevelItem} = Vector{ToplevelItem}(), line = 1, pos = 1) +function toplevelitems( + expr, text, # necessary + items::Vector{ToplevelItem} = Vector{ToplevelItem}(), line = 1, pos = 1; + mod::Union{Nothing, String} = nothing, # if given, don't enter into modules other than `mod` +) # binding bind = CSTParser.bindingof(expr) if bind !== nothing @@ -44,10 +47,10 @@ function toplevelitems(expr, text, items::Vector{ToplevelItem} = Vector{Toplevel ismultiplereturn(expr) && push!(items, ToplevelTupleH(expr, lines)) # look for more toplevel items in expr: - if shouldenter(expr) + if shouldenter(expr, mod) if expr.args !== nothing for arg in expr.args - toplevelitems(arg, text, items, line, pos) + toplevelitems(arg, text, items, line, pos; mod = mod) line += countlines(arg, text, pos) pos += arg.fullspan end @@ -56,11 +59,13 @@ function toplevelitems(expr, text, items::Vector{ToplevelItem} = Vector{Toplevel return items end -function shouldenter(expr::CSTParser.EXPR) +function shouldenter(expr::CSTParser.EXPR, mod::Union{Nothing, String}) !(scopeof(expr) !== nothing && !( expr.typ === CSTParser.FileH || - expr.typ === CSTParser.ModuleH || - expr.typ === CSTParser.BareModule || + (ismodule(expr) && shouldentermodule(expr, mod)) || isdoc(expr) )) end + +shouldentermodule(expr::CSTParser.EXPR, mod::Nothing) = true +shouldentermodule(expr::CSTParser.EXPR, mod::String) = expr.binding.name == mod From ed3e9e6ed0b76b58341f95fd59a4e552379c8d19 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sat, 2 Nov 2019 18:39:39 +0900 Subject: [PATCH 02/13] don't enter submodules in `modulefiles` --- src/modules.jl | 23 ++++++++++++----------- test/modules.jl | 6 +++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/modules.jl b/src/modules.jl index 803f258d..f9f8c2e1 100644 --- a/src/modules.jl +++ b/src/modules.jl @@ -113,7 +113,7 @@ const stdlib_names = Set([ function pkg_fileinfo(id::PkgId) uuid, name = id.uuid, id.name - # Try to find the matching cache file + # Try to find the matching cache file paths = Base.find_all_in_cache_path(id) sourcepath = Base.locate_package(id) for path in paths @@ -141,8 +141,8 @@ function parse_cache_header(f::IO) push!(modules, PkgId(uuid, sym) => build_id) end totbytes = read(f, Int64) # total bytes for file dependencies - # read the list of requirements - # and split the list into include and requires statements + # read the list of requirements + # and split the list into include and requires statements includes = Tuple{Module,String,Float64}[] requires = Pair{Module,PkgId}[] while true @@ -153,7 +153,7 @@ function parse_cache_header(f::IO) n1 = read(f, Int32) mod = (n1 == 0) ? Main : Base.root_module(modules[n1].first) if n1 != 0 - # determine the complete module path + # determine the complete module path while true n1 = read(f, Int32) totbytes -= 4 @@ -186,20 +186,21 @@ end # ------------------------ """ - included_files = modulefiles(entrypath::String)::Vector{String} + included_files = modulefiles(mod::String, entrypath::String)::Vector{String} -Returns all the files that can be reached via [`include`](@ref) calls from `entrypath`. -Note this function currently only looks for static toplevel calls (i.e. miss the calls - in not in toplevel scope), and can include files in the submodules as well. +Returns all the files in `mod` module that can be reached via [`include`](@ref) + calls from `entrypath`. +Note this function currently only looks for static toplevel calls (i.e. miss the + calls in non-toplevel scope). """ -function modulefiles(entrypath::String, files = Vector{String}()) +function modulefiles(mod::String, entrypath::String, files = Vector{String}()) isfile′(entrypath) || return files push!(files, entrypath) text = read(entrypath, String) parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text) + items = toplevelitems(parsed, text; mod = mod) for item in items if item isa ToplevelCall @@ -208,7 +209,7 @@ function modulefiles(entrypath::String, files = Vector{String}()) nextfile = expr.args[3].val nextentrypath = joinpath(dirname(entrypath), nextfile) isfile(nextentrypath) || continue - modulefiles(nextentrypath, files) + modulefiles(mod, nextentrypath, files) end end end diff --git a/test/modules.jl b/test/modules.jl index 7272f4d2..a3d14a69 100644 --- a/test/modules.jl +++ b/test/modules.jl @@ -34,15 +34,15 @@ @test_broken junkpath == modulefiles(Junk)[1] ## CSTPraser-based module file detection - let included_files = normpath.(modulefiles(joinpath′(atomjldir, "Atom.jl"))) + let included_files = normpath.(modulefiles("Atom", joinpath′(atomjldir, "Atom.jl"))) # finds all the files in Atom module except display/webio.jl for f in atommodfiles f == webiofile && continue @test f in included_files end - # can't exclude files in the submodules - @test_broken length(atommodfiles) == length(included_files) + # only finds files in a module -- exclude files in the submodules + @test length(atommodfiles) == length(included_files) # can't look for non-toplevel `include` calls @test_broken webiofile in included_files From b69107bd2fb21a727731068ee4be0aea125ea6aa Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sun, 3 Nov 2019 04:17:01 +0900 Subject: [PATCH 03/13] improve goto.jl - don't enter submodules in `modulefiles` - enable gotos within unsaved editors - rename refactor on goto.jl - correct lines in goto's secondary - add clearsymbols handler --- src/goto.jl | 152 +++++++++++++++++++++++++++---------------------- src/outline.jl | 6 +- src/utils.jl | 9 ++- 3 files changed, 96 insertions(+), 71 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index d360c0d0..29c9b343 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -17,13 +17,15 @@ handle("gotosymbol") do data gotosymbol( word, path, column, row, startRow, context, onlyGlobal, - mod, text + mod, text, ) end function gotosymbol( word, path = nothing, + # local context column = 1, row = 1, startrow = 0, context = "", onlyglobal = false, + # module context mod = "Main", text = "" ) try @@ -32,15 +34,15 @@ function gotosymbol( localitems = localgotoitem(word, path, column, row, startrow, context) isempty(localitems) || return Dict( :error => false, - :items => map(Dict, localitems) + :items => todict.(localitems) ) end # global goto - globalitems = globalgotoitems(word, mod, text, path) + globalitems = globalgotoitems(word, mod, path, text) isempty(globalitems) || return Dict( :error => false, - :items => map(Dict, globalitems), + :items => todict.(globalitems), ) catch err return Dict(:error => true) @@ -56,7 +58,8 @@ struct GotoItem secondary::String GotoItem(text, file, line = 0, secondary = "") = new(text, normpath(file), line, secondary) end -Dict(gotoitem::GotoItem) = Dict( + +todict(gotoitem::GotoItem) = Dict( :text => gotoitem.text, :file => gotoitem.file, :line => gotoitem.line, @@ -79,26 +82,26 @@ function localgotoitem(word, path, column, row, startrow, context) GotoItem(text, path, line) end end -localgotoitem(word, ::Nothing, column, row, startrow, context) = [] # when `path` is not destructured +localgotoitem(word, ::Nothing, column, row, startrow, context) = [] # when called from docpane/workspace ### global goto - bundles toplevel gotos & method gotos -function globalgotoitems(word, mod, text, path) - mod = getmodule(mod) +function globalgotoitems(word, mod, path, text) + m = getmodule(mod) # strip a dot-accessed module if exists identifiers = split(word, '.') head = string(identifiers[1]) - if head ≠ word && getfield′(mod, head) isa Module + if head ≠ word && getfield′(m, head) isa Module # if `head` is a module, update `word` and `mod` nextword = join(identifiers[2:end], '.') return globalgotoitems(nextword, head, text, path) end - val = getfield′(mod, word) + val = getfield′(m, word) val isa Module && return [GotoItem(val)] # module goto - toplevelitems = toplevelgotoitems(word, mod, text, path) + toplevelitems = toplevelgotoitems(word, mod, path, text) # append method gotos that are not caught by `toplevelgotoitems` ml = methods(val) @@ -119,16 +122,15 @@ end const PathItemsMaps = Dict{String, Vector{ToplevelItem}} const SYMBOLSCACHE = Dict{String, PathItemsMaps}() -function toplevelgotoitems(word, mod, text, path) - key = string(mod) - pathitemsmaps = if haskey(SYMBOLSCACHE, key) - SYMBOLSCACHE[key] +function toplevelgotoitems(word, mod, path, text) + pathitemsmaps = if haskey(SYMBOLSCACHE, mod) + SYMBOLSCACHE[mod] else - SYMBOLSCACHE[key] = searchtoplevelitems(mod, text, path) # caching + SYMBOLSCACHE[mod] = collecttoplevelitems(mod, path, text) # caching end ismacro(word) && (word = lstrip(word, '@')) - ret = Vector{GotoItem}() + ret = [] for (path, items) in pathitemsmaps for item in filter(item -> filtertoplevelitem(word, item), items) push!(ret, GotoItem(path, item)) @@ -138,51 +140,57 @@ function toplevelgotoitems(word, mod, text, path) end # entry method -function searchtoplevelitems(mod::Module, text::String, path::String) +function collecttoplevelitems(mod::String, path::String, text::String) pathitemsmaps = PathItemsMaps() - if mod == Main # for `Main` module, always use the passed text - _searchtoplevelitems(text, path, pathitemsmaps) + if mod == "Main" || isuntitled(path) + # for `Main` module and unsaved editors, always use CSTPraser-based approach + # with a given buffer text + _collecttoplevelitems!(mod, path, text, pathitemsmaps) else - _searchtoplevelitems(mod, pathitemsmaps) + _collecttoplevelitems!(mod, pathitemsmaps) end return pathitemsmaps end -# entry method when path is not deconstructured, e.g.: called from docpane/workspace -function searchtoplevelitems(mod::Module, text::String, path::Nothing) +# entry method when called from docpane/workspace +function collecttoplevelitems(mod::String, path::Nothing, text::String) pathitemsmaps = PathItemsMaps() - _searchtoplevelitems(mod, pathitemsmaps) + _collecttoplevelitems!(mod, pathitemsmaps) return pathitemsmaps end # sub entry method -function _searchtoplevelitems(mod::Module, pathitemsmaps::PathItemsMaps) - entrypath, paths = modulefiles(mod) # Revise-like approach - if entrypath !== nothing - for p in [entrypath; paths] - _searchtoplevelitems(p, pathitemsmaps) - end +function _collecttoplevelitems!(mod::String, pathitemsmaps::PathItemsMaps) + m = getmodule(mod) + entrypath, paths = modulefiles(m) + if entrypath !== nothing # Revise-like approach + _collecttoplevelitems!([entrypath; paths], pathitemsmaps) else # if Revise-like approach fails, fallback to CSTParser-based approach - path, line = moduledefinition(mod) - text = read(path, String) - _searchtoplevelitems(text, path, pathitemsmaps) + entrypath, line = moduledefinition(m) + _collecttoplevelitems!(mod, entrypath, pathitemsmaps) end end # module-walk via Revise-like approach -function _searchtoplevelitems(path::String, pathitemsmaps::PathItemsMaps) - text = read(path, String) - parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text) - pathitemsmap = path => items - push!(pathitemsmaps, pathitemsmap) +function _collecttoplevelitems!(paths::Vector{String}, pathitemsmaps::PathItemsMaps) + for path in paths + text = read(path, String) + parsed = CSTParser.parse(text, true) + items = toplevelitems(parsed, text) + push!(pathitemsmaps, path => items) + end end # module-walk based on CSTParser, looking for toplevel `installed` calls -function _searchtoplevelitems(text::String, path::String, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems!(mod::String, entrypath::String, pathitemsmaps::PathItemsMaps) + isfile′(entrypath) || return + text = read(entrypath, String) + _collecttoplevelitems!(mod, entrypath, text, pathitemsmaps) +end +function _collecttoplevelitems!(mod::String, entrypath::String, text::String, pathitemsmaps::PathItemsMaps) parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text) - push!(pathitemsmaps, path => items) + items = toplevelitems(parsed, text; mod = mod) + push!(pathitemsmaps, entrypath => items) # looking for toplevel `include` calls for item in items @@ -190,10 +198,9 @@ function _searchtoplevelitems(text::String, path::String, pathitemsmaps::PathIte expr = item.expr if isinclude(expr) nextfile = expr.args[3].val - nextpath = joinpath(dirname(path), nextfile) - isfile(nextpath) || continue - text = read(nextpath, String) - _searchtoplevelitems(text, nextpath, pathitemsmaps) + nextentrypath = joinpath(dirname(entrypath), nextfile) + isfile′(nextentrypath) || continue + _collecttoplevelitems!(mod, nextentrypath, pathitemsmaps) end end end @@ -222,34 +229,36 @@ function GotoItem(path::String, bind::ToplevelBinding) text = str_value(sig) end line = bind.lines.start - 1 - secondary = path * ":" * string(line) + secondary = string(path, ":", line + 1) GotoItem(text, path, line, secondary) end function GotoItem(path::String, tupleh::ToplevelTupleH) expr = tupleh.expr text = str_value(expr) line = tupleh.lines.start - 1 - secondary = path * ":" * string(line) + secondary = string(path, ":", line + 1) GotoItem(text, path, line, secondary) end -## update toplevel symbols +## update toplevel symbols cache # NOTE: handled by the `updateeditor` handler in outline.jl -function updatesymbols(text, mod, path, items) - if haskey(SYMBOLSCACHE, mod) - push!(SYMBOLSCACHE[mod], path => items) # don't try to walk in a module - else - # initialize the cache if there is no cache - SYMBOLSCACHE[mod] = searchtoplevelitems(getmodule(mod), text, path) +function updatesymbols(items, mod, path::Nothing, text) end # fallback case +function updatesymbols(items, mod, path::String, text) + # initialize the cache if there is no previous one + if !haskey(SYMBOLSCACHE, mod) + SYMBOLSCACHE[mod] = collecttoplevelitems(mod, path, text) end + push!(SYMBOLSCACHE[mod], path => items) end -## generate toplevel symbols +## generate toplevel symbols cache + handle("regeneratesymbols") do with_logger(JunoProgressLogger()) do regeneratesymbols() end + nothing end function regeneratesymbols() @@ -267,15 +276,13 @@ function regeneratesymbols() unloadedlen = length(unloaded) total = loadedlen + unloadedlen - for (i, mod) in enumerate(Base.loaded_modules_array()) + for (i, m) in enumerate(Base.loaded_modules_array()) try - modstr = string(mod) - modstr == "__PackagePrecompilationStatementModule" && continue # will cause error - pathitemsmap = PathItemsMaps() + mod = string(m) + mod == "__PackagePrecompilationStatementModule" && continue # will cause error - @logmsg -1 "Symbols: $modstr ($i / $total)" progress=i/total _id=id - _searchtoplevelitems(mod, pathitemsmap) - SYMBOLSCACHE[modstr] = pathitemsmap + @logmsg -1 "Symbols: $mod ($i / $total)" progress=i/total _id=id + SYMBOLSCACHE[mod] = collecttoplevelitems(mod, nothing, "") catch err @error err end @@ -283,12 +290,10 @@ function regeneratesymbols() for (i, pkg) in enumerate(unloaded) try + @logmsg -1 "Symbols: $pkg ($(i + loadedlen) / $total)" progress=(i+loadedlen)/total _id=id path = Base.find_package(pkg) - text = read(path, String) pathitemsmap = PathItemsMaps() - - @logmsg -1 "Symbols: $pkg ($(i + loadedlen) / $total)" progress=(i+loadedlen)/total _id=id - _searchtoplevelitems(text, path, pathitemsmap) + _collecttoplevelitems!(pkg, path, pathitemsmap) SYMBOLSCACHE[pkg] = pathitemsmap catch err @error err @@ -298,6 +303,19 @@ function regeneratesymbols() @info "Finished generating the symbols cache" progress=1 _id=id end +## clear toplevel symbols cache + +handle("clearsymbols") do + clearsymbols() + nothing +end + +function clearsymbols() + for key in keys(SYMBOLSCACHE) + delete!(SYMBOLSCACHE, key) + end +end + ## method goto methodgotoitems(ml) = map(GotoItem, aggregatemethods(ml)) diff --git a/src/outline.jl b/src/outline.jl index 3c7eb36a..6dc4fbc5 100644 --- a/src/outline.jl +++ b/src/outline.jl @@ -2,7 +2,7 @@ handle("updateeditor") do data @destruct [ text || "", mod || "Main", - path || "untitled", + path || nothing, updateSymbols || true ] = data @@ -14,13 +14,13 @@ handle("updateeditor") do data end # NOTE: update outline and symbols cache all in one go -function updateeditor(text, mod = "Main", path = "untitled", updateSymbols = true) +function updateeditor(text, mod = "Main", path = nothing, updateSymbols = true) parsed = CSTParser.parse(text, true) items = toplevelitems(parsed, text) # update symbols cache # ref: https://github.com/JunoLab/Juno.jl/issues/407 - updateSymbols && updatesymbols(text, mod, path, items) + updateSymbols && updatesymbols(items, mod, path, text) # return outline outline(items) diff --git a/src/utils.jl b/src/utils.jl index 59868fa0..d751ffd4 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -17,7 +17,14 @@ nonwritablefiles(files) = filter(!iswritablefile, files) include("path_matching.jl") -isuntitled(p) = occursin(r"^(\.\\|\./)?untitled-[\d\w]+(:\d+)?$", p) +""" + isuntitled(path::AbstractString) + +Checks if `path` represents an unsaved editor. +Usualy the string that follows `"untitled-"` is obtained from `editor.getBuffer().getId()`: + e.g. `path = "untitled-266305858c1298b906bed15ddad81cea"`. +""" +isuntitled(path::AbstractString) = occursin(r"^(\.\\|\./)?untitled-[\d\w]+(:\d+)?$", path) appendline(path, line) = line > 0 ? "$path:$line" : path From 4295ed345c57f62b72946e3a13a60c2184cb6146 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sun, 3 Nov 2019 05:56:07 +0900 Subject: [PATCH 04/13] strip parent module prefixes --- src/goto.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index 29c9b343..6b41d907 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -142,31 +142,30 @@ end # entry method function collecttoplevelitems(mod::String, path::String, text::String) pathitemsmaps = PathItemsMaps() - if mod == "Main" || isuntitled(path) + return if mod == "Main" || isuntitled(path) # for `Main` module and unsaved editors, always use CSTPraser-based approach # with a given buffer text _collecttoplevelitems!(mod, path, text, pathitemsmaps) else _collecttoplevelitems!(mod, pathitemsmaps) end - return pathitemsmaps end # entry method when called from docpane/workspace function collecttoplevelitems(mod::String, path::Nothing, text::String) pathitemsmaps = PathItemsMaps() _collecttoplevelitems!(mod, pathitemsmaps) - return pathitemsmaps end # sub entry method function _collecttoplevelitems!(mod::String, pathitemsmaps::PathItemsMaps) m = getmodule(mod) entrypath, paths = modulefiles(m) - if entrypath !== nothing # Revise-like approach + return if entrypath !== nothing # Revise-like approach _collecttoplevelitems!([entrypath; paths], pathitemsmaps) else # if Revise-like approach fails, fallback to CSTParser-based approach entrypath, line = moduledefinition(m) + mod = string(last(split(mod, '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` _collecttoplevelitems!(mod, entrypath, pathitemsmaps) end end @@ -179,6 +178,7 @@ function _collecttoplevelitems!(paths::Vector{String}, pathitemsmaps::PathItemsM items = toplevelitems(parsed, text) push!(pathitemsmaps, path => items) end + pathitemsmaps end # module-walk based on CSTParser, looking for toplevel `installed` calls @@ -204,6 +204,8 @@ function _collecttoplevelitems!(mod::String, entrypath::String, text::String, pa end end end + + pathitemsmaps end filtertoplevelitem(word, item::ToplevelItem) = false @@ -292,9 +294,7 @@ function regeneratesymbols() try @logmsg -1 "Symbols: $pkg ($(i + loadedlen) / $total)" progress=(i+loadedlen)/total _id=id path = Base.find_package(pkg) - pathitemsmap = PathItemsMaps() - _collecttoplevelitems!(pkg, path, pathitemsmap) - SYMBOLSCACHE[pkg] = pathitemsmap + SYMBOLSCACHE[pkg] = _collecttoplevelitems!(pkg, path, PathItemsMaps()) catch err @error err end From c7c151485ffd8b92e68d18220d7e36a42ff662f7 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sun, 3 Nov 2019 15:21:05 +0900 Subject: [PATCH 05/13] update tests and fix broken tests :tada: --- test/goto.jl | 88 +++++++++++++++++++++++++++------------------------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/test/goto.jl b/test/goto.jl index 129483ec..513534a9 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -1,4 +1,6 @@ @testset "goto symbols" begin + using Atom: todict + @testset "goto local symbols" begin let str = """ function localgotoitem(word, path, column, row, startRow, context) # L0 @@ -15,7 +17,7 @@ end # L11 end # L12 """, - localgotoitem(word, line) = Atom.localgotoitem(word, "path", Inf, line + 1, 0, str)[1] |> Dict + localgotoitem(word, line) = Atom.localgotoitem(word, "path", Inf, line + 1, 0, str)[1] |> todict let item = localgotoitem("row", 2) @test item[:line] === 0 @@ -35,7 +37,7 @@ return val end """, - localgotoitem(word, line) = Atom.localgotoitem(word, "path", Inf, line + 1, 0, str)[1] |> Dict + localgotoitem(word, line) = Atom.localgotoitem(word, "path", Inf, line + 1, 0, str)[1] |> todict @test localgotoitem("expr.args", 1)[:line] === 0 @test localgotoitem("bind.val", 2)[:line] === 1 @@ -47,17 +49,17 @@ @testset "goto global symbols" begin using Atom: globalgotoitems, toplevelgotoitems, SYMBOLSCACHE, - regeneratesymbols, methodgotoitems + clearsymbols, regeneratesymbols, methodgotoitems ## strip a dot-accessed modules let path = joinpath′(@__DIR__, "..", "src", "comm.jl") text = read(path, String) - items = Dict.(globalgotoitems("Atom.handlers", "Atom", text, path)) + items = todict.(globalgotoitems("Atom.handlers", "Atom", path, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "handlers" - items = Dict.(globalgotoitems("Main.Atom.handlers", "Atom", text, path)) + items = todict.(globalgotoitems("Main.Atom.handlers", "Atom", path, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "handlers" @@ -65,20 +67,20 @@ # can access the non-exported (non-method) bindings in the other module path = joinpath′(@__DIR__, "..", "src", "goto.jl") text = read(@__FILE__, String) - items = Dict.(globalgotoitems("Atom.SYMBOLSCACHE", "Main", text, @__FILE__)) + items = todict.(globalgotoitems("Atom.SYMBOLSCACHE", "Main", @__FILE__, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "SYMBOLSCACHE" end @testset "goto modules" begin - let item = globalgotoitems("Atom", "Main", "", nothing)[1] - @test item.file == joinpath′(atomjldir, "Atom.jl") - @test item.line == 3 + let item = globalgotoitems("Atom", "Main", nothing, "")[1] |> todict + @test item[:file] == joinpath′(atomjldir, "Atom.jl") + @test item[:line] == 3 end - let item = globalgotoitems("Junk2", "Main.Junk", "", nothing)[1] - @test item.file == joinpath′(junkpath) - @test item.line == 14 + let item = globalgotoitems("Junk2", "Main.Junk", nothing, "")[1] |> todict + @test item[:file] == joinpath′(junkpath) + @test item[:line] == 14 end end @@ -86,55 +88,51 @@ ## where Revise approach works, i.e.: precompiled modules let path = joinpath′(atomjldir, "comm.jl") text = read(path, String) - mod = Atom - key = "Atom" + mod = "Atom" word = "handlers" # basic - let items = toplevelgotoitems(word, mod, text, path) .|> Dict + let items = todict.(toplevelgotoitems(word, mod, path, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == word end # check caching works - @test haskey(SYMBOLSCACHE, key) + @test haskey(SYMBOLSCACHE, mod) # check the Revise-like approach finds all files in Atom module - @test length(SYMBOLSCACHE[key]) == length(atommodfiles) + @test length(SYMBOLSCACHE[mod]) == length(atommodfiles) # when `path` isn't given, i.e. via docpane / workspace - let items = toplevelgotoitems(word, mod, "", nothing) .|> Dict + let items = todict.(toplevelgotoitems(word, mod, nothing, "")) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == word end # same as above, but without any previous cache -- falls back to CSTPraser-based module-walk - delete!(SYMBOLSCACHE, key) + delete!(SYMBOLSCACHE, mod) - let items = toplevelgotoitems(word, mod, "", nothing) .|> Dict + let items = toplevelgotoitems(word, mod, nothing, "") .|> todict @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == word end # check CSTPraser-based module-walk finds all the included files - # currently broken: - # - files in submodules are included - # - webio.jl is excluded since `include("webio.jl")` is a toplevel call - @test_broken length(SYMBOLSCACHE[key]) == length(atommoddir) + # NOTE: webio.jl is excluded since `include("webio.jl")` is a toplevel call + @test length(SYMBOLSCACHE[mod]) == length(atommodfiles) end ## where the Revise-like approach doesn't work, e.g. non-precompiled modules let path = junkpath text = read(path, String) - mod = Main.Junk - key = "Main.Junk" + mod = "Main.Junk" word = "toplevelval" # basic - let items = toplevelgotoitems(word, mod, text, path) .|> Dict + let items = toplevelgotoitems(word, mod, path, text) .|> todict @test !isempty(items) @test items[1][:file] == path @test items[1][:line] == 16 @@ -142,10 +140,10 @@ end # check caching works - @test haskey(Atom.SYMBOLSCACHE, key) + @test haskey(Atom.SYMBOLSCACHE, mod) # when `path` isn't given, i.e.: via docpane / workspace - let items = toplevelgotoitems(word, mod, "", nothing) .|> Dict + let items = toplevelgotoitems(word, mod, nothing, "") .|> todict @test !isempty(items) @test items[1][:file] == path @test items[1][:line] == 16 @@ -155,16 +153,16 @@ end @testset "updating toplevel symbols" begin - mod = "Main.Junk" - path = junkpath - text = read(path, String) - function updatesymbols(mod, text, path) + function updatesymbols(mod, path, text) parsed = CSTParser.parse(text, true) items = Atom.toplevelitems(parsed, text) - Atom.updatesymbols(text, mod, path, items) + Atom.updatesymbols(items, mod, path, text) end # check there is no cache before updating + mod = "Main.Junk" + path = junkpath + text = read(path, String) @test filter(SYMBOLSCACHE[mod][path]) do item Atom.str_value(item.expr) == "toplevelval2" end |> isempty @@ -174,27 +172,27 @@ newtext = join(originallines[1:end - 1], '\n') word = "toplevelval2" newtext *= "\n$word = :youshoulderaseme\nend" - updatesymbols(mod, newtext, path) + updatesymbols(mod, path, newtext) # check the cache is updated @test filter(SYMBOLSCACHE[mod][path]) do item Atom.str_value(item.expr) == word end |> !isempty - let items = toplevelgotoitems(word, mod, newtext, path) .|> Dict + let items = toplevelgotoitems(word, mod, path, newtext) .|> todict @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "toplevelval2" end # re-update the cache - updatesymbols(mod, text, path) + updatesymbols(mod, path, text) @test filter(SYMBOLSCACHE[mod][path]) do item Atom.str_value(item.expr) == word end |> isempty end - @testset "regenerating symbols" begin + @testset "regenerating toplevel symbols" begin regeneratesymbols() @test haskey(SYMBOLSCACHE, "Base") @@ -203,6 +201,12 @@ @test toplevelgotoitems("hello", "Example", "", nothing) |> !isempty end + @testset "clear toplevel symbols" begin + clearsymbols() + + @test length(keys(SYMBOLSCACHE)) === 0 + end + @testset "goto methods" begin ## basic let ms = methods(Atom.handlemsg) @@ -212,7 +216,7 @@ ## aggregate methods with default params @eval Main function funcwithdefaultargs(args, defarg = "default") end - let items = methodgotoitems(methods(funcwithdefaultargs)) .|> Dict + let items = methodgotoitems(methods(funcwithdefaultargs)) .|> todict # should be handled as an unique method @test length(items) === 1 # show a method with full arguments @@ -221,7 +225,7 @@ @eval Main function funcwithdefaultargs(args::String, defarg = "default") end - let items = methodgotoitems(methods(funcwithdefaultargs)) .|> Dict + let items = methodgotoitems(methods(funcwithdefaultargs)) .|> todict # should be handled as different methods @test length(items) === 2 # show methods with full arguments @@ -231,13 +235,13 @@ end ## both the original methods and the toplevel bindings that are overloaded in a context module should be shown - let items = globalgotoitems("isconst", "Main.Junk", "", nothing) + let items = globalgotoitems("isconst", "Main.Junk", nothing, "") @test length(items) === 2 @test "isconst(m::Module, s::Symbol)" in map(item -> item.text, items) # from Base @test "Base.isconst(::JunkType)" in map(item -> item.text, items) # from Junk end ## don't error on the fallback case - @test globalgotoitems("word", "Main", "", nothing) == [] + @test globalgotoitems("word", "Main", nothing, "") == [] end end From 1e34d3e1fc2e08689d0c535863445e9f76f1dd16 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sun, 3 Nov 2019 15:57:59 +0900 Subject: [PATCH 06/13] fix symbols search in Main module and add tests for that --- src/goto.jl | 10 ++++------ test/goto.jl | 13 +++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index 6b41d907..8b2bb15a 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -1,5 +1,3 @@ -using CSTParser - handle("gotosymbol") do data @destruct [ word, @@ -144,8 +142,8 @@ function collecttoplevelitems(mod::String, path::String, text::String) pathitemsmaps = PathItemsMaps() return if mod == "Main" || isuntitled(path) # for `Main` module and unsaved editors, always use CSTPraser-based approach - # with a given buffer text - _collecttoplevelitems!(mod, path, text, pathitemsmaps) + # with a given buffer text, and don't check module validity + _collecttoplevelitems!(nothing, path, text, pathitemsmaps) else _collecttoplevelitems!(mod, pathitemsmaps) end @@ -182,12 +180,12 @@ function _collecttoplevelitems!(paths::Vector{String}, pathitemsmaps::PathItemsM end # module-walk based on CSTParser, looking for toplevel `installed` calls -function _collecttoplevelitems!(mod::String, entrypath::String, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, pathitemsmaps::PathItemsMaps) isfile′(entrypath) || return text = read(entrypath, String) _collecttoplevelitems!(mod, entrypath, text, pathitemsmaps) end -function _collecttoplevelitems!(mod::String, entrypath::String, text::String, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, text::String, pathitemsmaps::PathItemsMaps) parsed = CSTParser.parse(text, true) items = toplevelitems(parsed, text; mod = mod) push!(pathitemsmaps, entrypath => items) diff --git a/test/goto.jl b/test/goto.jl index 513534a9..194651d6 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -150,6 +150,19 @@ @test items[1][:text] == word end end + + ## `Main` module -- use a passed buffer text + let path = joinpath′(@__DIR__, "runtests.jl") + text = read(path, String) + mod = "Main" + word = "atomjldir" + + items = toplevelgotoitems(word, mod, path, text) .|> todict + @test !isempty(items) + @test items[1][:file] == path + @test items[1][:line] == 5 + @test items[1][:text] == word + end end @testset "updating toplevel symbols" begin From d92fa94e787bbceeddec4343824ede1ff39ba566 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sun, 3 Nov 2019 16:14:49 +0900 Subject: [PATCH 07/13] more tests for module validations within `toplevelitems` (some broken) --- test/fixtures/Junk.jl | 12 +----------- test/fixtures/SubJunks.jl | 16 ++++++++++++++++ test/goto.jl | 33 +++++++++++++++++++++++---------- test/modules.jl | 12 ++++++------ test/runtests.jl | 3 ++- test/static/static.jl | 7 +++---- test/static/toplevel.jl | 26 ++++++++++++++++++++++++++ test/workspace.jl | 2 +- 8 files changed, 78 insertions(+), 33 deletions(-) create mode 100644 test/fixtures/SubJunks.jl create mode 100644 test/static/toplevel.jl diff --git a/test/fixtures/Junk.jl b/test/fixtures/Junk.jl index 16e731a2..0687404a 100644 --- a/test/fixtures/Junk.jl +++ b/test/fixtures/Junk.jl @@ -12,22 +12,12 @@ macro immacro(expr) end end -module Junk2 end - const toplevelval = "you should jump to me !" # mock overloaded method struct JunkType end Base.isconst(::JunkType) = false -"""im a doc in Junk""" -const imwithdoc = nothing - -baremodule BareJunk - -"""im a doc in BareJunk""" -const imwithdoc = nothing - -end +include("SubJunks.jl") end diff --git a/test/fixtures/SubJunks.jl b/test/fixtures/SubJunks.jl new file mode 100644 index 00000000..77bbd67b --- /dev/null +++ b/test/fixtures/SubJunks.jl @@ -0,0 +1,16 @@ +"""im a doc in Junk""" +const imwithdoc = nothing + +module SubJunk + +"""im a doc in SubJunk""" +const imwithdoc = nothing + +end + +baremodule BareJunk + +"""im a doc in BareJunk""" +const imwithdoc = nothing + +end diff --git a/test/goto.jl b/test/goto.jl index 194651d6..440a0ace 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -78,21 +78,20 @@ @test item[:file] == joinpath′(atomjldir, "Atom.jl") @test item[:line] == 3 end - let item = globalgotoitems("Junk2", "Main.Junk", nothing, "")[1] |> todict - @test item[:file] == joinpath′(junkpath) - @test item[:line] == 14 + let item = globalgotoitems("SubJunk", "Main.Junk", nothing, "")[1] |> todict + @test item[:file] == subjunkspath + @test item[:line] == 3 end end @testset "goto toplevel symbols" begin ## where Revise approach works, i.e.: precompiled modules let path = joinpath′(atomjldir, "comm.jl") - text = read(path, String) mod = "Atom" word = "handlers" # basic - let items = todict.(toplevelgotoitems(word, mod, path, text)) + let items = todict.(toplevelgotoitems(word, mod, path, "")) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == word @@ -127,15 +126,14 @@ ## where the Revise-like approach doesn't work, e.g. non-precompiled modules let path = junkpath - text = read(path, String) mod = "Main.Junk" word = "toplevelval" - # basic - let items = toplevelgotoitems(word, mod, path, text) .|> todict + # basic -- no need to pass a buffer text + let items = toplevelgotoitems(word, mod, path, "") .|> todict @test !isempty(items) @test items[1][:file] == path - @test items[1][:line] == 16 + @test items[1][:line] == 14 @test items[1][:text] == word end @@ -146,7 +144,22 @@ let items = toplevelgotoitems(word, mod, nothing, "") .|> todict @test !isempty(items) @test items[1][:file] == path - @test items[1][:line] == 16 + @test items[1][:line] == 14 + @test items[1][:text] == word + end + end + + ## don't include bindings outside of a module + let path = subjunkspath + text = read(subjunkspath, String) + mod = "Main.Junk.SubJunk" + word = "imwithdoc" + + items = toplevelgotoitems(word, mod, path, text) .|> todict + @test_broken length(items) === 1 # broken since it finds `imwithdoc` in `Junk` as well + if length(items) === 1 + @test items[1][:file] == path + @test items[1][:line] == 6 @test items[1][:text] == word end end diff --git a/test/modules.jl b/test/modules.jl index a3d14a69..8e92eb34 100644 --- a/test/modules.jl +++ b/test/modules.jl @@ -3,16 +3,16 @@ using Atom: moduledefinition let (path, line) = moduledefinition(Atom) - @test path == joinpath′(@__DIR__, "..", "src", "Atom.jl") + @test path == atommodfile @test line == 4 end let (path, line) = moduledefinition(Junk) - @test path == joinpath′(@__DIR__, "fixtures", "Junk.jl") + @test path == junkpath @test line == 1 end - let (path, line) = moduledefinition(Junk.Junk2) - @test path == joinpath′(@__DIR__, "fixtures", "Junk.jl") - @test line == 15 + let (path, line) = moduledefinition(Junk.SubJunk) + @test path == subjunkspath + @test line == 4 end end @@ -34,7 +34,7 @@ @test_broken junkpath == modulefiles(Junk)[1] ## CSTPraser-based module file detection - let included_files = normpath.(modulefiles("Atom", joinpath′(atomjldir, "Atom.jl"))) + let included_files = normpath.(modulefiles("Atom", atommodfile)) # finds all the files in Atom module except display/webio.jl for f in atommodfiles f == webiofile && continue diff --git a/test/runtests.jl b/test/runtests.jl index 27b4336e..26b6dbe0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,7 +4,7 @@ using Atom, Test, JSON, Logging, CSTParser joinpath′(files...) = Atom.fullpath(joinpath(files...)) atomjldir = joinpath′(@__DIR__, "..", "src") - +atommodfile = joinpath′(atomjldir, "Atom.jl") webiofile = joinpath′(atomjldir, "display", "webio.jl") # files in `Atom` module (except files in its submodules) @@ -42,6 +42,7 @@ readmsg() = JSON.parse(String(take!(Atom.sock))) # mock Module junkpath = joinpath′(@__DIR__, "fixtures", "Junk.jl") +subjunkspath = joinpath′(@__DIR__, "fixtures", "SubJunks.jl") include(junkpath) # basics diff --git a/test/static/static.jl b/test/static/static.jl index 0a32b7b5..2b720dd7 100644 --- a/test/static/static.jl +++ b/test/static/static.jl @@ -1,8 +1,7 @@ @testset "static analysis" begin - # TODO - # @testset "toplevel items" begin - # include("toplevel.jl") - # end + @testset "toplevel items" begin + include("toplevel.jl") + end @testset "local bindings" begin include("local.jl") diff --git a/test/static/toplevel.jl b/test/static/toplevel.jl new file mode 100644 index 00000000..e8174229 --- /dev/null +++ b/test/static/toplevel.jl @@ -0,0 +1,26 @@ +@testset "module validation" begin + using Atom: toplevelitems + + path = subjunkspath + text = read(path, String) + parsed = CSTParser.parse(text, true) + + # basic -- finds every toplevel items when `mod` options is `nothing` (default) + @test filter(toplevelitems(parsed, text; mod = nothing)) do item + item isa Atom.ToplevelBinding && + item.bind.name == "imwithdoc" + end |> length === 3 + + # don't enter non-target modules, e.g.: submodules + @test filter(toplevelitems(parsed, text; mod = "Junk")) do item + item isa Atom.ToplevelBinding && + item.bind.name == "imwithdoc" + end |> length === 1 # only `imwithdoc` in Junk module + + # don't include items outside of a module + # FIX: currently broken -- include `imwithdoc` in Junk module as well + @test_broken filter(toplevelitems(parsed, text; mod = "SubJunk")) do item + item isa Atom.ToplevelBinding && + item.bind.name == "imwithdoc" + end |> length === 1 # only `imwithdoc` in Junk module +end diff --git a/test/workspace.jl b/test/workspace.jl index 63b450aa..445eb48b 100644 --- a/test/workspace.jl +++ b/test/workspace.jl @@ -19,7 +19,7 @@ end # recoginise submodule - let items = filter(i -> i[:name] == :Junk2, items) + let items = filter(i -> i[:name] == :SubJunk, items) @test !isempty(items) @test items[1][:type] == "module" @test items[1][:icon] == "icon-package" From 948677357166f0058523bcf2e26b653d22a97220 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Sun, 3 Nov 2019 16:41:53 +0900 Subject: [PATCH 08/13] improve fallback case tests --- test/completions.jl | 2 +- test/datatip.jl | 2 +- test/goto.jl | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/completions.jl b/test/completions.jl index 4c7eddfb..ce0cab44 100644 --- a/test/completions.jl +++ b/test/completions.jl @@ -163,5 +163,5 @@ end end # don't error on fallback case - @test Atom.localcompletions("", 1, 1) == [] + @test_nowarn @test Atom.localcompletions("", 1, 1) == [] end diff --git a/test/datatip.jl b/test/datatip.jl index f00c6359..a7ba5ecd 100644 --- a/test/datatip.jl +++ b/test/datatip.jl @@ -34,7 +34,7 @@ end # don't error on fallback case - @test Atom.localdatatip("word", 1, 1, 0, "") == [] + @test_nowarn @test Atom.localdatatip("word", 1, 1, 0, "") == [] end @testset "code block search" begin diff --git a/test/goto.jl b/test/goto.jl index 440a0ace..d9adfbf4 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -44,7 +44,7 @@ end # don't error on fallback case - @test Atom.localgotoitem("word", nothing, 1, 1, 0, "") == [] + @test_nowarn @test Atom.localgotoitem("word", nothing, 1, 1, 0, "") == [] end @testset "goto global symbols" begin @@ -216,6 +216,9 @@ @test filter(SYMBOLSCACHE[mod][path]) do item Atom.str_value(item.expr) == word end |> isempty + + # don't error on fallback case + @test_nowarn @test updatesymbols(mod, nothing, text) === nothing end @testset "regenerating toplevel symbols" begin @@ -268,6 +271,6 @@ end ## don't error on the fallback case - @test globalgotoitems("word", "Main", nothing, "") == [] + @test_nowarn @test globalgotoitems("word", "Main", nothing, "") == [] end end From 21f212e6c3aa916b66590095879a46e06cdc9f29 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Mon, 4 Nov 2019 17:27:50 +0900 Subject: [PATCH 09/13] canonical module keys in SYMBOLSCACHE --- src/goto.jl | 54 +++++++++++++++++++++++++++--------------------- test/goto.jl | 51 ++++++++++++++++++++++++--------------------- test/runtests.jl | 2 +- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index 8b2bb15a..6d24de91 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -37,7 +37,7 @@ function gotosymbol( end # global goto - globalitems = globalgotoitems(word, mod, path, text) + globalitems = globalgotoitems(word, getmodule(mod), path, text) isempty(globalitems) || return Dict( :error => false, :items => todict.(globalitems), @@ -85,18 +85,16 @@ localgotoitem(word, ::Nothing, column, row, startrow, context) = [] # when calle ### global goto - bundles toplevel gotos & method gotos function globalgotoitems(word, mod, path, text) - m = getmodule(mod) - # strip a dot-accessed module if exists identifiers = split(word, '.') head = string(identifiers[1]) - if head ≠ word && getfield′(m, head) isa Module + if head ≠ word && (nextmod = getfield′(mod, head)) isa Module # if `head` is a module, update `word` and `mod` nextword = join(identifiers[2:end], '.') - return globalgotoitems(nextword, head, text, path) + return globalgotoitems(nextword, nextmod, text, path) end - val = getfield′(m, word) + val = getfield′(mod, word) val isa Module && return [GotoItem(val)] # module goto toplevelitems = toplevelgotoitems(word, mod, path, text) @@ -118,13 +116,24 @@ end ## toplevel goto const PathItemsMaps = Dict{String, Vector{ToplevelItem}} + +""" + Atom.SYMBOLSCACHE + +"module" (`String`) ⟶ "path" (`String`) ⟶ "symbols" (`Vector{ToplevelItem}`) map + +!!! note + "module" should be canonical, i.e.: should be identical to names that are + constructed from `string(mod::Module)`. +""" const SYMBOLSCACHE = Dict{String, PathItemsMaps}() function toplevelgotoitems(word, mod, path, text) - pathitemsmaps = if haskey(SYMBOLSCACHE, mod) - SYMBOLSCACHE[mod] + key = string(mod) + pathitemsmaps = if haskey(SYMBOLSCACHE, key) + SYMBOLSCACHE[key] else - SYMBOLSCACHE[mod] = collecttoplevelitems(mod, path, text) # caching + SYMBOLSCACHE[key] = collecttoplevelitems(mod, path, text) # caching end ismacro(word) && (word = lstrip(word, '@')) @@ -138,9 +147,9 @@ function toplevelgotoitems(word, mod, path, text) end # entry method -function collecttoplevelitems(mod::String, path::String, text::String) +function collecttoplevelitems(mod::Module, path::String, text::String) pathitemsmaps = PathItemsMaps() - return if mod == "Main" || isuntitled(path) + return if mod == Main || isuntitled(path) # for `Main` module and unsaved editors, always use CSTPraser-based approach # with a given buffer text, and don't check module validity _collecttoplevelitems!(nothing, path, text, pathitemsmaps) @@ -150,20 +159,19 @@ function collecttoplevelitems(mod::String, path::String, text::String) end # entry method when called from docpane/workspace -function collecttoplevelitems(mod::String, path::Nothing, text::String) +function collecttoplevelitems(mod::Module, path::Nothing, text::String) pathitemsmaps = PathItemsMaps() _collecttoplevelitems!(mod, pathitemsmaps) end # sub entry method -function _collecttoplevelitems!(mod::String, pathitemsmaps::PathItemsMaps) - m = getmodule(mod) - entrypath, paths = modulefiles(m) +function _collecttoplevelitems!(mod::Module, pathitemsmaps::PathItemsMaps) + entrypath, paths = modulefiles(mod) return if entrypath !== nothing # Revise-like approach _collecttoplevelitems!([entrypath; paths], pathitemsmaps) else # if Revise-like approach fails, fallback to CSTParser-based approach - entrypath, line = moduledefinition(m) - mod = string(last(split(mod, '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` + entrypath, line = moduledefinition(mod) + mod = string(last(split(string(mod), '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` _collecttoplevelitems!(mod, entrypath, pathitemsmaps) end end @@ -247,7 +255,7 @@ function updatesymbols(items, mod, path::Nothing, text) end # fallback case function updatesymbols(items, mod, path::String, text) # initialize the cache if there is no previous one if !haskey(SYMBOLSCACHE, mod) - SYMBOLSCACHE[mod] = collecttoplevelitems(mod, path, text) + SYMBOLSCACHE[mod] = collecttoplevelitems(getmodule(mod), path, text) end push!(SYMBOLSCACHE[mod], path => items) end @@ -276,13 +284,13 @@ function regeneratesymbols() unloadedlen = length(unloaded) total = loadedlen + unloadedlen - for (i, m) in enumerate(Base.loaded_modules_array()) + for (i, mod) in enumerate(Base.loaded_modules_array()) try - mod = string(m) - mod == "__PackagePrecompilationStatementModule" && continue # will cause error + key = string(mod) + key == "__PackagePrecompilationStatementModule" && continue # will cause error - @logmsg -1 "Symbols: $mod ($i / $total)" progress=i/total _id=id - SYMBOLSCACHE[mod] = collecttoplevelitems(mod, nothing, "") + @logmsg -1 "Symbols: $key ($i / $total)" progress=i/total _id=id + SYMBOLSCACHE[key] = collecttoplevelitems(mod, nothing, "") catch err @error err end diff --git a/test/goto.jl b/test/goto.jl index d9adfbf4..0a420947 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -55,11 +55,11 @@ let path = joinpath′(@__DIR__, "..", "src", "comm.jl") text = read(path, String) - items = todict.(globalgotoitems("Atom.handlers", "Atom", path, text)) + items = todict.(globalgotoitems("Atom.handlers", Atom, path, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "handlers" - items = todict.(globalgotoitems("Main.Atom.handlers", "Atom", path, text)) + items = todict.(globalgotoitems("Main.Atom.handlers", Atom, path, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "handlers" @@ -67,18 +67,18 @@ # can access the non-exported (non-method) bindings in the other module path = joinpath′(@__DIR__, "..", "src", "goto.jl") text = read(@__FILE__, String) - items = todict.(globalgotoitems("Atom.SYMBOLSCACHE", "Main", @__FILE__, text)) + items = todict.(globalgotoitems("Atom.SYMBOLSCACHE", Main, @__FILE__, text)) @test !isempty(items) @test items[1][:file] == path @test items[1][:text] == "SYMBOLSCACHE" end @testset "goto modules" begin - let item = globalgotoitems("Atom", "Main", nothing, "")[1] |> todict + let item = globalgotoitems("Atom", Main, nothing, "")[1] |> todict @test item[:file] == joinpath′(atomjldir, "Atom.jl") @test item[:line] == 3 end - let item = globalgotoitems("SubJunk", "Main.Junk", nothing, "")[1] |> todict + let item = globalgotoitems("SubJunk", Junk, nothing, "")[1] |> todict @test item[:file] == subjunkspath @test item[:line] == 3 end @@ -87,7 +87,8 @@ @testset "goto toplevel symbols" begin ## where Revise approach works, i.e.: precompiled modules let path = joinpath′(atomjldir, "comm.jl") - mod = "Atom" + mod = Atom + key = "Atom" word = "handlers" # basic @@ -98,10 +99,10 @@ end # check caching works - @test haskey(SYMBOLSCACHE, mod) + @test haskey(SYMBOLSCACHE, key) # check the Revise-like approach finds all files in Atom module - @test length(SYMBOLSCACHE[mod]) == length(atommodfiles) + @test length(SYMBOLSCACHE[key]) == length(atommodfiles) # when `path` isn't given, i.e. via docpane / workspace let items = todict.(toplevelgotoitems(word, mod, nothing, "")) @@ -111,7 +112,7 @@ end # same as above, but without any previous cache -- falls back to CSTPraser-based module-walk - delete!(SYMBOLSCACHE, mod) + delete!(SYMBOLSCACHE, key) let items = toplevelgotoitems(word, mod, nothing, "") .|> todict @test !isempty(items) @@ -121,12 +122,13 @@ # check CSTPraser-based module-walk finds all the included files # NOTE: webio.jl is excluded since `include("webio.jl")` is a toplevel call - @test length(SYMBOLSCACHE[mod]) == length(atommodfiles) + @test length(SYMBOLSCACHE[key]) == length(atommodfiles) end ## where the Revise-like approach doesn't work, e.g. non-precompiled modules let path = junkpath - mod = "Main.Junk" + mod = Main.Junk + key = "Main.Junk" word = "toplevelval" # basic -- no need to pass a buffer text @@ -138,7 +140,7 @@ end # check caching works - @test haskey(Atom.SYMBOLSCACHE, mod) + @test haskey(Atom.SYMBOLSCACHE, key) # when `path` isn't given, i.e.: via docpane / workspace let items = toplevelgotoitems(word, mod, nothing, "") .|> todict @@ -152,7 +154,7 @@ ## don't include bindings outside of a module let path = subjunkspath text = read(subjunkspath, String) - mod = "Main.Junk.SubJunk" + mod = Main.Junk.SubJunk word = "imwithdoc" items = toplevelgotoitems(word, mod, path, text) .|> todict @@ -167,7 +169,7 @@ ## `Main` module -- use a passed buffer text let path = joinpath′(@__DIR__, "runtests.jl") text = read(path, String) - mod = "Main" + mod = Main word = "atomjldir" items = toplevelgotoitems(word, mod, path, text) .|> todict @@ -186,10 +188,11 @@ end # check there is no cache before updating - mod = "Main.Junk" + mod = Main.Junk + key = "Main.Junk" path = junkpath text = read(path, String) - @test filter(SYMBOLSCACHE[mod][path]) do item + @test filter(SYMBOLSCACHE[key][path]) do item Atom.str_value(item.expr) == "toplevelval2" end |> isempty @@ -198,10 +201,10 @@ newtext = join(originallines[1:end - 1], '\n') word = "toplevelval2" newtext *= "\n$word = :youshoulderaseme\nend" - updatesymbols(mod, path, newtext) + updatesymbols(key, path, newtext) # check the cache is updated - @test filter(SYMBOLSCACHE[mod][path]) do item + @test filter(SYMBOLSCACHE[key][path]) do item Atom.str_value(item.expr) == word end |> !isempty @@ -212,13 +215,13 @@ end # re-update the cache - updatesymbols(mod, path, text) - @test filter(SYMBOLSCACHE[mod][path]) do item + updatesymbols(key, path, text) + @test filter(SYMBOLSCACHE[key][path]) do item Atom.str_value(item.expr) == word end |> isempty # don't error on fallback case - @test_nowarn @test updatesymbols(mod, nothing, text) === nothing + @test_nowarn @test updatesymbols(key, nothing, text) === nothing end @testset "regenerating toplevel symbols" begin @@ -227,7 +230,7 @@ @test haskey(SYMBOLSCACHE, "Base") @test length(keys(SYMBOLSCACHE["Base"])) > 100 @test haskey(SYMBOLSCACHE, "Example") # cache symbols even if not loaded - @test toplevelgotoitems("hello", "Example", "", nothing) |> !isempty + @test toplevelgotoitems("hello", Example, "", nothing) |> !isempty end @testset "clear toplevel symbols" begin @@ -264,13 +267,13 @@ end ## both the original methods and the toplevel bindings that are overloaded in a context module should be shown - let items = globalgotoitems("isconst", "Main.Junk", nothing, "") + let items = globalgotoitems("isconst", Main.Junk, nothing, "") @test length(items) === 2 @test "isconst(m::Module, s::Symbol)" in map(item -> item.text, items) # from Base @test "Base.isconst(::JunkType)" in map(item -> item.text, items) # from Junk end ## don't error on the fallback case - @test_nowarn @test globalgotoitems("word", "Main", nothing, "") == [] + @test_nowarn @test globalgotoitems("word", Main, nothing, "") == [] end end diff --git a/test/runtests.jl b/test/runtests.jl index 26b6dbe0..c445c69b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,4 +1,4 @@ -using Atom, Test, JSON, Logging, CSTParser +using Atom, Test, JSON, Logging, CSTParser, Example joinpath′(files...) = Atom.fullpath(joinpath(files...)) From 99fed3814f0ab97831fa9af735f6d3276337088e Mon Sep 17 00:00:00 2001 From: aviatesk Date: Mon, 4 Nov 2019 22:00:33 +0900 Subject: [PATCH 10/13] don't enter into submodules within Revise-based goto --- src/goto.jl | 8 ++++---- test/static/toplevel.jl | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index 6d24de91..1c9f40af 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -168,7 +168,8 @@ end function _collecttoplevelitems!(mod::Module, pathitemsmaps::PathItemsMaps) entrypath, paths = modulefiles(mod) return if entrypath !== nothing # Revise-like approach - _collecttoplevelitems!([entrypath; paths], pathitemsmaps) + mod = string(last(split(string(mod), '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` + _collecttoplevelitems!(mod, [entrypath; paths], pathitemsmaps) else # if Revise-like approach fails, fallback to CSTParser-based approach entrypath, line = moduledefinition(mod) mod = string(last(split(string(mod), '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` @@ -177,11 +178,10 @@ function _collecttoplevelitems!(mod::Module, pathitemsmaps::PathItemsMaps) end # module-walk via Revise-like approach -function _collecttoplevelitems!(paths::Vector{String}, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems!(mod::Union{Nothing, String}, paths::Vector{String}, pathitemsmaps::PathItemsMaps) for path in paths text = read(path, String) - parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text) + items = toplevelitems(text; mod = mod) push!(pathitemsmaps, path => items) end pathitemsmaps diff --git a/test/static/toplevel.jl b/test/static/toplevel.jl index e8174229..f2800185 100644 --- a/test/static/toplevel.jl +++ b/test/static/toplevel.jl @@ -15,12 +15,12 @@ @test filter(toplevelitems(parsed, text; mod = "Junk")) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" - end |> length === 1 # only `imwithdoc` in Junk module + end |> length === 1 # should only find the `imwithdoc` in Junk module # don't include items outside of a module # FIX: currently broken -- include `imwithdoc` in Junk module as well @test_broken filter(toplevelitems(parsed, text; mod = "SubJunk")) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" - end |> length === 1 # only `imwithdoc` in Junk module + end |> length === 1 # should only find the `imwithdoc` in SubJunk module end From 904cf6a3e454cf48518d6552e3e660d2ab6f4801 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Mon, 4 Nov 2019 22:02:26 +0900 Subject: [PATCH 11/13] better `toplevelitems` API --- src/goto.jl | 3 +-- src/modules.jl | 3 +-- src/outline.jl | 3 +-- src/static/toplevel.jl | 22 ++++++++++++++++++---- test/goto.jl | 3 +-- test/outline.jl | 6 +----- test/static/toplevel.jl | 6 +++--- 7 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index 1c9f40af..0d5a40a3 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -194,8 +194,7 @@ function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, _collecttoplevelitems!(mod, entrypath, text, pathitemsmaps) end function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, text::String, pathitemsmaps::PathItemsMaps) - parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text; mod = mod) + items = toplevelitems(text; mod = mod) push!(pathitemsmaps, entrypath => items) # looking for toplevel `include` calls diff --git a/src/modules.jl b/src/modules.jl index f9f8c2e1..b2261707 100644 --- a/src/modules.jl +++ b/src/modules.jl @@ -199,8 +199,7 @@ function modulefiles(mod::String, entrypath::String, files = Vector{String}()) push!(files, entrypath) text = read(entrypath, String) - parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text; mod = mod) + items = toplevelitems(text; mod = mod) for item in items if item isa ToplevelCall diff --git a/src/outline.jl b/src/outline.jl index 6dc4fbc5..c47af17e 100644 --- a/src/outline.jl +++ b/src/outline.jl @@ -15,8 +15,7 @@ end # NOTE: update outline and symbols cache all in one go function updateeditor(text, mod = "Main", path = nothing, updateSymbols = true) - parsed = CSTParser.parse(text, true) - items = toplevelitems(parsed, text) + items = toplevelitems(text) # update symbols cache # ref: https://github.com/JunoLab/Juno.jl/issues/407 diff --git a/src/static/toplevel.jl b/src/static/toplevel.jl index c59f2db6..8f0596a1 100644 --- a/src/static/toplevel.jl +++ b/src/static/toplevel.jl @@ -24,10 +24,24 @@ struct ToplevelTupleH <: ToplevelItem lines::UnitRange{Int} end -function toplevelitems( - expr, text, # necessary +""" + toplevelitems(text; kwargs...) + +Returns [`ToplevelItem`](@ref)s in `text`. + +keyword arguments: +- `mod::Union{Nothing, String}`: if not `nothing` don't return items within modules + other than `mod`, otherwise enter into every module. +""" +function toplevelitems(text; kwargs...) + parsed = CSTParser.parse(text, true) + _toplevelitems(text, parsed; kwargs...) +end + +function _toplevelitems( + text, expr, items::Vector{ToplevelItem} = Vector{ToplevelItem}(), line = 1, pos = 1; - mod::Union{Nothing, String} = nothing, # if given, don't enter into modules other than `mod` + mod::Union{Nothing, String} = nothing, ) # binding bind = CSTParser.bindingof(expr) @@ -50,7 +64,7 @@ function toplevelitems( if shouldenter(expr, mod) if expr.args !== nothing for arg in expr.args - toplevelitems(arg, text, items, line, pos; mod = mod) + _toplevelitems(text, arg, items, line, pos; mod = mod) line += countlines(arg, text, pos) pos += arg.fullspan end diff --git a/test/goto.jl b/test/goto.jl index 0a420947..5df1e6a6 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -182,8 +182,7 @@ @testset "updating toplevel symbols" begin function updatesymbols(mod, path, text) - parsed = CSTParser.parse(text, true) - items = Atom.toplevelitems(parsed, text) + items = Atom.toplevelitems(text) Atom.updatesymbols(items, mod, path, text) end diff --git a/test/outline.jl b/test/outline.jl index 1e36530b..2d5bc78e 100644 --- a/test/outline.jl +++ b/test/outline.jl @@ -1,9 +1,5 @@ @testset "outline" begin - function outline(str) - parsed = CSTParser.parse(str, true) - items = Atom.toplevelitems(parsed, str) - Atom.outline(items) - end + outline(text) = Atom.outline(Atom.toplevelitems(text)) let str = """ module Foo diff --git a/test/static/toplevel.jl b/test/static/toplevel.jl index f2800185..c3ec0f3e 100644 --- a/test/static/toplevel.jl +++ b/test/static/toplevel.jl @@ -6,20 +6,20 @@ parsed = CSTParser.parse(text, true) # basic -- finds every toplevel items when `mod` options is `nothing` (default) - @test filter(toplevelitems(parsed, text; mod = nothing)) do item + @test filter(toplevelitems(text; mod = nothing)) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" end |> length === 3 # don't enter non-target modules, e.g.: submodules - @test filter(toplevelitems(parsed, text; mod = "Junk")) do item + @test filter(toplevelitems(text; mod = "Junk")) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" end |> length === 1 # should only find the `imwithdoc` in Junk module # don't include items outside of a module # FIX: currently broken -- include `imwithdoc` in Junk module as well - @test_broken filter(toplevelitems(parsed, text; mod = "SubJunk")) do item + @test_broken filter(toplevelitems(text; mod = "SubJunk")) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" end |> length === 1 # should only find the `imwithdoc` in SubJunk module From c0ec452f834ed2646eb5246ad80c71c018c71508 Mon Sep 17 00:00:00 2001 From: aviatesk Date: Tue, 5 Nov 2019 03:41:41 +0900 Subject: [PATCH 12/13] implements https://github.com/JunoLab/Atom.jl/pull/215#issuecomment-549272498 - fixes two broken test cases --- src/goto.jl | 47 +++++++++++++++++++++++++---------------- src/modules.jl | 7 +++--- src/outline.jl | 6 ++---- src/static/toplevel.jl | 21 ++++++++++++------ src/utils.jl | 3 +++ test/goto.jl | 9 ++------ test/static/toplevel.jl | 10 ++++----- 7 files changed, 59 insertions(+), 44 deletions(-) diff --git a/src/goto.jl b/src/goto.jl index 0d5a40a3..e4140df4 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -97,13 +97,13 @@ function globalgotoitems(word, mod, path, text) val = getfield′(mod, word) val isa Module && return [GotoItem(val)] # module goto - toplevelitems = toplevelgotoitems(word, mod, path, text) + items = toplevelgotoitems(word, mod, path, text) # append method gotos that are not caught by `toplevelgotoitems` ml = methods(val) - files = map(item -> item.file, toplevelitems) + files = map(item -> item.file, items) methoditems = filter!(item -> item.file ∉ files, methodgotoitems(ml)) - append!(toplevelitems, methoditems) + append!(items, methoditems) end ## module goto @@ -168,33 +168,36 @@ end function _collecttoplevelitems!(mod::Module, pathitemsmaps::PathItemsMaps) entrypath, paths = modulefiles(mod) return if entrypath !== nothing # Revise-like approach - mod = string(last(split(string(mod), '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` - _collecttoplevelitems!(mod, [entrypath; paths], pathitemsmaps) + _collecttoplevelitems!(stripdotprefixes(string(mod)), entrypath, paths, pathitemsmaps) else # if Revise-like approach fails, fallback to CSTParser-based approach entrypath, line = moduledefinition(mod) - mod = string(last(split(string(mod), '.'))) # strip parent module prefixes e.g.: `"Main.Junk"` - _collecttoplevelitems!(mod, entrypath, pathitemsmaps) + _collecttoplevelitems!(stripdotprefixes(string(mod)), entrypath, pathitemsmaps) end end # module-walk via Revise-like approach -function _collecttoplevelitems!(mod::Union{Nothing, String}, paths::Vector{String}, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, paths::Vector{String}, pathitemsmaps::PathItemsMaps) + # ignore toplevel items outside of `mod` + items = toplevelitems(read(entrypath, String); mod = mod) + push!(pathitemsmaps, entrypath => items) + for path in paths - text = read(path, String) - items = toplevelitems(text; mod = mod) + # collect symbols in included files (always in `mod`) + items = toplevelitems(read(path, String); mod = mod, inmod = true) push!(pathitemsmaps, path => items) end + pathitemsmaps end # module-walk based on CSTParser, looking for toplevel `installed` calls -function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, pathitemsmaps::PathItemsMaps; inmod = false) isfile′(entrypath) || return text = read(entrypath, String) - _collecttoplevelitems!(mod, entrypath, text, pathitemsmaps) + _collecttoplevelitems!(mod, entrypath, text, pathitemsmaps; inmod = inmod) end -function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, text::String, pathitemsmaps::PathItemsMaps) - items = toplevelitems(text; mod = mod) +function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, text::String, pathitemsmaps::PathItemsMaps; inmod = false) + items = toplevelitems(text; mod = mod, inmod = inmod) push!(pathitemsmaps, entrypath => items) # looking for toplevel `include` calls @@ -205,7 +208,8 @@ function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, nextfile = expr.args[3].val nextentrypath = joinpath(dirname(entrypath), nextfile) isfile′(nextentrypath) || continue - _collecttoplevelitems!(mod, nextentrypath, pathitemsmaps) + # `nextentrypath` is always in `mod` + _collecttoplevelitems!(mod, nextentrypath, pathitemsmaps; inmod = true) end end end @@ -250,12 +254,19 @@ end ## update toplevel symbols cache # NOTE: handled by the `updateeditor` handler in outline.jl -function updatesymbols(items, mod, path::Nothing, text) end # fallback case -function updatesymbols(items, mod, path::String, text) +function updatesymbols(mod, path::Nothing, text) end # fallback case +function updatesymbols(mod, path::String, text) + m = getmodule(mod) + # initialize the cache if there is no previous one if !haskey(SYMBOLSCACHE, mod) - SYMBOLSCACHE[mod] = collecttoplevelitems(getmodule(mod), path, text) + SYMBOLSCACHE[mod] = collecttoplevelitems(m, path, text) end + + # ignore toplevel items outside of `mod` when `path` is an entry file + entrypath, _ = moduledefinition(m) + inmod = path != entrypath + items = toplevelitems(text; mod = stripdotprefixes(mod), inmod = inmod) push!(SYMBOLSCACHE[mod], path => items) end diff --git a/src/modules.jl b/src/modules.jl index b2261707..d3707545 100644 --- a/src/modules.jl +++ b/src/modules.jl @@ -193,13 +193,13 @@ Returns all the files in `mod` module that can be reached via [`include`](@ref) Note this function currently only looks for static toplevel calls (i.e. miss the calls in non-toplevel scope). """ -function modulefiles(mod::String, entrypath::String, files = Vector{String}()) +function modulefiles(mod::String, entrypath::String, files = Vector{String}(); inmod = false) isfile′(entrypath) || return files push!(files, entrypath) text = read(entrypath, String) - items = toplevelitems(text; mod = mod) + items = toplevelitems(text; mod = mod, inmod = inmod) for item in items if item isa ToplevelCall @@ -208,7 +208,8 @@ function modulefiles(mod::String, entrypath::String, files = Vector{String}()) nextfile = expr.args[3].val nextentrypath = joinpath(dirname(entrypath), nextfile) isfile(nextentrypath) || continue - modulefiles(mod, nextentrypath, files) + # `nextentrypath` is always in `mod` + modulefiles(mod, nextentrypath, files; inmod = true) end end end diff --git a/src/outline.jl b/src/outline.jl index c47af17e..d23c8493 100644 --- a/src/outline.jl +++ b/src/outline.jl @@ -15,14 +15,12 @@ end # NOTE: update outline and symbols cache all in one go function updateeditor(text, mod = "Main", path = nothing, updateSymbols = true) - items = toplevelitems(text) - # update symbols cache # ref: https://github.com/JunoLab/Juno.jl/issues/407 - updateSymbols && updatesymbols(items, mod, path, text) + updateSymbols && updatesymbols(mod, path, text) # return outline - outline(items) + outline(toplevelitems(text)) end function outline(items) diff --git a/src/static/toplevel.jl b/src/static/toplevel.jl index 8f0596a1..abfff75d 100644 --- a/src/static/toplevel.jl +++ b/src/static/toplevel.jl @@ -25,13 +25,14 @@ struct ToplevelTupleH <: ToplevelItem end """ - toplevelitems(text; kwargs...) + toplevelitems(text; kwargs...)::Vector{ToplevelItem} -Returns [`ToplevelItem`](@ref)s in `text`. +Finds and returns toplevel "item"s (call and binding) in `text`. keyword arguments: - `mod::Union{Nothing, String}`: if not `nothing` don't return items within modules other than `mod`, otherwise enter into every module. +- `inmod::Bool`: if `true`, don't include toplevel items until it enters into `mod`. """ function toplevelitems(text; kwargs...) parsed = CSTParser.parse(text, true) @@ -42,10 +43,13 @@ function _toplevelitems( text, expr, items::Vector{ToplevelItem} = Vector{ToplevelItem}(), line = 1, pos = 1; mod::Union{Nothing, String} = nothing, + inmod::Bool = false, ) + shouldadd = mod === nothing || inmod + # binding bind = CSTParser.bindingof(expr) - if bind !== nothing + if bind !== nothing && shouldadd lines = line:line+countlines(expr, text, pos, false) push!(items, ToplevelBinding(expr, bind, lines)) end @@ -53,18 +57,23 @@ function _toplevelitems( lines = line:line+countlines(expr, text, pos, false) # toplevel call - if iscallexpr(expr) + if iscallexpr(expr) && shouldadd push!(items, ToplevelCall(expr, lines, str_value_as_is(expr, text, pos))) end # destructure multiple returns - ismultiplereturn(expr) && push!(items, ToplevelTupleH(expr, lines)) + if ismultiplereturn(expr) && shouldadd + push!(items, ToplevelTupleH(expr, lines)) + end # look for more toplevel items in expr: if shouldenter(expr, mod) if expr.args !== nothing + if ismodule(expr) && shouldentermodule(expr, mod) + inmod = true + end for arg in expr.args - _toplevelitems(text, arg, items, line, pos; mod = mod) + _toplevelitems(text, arg, items, line, pos; mod = mod, inmod = inmod) line += countlines(arg, text, pos) pos += arg.fullspan end diff --git a/src/utils.jl b/src/utils.jl index d751ffd4..1c14fe21 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -146,6 +146,9 @@ end shortstr(val) = strlimit(string(val), 20) +"""used to strip parent module prefixes e.g.: `"Main.Junk" ⟶ "Junk"`""" +stripdotprefixes(str::AbstractString) = string(last(split(str, '.'))) + """ Undefined diff --git a/test/goto.jl b/test/goto.jl index 5df1e6a6..55a71218 100644 --- a/test/goto.jl +++ b/test/goto.jl @@ -48,7 +48,7 @@ end @testset "goto global symbols" begin - using Atom: globalgotoitems, toplevelgotoitems, SYMBOLSCACHE, + using Atom: globalgotoitems, toplevelgotoitems, SYMBOLSCACHE, updatesymbols, clearsymbols, regeneratesymbols, methodgotoitems ## strip a dot-accessed modules @@ -158,7 +158,7 @@ word = "imwithdoc" items = toplevelgotoitems(word, mod, path, text) .|> todict - @test_broken length(items) === 1 # broken since it finds `imwithdoc` in `Junk` as well + @test length(items) === 1 if length(items) === 1 @test items[1][:file] == path @test items[1][:line] == 6 @@ -181,11 +181,6 @@ end @testset "updating toplevel symbols" begin - function updatesymbols(mod, path, text) - items = Atom.toplevelitems(text) - Atom.updatesymbols(items, mod, path, text) - end - # check there is no cache before updating mod = Main.Junk key = "Main.Junk" diff --git a/test/static/toplevel.jl b/test/static/toplevel.jl index c3ec0f3e..bea6ca1c 100644 --- a/test/static/toplevel.jl +++ b/test/static/toplevel.jl @@ -3,23 +3,21 @@ path = subjunkspath text = read(path, String) - parsed = CSTParser.parse(text, true) - # basic -- finds every toplevel items when `mod` options is `nothing` (default) - @test filter(toplevelitems(text; mod = nothing)) do item + # basic -- finds every toplevel items with default arguments + @test filter(toplevelitems(text)) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" end |> length === 3 # don't enter non-target modules, e.g.: submodules - @test filter(toplevelitems(text; mod = "Junk")) do item + @test filter(toplevelitems(text; mod = "Junk", inmod = true)) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" end |> length === 1 # should only find the `imwithdoc` in Junk module # don't include items outside of a module - # FIX: currently broken -- include `imwithdoc` in Junk module as well - @test_broken filter(toplevelitems(text; mod = "SubJunk")) do item + @test filter(toplevelitems(text; mod = "SubJunk", inmod = false)) do item item isa Atom.ToplevelBinding && item.bind.name == "imwithdoc" end |> length === 1 # should only find the `imwithdoc` in SubJunk module From e1752330ea9228a338c3da42c096caabe6cd454b Mon Sep 17 00:00:00 2001 From: aviatesk Date: Tue, 5 Nov 2019 11:50:37 +0900 Subject: [PATCH 13/13] minor refactors --- src/Atom.jl | 8 ++++---- src/goto.jl | 46 ++++++++++++++++++++++------------------------ 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/Atom.jl b/src/Atom.jl index d77ed850..a7a4e78a 100644 --- a/src/Atom.jl +++ b/src/Atom.jl @@ -22,12 +22,12 @@ function __init__() # HACK: overloading this allows us to open remote files InteractiveUtils.eval(quote - function InteractiveUtils.edit(path::AbstractString, line::Integer=0) + function InteractiveUtils.edit(path::AbstractString, line::Integer = 0) if endswith(path, ".jl") - f = Base.find_source_file(path) - f !== nothing && (path = f) + f = Base.find_source_file(path) + f !== nothing && (path = f) end - $(msg)("openFile", Base.abspath(path), line-1) + $(msg)("openFile", Base.abspath(path), line - 1) end end) end diff --git a/src/goto.jl b/src/goto.jl index e4140df4..3c37ded8 100644 --- a/src/goto.jl +++ b/src/goto.jl @@ -120,7 +120,7 @@ const PathItemsMaps = Dict{String, Vector{ToplevelItem}} """ Atom.SYMBOLSCACHE -"module" (`String`) ⟶ "path" (`String`) ⟶ "symbols" (`Vector{ToplevelItem}`) map +"module" (`String`) ⟶ "path" (`String`) ⟶ "symbols" (`Vector{ToplevelItem}`) map. !!! note "module" should be canonical, i.e.: should be identical to names that are @@ -146,43 +146,41 @@ function toplevelgotoitems(word, mod, path, text) return ret end -# entry method +# entry methods function collecttoplevelitems(mod::Module, path::String, text::String) - pathitemsmaps = PathItemsMaps() return if mod == Main || isuntitled(path) # for `Main` module and unsaved editors, always use CSTPraser-based approach # with a given buffer text, and don't check module validity - _collecttoplevelitems!(nothing, path, text, pathitemsmaps) + __collecttoplevelitems(nothing, path, text) else - _collecttoplevelitems!(mod, pathitemsmaps) + _collecttoplevelitems(mod) end end +# when `path === nothing`, e.g.: called from docpane/workspace +collecttoplevelitems(mod::Module, path::Nothing, text::String) = _collecttoplevelitems(mod) -# entry method when called from docpane/workspace -function collecttoplevelitems(mod::Module, path::Nothing, text::String) - pathitemsmaps = PathItemsMaps() - _collecttoplevelitems!(mod, pathitemsmaps) -end - -# sub entry method -function _collecttoplevelitems!(mod::Module, pathitemsmaps::PathItemsMaps) +function _collecttoplevelitems(mod::Module) entrypath, paths = modulefiles(mod) return if entrypath !== nothing # Revise-like approach - _collecttoplevelitems!(stripdotprefixes(string(mod)), entrypath, paths, pathitemsmaps) + __collecttoplevelitems(stripdotprefixes(string(mod)), [entrypath; paths]) else # if Revise-like approach fails, fallback to CSTParser-based approach entrypath, line = moduledefinition(mod) - _collecttoplevelitems!(stripdotprefixes(string(mod)), entrypath, pathitemsmaps) + __collecttoplevelitems(stripdotprefixes(string(mod)), entrypath) end end # module-walk via Revise-like approach -function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, paths::Vector{String}, pathitemsmaps::PathItemsMaps) +function __collecttoplevelitems(mod::Union{Nothing, String}, paths::Vector{String}) + pathitemsmaps = PathItemsMaps() + + entrypath, paths = paths[1], paths[2:end] + # ignore toplevel items outside of `mod` items = toplevelitems(read(entrypath, String); mod = mod) push!(pathitemsmaps, entrypath => items) + # collect symbols in included files (always in `mod`) for path in paths - # collect symbols in included files (always in `mod`) items = toplevelitems(read(path, String); mod = mod, inmod = true) push!(pathitemsmaps, path => items) end @@ -190,13 +188,13 @@ function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, pathitemsmaps end -# module-walk based on CSTParser, looking for toplevel `installed` calls -function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, pathitemsmaps::PathItemsMaps; inmod = false) +# module-walk based on CSTParser, looking for toplevel `included` calls +function __collecttoplevelitems(mod::Union{Nothing, String}, entrypath::String, pathitemsmaps::PathItemsMaps = PathItemsMaps(); inmod = false) isfile′(entrypath) || return text = read(entrypath, String) - _collecttoplevelitems!(mod, entrypath, text, pathitemsmaps; inmod = inmod) + __collecttoplevelitems(mod, entrypath, text, pathitemsmaps; inmod = inmod) end -function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, text::String, pathitemsmaps::PathItemsMaps; inmod = false) +function __collecttoplevelitems(mod::Union{Nothing, String}, entrypath::String, text::String, pathitemsmaps::PathItemsMaps = PathItemsMaps(); inmod = false) items = toplevelitems(text; mod = mod, inmod = inmod) push!(pathitemsmaps, entrypath => items) @@ -209,7 +207,7 @@ function _collecttoplevelitems!(mod::Union{Nothing, String}, entrypath::String, nextentrypath = joinpath(dirname(entrypath), nextfile) isfile′(nextentrypath) || continue # `nextentrypath` is always in `mod` - _collecttoplevelitems!(mod, nextentrypath, pathitemsmaps; inmod = true) + __collecttoplevelitems(mod, nextentrypath, pathitemsmaps; inmod = true) end end end @@ -300,7 +298,7 @@ function regeneratesymbols() key == "__PackagePrecompilationStatementModule" && continue # will cause error @logmsg -1 "Symbols: $key ($i / $total)" progress=i/total _id=id - SYMBOLSCACHE[key] = collecttoplevelitems(mod, nothing, "") + SYMBOLSCACHE[key] = _collecttoplevelitems(mod) catch err @error err end @@ -310,7 +308,7 @@ function regeneratesymbols() try @logmsg -1 "Symbols: $pkg ($(i + loadedlen) / $total)" progress=(i+loadedlen)/total _id=id path = Base.find_package(pkg) - SYMBOLSCACHE[pkg] = _collecttoplevelitems!(pkg, path, PathItemsMaps()) + SYMBOLSCACHE[pkg] = __collecttoplevelitems(pkg, path) catch err @error err end