diff --git a/docs/languages.md b/docs/languages.md index af6324ff..a07cc0dd 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -84,7 +84,7 @@ on interface nodes stores the expected method set for implementation matching. | Dart | Full | Full | Classes/Enums/Mixins/Extensions | Abstract interface | Full | Full | Full | | OCaml | Full | Full (class) | Types/Modules | Module types | open | Full | Full | | Lua | Full | Full (M.func/M:method) | - | - | require() | Full | Full | -| Julia | Full (long + short form, `where` syntax) | Full (qualified `Base.show`, operators) | Structs/abstract/primitive + fields | - | Full (`using`/`import`/`include`, selective lists) | Full (incl. broadcast, macro calls) | `const` | +| Julia | Full (long + short form, `where` syntax) | Full (qualified `Base.show`, operators) | Structs/abstract/primitive + fields | - | Full (`using`/`import`/`include`, selective lists incl. macros/operators) | Full (incl. broadcast, macro calls, `Vector{Int}` constructors) | `const` (with `member_of`) | ### Rust specifics @@ -113,7 +113,11 @@ Recent extraction refinements (each covered by a per-feature CI golden): Java `@ `module` / `baremodule` index as `KindType` nodes (the graph's `KindModule` is reserved for ecosystem packages) and carry the module's `export` list in -`Meta["exports"]`; definitions inside a module get `member_of` edges to it. +`Meta["exports"]` — including exported macros and operators, recorded +verbatim (`export @m, ⊗` records `@m` and `⊗`) — and the Julia 1.11 +`public` list in `Meta["public"]` (public-without-reexport, operators +included); definitions, constants and +nested modules inside a module get `member_of` edges to it. Node ids stay flat — the enclosing module rides on `Meta["scope_mod"]`, the Rust `mod` convention — and two definitions that would collide on one id (`f` in module `A` and `f` in module `B`) separate through the shared @@ -139,18 +143,53 @@ the unaliased spelling produces. Nothing in the resolver reads the import `Meta` — the edge targets are the consumable surface. All imports including `include("file.jl")` target `unresolved::import::`. Calls attribute to the enclosing function-like definition (long form, short -form, macro, or nested closure) and cover qualified (`Mod.f`) and broadcast -(`f.(x)`, `Meta["broadcast"]`) callees plus macro invocations -(`Meta["macro"]`). Operator calls (`a + b`) are not emitted — dispatch on -operators is not statically attributable. Docstrings attach as +form, macro, or nested closure) and cover qualified (`Mod.f`), +parametric-constructor (`Vector{Int}(xs)` → `unresolved::Vector{Int}`), and +broadcast (`f.(x)`, `Meta["broadcast"]`) callees, plus bare and +module-qualified macro invocations (`Base.@time` → `unresolved::Base.time`, +`Meta["macro"]`). A callee chained onto a call result (`get(cfg).run(x)`) +has no decodable receiver and records its method name (`unresolved::run`); +quoted operators normalise (`Base.:+` and `Base.:(==)` → +`unresolved::Base.+` / `unresolved::Base.==`). Docstrings attach as `Meta["doc"]` to long and short definitions, types, modules and constants, but only when the string sits immediately above the documented object — the adjacency Julia itself enforces, where a blank line or an own-line comment detaches the string and leaves the definition undocumented. A string at the top of a function body is executable code, not -documentation. The stored text is the first PROSE paragraph, skipping the +documentation. The explicit `@doc "text" object` form (which is what +Julia lowers every docstring to) attaches the same way, with the text +taken from inside the macro call. The stored text is the first PROSE +paragraph, skipping the indented signature block Julia's convention opens a docstring with. +What is **not** covered: + +- **Calls inside anonymous functions and do-blocks** + (`double = x -> f(x)`, `map(xs) do y g(y) end`) attribute to the + ENCLOSING function — source locality outranks a closure node for the + graph's consumers, so the closure itself mints no node. A short-form + definition nested in a block (`nested() = 1`) is still its own node. +- **`@enum` members** are not extracted — the macro generates the enum + type and its member constants at runtime. +- **`@.`** records no macro edge (a target named `.` is meaningless); + calls inside its arguments edge normally. +- **Operator calls in infix position** (`a + b`) — dispatch on operators is + not statically attributable. Explicit call syntax (`Base.:+(a, b)`) is a + normal call. +- **Typed constant declarations** (`const X::Int = 1`) mint no variable node. +- **Parametric constructor definitions** (`Box{T}(x) where T = …`) are + extracted as plain functions named `Box{T}` — not with the + `.` constructor spelling, and not bound to `Box`. +- **Callable-object definitions** (`(f::Box)(x) = …`) are not extracted. +- **Calls inside string interpolation** (`"$(f(x))"`) are not walked. +- **Two methods of one name on one physical line** + (`g(x) = h(x); g(y) = k(y)`) collapse onto one node — line numbers cannot + separate them — though each body's call edges are preserved. +- **Call and macro edges are extraction-side facts**: targets are + `unresolved::` names, and binding them to definitions in another file — + including qualified calls into another file's module, and constructor + call sites to `.` — is resolver work, not attempted here. + ## Data, config, build | Language | Extensions | What it extracts | diff --git a/internal/indexer/extractor_version.go b/internal/indexer/extractor_version.go index 76742b4d..d01d6260 100644 --- a/internal/indexer/extractor_version.go +++ b/internal/indexer/extractor_version.go @@ -45,7 +45,7 @@ var extractorVersions = map[string]int{ "go": 3, // generic instantiations are marked so indexing a func value cannot bind (was: generic calls emit call edges) "cpp": 2, // templated and namespace-qualified calls emit call edges "swift": 2, // generic calls and ordinary member calls emit call edges - "julia": 2, // bespoke tree-sitter extractor replaces the regex extractor (fields, exports, qualified methods, broadcast/macro calls) + "julia": 3, // callee and macro decoding from CST children (chained/parametric callees, Base.@time, Base.:(==)), macro/operator exports, member_of for module consts and nested modules, @doc and public metadata (was: bespoke tree-sitter extractor replaced the regex extractor) } // extractorSaltExtLang maps a lower-case file extension to the language diff --git a/internal/indexer/extractor_version_test.go b/internal/indexer/extractor_version_test.go index 19e70e0b..6551c502 100644 --- a/internal/indexer/extractor_version_test.go +++ b/internal/indexer/extractor_version_test.go @@ -206,6 +206,14 @@ func TestStaleLangsDetection(t *testing.T) { snapshotJSON(t, nil, "julia")); !reflect.DeepEqual(got, []string{"julia"}) { t.Errorf("previous-release snapshot without julia = %v, want [julia]", got) } + // A store extracted by the previous Julia extractor version must + // re-extract unchanged .jl files too: the callee decoder, macro + // and export handling, containment edges, and docstring metadata + // all changed what the graph records without any content change. + if got := ExtractorVersionStaleLangs( + snapshotJSON(t, map[string]int{"julia": 2})); !reflect.DeepEqual(got, []string{"julia"}) { + t.Errorf("snapshot stored at the previous julia version = %v, want [julia]", got) + } if got := extractorVersionsSnapshot()[postExtractionPolicySnapshotKey]; got != postExtractionPolicyVersion { t.Errorf("persisted policy epoch = %d, want %d", got, postExtractionPolicyVersion) } @@ -238,7 +246,7 @@ func TestStaleLangsDetection(t *testing.T) { // The Merkle half of the Julia bump: without the .jl → julia // mapping the leaf salt stays empty and Merkle mode misses the // bump the same way the mtime path did. - if got, want := merkleSaltFor("src/model.jl"), policySalt+"|julia@2"; got != want { + if got, want := merkleSaltFor("src/model.jl"), policySalt+"|julia@3"; got != want { t.Errorf("Julia extractor salt = %q, want %q", got, want) } }) diff --git a/internal/parser/languages/julia.go b/internal/parser/languages/julia.go index 75a75f5a..ed92c247 100644 --- a/internal/parser/languages/julia.go +++ b/internal/parser/languages/julia.go @@ -27,7 +27,8 @@ import ( // (`<: Living` → EdgeExtends), plus struct fields (KindField), // including the `x::T = default` form `Base.@kwdef` requires // - `module` / `baremodule` — KindType node whose Meta carries the -// module's `export` list; definitions inside get EdgeMemberOf +// module's `export` list; definitions, constants and nested modules +// inside get EdgeMemberOf // - `const X = ...` constants (KindVariable) // // Node ids stay flat (`::`, `::.`) as in @@ -56,6 +57,9 @@ import ( // Docstrings — a string literal on the line DIRECTLY above a definition, // which is the adjacency Julia itself requires — attach as Meta["doc"], // on long and short definitions, types, modules and constants alike. +// The explicit `@doc "text" object` / `Core.@doc "text" object` form +// attaches the same way, with the string taken from inside the macro +// call. type JuliaExtractor struct { lang *sitter.Language } @@ -113,6 +117,13 @@ type juliaWalkState struct { // several modules giving the same short nickname to different // packages. importAliases map[string]string + // modules maps a lexical scope + module name to the minted module + // id, so `module M … end; function M.f() … end` binds the qualified + // method to the real module node instead of an invented + // unresolved::M. Keyed like the type and alias tables — modules are + // KindType nodes, indistinguishable from structs by kind, so a + // receiver that is a module needs its own table to resolve through. + modules map[string]string } func (e *JuliaExtractor) Extract(filePath string, src []byte) (*parser.ExtractionResult, error) { @@ -141,6 +152,7 @@ func (e *JuliaExtractor) Extract(filePath string, src []byte) (*parser.Extractio types: map[string]string{}, declaredTypes: map[string]bool{}, importAliases: map[string]string{}, + modules: map[string]string{}, } juliaPrescan(root, src, "", st) e.walk(root, src, juliaScope{}, st) @@ -236,6 +248,9 @@ func (e *JuliaExtractor) walkFrom(n *sitter.Node, src []byte, scope juliaScope, case "export_statement": e.handleExport(c, src, scope, st) + case "public_statement": + e.handlePublic(c, src, scope, st) + case "call_expression", "broadcast_call_expression": e.handleCall(c, src, scope, st) e.walk(c, src, scope, st) @@ -253,10 +268,68 @@ func (e *JuliaExtractor) walkFrom(n *sitter.Node, src []byte, scope juliaScope, // walkMacroArgs walks a macro call's arguments, carrying a docstring that // sat above the macro CALL into the definition it wraps. +// juliaDocMacroArg reports the docstring carried INSIDE an explicit +// `@doc "text" object` / `Core.@doc "text" object` call. Julia lowers +// every docstring — triple-quoted or explicit — through Core.@doc, so +// the string beside the object in the macro call is that object's +// documentation, not an argument of anything. Returns false when the +// call is not a doc form or carries no string. +func juliaDocMacroArg(n *sitter.Node, src []byte) (string, bool) { + var args *sitter.Node + isDoc := false + for c := range n.NamedChildren() { + switch c.Type() { + case "macro_identifier": + for m := range c.NamedChildren() { + if m.Type() == "identifier" && m.Content(src) == "doc" { + isDoc = true + } + } + case "field_expression": + count := int(c.NamedChildCount()) + if count < 2 { + continue + } + // Only the standard-library documentation macros lower a + // docstring: `Core.@doc` and `Base.@doc` (bare `@doc` is the + // macro_identifier case above). A user macro that merely ends + // in `.@doc`, like `Foo.@doc`, is something else and must not + // hijack the docstring slot. + base, prop := c.NamedChild(0), c.NamedChild(count-1) + if base.Type() != "identifier" { + continue + } + if mod := base.Content(src); mod != "Core" && mod != "Base" { + continue + } + for m := range prop.NamedChildren() { + if m.Type() == "identifier" && m.Content(src) == "doc" { + isDoc = true + } + } + case "macro_argument_list": + args = c + } + } + if !isDoc || args == nil { + return "", false + } + for a := range args.NamedChildren() { + if a.Type() == "string_literal" { + return juliaDocText(a, src), true + } + return "", false + } + return "", false +} + // `Base.@kwdef struct S ... end` is a documented struct whose docstring // attaches to the wrapper, so stopping at the macro boundary would leave // the single most common documented struct form undocumented. func (e *JuliaExtractor) walkMacroArgs(n *sitter.Node, src []byte, scope juliaScope, st *juliaWalkState, doc string) { + if inner, ok := juliaDocMacroArg(n, src); ok && doc == "" { + doc = inner + } for c := range n.NamedChildren() { if doc == "" || c.Type() != "macro_argument_list" { e.walk(c, src, scope, st) @@ -270,7 +343,37 @@ func (e *JuliaExtractor) walkMacroArgs(n *sitter.Node, src []byte, scope juliaSc e.handleFunction(a, src, scope, st, doc) case "module_definition": e.handleModule(a, src, scope, st, doc) + case "assignment": + // `@doc "text" f(x) = x` documents a short-form + // definition, which arrives as a plain assignment. + e.handleAssignment(a, src, scope, st, false, doc) + case "const_statement": + // `@doc "text" const X = 1` documents a constant, which + // arrives wrapped in a const_statement — dispatch its + // inner assignment AS const, the shape walkFrom gives a + // top-level constant, so neither the constant nor its doc + // is dropped. + for inner := range a.NamedChildren() { + if inner.Type() == "assignment" { + e.handleAssignment(inner, src, scope, st, true, doc) + } + } default: + // Walk a macro argument the way the generic walker + // would, dispatching the argument's own kind before + // its children: walk() visits a node's children but + // never the node itself, so a call_expression argument + // was never shown to handleCall. `@everywhere + // include("f.jl")` under a docstring is the shape that + // loses its edge — call edges otherwise need an + // enclosing function, which a documented (module- or + // file-level) macro call never has. + switch a.Type() { + case "call_expression", "broadcast_call_expression": + e.handleCall(a, src, scope, st) + case "macrocall_expression": + e.handleMacroCall(a, src, scope, st) + } e.walk(a, src, scope, st) continue } @@ -311,10 +414,23 @@ func (e *JuliaExtractor) handleModule(n *sitter.Node, src []byte, scope juliaSco } st.result.Nodes = append(st.result.Nodes, node) st.nodes[id] = node + // Record the module by lexical scope so a later qualified + // method (`function M.f()`) in the same file resolves it as + // the receiver's owner rather than inventing unresolved::M. + st.modules[juliaTypeKey(scope.modulePath, name)] = id st.result.Edges = append(st.result.Edges, &graph.Edge{ From: st.fileNode.ID, To: id, Kind: graph.EdgeDefines, FilePath: st.filePath, Line: line, }) + // A nested module belongs to its parent just as any other + // resident does; without the edge, a traversal from Outer + // stops at functions and types and never reaches Inner. + if scope.moduleID != "" { + st.result.Edges = append(st.result.Edges, &graph.Edge{ + From: id, To: scope.moduleID, Kind: graph.EdgeMemberOf, + FilePath: st.filePath, Line: line, + }) + } } inner.moduleID = id inner.modulePath = name @@ -538,32 +654,183 @@ func (e *JuliaExtractor) handleType(n *sitter.Node, src []byte, scope juliaScope e.walk(n, src, inner, st) } -// juliaCalleeName decodes a call callee: bare identifier or qualified -// field_expression (`Base.show`, `Base.:+`). Returns name, receiver. +// juliaUnwrappedName decodes a single possibly-wrapped name down to its +// plain spelling. An operator callee can wear two wrappers: `:+` is a +// quote_expression around the operator and `:(==)` a quote_expression +// around a parenthesized_expression around it, while a bare `(==)` callee +// wears only the parenthesized one. Returns "" when the node is not a +// name in any of these shapes. +func juliaUnwrappedName(n *sitter.Node, src []byte) string { + for n != nil { + switch n.Type() { + case "identifier", "operator": + return n.Content(src) + case "quote_expression", "parenthesized_expression": + n = n.NamedChild(0) + default: + return "" + } + } + return "" +} + +// juliaCalleeName decodes a call callee: bare identifiers, qualified +// field_expressions (`Base.show`, `A.B.c`), quoted operators (`:+`, +// `:(==)`), and parametrized constructors (`Vector{Int}`). The +// field_expression is decoded from its base and property CHILDREN, never +// from source text: text split on the last dot dragged a chained +// callee's arguments (`get(cfg).run`) and even a line break inside a +// multi-line chain into the call target. A base that is not itself a +// dotted name leaves only the property decodable, so the callee degrades +// to its bare method name — the only part a resolver could ever match. +// Returns name, receiver. func juliaCalleeName(n *sitter.Node, src []byte) (name, receiver string) { if n == nil { return "", "" } switch n.Type() { - case "identifier": + case "identifier", "operator": return n.Content(src), "" + case "quote_expression", "parenthesized_expression": // bare `:+` / `(==)` callee + return juliaUnwrappedName(n, src), "" + case "parametrized_type_expression": // `Vector{Int}(xs)` + return juliaParametrizedCallee(n, src) case "field_expression": - full := n.Content(src) - idx := strings.LastIndex(full, ".") - if idx <= 0 { - return strings.TrimPrefix(full, ":"), "" + count := int(n.NamedChildCount()) + if count < 2 { + return "", "" } - receiver = full[:idx] - name = strings.TrimPrefix(full[idx+1:], ":") // Base.:+ → + - return name, receiver - case "quote_expression": // bare operator callee, e.g. `:+` - if inner := n.NamedChild(0); inner != nil { - return inner.Content(src), "" + name = juliaUnwrappedName(n.NamedChild(count-1), src) + if name == "" { + return "", "" + } + switch base := n.NamedChild(0); base.Type() { + case "identifier": + return name, base.Content(src) + case "field_expression": + inner, recv := juliaCalleeName(base, src) + if inner == "" { + return "", "" + } + if recv == "" { + return name, inner + } + return name, recv + "." + inner + default: + // `get(cfg).run(x)` — the base is a call, not a name, so + // only the method name is decodable. + return name, "" } } return "", "" } +// juliaParametrizedCallee decodes a `Vector{Int}(xs)`-style constructor +// callee: the head name — possibly qualified, as `Base.Vector{Int}` — +// followed by the literal type parameters, which are part of the +// constructor's name the way Julia prints it. The curly list is rebuilt +// from its children so a parameter list broken across lines cannot leak +// a newline into the target. +func juliaParametrizedCallee(n *sitter.Node, src []byte) (name, receiver string) { + head := n.NamedChild(0) + if head == nil { + return "", "" + } + switch head.Type() { + case "identifier": + name = head.Content(src) + case "field_expression": + inner, recv := juliaCalleeName(head, src) + if inner == "" { + return "", "" + } + name, receiver = inner, recv + default: + return "", "" + } + var params []string + for i, count := 1, int(n.NamedChildCount()); i < count; i++ { + if c := n.NamedChild(i); c != nil && c.Type() == "curly_expression" { + for j, jcount := 0, int(c.NamedChildCount()); j < jcount; j++ { + if p := c.NamedChild(j); p != nil { + // Canonicalise nested type parameters so a nested `{…}` + // split across lines cannot leak a newline into the + // target. + params = append(params, juliaCanonType(p, src)) + } + } + } + } + if len(params) > 0 { + name += "{" + strings.Join(params, ",") + "}" + } + return name, receiver +} + +// juliaMacroReceiver decodes the receiver of a module-qualified macro +// call (`Base.@time`, `Base.Threads.@threads`) into its dotted spelling, +// from children rather than source text so a chain of any depth keeps its +// full qualification. Returns "" when the receiver is not a name. +func juliaMacroReceiver(n *sitter.Node, src []byte) string { + switch n.Type() { + case "identifier": + return n.Content(src) + case "field_expression": + name, recv := juliaCalleeName(n, src) + if name == "" { + return "" + } + if recv == "" { + return name + } + return recv + "." + name + } + return "" +} + +// juliaCanonType renders a type expression to a single-line canonical +// spelling, rebuilding any nested `{…}` parameter list from its children +// so source formatting — inner spaces, or a parameter list split across +// lines — cannot leak into a constructor-callee target. +func juliaCanonType(n *sitter.Node, src []byte) string { + if n == nil { + return "" + } + switch n.Type() { + case "parametrized_type_expression": + head := juliaCanonType(n.NamedChild(0), src) + var params []string + for i, count := 1, int(n.NamedChildCount()); i < count; i++ { + c := n.NamedChild(i) + if c == nil || c.Type() != "curly_expression" { + continue + } + for j, jcount := 0, int(c.NamedChildCount()); j < jcount; j++ { + if p := c.NamedChild(j); p != nil { + params = append(params, juliaCanonType(p, src)) + } + } + } + if len(params) > 0 { + return head + "{" + strings.Join(params, ",") + "}" + } + return head + case "field_expression": + name, recv := juliaCalleeName(n, src) + if name == "" { + return strings.Join(strings.Fields(n.Content(src)), "") + } + if recv == "" { + return name + } + return recv + "." + name + default: + // identifier, operator, a literal type parameter — collapse any + // internal whitespace so a line break cannot survive. + return strings.Join(strings.Fields(n.Content(src)), "") + } +} + // juliaSignatureCall peels the wrappers a definition head can carry until // it reaches the call_expression that names the definition. Three wrappers // occur, and they nest in either order: @@ -673,6 +940,15 @@ func (e *JuliaExtractor) handleAssignment(n *sitter.Node, src []byte, scope juli From: st.fileNode.ID, To: id, Kind: graph.EdgeDefines, FilePath: st.filePath, Line: line, }) + // A constant inside a module belongs to it the same way a + // function does — the edge is what a traversal from the + // module reaches; scope_mod on Meta is not traversable. + if scope.moduleID != "" { + st.result.Edges = append(st.result.Edges, &graph.Edge{ + From: id, To: scope.moduleID, Kind: graph.EdgeMemberOf, + FilePath: st.filePath, Line: line, + }) + } } } @@ -712,13 +988,27 @@ func (e *JuliaExtractor) emitCallable( kind := graph.KindMethod nodeName := name isCtor := false - var baseID, ownerID, ownerName string + var baseID, ownerID, ownerTarget, ownerName string switch { case receiver != "": - ownerID, ownerName = st.filePath+"::"+receiver, receiver + ownerName = receiver if id, _, ok := st.lookupType(scope.modulePath, receiver); ok { - ownerID = id + ownerID, ownerTarget = id, id + } else if id, ok := st.modules[juliaTypeKey(scope.modulePath, receiver)]; ok { + // `module M … end; function M.f() … end` — the receiver names + // a module this file declares, so member_of reaches the real + // module node instead of an invented unresolved::M. + ownerID, ownerTarget = id, id + } else { + // `function Base.show` extends a module this file does not + // declare, so no node carries the receiver's name — a + // member_of to the node-shaped id would claim a resident + // of the graph that does not exist. The method's own id + // stays flat; the edge target becomes self-describing, the + // same honesty extends and call edges already carry. + ownerID = st.filePath + "::" + receiver + ownerTarget = "unresolved::" + receiver } baseID = ownerID + "." + name @@ -745,7 +1035,7 @@ func (e *JuliaExtractor) emitCallable( } if typeID != "" { baseID = typeID + "." - ownerID, ownerName = typeID, typeName + ownerID, ownerTarget, ownerName = typeID, typeID, typeName nodeName = typeName + "." isCtor = true } else { @@ -803,7 +1093,7 @@ func (e *JuliaExtractor) emitCallable( }) if ownerID != "" { st.result.Edges = append(st.result.Edges, &graph.Edge{ - From: id, To: ownerID, Kind: graph.EdgeMemberOf, + From: id, To: ownerTarget, Kind: graph.EdgeMemberOf, FilePath: st.filePath, Line: line, }) } @@ -1037,6 +1327,19 @@ func (e *JuliaExtractor) handleImport(n *sitter.Node, src []byte, st *juliaWalkS // module node's Meta (Julia export lists are only meaningful inside // modules). func (e *JuliaExtractor) handleExport(n *sitter.Node, src []byte, scope juliaScope, st *juliaWalkState) { + e.recordModuleNames(n, src, scope, st, "exports") +} + +// handlePublic records a Julia 1.11 `public` list the same way. Public +// names are visible API WITHOUT being re-exported, so they ride on their +// own Meta key next to the export list. +func (e *JuliaExtractor) handlePublic(n *sitter.Node, src []byte, scope juliaScope, st *juliaWalkState) { + e.recordModuleNames(n, src, scope, st, "public") +} + +// recordModuleNames collects the names named by an export or public +// statement onto the enclosing module node's Meta under key. +func (e *JuliaExtractor) recordModuleNames(n *sitter.Node, src []byte, scope juliaScope, st *juliaWalkState, key string) { if scope.moduleID == "" { return } @@ -1046,7 +1349,14 @@ func (e *JuliaExtractor) handleExport(n *sitter.Node, src []byte, scope juliaSco } names := []string{} for c := range n.NamedChildren() { - if c.Type() == "identifier" { + // `export apply, @m, ⊗` exports a function, a macro and an + // operator: a macro name is a macro_identifier node and an + // operator an operator node — the same distinction import + // lists already decode (`using Base: @time, +`). Record them + // verbatim (@m, ⊗) so the module's public surface keeps its + // macro and operator names. + switch c.Type() { + case "identifier", "operator", "macro_identifier": names = append(names, c.Content(src)) } } @@ -1056,8 +1366,8 @@ func (e *JuliaExtractor) handleExport(n *sitter.Node, src []byte, scope juliaSco if node.Meta == nil { node.Meta = map[string]any{} } - prev, _ := node.Meta["exports"].([]string) - node.Meta["exports"] = append(prev, names...) + prev, _ := node.Meta[key].([]string) + node.Meta[key] = append(prev, names...) } // handleCall emits EdgeCalls from the enclosing function to the callee @@ -1121,25 +1431,72 @@ func (e *JuliaExtractor) handleCall(n *sitter.Node, src []byte, scope juliaScope st.result.Edges = append(st.result.Edges, edge) } -// handleMacroCall attributes `@macroname ...` invocations to the -// enclosing function as calls to the bare macro name. +// handleMacroCall attributes `@macroname ...` and `Mod.@macroname ...` +// invocations to the enclosing function as EdgeCalls with macro:true +// meta. The macro's name lives in a disjoint namespace from types and +// functions, so the target carries the bare name (time, not @time) with +// the module as receiver — the same receiver.name spelling qualified +// call callees use. func (e *JuliaExtractor) handleMacroCall(n *sitter.Node, src []byte, scope juliaScope, st *juliaWalkState) { if scope.functionID == "" { return } + emit := func(target string) { + st.result.Edges = append(st.result.Edges, &graph.Edge{ + From: scope.functionID, To: "unresolved::" + target, + Kind: graph.EdgeCalls, + Meta: map[string]any{"macro": true}, + FilePath: st.filePath, Line: int(n.StartPoint().Row) + 1, + }) + } for c := range n.NamedChildren() { - if c.Type() != "macro_identifier" { - continue - } - for m := range c.NamedChildren() { - if m.Type() == "identifier" { - st.result.Edges = append(st.result.Edges, &graph.Edge{ - From: scope.functionID, To: "unresolved::" + m.Content(src), - Kind: graph.EdgeCalls, - Meta: map[string]any{"macro": true}, - FilePath: st.filePath, Line: int(n.StartPoint().Row) + 1, - }) + switch c.Type() { + case "macro_identifier": // `@time x` + for m := range c.NamedChildren() { + if m.Type() == "identifier" { + emit(m.Content(src)) + } + } + case "field_expression": // `Base.@time x`, `Base.Threads.@threads x` + count := int(c.NamedChildCount()) + if count < 2 { + continue + } + prop, base := c.NamedChild(count-1), c.NamedChild(0) + if prop.Type() != "macro_identifier" { + continue + } + // The receiver can be a bare module (`Base`) or a dotted + // chain (`Base.Threads`), decoded from its children to any + // depth so a multi-segment qualifier keeps its module instead + // of losing the edge. + receiver := juliaMacroReceiver(base, src) + if receiver == "" { + continue + } + name := "" + for m := range prop.NamedChildren() { + if m.Type() == "identifier" { + name = m.Content(src) + } + } + if name == "" { + continue + } + // `import Foo as F` then `F.@spawn ...`: name the module, not + // the file-local nickname, exactly as a qualified call callee + // does. An alias binds a single name, so only the leading + // segment of a dotted receiver can be one. + head, rest, dotted := strings.Cut(receiver, ".") + if module, ok := st.importAliases[juliaTypeKey(scope.modulePath, head)]; ok { + head = module + } + if dotted { + receiver = head + "." + rest + } else { + receiver = head } + emit(receiver + "." + name) } } } diff --git a/internal/parser/languages/julia_test.go b/internal/parser/languages/julia_test.go index e907c765..d8b327e5 100644 --- a/internal/parser/languages/julia_test.go +++ b/internal/parser/languages/julia_test.go @@ -796,6 +796,30 @@ import Base: + as plus, - "and the selected names are still recorded on it") } +// An export list can export a macro (`export @m`) and an operator +// (`export ⊗`) — a macro_identifier node and an operator node, the same +// distinction import selections already decode — so an identifier-only +// scan dropped them from the module's recorded public surface. +func TestJuliaExtractor_MacroAndOperatorExports(t *testing.T) { + src := []byte(`module Ops +export apply, @m, ⊗ +end +`) + res, err := NewJuliaExtractor().Extract("exports.jl", src) + require.NoError(t, err) + + mod := map[string]*graph.Node{} + for _, n := range res.Nodes { + mod[n.ID] = n + } + m := mod["exports.jl::Ops"] + require.NotNil(t, m) + exports, ok := m.Meta["exports"].([]string) + require.True(t, ok, "module Meta exports missing") + assert.ElementsMatch(t, []string{"apply", "@m", "⊗"}, exports, + "a module's exported macros and operators are part of its public surface") +} + func TestJuliaExtractor_Calls(t *testing.T) { src := []byte(`module Calls @@ -868,6 +892,377 @@ end } } +// A chained callee's base is a call, not a name: `get(cfg).run(x)` used +// to reach the graph as `unresolved::get(cfg).run` — arguments and all — +// because the callee was decoded by splitting its source text on the +// last dot, and a chain broken across lines even carried the line break +// into the target. Decoding the field_expression's children instead +// degrades the callee to its method name, the only part a resolver +// could ever match, while a genuinely dotted base (A.B.c) keeps its +// full qualification. +func TestJuliaExtractor_ChainedCalleeDegradesToMethodName(t *testing.T) { + src := []byte(`launch(cfg) = get(cfg).run(x) + +wrapped(x) = foo(x, + 1, +).bar(y) + +deep(x) = A.B.c(x) +`) + res, err := NewJuliaExtractor().Extract("chain.jl", src) + require.NoError(t, err) + + calls := map[string]bool{} + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeCalls { + calls[ed.From+" -> "+ed.To] = true + } + require.NotContains(t, ed.To, "\n", "a target must never carry a line break") + require.NotContains(t, ed.To, "(", "a target must never carry argument text") + } + require.True(t, calls["chain.jl::launch -> unresolved::run"], + "a chained callee degrades to its method name, not its argument text") + require.True(t, calls["chain.jl::wrapped -> unresolved::bar"], + "a chain broken across lines must not leak the break into the target") + require.True(t, calls["chain.jl::deep -> unresolved::A.B.c"], + "a dotted base keeps its full qualification") +} + +// `Base.:(==)(a, b)` used to normalise its callee to `(==)` — the +// parenthesised quote survived the text trim — while `Base.:+` trimmed +// to `+`. Both spellings name the same operator, so both must decode to +// the operator's own name; a bare `(==)(a, b)` callee, which wears only +// the parentheses, does too. +func TestJuliaExtractor_QuotedOperatorCallee(t *testing.T) { + src := []byte(`same(a, b) = Base.:(==)(a, b) +plus(a, b) = Base.:+(a, b) +plain(a, b) = (==)(a, b) +`) + res, err := NewJuliaExtractor().Extract("op.jl", src) + require.NoError(t, err) + + calls := map[string]bool{} + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeCalls { + calls[ed.From+" -> "+ed.To] = true + } + } + require.True(t, calls["op.jl::same -> unresolved::Base.=="], + "`:(==)` must normalise to the operator name like `:+` does") + require.True(t, calls["op.jl::plus -> unresolved::Base.+"], + "`:+` keeps its existing normalisation") + require.True(t, calls["op.jl::plain -> unresolved::=="], + "a bare parenthesised operator callee decodes too") +} + +// Constructing a parametric type is one of the most common calls in real +// Julia, and its callee is a parametrized_type_expression, not an +// identifier — a decoder with no case for it dropped the edge, so +// `Vector{Int}(xs)` vanished from the call graph. The edge names the +// constructor the way Julia prints it, parameters included, and a +// qualified head keeps its module. +func TestJuliaExtractor_ParametrizedConstructorCallee(t *testing.T) { + src := []byte(`build(xs) = Vector{Int}(xs) +qualified(xs) = Base.Vector{Int}(xs) +table(xs) = Dict{String,Int}(xs) +`) + res, err := NewJuliaExtractor().Extract("param.jl", src) + require.NoError(t, err) + + calls := map[string]bool{} + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeCalls { + calls[ed.From+" -> "+ed.To] = true + } + require.NotContains(t, ed.To, "\n", "a target must never carry a line break") + } + require.True(t, calls["param.jl::build -> unresolved::Vector{Int}"], + "a parametric constructor call is a call edge") + require.True(t, calls["param.jl::qualified -> unresolved::Base.Vector{Int}"], + "a qualified parametric head keeps its module") + require.True(t, calls["param.jl::table -> unresolved::Dict{String,Int}"], + "multi-parameter constructors carry their parameters") +} + +// A module-qualified macro call nests its macro_identifier under a +// field_expression (Base.@time), so a scan that matches only a direct +// macro_identifier child recorded the inner helper call but never the +// macro's own edge. The qualified form must record both, with the +// module as receiver — the same spelling qualified call callees use — +// and the import-alias rewrite applies to it just as it does to calls. +func TestJuliaExtractor_QualifiedMacroCall(t *testing.T) { + src := []byte(`module M +import Foo as F + +function work(xs) + Base.@time helper(xs) + F.@spawn helper(xs) +end +end +`) + res, err := NewJuliaExtractor().Extract("qmac.jl", src) + require.NoError(t, err) + + type call struct { + to string + meta map[string]any + } + calls := map[string][]call{} + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeCalls { + meta := map[string]any{} + for k, v := range ed.Meta { + meta[k] = v + } + calls[ed.From] = append(calls[ed.From], call{ed.To, meta}) + } + } + var sawBase, sawAliased, sawHelper bool + for _, c := range calls["qmac.jl::work"] { + switch c.to { + case "unresolved::Base.time": + sawBase = c.meta["macro"] == true + case "unresolved::Foo.spawn": + sawAliased = c.meta["macro"] == true + case "unresolved::helper": + sawHelper = true + } + } + assert.True(t, sawBase, "Base.@time needs its macro edge, receiver included") + assert.True(t, sawAliased, "an aliased module qualifies the macro edge, as it does for calls") + assert.True(t, sawHelper, "the inner call of a qualified macro keeps its own edge") +} + +// A docstring above a macro call switches walkMacroArgs into its +// doc-carrying loop, which dispatched definition arguments to their +// handlers but walked every other argument with walk() — and walk() +// visits a node's children, never the node itself. A call_expression +// argument therefore never reached handleCall; since ordinary call +// edges need an enclosing function, which a documented module-level +// macro call never has, the one observable loss is the include() that +// loads a file on every worker. +func TestJuliaExtractor_DocumentedMacroArgumentsKeepCalls(t *testing.T) { + src := []byte(`module M +"""load helpers on every worker""" +@everywhere include("helpers.jl") +end +`) + res, err := NewJuliaExtractor().Extract("everywhere.jl", src) + require.NoError(t, err) + + var got bool + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeImports && ed.To == "unresolved::import::helpers.jl" { + got = true + } + } + assert.True(t, got, "an include() in a documented macro argument keeps its import edge") + + // The undocumented form must keep working identically. + plain, err := NewJuliaExtractor().Extract("plain.jl", []byte("module M\n@everywhere include(\"more.jl\")\nend\n")) + require.NoError(t, err) + var plainGot bool + for _, ed := range plain.Edges { + if ed.Kind == graph.EdgeImports && ed.To == "unresolved::import::more.jl" { + plainGot = true + } + } + assert.True(t, plainGot, "the undocumented form is unchanged") +} + +// Functions, types and fields inside a module carry a member_of edge to +// it, but a constant and a nested module carried only scope_mod on +// Meta — recorded, yet unreachable from a traversal of the module. A +// constant belongs to its module and a nested module to its parent +// through the same edge every other resident uses; at the top level +// (no enclosing module) neither gets one. +func TestJuliaExtractor_ConstAndNestedModuleContainment(t *testing.T) { + src := []byte(`module Outer +const X = 1 +module Inner +const Y = 2 +f() = 1 +end +end + +const TOP = 3 +`) + res, err := NewJuliaExtractor().Extract("own.jl", src) + require.NoError(t, err) + + nodes := map[string]*graph.Node{} + for _, n := range res.Nodes { + nodes[n.ID] = n + } + require.NotNil(t, nodes["own.jl::Outer"]) + require.NotNil(t, nodes["own.jl::X"]) + inner := nodes["own.jl::Inner"] + require.NotNil(t, inner) + assert.Equal(t, "Outer", inner.Meta["scope_mod"], "scope_mod keeps recording the lexical path") + require.NotNil(t, nodes["own.jl::Y"]) + require.NotNil(t, nodes["own.jl::TOP"], "a top-level constant still mints its variable node") + + owners := juliaOwners(res.Edges) + assert.True(t, owners["own.jl::X"]["own.jl::Outer"], + "a module-level constant belongs to its module") + assert.True(t, owners["own.jl::Inner"]["own.jl::Outer"], + "a nested module belongs to its parent") + assert.True(t, owners["own.jl::Y"]["own.jl::Inner"], + "a constant in the inner module belongs to the inner module") + assert.True(t, owners["own.jl::f"]["own.jl::Inner"], + "the pre-existing function containment is unchanged") + assert.Empty(t, owners["own.jl::TOP"], + "a top-level constant has no enclosing module to belong to") + assert.Empty(t, owners["own.jl::Outer"], + "a top-level module has no enclosing module to belong to") +} + +// A qualified definition's member_of names its receiver. When the +// receiver is a type in the same file the edge targets that node — but +// `function Base.show` extends a module this file does not declare, and +// a member_of to a node-shaped id that has no node claims a resident of +// the graph that does not exist. Such a target must be self-describing, +// the way extends and call edges already are: prefixed unresolved::, +// while the method's own id stays flat. +func TestJuliaExtractor_ExternalReceiverMemberOfIsUnresolved(t *testing.T) { + src := []byte(`function Base.show(io, x) + nothing +end + +struct Box + v::Int +end + +function Box.f(x) + x +end +`) + res, err := NewJuliaExtractor().Extract("ext.jl", src) + require.NoError(t, err) + + owners := juliaOwners(res.Edges) + assert.True(t, owners["ext.jl::Base.show"]["unresolved::Base"], + "a receiver no node in the file declares must not be minted as a node id") + nodes := map[string]bool{} + for _, n := range res.Nodes { + nodes[n.ID] = true + } + assert.False(t, nodes["ext.jl::Base"], + "the external receiver itself must not become a phantom node") + assert.True(t, owners["ext.jl::Box.f"]["ext.jl::Box"], + "an in-file receiver still targets the real type node") +} + +// A docstring and the other Meta a definition carries are not in +// competition: a qualified method keeps its doc next to its receiver, +// and a macro keeps its doc next to its macro flag. Docstring handling +// builds the node's Meta in one place, so no key can evict another. +func TestJuliaExtractor_DocstringSurvivesOtherMeta(t *testing.T) { + src := []byte(`module M + +"""Render p compactly.""" +function Base.show(io, p) + nothing +end + +"""Build a point.""" +macro point(x) + x +end + +end +`) + res, err := NewJuliaExtractor().Extract("meta.jl", src) + require.NoError(t, err) + + nodes := map[string]*graph.Node{} + for _, n := range res.Nodes { + nodes[n.ID] = n + } + + show := nodes["meta.jl::Base.show"] + require.NotNil(t, show, "the qualified method must exist") + assert.Equal(t, "Base", show.Meta["receiver"]) + assert.Equal(t, "Render p compactly.", show.Meta["doc"], + "a docstring must survive a receiver already on Meta") + + pt := nodes["meta.jl::point"] + require.NotNil(t, pt, "the macro must exist") + assert.Equal(t, true, pt.Meta["macro"]) + assert.Equal(t, "Build a point.", pt.Meta["doc"], + "a docstring must survive the macro flag already on Meta") +} + +// Julia lowers EVERY docstring — triple-quoted or explicit — through +// Core.@doc, so `@doc "text" obj` is the same documentation mechanism as +// a string above the object, and the string sits INSIDE the macro call +// where the pending-doc walk never looks. Every documented-object shape +// accepts the form: short-form definitions, long-form functions, macros, +// structs. An orphan `@doc "text"` with no object attaches nothing. +func TestJuliaExtractor_ExplicitDocMacroAttaches(t *testing.T) { + src := []byte(`@doc "Short doc." pd(x) = x + +Core.@doc "Long doc." function cd(x) + help(x) +end + +@doc "Struct doc." struct DS + v::Int +end + +@doc "Macro doc." macro dm(x) + x +end + +@doc "Orphan." +`) + res, err := NewJuliaExtractor().Extract("docm.jl", src) + require.NoError(t, err) + + docs := map[string]string{} + for _, n := range res.Nodes { + if d, ok := n.Meta["doc"].(string); ok { + docs[n.ID] = d + } + } + assert.Equal(t, "Short doc.", docs["docm.jl::pd"], "the explicit form documents a short-form definition") + assert.Equal(t, "Long doc.", docs["docm.jl::cd"], "the qualified Core.@doc form documents a long-form definition") + assert.Equal(t, "Struct doc.", docs["docm.jl::DS"], "the explicit form documents a struct") + assert.Equal(t, "Macro doc.", docs["docm.jl::dm"], "the explicit form documents a macro") + _, orphan := docs["docm.jl::Orphan."] + assert.False(t, orphan, "an @doc with no object must not mint a phantom node") +} + +// Julia 1.11 `public` names API that is visible WITHOUT being +// re-exported — a different contract from `export`, so it rides on its +// own Meta key next to the export list. The statement's children are the +// same node kinds exports accept, operators included (`public +`). +func TestJuliaExtractor_PublicStatementRecorded(t *testing.T) { + src := []byte(`module Pubbed +export kept +public shown, also_shown, + + +kept() = 1 +shown() = 2 +end +`) + res, err := NewJuliaExtractor().Extract("public.jl", src) + require.NoError(t, err) + + mod := map[string]*graph.Node{} + for _, n := range res.Nodes { + mod[n.ID] = n + } + m := mod["public.jl::Pubbed"] + require.NotNil(t, m) + exports, ok := m.Meta["exports"].([]string) + require.True(t, ok, "exports stay recorded") + assert.ElementsMatch(t, []string{"kept"}, exports) + public, ok := m.Meta["public"].([]string) + require.True(t, ok, "public names missing") + assert.ElementsMatch(t, []string{"shown", "also_shown", "+"}, public) +} + func TestJuliaExtractor_ConstAndDocstrings(t *testing.T) { src := []byte(`""" Circle radius helpers. @@ -1068,3 +1463,144 @@ end assert.False(t, ok, "%s must not be documented, got %q", id, docs[id]) } } + +// A qualified method whose receiver is a module DECLARED IN THE SAME FILE +// belongs to that module's node: `module M … end; function M.f() … end` +// must member_of the real `file::M`, not an invented `unresolved::M`. +// Modules are KindType nodes, indistinguishable from structs by kind, so +// the receiver resolves through their own table after the type lookup +// misses. +func TestJuliaExtractor_SameFileModuleMethodOwner(t *testing.T) { + src := []byte(`module M +greet() = 1 +end + +function M.f(x) + helper(x) +end + +module Outer +module Inner +end +function Inner.g(x) + x +end +end +`) + res, err := NewJuliaExtractor().Extract("modm.jl", src) + require.NoError(t, err) + + nodes := map[string]bool{} + for _, n := range res.Nodes { + nodes[n.ID] = true + } + require.True(t, nodes["modm.jl::M"], "the module node must exist") + + owners := juliaOwners(res.Edges) + assert.True(t, owners["modm.jl::M.f"]["modm.jl::M"], + "a method on a same-file module belongs to that module's node") + assert.False(t, owners["modm.jl::M.f"]["unresolved::M"], + "it must not fall back to an invented unresolved receiver") + assert.True(t, owners["modm.jl::Inner.g"]["modm.jl::Inner"], + "a nested module receiver resolves in its own lexical scope") +} + +// Julia lowers a documented constant through the same @doc mechanism: +// `@doc "text" const X = 1`. The object is a const_statement, not a bare +// assignment, so walkMacroArgs has to dispatch it AS const or both the +// constant and its documentation vanish. +func TestJuliaExtractor_ExplicitDocOnConstant(t *testing.T) { + src := []byte(`@doc "Tuning knob." const K = 42 +`) + res, err := NewJuliaExtractor().Extract("dc.jl", src) + require.NoError(t, err) + + var k *graph.Node + for _, n := range res.Nodes { + if n.ID == "dc.jl::K" { + k = n + } + } + require.NotNil(t, k, "the documented constant must mint its variable node") + assert.Equal(t, graph.KindVariable, k.Kind) + assert.Equal(t, "Tuning knob.", k.Meta["doc"], + "the constant keeps the docstring the explicit @doc form carries") +} + +// Only bare `@doc`, `Core.@doc`, and `Base.@doc` lower a docstring. A user +// macro that merely ends in `.@doc` — `Foo.@doc` — is unrelated and must +// not attach its string as documentation; the standard forms still do, so +// the restriction is not over-broad. +func TestJuliaExtractor_ForeignDocMacroDoesNotAttach(t *testing.T) { + src := []byte(`Foo.@doc "not a docstring" function g(x) + x +end + +Core.@doc "real doc" function h(x) + x +end +`) + res, err := NewJuliaExtractor().Extract("fd.jl", src) + require.NoError(t, err) + + docs := map[string]string{} + for _, n := range res.Nodes { + if d, ok := n.Meta["doc"].(string); ok { + docs[n.ID] = d + } + } + _, gDoc := docs["fd.jl::g"] + assert.False(t, gDoc, "Foo.@doc must not hijack the docstring slot") + assert.Equal(t, "real doc", docs["fd.jl::h"], + "Core.@doc still attaches, so the restriction is not over-broad") +} + +// A module-qualified macro receiver can be a dotted chain, not just a bare +// module: `Base.Threads.@spawn`, `A.B.@m`. Matching only a single +// identifier base dropped these edges; the receiver is decoded from its +// children to any depth, keeping the full qualification. +func TestJuliaExtractor_MultiSegmentQualifiedMacroCall(t *testing.T) { + src := []byte(`function work(xs) + Base.Threads.@spawn compute(xs) + A.B.@m(xs) +end +`) + res, err := NewJuliaExtractor().Extract("mseg.jl", src) + require.NoError(t, err) + + calls := map[string]bool{} + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeCalls { + calls[ed.From+" -> "+ed.To] = true + } + } + assert.True(t, calls["mseg.jl::work -> unresolved::Base.Threads.spawn"], + "a two-segment qualifier keeps its full module path") + assert.True(t, calls["mseg.jl::work -> unresolved::A.B.m"], + "an any-depth qualified macro receiver decodes") +} + +// A parametric constructor callee can nest — `Vector{Tuple{Int,String}}` — +// and a nested parameter list may be broken across lines. Canonicalising +// only the outer list left the inner `{…}` as raw source, so a newline in +// it leaked into the unresolved target. Every level is rebuilt from +// children. +func TestJuliaExtractor_NestedParametrizedConstructorCallee(t *testing.T) { + src := []byte("build(x) = Vector{Tuple{Int,\n String}}(x)\n" + + "nested(x) = Dict{String,Vector{Int}}(x)\n") + res, err := NewJuliaExtractor().Extract("np.jl", src) + require.NoError(t, err) + + calls := map[string]bool{} + for _, ed := range res.Edges { + if ed.Kind == graph.EdgeCalls { + calls[ed.From+" -> "+ed.To] = true + } + require.NotContains(t, ed.To, "\n", "a target must never carry a line break") + require.NotContains(t, ed.To, " ", "a canonical type target carries no whitespace") + } + assert.True(t, calls["np.jl::build -> unresolved::Vector{Tuple{Int,String}}"], + "a nested parametric callee is canonicalised at every level") + assert.True(t, calls["np.jl::nested -> unresolved::Dict{String,Vector{Int}}"], + "a nested type parameter keeps its own parameters") +}