From 06fe7b151808b7c952507124f2fc11a8fb7ceead Mon Sep 17 00:00:00 2001 From: Nicolas Laval Date: Thu, 20 Aug 2026 19:45:56 +0200 Subject: [PATCH 1/5] Support component-wide operands in prov-v2 --- vtl-prov/specs/20260729_02_work-breakdown.md | 7 +- .../fr/insee/vtl/prov2/ProvenanceVisitor.java | 104 +++++++++++++++--- .../insee/vtl/prov2/SupportCheckVisitor.java | 35 +++++- 3 files changed, 127 insertions(+), 19 deletions(-) diff --git a/vtl-prov/specs/20260729_02_work-breakdown.md b/vtl-prov/specs/20260729_02_work-breakdown.md index 9fd476b27..f943b9aa3 100644 --- a/vtl-prov/specs/20260729_02_work-breakdown.md +++ b/vtl-prov/specs/20260729_02_work-breakdown.md @@ -12,7 +12,7 @@ progress paragraph. |------|----|------------|-------------| | [x] | 1 | Corpus harness (DOT import, `GraphAssert`, golden self-check) | self-check | | [x] | 2 | `VtlBaseVisitor` + structure oracle (run-once) + identity assignment; `ProvenanceExtractor` entry | 01 | -| [ ] | 3 | Component-wise dataset ops | 02, 13 | +| [x] | 3 | Component-wise dataset ops (`+`, `*`, …; scalar literals not nodes) | 02, 13 | | [ ] | 4 | Expression nodes (calc) | 03 | | [ ] | 5 | Condition edges (filter, sub) | 04, 14 | | [ ] | 6 | Projection & rename | 05, 06 | @@ -71,8 +71,9 @@ directive parser later migrates to `vtl-test-utils` (spec 20260729_01 §6/§8). Richer RDF than today's triples is a later view, not PR-15. **PR-2** ships `ProvenanceExtractor` → `SupportCheckVisitor` / -`ProvenanceVisitor` (identity `:=` / `<-` only). Later PRs add `visit*` methods -on the same visitor. +`ProvenanceVisitor` (identity `:=` / `<-`). **PR-3** extends the same visitors +for component-wise dataset arithmetic (corpus `02`, `13`). Later PRs add more +`visit*` methods. ## Embedded decisions (flag if you disagree) diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java index 4af34b72f..f5228b232 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java @@ -2,11 +2,13 @@ import fr.insee.vtl.antlr.runtime.CharStream; import fr.insee.vtl.antlr.runtime.ParserRuleContext; +import fr.insee.vtl.antlr.runtime.Token; import fr.insee.vtl.antlr.runtime.misc.Interval; import fr.insee.vtl.model.Structured.Component; import fr.insee.vtl.model.Structured.DataStructure; import fr.insee.vtl.parser.VtlParser; import fr.insee.vtl.prov.utils.VTLTypes; +import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -16,9 +18,10 @@ * SupportCheckVisitor} for the shared {@code unsupported: …} surface; mutates a shared {@link * ProvGraph}. * - *

{@code T = Void}: the graph is the artifact; parse {@code ctx} plus run state ({@code - * versions}, oracle, {@code lastResultId}) carry what each visit needs. {@code lastResultId} is the - * dataset/expression node produced by the expression subtree just visited. + *

{@code T = Void}: the graph is the artifact; parse {@code ctx} plus run state carry what each + * visit needs. After visiting an expression: {@code lastOp == null} means identity ({@code + * lastResultId} is the source dataset); otherwise {@code lastOp} + {@code lastOperandIds} describe + * a component-wise dataset op whose result node is created by the enclosing assignment. */ final class ProvenanceVisitor extends SupportCheckVisitor { @@ -26,8 +29,16 @@ final class ProvenanceVisitor extends SupportCheckVisitor { private final StructureOracle oracle; private final Map versions = new LinkedHashMap<>(); private int stmtIndex; + + /** Dataset id produced by the last varId (identity) expression. */ private String lastResultId; + /** Operator of the last component-wise expression ({@code +}, {@code *}, …), or null. */ + private String lastOp; + + /** Dataset operand ids of the last component-wise expression (literals omitted). */ + private List lastOperandIds = List.of(); + ProvenanceVisitor(ProvGraph graph, StructureOracle oracle, List inputs) { this.graph = graph; this.oracle = oracle; @@ -56,31 +67,90 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { throw new IllegalStateException("unknown dataset " + name); } lastResultId = id; + lastOp = null; + lastOperandIds = List.of(id); + return null; + } + + @Override + public Void visitArithmeticExpr(VtlParser.ArithmeticExprContext ctx) { + return binaryArithmetic(ctx.left, ctx.right, ctx.op); + } + + @Override + public Void visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext ctx) { + return binaryArithmetic(ctx.left, ctx.right, ctx.op); + } + + private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right, Token op) { + List operands = new ArrayList<>(2); + String leftId = datasetOperand(left); + if (leftId != null) { + operands.add(leftId); + } + String rightId = datasetOperand(right); + if (rightId != null) { + operands.add(rightId); + } + if (operands.isEmpty()) { + throw unsupported("scalar"); + } + lastOperandIds = List.copyOf(operands); + lastOp = op.getText(); + lastResultId = null; return null; } + /** {@code null} if the operand is a scalar literal (not a provenance node). */ + private String datasetOperand(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = unwrap(expr); + if (current instanceof VtlParser.VarIdExprContext) { + visit(current); + return lastResultId; + } + if (current instanceof VtlParser.ConstantExprContext) { + return null; + } + throw unsupported("arithmetic"); + } + private Void assign(String out, VtlParser.ExprContext expr) { stmtIndex++; visit(expr); - String srcId = lastResultId; - if (srcId == null) { - throw new IllegalStateException("assignment RHS produced no result id"); - } String outId = out + "@" + stmtIndex; DataStructure outStructure = oracle.requireDataset(out); addDataset(outId, outStructure, text(expr)); - graph.addEdge(outId, srcId, Map.of("op", "assign")); - for (Component component : outStructure.values()) { - graph.addEdge( - outId + "." + component.getName(), - srcId + "." + component.getName(), - Map.of("op", "assign")); + + if (lastOp == null) { + if (lastResultId == null) { + throw unsupported("scalar"); + } + linkComponentWise(outId, outStructure, List.of(lastResultId), "assign"); + } else { + linkComponentWise(outId, outStructure, lastOperandIds, lastOp); } + versions.put(out, outId); lastResultId = outId; + lastOp = null; + lastOperandIds = List.of(outId); return null; } + private void linkComponentWise( + String outId, DataStructure outStructure, List operandIds, String op) { + Map edge = Map.of("op", op); + for (String operandId : operandIds) { + graph.addEdge(outId, operandId, edge); + } + for (Component component : outStructure.values()) { + String outVar = outId + "." + component.getName(); + for (String operandId : operandIds) { + graph.addEdge(outVar, operandId + "." + component.getName(), edge); + } + } + } + private void addDataset(String id, DataStructure structure, String src) { if (graph.vertices().containsKey(id)) { return; @@ -101,6 +171,14 @@ private void addDataset(String id, DataStructure structure, String src) { } } + private static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = expr; + while (current instanceof VtlParser.ParenthesisExprContext parenthesis) { + current = parenthesis.expr(); + } + return current; + } + private static String text(ParserRuleContext ctx) { CharStream input = ctx.getStart().getInputStream(); return input.getText(Interval.of(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex())); diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java index 7dd06168a..9e44c94b6 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java @@ -7,6 +7,9 @@ /** * Grammar-only support gate: throws {@code unsupported: …} before the structure oracle runs, so the * corpus backlog stays explicit even when the engine cannot eval the script. + * + *

PR-3: binary arithmetic / concat with leaf operands (varId or constant). Nested arithmetic and + * unary stay unsupported for now. */ class SupportCheckVisitor extends VtlBaseVisitor { @@ -50,12 +53,12 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { @Override public Void visitArithmeticExpr(VtlParser.ArithmeticExprContext ctx) { - throw unsupported("arithmetic"); + return binaryArithmetic(ctx.left, ctx.right); } @Override public Void visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext ctx) { - throw unsupported("arithmetic"); + return binaryArithmetic(ctx.left, ctx.right); } @Override @@ -78,7 +81,33 @@ public Void visitConstantExpr(VtlParser.ConstantExprContext ctx) { throw unsupported("scalar"); } - private static UnsupportedOperationException unsupported(String what) { + private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right) { + leafOperand(left); + leafOperand(right); + return null; + } + + /** Dataset name or scalar literal; nested ops deferred. */ + private void leafOperand(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = unwrap(expr); + if (current instanceof VtlParser.VarIdExprContext) { + return; + } + if (current instanceof VtlParser.ConstantExprContext) { + return; + } + throw unsupported("arithmetic"); + } + + private static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = expr; + while (current instanceof VtlParser.ParenthesisExprContext parenthesis) { + current = parenthesis.expr(); + } + return current; + } + + static UnsupportedOperationException unsupported(String what) { return new UnsupportedOperationException("unsupported: " + what); } } From 7b21cdac22cc02bac603fb1356f12797d8f3b51c Mon Sep 17 00:00:00 2001 From: Nicolas Laval Date: Thu, 20 Aug 2026 20:01:39 +0200 Subject: [PATCH 2/5] Support calc in prov-v2 --- vtl-prov/specs/20260729_02_work-breakdown.md | 8 +- .../fr/insee/vtl/prov2/ProvenanceVisitor.java | 87 +++++++++++++++++-- .../insee/vtl/prov2/SupportCheckVisitor.java | 40 ++++++++- 3 files changed, 119 insertions(+), 16 deletions(-) diff --git a/vtl-prov/specs/20260729_02_work-breakdown.md b/vtl-prov/specs/20260729_02_work-breakdown.md index f943b9aa3..2829b4d20 100644 --- a/vtl-prov/specs/20260729_02_work-breakdown.md +++ b/vtl-prov/specs/20260729_02_work-breakdown.md @@ -13,7 +13,7 @@ progress paragraph. | [x] | 1 | Corpus harness (DOT import, `GraphAssert`, golden self-check) | self-check | | [x] | 2 | `VtlBaseVisitor` + structure oracle (run-once) + identity assignment; `ProvenanceExtractor` entry | 01 | | [x] | 3 | Component-wise dataset ops (`+`, `*`, …; scalar literals not nodes) | 02, 13 | -| [ ] | 4 | Expression nodes (calc) | 03 | +| [x] | 4 | Expression nodes (calc) | 03 | | [ ] | 5 | Condition edges (filter, sub) | 04, 14 | | [ ] | 6 | Projection & rename | 05, 06 | | [ ] | 7 | Clause chaining + anonymous intermediates | chain-filter-calc | @@ -71,9 +71,9 @@ directive parser later migrates to `vtl-test-utils` (spec 20260729_01 §6/§8). Richer RDF than today's triples is a later view, not PR-15. **PR-2** ships `ProvenanceExtractor` → `SupportCheckVisitor` / -`ProvenanceVisitor` (identity `:=` / `<-`). **PR-3** extends the same visitors -for component-wise dataset arithmetic (corpus `02`, `13`). Later PRs add more -`visit*` methods. +`ProvenanceVisitor` (identity `:=` / `<-`). **PR-3** adds component-wise dataset +arithmetic (`02`, `13`). **PR-4** adds single-clause `calc` + expression nodes +(`03`). Later PRs add more `visit*` methods. ## Embedded decisions (flag if you disagree) diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java index f5228b232..52a68f6c1 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java @@ -6,12 +6,15 @@ import fr.insee.vtl.antlr.runtime.misc.Interval; import fr.insee.vtl.model.Structured.Component; import fr.insee.vtl.model.Structured.DataStructure; +import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; import fr.insee.vtl.prov.utils.VTLTypes; import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * Grammar-driven provenance walk ({@code VtlBaseVisitor}). Extends {@link @@ -20,8 +23,8 @@ * *

{@code T = Void}: the graph is the artifact; parse {@code ctx} plus run state carry what each * visit needs. After visiting an expression: {@code lastOp == null} means identity ({@code - * lastResultId} is the source dataset); otherwise {@code lastOp} + {@code lastOperandIds} describe - * a component-wise dataset op whose result node is created by the enclosing assignment. + * lastResultId} is the source dataset); otherwise {@code lastOp} + operands / calc items describe a + * dataset op whose result node is created by the enclosing assignment. */ final class ProvenanceVisitor extends SupportCheckVisitor { @@ -33,12 +36,15 @@ final class ProvenanceVisitor extends SupportCheckVisitor { /** Dataset id produced by the last varId (identity) expression. */ private String lastResultId; - /** Operator of the last component-wise expression ({@code +}, {@code *}, …), or null. */ + /** Operator of the last dataset expression ({@code +}, {@code *}, {@code calc}, …), or null. */ private String lastOp; /** Dataset operand ids of the last component-wise expression (literals omitted). */ private List lastOperandIds = List.of(); + /** Calc outputs: component name → expression node id. Empty unless {@code lastOp} is calc. */ + private Map lastCalcExprs = Map.of(); + ProvenanceVisitor(ProvGraph graph, StructureOracle oracle, List inputs) { this.graph = graph; this.oracle = oracle; @@ -69,6 +75,7 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { lastResultId = id; lastOp = null; lastOperandIds = List.of(id); + lastCalcExprs = Map.of(); return null; } @@ -82,6 +89,44 @@ public Void visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext return binaryArithmetic(ctx.left, ctx.right, ctx.op); } + @Override + public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { + visit(ctx.expr()); + if (lastOp != null || lastResultId == null) { + throw unsupported("clause"); + } + String srcId = lastResultId; + VtlParser.CalcClauseContext calc = ctx.datasetClause().calcClause(); + if (calc != null) { + return applyCalc(srcId, calc); + } + throw unsupported("clause"); + } + + private Void applyCalc(String srcId, VtlParser.CalcClauseContext calc) { + Map calcExprs = new LinkedHashMap<>(); + int exprSeq = 0; + for (VtlParser.CalcClauseItemContext item : calc.calcClauseItem()) { + exprSeq++; + String component = item.componentID().getText(); + VtlParser.ExprContext rhs = item.expr(); + String exprId = "e" + stmtIndex + "." + exprSeq; + Map attrs = new LinkedHashMap<>(); + attrs.put("kind", "expression"); + attrs.put("src", text(rhs)); + graph.addVertex(exprId, attrs); + for (String ref : componentRefs(rhs)) { + graph.addEdge(exprId, srcId + "." + ref, Map.of()); + } + calcExprs.put(component, exprId); + } + lastOp = "calc"; + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = Map.copyOf(calcExprs); + return null; + } + private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right, Token op) { List operands = new ArrayList<>(2); String leftId = datasetOperand(left); @@ -98,6 +143,7 @@ private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext lastOperandIds = List.copyOf(operands); lastOp = op.getText(); lastResultId = null; + lastCalcExprs = Map.of(); return null; } @@ -126,6 +172,8 @@ private Void assign(String out, VtlParser.ExprContext expr) { throw unsupported("scalar"); } linkComponentWise(outId, outStructure, List.of(lastResultId), "assign"); + } else if ("calc".equals(lastOp)) { + linkCalc(outId, outStructure, lastResultId, lastCalcExprs); } else { linkComponentWise(outId, outStructure, lastOperandIds, lastOp); } @@ -134,9 +182,25 @@ private Void assign(String out, VtlParser.ExprContext expr) { lastResultId = outId; lastOp = null; lastOperandIds = List.of(outId); + lastCalcExprs = Map.of(); return null; } + private void linkCalc( + String outId, DataStructure outStructure, String srcId, Map calcExprs) { + Map edge = Map.of("op", "calc"); + graph.addEdge(outId, srcId, edge); + for (Component component : outStructure.values()) { + String outVar = outId + "." + component.getName(); + String exprId = calcExprs.get(component.getName()); + if (exprId != null) { + graph.addEdge(outVar, exprId, edge); + } else { + graph.addEdge(outVar, srcId + "." + component.getName(), edge); + } + } + } + private void linkComponentWise( String outId, DataStructure outStructure, List operandIds, String op) { Map edge = Map.of("op", op); @@ -171,12 +235,17 @@ private void addDataset(String id, DataStructure structure, String src) { } } - private static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { - VtlParser.ExprContext current = expr; - while (current instanceof VtlParser.ParenthesisExprContext parenthesis) { - current = parenthesis.expr(); - } - return current; + /** Component names referenced in a calc RHS (not dataset bindings). */ + private static Set componentRefs(VtlParser.ExprContext expr) { + Set refs = new LinkedHashSet<>(); + new VtlBaseVisitor() { + @Override + public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { + refs.add(ctx.varID().getText()); + return null; + } + }.visit(expr); + return refs; } private static String text(ParserRuleContext ctx) { diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java index 9e44c94b6..b4069f994 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java @@ -8,8 +8,8 @@ * Grammar-only support gate: throws {@code unsupported: …} before the structure oracle runs, so the * corpus backlog stays explicit even when the engine cannot eval the script. * - *

PR-3: binary arithmetic / concat with leaf operands (varId or constant). Nested arithmetic and - * unary stay unsupported for now. + *

Supported so far: identity assign, binary dataset arithmetic (leaf operands), single {@code + * calc} clause on a dataset varId (RHS = component refs / literals / binary arithmetic). */ class SupportCheckVisitor extends VtlBaseVisitor { @@ -68,6 +68,14 @@ public Void visitUnaryExpr(VtlParser.UnaryExprContext ctx) { @Override public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { + datasetVarId(ctx.expr()); + VtlParser.CalcClauseContext calc = ctx.datasetClause().calcClause(); + if (calc != null) { + for (VtlParser.CalcClauseItemContext item : calc.calcClauseItem()) { + calcRhs(item.expr()); + } + return null; + } throw unsupported("clause"); } @@ -99,7 +107,33 @@ private void leafOperand(VtlParser.ExprContext expr) { throw unsupported("arithmetic"); } - private static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { + private void datasetVarId(VtlParser.ExprContext expr) { + if (!(unwrap(expr) instanceof VtlParser.VarIdExprContext)) { + throw unsupported("clause"); + } + } + + /** Component-level calc RHS (not dataset-level arithmetic). */ + private void calcRhs(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = unwrap(expr); + if (current instanceof VtlParser.VarIdExprContext + || current instanceof VtlParser.ConstantExprContext) { + return; + } + if (current instanceof VtlParser.ArithmeticExprContext arithmetic) { + calcRhs(arithmetic.left); + calcRhs(arithmetic.right); + return; + } + if (current instanceof VtlParser.ArithmeticExprOrConcatContext arithmetic) { + calcRhs(arithmetic.left); + calcRhs(arithmetic.right); + return; + } + throw unsupported("clause"); + } + + static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { VtlParser.ExprContext current = expr; while (current instanceof VtlParser.ParenthesisExprContext parenthesis) { current = parenthesis.expr(); From 28d3406ac4c4e7fb4f09e86546c0496d6ba88d5f Mon Sep 17 00:00:00 2001 From: Nicolas Laval Date: Thu, 20 Aug 2026 20:05:49 +0200 Subject: [PATCH 3/5] Support filter/sub in prov-v2 --- vtl-prov/specs/20260729_02_work-breakdown.md | 5 +- .../fr/insee/vtl/prov2/ProvenanceVisitor.java | 115 ++++++++++++++---- .../insee/vtl/prov2/SupportCheckVisitor.java | 25 +++- 3 files changed, 118 insertions(+), 27 deletions(-) diff --git a/vtl-prov/specs/20260729_02_work-breakdown.md b/vtl-prov/specs/20260729_02_work-breakdown.md index 2829b4d20..e7c9e5cb9 100644 --- a/vtl-prov/specs/20260729_02_work-breakdown.md +++ b/vtl-prov/specs/20260729_02_work-breakdown.md @@ -14,7 +14,7 @@ progress paragraph. | [x] | 2 | `VtlBaseVisitor` + structure oracle (run-once) + identity assignment; `ProvenanceExtractor` entry | 01 | | [x] | 3 | Component-wise dataset ops (`+`, `*`, …; scalar literals not nodes) | 02, 13 | | [x] | 4 | Expression nodes (calc) | 03 | -| [ ] | 5 | Condition edges (filter, sub) | 04, 14 | +| [x] | 5 | Condition edges (filter, sub) | 04, 14 | | [ ] | 6 | Projection & rename | 05, 06 | | [ ] | 7 | Clause chaining + anonymous intermediates | chain-filter-calc | | [ ] | 8 | Aggr | 07 | @@ -73,7 +73,8 @@ Richer RDF than today's triples is a later view, not PR-15. **PR-2** ships `ProvenanceExtractor` → `SupportCheckVisitor` / `ProvenanceVisitor` (identity `:=` / `<-`). **PR-3** adds component-wise dataset arithmetic (`02`, `13`). **PR-4** adds single-clause `calc` + expression nodes -(`03`). Later PRs add more `visit*` methods. +(`03`). **PR-5** adds `filter` / `sub` with `role=condition` (`04`, `14`). Later +PRs add more `visit*` methods. ## Embedded decisions (flag if you disagree) diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java index 52a68f6c1..6edf01c47 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java @@ -23,8 +23,8 @@ * *

{@code T = Void}: the graph is the artifact; parse {@code ctx} plus run state carry what each * visit needs. After visiting an expression: {@code lastOp == null} means identity ({@code - * lastResultId} is the source dataset); otherwise {@code lastOp} + operands / calc items describe a - * dataset op whose result node is created by the enclosing assignment. + * lastResultId} is the source dataset); otherwise {@code lastOp} + operands / calc / condition + * exprs describe a dataset op whose result node is created by the enclosing assignment. */ final class ProvenanceVisitor extends SupportCheckVisitor { @@ -32,11 +32,12 @@ final class ProvenanceVisitor extends SupportCheckVisitor { private final StructureOracle oracle; private final Map versions = new LinkedHashMap<>(); private int stmtIndex; + private int exprSeq; /** Dataset id produced by the last varId (identity) expression. */ private String lastResultId; - /** Operator of the last dataset expression ({@code +}, {@code *}, {@code calc}, …), or null. */ + /** Operator of the last dataset expression ({@code +}, {@code calc}, {@code filter}, …). */ private String lastOp; /** Dataset operand ids of the last component-wise expression (literals omitted). */ @@ -45,6 +46,9 @@ final class ProvenanceVisitor extends SupportCheckVisitor { /** Calc outputs: component name → expression node id. Empty unless {@code lastOp} is calc. */ private Map lastCalcExprs = Map.of(); + /** Condition expression node ids ({@code filter}/{@code sub}). */ + private List lastConditionExprIds = List.of(); + ProvenanceVisitor(ProvGraph graph, StructureOracle oracle, List inputs) { this.graph = graph; this.oracle = oracle; @@ -72,10 +76,9 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { if (id == null) { throw new IllegalStateException("unknown dataset " + name); } + clearExprState(); lastResultId = id; - lastOp = null; lastOperandIds = List.of(id); - lastCalcExprs = Map.of(); return null; } @@ -96,34 +99,61 @@ public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { throw unsupported("clause"); } String srcId = lastResultId; - VtlParser.CalcClauseContext calc = ctx.datasetClause().calcClause(); - if (calc != null) { - return applyCalc(srcId, calc); + VtlParser.DatasetClauseContext clause = ctx.datasetClause(); + if (clause.calcClause() != null) { + return applyCalc(srcId, clause.calcClause()); + } + if (clause.filterClause() != null) { + return applyFilter(srcId, clause.filterClause()); + } + if (clause.subspaceClause() != null) { + return applySub(srcId, clause.subspaceClause()); } throw unsupported("clause"); } private Void applyCalc(String srcId, VtlParser.CalcClauseContext calc) { Map calcExprs = new LinkedHashMap<>(); - int exprSeq = 0; for (VtlParser.CalcClauseItemContext item : calc.calcClauseItem()) { - exprSeq++; String component = item.componentID().getText(); VtlParser.ExprContext rhs = item.expr(); - String exprId = "e" + stmtIndex + "." + exprSeq; - Map attrs = new LinkedHashMap<>(); - attrs.put("kind", "expression"); - attrs.put("src", text(rhs)); - graph.addVertex(exprId, attrs); - for (String ref : componentRefs(rhs)) { - graph.addEdge(exprId, srcId + "." + ref, Map.of()); - } + String exprId = nextExprId(); + addExpression(exprId, text(rhs), srcId, componentRefs(rhs)); calcExprs.put(component, exprId); } lastOp = "calc"; lastResultId = srcId; lastOperandIds = List.of(srcId); lastCalcExprs = Map.copyOf(calcExprs); + lastConditionExprIds = List.of(); + return null; + } + + private Void applyFilter(String srcId, VtlParser.FilterClauseContext filter) { + VtlParser.ExprContext predicate = filter.expr(); + String exprId = nextExprId(); + addExpression(exprId, text(predicate), srcId, componentRefs(predicate)); + lastOp = "filter"; + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = Map.of(); + lastConditionExprIds = List.of(exprId); + return null; + } + + private Void applySub(String srcId, VtlParser.SubspaceClauseContext sub) { + List conditionIds = new ArrayList<>(); + for (VtlParser.SubspaceClauseItemContext item : sub.subspaceClauseItem()) { + String exprId = nextExprId(); + String component = item.componentID().getText(); + addExpression(exprId, text(item), srcId, Set.of(component)); + conditionIds.add(exprId); + } + lastOp = "sub"; + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = Map.of(); + lastConditionExprIds = List.copyOf(conditionIds); return null; } @@ -144,6 +174,7 @@ private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext lastOp = op.getText(); lastResultId = null; lastCalcExprs = Map.of(); + lastConditionExprIds = List.of(); return null; } @@ -162,6 +193,7 @@ private String datasetOperand(VtlParser.ExprContext expr) { private Void assign(String out, VtlParser.ExprContext expr) { stmtIndex++; + exprSeq = 0; visit(expr); String outId = out + "@" + stmtIndex; DataStructure outStructure = oracle.requireDataset(out); @@ -174,15 +206,16 @@ private Void assign(String out, VtlParser.ExprContext expr) { linkComponentWise(outId, outStructure, List.of(lastResultId), "assign"); } else if ("calc".equals(lastOp)) { linkCalc(outId, outStructure, lastResultId, lastCalcExprs); + } else if ("filter".equals(lastOp) || "sub".equals(lastOp)) { + linkConditionClause(outId, outStructure, lastResultId, lastConditionExprIds, lastOp); } else { linkComponentWise(outId, outStructure, lastOperandIds, lastOp); } versions.put(out, outId); + clearExprState(); lastResultId = outId; - lastOp = null; lastOperandIds = List.of(outId); - lastCalcExprs = Map.of(); return null; } @@ -201,6 +234,25 @@ private void linkCalc( } } + private void linkConditionClause( + String outId, + DataStructure outStructure, + String srcId, + List conditionExprIds, + String op) { + Map edge = Map.of("op", op); + Map condition = new LinkedHashMap<>(); + condition.put("op", op); + condition.put("role", "condition"); + graph.addEdge(outId, srcId, edge); + for (String exprId : conditionExprIds) { + graph.addEdge(outId, exprId, condition); + } + for (Component component : outStructure.values()) { + graph.addEdge(outId + "." + component.getName(), srcId + "." + component.getName(), edge); + } + } + private void linkComponentWise( String outId, DataStructure outStructure, List operandIds, String op) { Map edge = Map.of("op", op); @@ -215,6 +267,27 @@ private void linkComponentWise( } } + private void addExpression(String exprId, String src, String datasetId, Set refs) { + Map attrs = new LinkedHashMap<>(); + attrs.put("kind", "expression"); + attrs.put("src", src); + graph.addVertex(exprId, attrs); + for (String ref : refs) { + graph.addEdge(exprId, datasetId + "." + ref, Map.of()); + } + } + + private String nextExprId() { + exprSeq++; + return "e" + stmtIndex + "." + exprSeq; + } + + private void clearExprState() { + lastOp = null; + lastCalcExprs = Map.of(); + lastConditionExprIds = List.of(); + } + private void addDataset(String id, DataStructure structure, String src) { if (graph.vertices().containsKey(id)) { return; @@ -235,7 +308,7 @@ private void addDataset(String id, DataStructure structure, String src) { } } - /** Component names referenced in a calc RHS (not dataset bindings). */ + /** Component names referenced in a scalar expression (not dataset bindings). */ private static Set componentRefs(VtlParser.ExprContext expr) { Set refs = new LinkedHashSet<>(); new VtlBaseVisitor() { diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java index b4069f994..d45a9235b 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java @@ -9,7 +9,7 @@ * corpus backlog stays explicit even when the engine cannot eval the script. * *

Supported so far: identity assign, binary dataset arithmetic (leaf operands), single {@code - * calc} clause on a dataset varId (RHS = component refs / literals / binary arithmetic). + * calc} / {@code filter} / {@code sub} clause on a dataset varId. */ class SupportCheckVisitor extends VtlBaseVisitor { @@ -69,13 +69,20 @@ public Void visitUnaryExpr(VtlParser.UnaryExprContext ctx) { @Override public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { datasetVarId(ctx.expr()); - VtlParser.CalcClauseContext calc = ctx.datasetClause().calcClause(); - if (calc != null) { - for (VtlParser.CalcClauseItemContext item : calc.calcClauseItem()) { + VtlParser.DatasetClauseContext clause = ctx.datasetClause(); + if (clause.calcClause() != null) { + for (VtlParser.CalcClauseItemContext item : clause.calcClause().calcClauseItem()) { calcRhs(item.expr()); } return null; } + if (clause.filterClause() != null) { + scalarPredicate(clause.filterClause().expr()); + return null; + } + if (clause.subspaceClause() != null) { + return null; + } throw unsupported("clause"); } @@ -133,6 +140,16 @@ private void calcRhs(VtlParser.ExprContext expr) { throw unsupported("clause"); } + /** Filter predicates may use functions/comparisons; reject nested dataset clauses only. */ + private void scalarPredicate(VtlParser.ExprContext expr) { + new VtlBaseVisitor() { + @Override + public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { + throw unsupported("clause"); + } + }.visit(expr); + } + static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { VtlParser.ExprContext current = expr; while (current instanceof VtlParser.ParenthesisExprContext parenthesis) { From 7e859a2b3b3f24478c2772f08e3807bb08418574 Mon Sep 17 00:00:00 2001 From: Nicolas Laval Date: Thu, 20 Aug 2026 20:12:15 +0200 Subject: [PATCH 4/5] Support projection/rename in prov-v2 --- vtl-prov/specs/20260729_02_work-breakdown.md | 7 +-- .../fr/insee/vtl/prov2/ProvenanceVisitor.java | 50 +++++++++++++++++++ .../insee/vtl/prov2/SupportCheckVisitor.java | 6 ++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/vtl-prov/specs/20260729_02_work-breakdown.md b/vtl-prov/specs/20260729_02_work-breakdown.md index e7c9e5cb9..653d732ae 100644 --- a/vtl-prov/specs/20260729_02_work-breakdown.md +++ b/vtl-prov/specs/20260729_02_work-breakdown.md @@ -15,7 +15,7 @@ progress paragraph. | [x] | 3 | Component-wise dataset ops (`+`, `*`, …; scalar literals not nodes) | 02, 13 | | [x] | 4 | Expression nodes (calc) | 03 | | [x] | 5 | Condition edges (filter, sub) | 04, 14 | -| [ ] | 6 | Projection & rename | 05, 06 | +| [x] | 6 | Projection & rename | 05, 06 | | [ ] | 7 | Clause chaining + anonymous intermediates | chain-filter-calc | | [ ] | 8 | Aggr | 07 | | [ ] | 9 | Join | 08 | @@ -73,8 +73,9 @@ Richer RDF than today's triples is a later view, not PR-15. **PR-2** ships `ProvenanceExtractor` → `SupportCheckVisitor` / `ProvenanceVisitor` (identity `:=` / `<-`). **PR-3** adds component-wise dataset arithmetic (`02`, `13`). **PR-4** adds single-clause `calc` + expression nodes -(`03`). **PR-5** adds `filter` / `sub` with `role=condition` (`04`, `14`). Later -PRs add more `visit*` methods. +(`03`). **PR-5** adds `filter` / `sub` with `role=condition` (`04`, `14`). +**PR-6** adds `keep`/`drop` / `rename` (`05`, `06`). Later PRs add more +`visit*` methods. ## Embedded decisions (flag if you disagree) diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java index 6edf01c47..0dcb577fb 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java @@ -46,6 +46,9 @@ final class ProvenanceVisitor extends SupportCheckVisitor { /** Calc outputs: component name → expression node id. Empty unless {@code lastOp} is calc. */ private Map lastCalcExprs = Map.of(); + /** Rename map: output component name → source component name. */ + private Map lastRenameFrom = Map.of(); + /** Condition expression node ids ({@code filter}/{@code sub}). */ private List lastConditionExprIds = List.of(); @@ -109,6 +112,12 @@ public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { if (clause.subspaceClause() != null) { return applySub(srcId, clause.subspaceClause()); } + if (clause.keepOrDropClause() != null) { + return applyKeepOrDrop(srcId, clause.keepOrDropClause()); + } + if (clause.renameClause() != null) { + return applyRename(srcId, clause.renameClause()); + } throw unsupported("clause"); } @@ -125,6 +134,7 @@ private Void applyCalc(String srcId, VtlParser.CalcClauseContext calc) { lastResultId = srcId; lastOperandIds = List.of(srcId); lastCalcExprs = Map.copyOf(calcExprs); + lastRenameFrom = Map.of(); lastConditionExprIds = List.of(); return null; } @@ -137,6 +147,7 @@ private Void applyFilter(String srcId, VtlParser.FilterClauseContext filter) { lastResultId = srcId; lastOperandIds = List.of(srcId); lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); lastConditionExprIds = List.of(exprId); return null; } @@ -153,10 +164,35 @@ private Void applySub(String srcId, VtlParser.SubspaceClauseContext sub) { lastResultId = srcId; lastOperandIds = List.of(srcId); lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); lastConditionExprIds = List.copyOf(conditionIds); return null; } + private Void applyKeepOrDrop(String srcId, VtlParser.KeepOrDropClauseContext keepOrDrop) { + lastOp = keepOrDrop.op.getText(); + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); + lastConditionExprIds = List.of(); + return null; + } + + private Void applyRename(String srcId, VtlParser.RenameClauseContext rename) { + Map renames = new LinkedHashMap<>(); + for (VtlParser.RenameClauseItemContext item : rename.renameClauseItem()) { + renames.put(item.toName.getText(), item.fromName.getText()); + } + lastOp = "rename"; + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = Map.of(); + lastRenameFrom = Map.copyOf(renames); + lastConditionExprIds = List.of(); + return null; + } + private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right, Token op) { List operands = new ArrayList<>(2); String leftId = datasetOperand(left); @@ -174,6 +210,7 @@ private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext lastOp = op.getText(); lastResultId = null; lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); lastConditionExprIds = List.of(); return null; } @@ -208,6 +245,8 @@ private Void assign(String out, VtlParser.ExprContext expr) { linkCalc(outId, outStructure, lastResultId, lastCalcExprs); } else if ("filter".equals(lastOp) || "sub".equals(lastOp)) { linkConditionClause(outId, outStructure, lastResultId, lastConditionExprIds, lastOp); + } else if ("rename".equals(lastOp)) { + linkRename(outId, outStructure, lastResultId, lastRenameFrom); } else { linkComponentWise(outId, outStructure, lastOperandIds, lastOp); } @@ -253,6 +292,16 @@ private void linkConditionClause( } } + private void linkRename( + String outId, DataStructure outStructure, String srcId, Map renameFrom) { + Map edge = Map.of("op", "rename"); + graph.addEdge(outId, srcId, edge); + for (Component component : outStructure.values()) { + String srcComponent = renameFrom.getOrDefault(component.getName(), component.getName()); + graph.addEdge(outId + "." + component.getName(), srcId + "." + srcComponent, edge); + } + } + private void linkComponentWise( String outId, DataStructure outStructure, List operandIds, String op) { Map edge = Map.of("op", op); @@ -285,6 +334,7 @@ private String nextExprId() { private void clearExprState() { lastOp = null; lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); lastConditionExprIds = List.of(); } diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java index d45a9235b..fa6c4b406 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java @@ -9,7 +9,8 @@ * corpus backlog stays explicit even when the engine cannot eval the script. * *

Supported so far: identity assign, binary dataset arithmetic (leaf operands), single {@code - * calc} / {@code filter} / {@code sub} clause on a dataset varId. + * calc} / {@code filter} / {@code sub} / {@code keep}/{@code drop} / {@code rename} clause on a + * dataset varId. */ class SupportCheckVisitor extends VtlBaseVisitor { @@ -83,6 +84,9 @@ public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { if (clause.subspaceClause() != null) { return null; } + if (clause.keepOrDropClause() != null || clause.renameClause() != null) { + return null; + } throw unsupported("clause"); } From 4131bcd3c269e3e6ac2fe56088bc74aeb22290c2 Mon Sep 17 00:00:00 2001 From: Nicolas Laval Date: Thu, 20 Aug 2026 20:19:00 +0200 Subject: [PATCH 5/5] Minor refactoring for prov-v2 --- .../fr/insee/vtl/prov2/ProvenanceVisitor.java | 168 ++++++++---------- .../insee/vtl/prov2/SupportCheckVisitor.java | 45 ++--- 2 files changed, 95 insertions(+), 118 deletions(-) diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java index 0dcb577fb..cbd5a223e 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java @@ -21,10 +21,10 @@ * SupportCheckVisitor} for the shared {@code unsupported: …} surface; mutates a shared {@link * ProvGraph}. * - *

{@code T = Void}: the graph is the artifact; parse {@code ctx} plus run state carry what each - * visit needs. After visiting an expression: {@code lastOp == null} means identity ({@code - * lastResultId} is the source dataset); otherwise {@code lastOp} + operands / calc / condition - * exprs describe a dataset op whose result node is created by the enclosing assignment. + *

{@code T = Void}: the graph is the artifact. After visiting an expression, run state describes + * what the enclosing assignment materializes: identity ({@code lastOp == null}), component-wise ops + * ({@code +}, {@code *}, {@code keep}, …), or clause ops ({@code calc}, {@code filter}, {@code + * sub}, {@code rename}). */ final class ProvenanceVisitor extends SupportCheckVisitor { @@ -34,22 +34,11 @@ final class ProvenanceVisitor extends SupportCheckVisitor { private int stmtIndex; private int exprSeq; - /** Dataset id produced by the last varId (identity) expression. */ private String lastResultId; - - /** Operator of the last dataset expression ({@code +}, {@code calc}, {@code filter}, …). */ private String lastOp; - - /** Dataset operand ids of the last component-wise expression (literals omitted). */ private List lastOperandIds = List.of(); - - /** Calc outputs: component name → expression node id. Empty unless {@code lastOp} is calc. */ private Map lastCalcExprs = Map.of(); - - /** Rename map: output component name → source component name. */ private Map lastRenameFrom = Map.of(); - - /** Condition expression node ids ({@code filter}/{@code sub}). */ private List lastConditionExprIds = List.of(); ProvenanceVisitor(ProvGraph graph, StructureOracle oracle, List inputs) { @@ -130,53 +119,28 @@ private Void applyCalc(String srcId, VtlParser.CalcClauseContext calc) { addExpression(exprId, text(rhs), srcId, componentRefs(rhs)); calcExprs.put(component, exprId); } - lastOp = "calc"; - lastResultId = srcId; - lastOperandIds = List.of(srcId); - lastCalcExprs = Map.copyOf(calcExprs); - lastRenameFrom = Map.of(); - lastConditionExprIds = List.of(); - return null; + return finishUnaryOp("calc", srcId, Map.copyOf(calcExprs), Map.of(), List.of()); } private Void applyFilter(String srcId, VtlParser.FilterClauseContext filter) { VtlParser.ExprContext predicate = filter.expr(); String exprId = nextExprId(); addExpression(exprId, text(predicate), srcId, componentRefs(predicate)); - lastOp = "filter"; - lastResultId = srcId; - lastOperandIds = List.of(srcId); - lastCalcExprs = Map.of(); - lastRenameFrom = Map.of(); - lastConditionExprIds = List.of(exprId); - return null; + return finishUnaryOp("filter", srcId, Map.of(), Map.of(), List.of(exprId)); } private Void applySub(String srcId, VtlParser.SubspaceClauseContext sub) { List conditionIds = new ArrayList<>(); for (VtlParser.SubspaceClauseItemContext item : sub.subspaceClauseItem()) { String exprId = nextExprId(); - String component = item.componentID().getText(); - addExpression(exprId, text(item), srcId, Set.of(component)); + addExpression(exprId, text(item), srcId, Set.of(item.componentID().getText())); conditionIds.add(exprId); } - lastOp = "sub"; - lastResultId = srcId; - lastOperandIds = List.of(srcId); - lastCalcExprs = Map.of(); - lastRenameFrom = Map.of(); - lastConditionExprIds = List.copyOf(conditionIds); - return null; + return finishUnaryOp("sub", srcId, Map.of(), Map.of(), List.copyOf(conditionIds)); } private Void applyKeepOrDrop(String srcId, VtlParser.KeepOrDropClauseContext keepOrDrop) { - lastOp = keepOrDrop.op.getText(); - lastResultId = srcId; - lastOperandIds = List.of(srcId); - lastCalcExprs = Map.of(); - lastRenameFrom = Map.of(); - lastConditionExprIds = List.of(); - return null; + return finishUnaryOp(keepOrDrop.op.getText(), srcId); } private Void applyRename(String srcId, VtlParser.RenameClauseContext rename) { @@ -184,13 +148,7 @@ private Void applyRename(String srcId, VtlParser.RenameClauseContext rename) { for (VtlParser.RenameClauseItemContext item : rename.renameClauseItem()) { renames.put(item.toName.getText(), item.fromName.getText()); } - lastOp = "rename"; - lastResultId = srcId; - lastOperandIds = List.of(srcId); - lastCalcExprs = Map.of(); - lastRenameFrom = Map.copyOf(renames); - lastConditionExprIds = List.of(); - return null; + return finishUnaryOp("rename", srcId, Map.of(), Map.copyOf(renames), List.of()); } private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right, Token op) { @@ -206,12 +164,9 @@ private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext if (operands.isEmpty()) { throw unsupported("scalar"); } - lastOperandIds = List.copyOf(operands); + clearExprState(); lastOp = op.getText(); - lastResultId = null; - lastCalcExprs = Map.of(); - lastRenameFrom = Map.of(); - lastConditionExprIds = List.of(); + lastOperandIds = List.copyOf(operands); return null; } @@ -235,32 +190,34 @@ private Void assign(String out, VtlParser.ExprContext expr) { String outId = out + "@" + stmtIndex; DataStructure outStructure = oracle.requireDataset(out); addDataset(outId, outStructure, text(expr)); + linkAssignment(outId, outStructure); + versions.put(out, outId); + clearExprState(); + lastResultId = outId; + lastOperandIds = List.of(outId); + return null; + } + private void linkAssignment(String outId, DataStructure outStructure) { if (lastOp == null) { if (lastResultId == null) { throw unsupported("scalar"); } linkComponentWise(outId, outStructure, List.of(lastResultId), "assign"); - } else if ("calc".equals(lastOp)) { - linkCalc(outId, outStructure, lastResultId, lastCalcExprs); - } else if ("filter".equals(lastOp) || "sub".equals(lastOp)) { - linkConditionClause(outId, outStructure, lastResultId, lastConditionExprIds, lastOp); - } else if ("rename".equals(lastOp)) { - linkRename(outId, outStructure, lastResultId, lastRenameFrom); - } else { - linkComponentWise(outId, outStructure, lastOperandIds, lastOp); + return; + } + switch (lastOp) { + case "calc" -> linkCalc(outId, outStructure, lastResultId, lastCalcExprs); + case "filter", "sub" -> + linkConditionClause(outId, outStructure, lastResultId, lastConditionExprIds, lastOp); + case "rename" -> linkRename(outId, outStructure, lastResultId, lastRenameFrom); + default -> linkComponentWise(outId, outStructure, lastOperandIds, lastOp); } - - versions.put(out, outId); - clearExprState(); - lastResultId = outId; - lastOperandIds = List.of(outId); - return null; } private void linkCalc( String outId, DataStructure outStructure, String srcId, Map calcExprs) { - Map edge = Map.of("op", "calc"); + Map edge = opEdge("calc"); graph.addEdge(outId, srcId, edge); for (Component component : outStructure.values()) { String outVar = outId + "." + component.getName(); @@ -279,22 +236,19 @@ private void linkConditionClause( String srcId, List conditionExprIds, String op) { - Map edge = Map.of("op", op); - Map condition = new LinkedHashMap<>(); - condition.put("op", op); + Map edge = opEdge(op); + Map condition = new LinkedHashMap<>(edge); condition.put("role", "condition"); graph.addEdge(outId, srcId, edge); for (String exprId : conditionExprIds) { graph.addEdge(outId, exprId, condition); } - for (Component component : outStructure.values()) { - graph.addEdge(outId + "." + component.getName(), srcId + "." + component.getName(), edge); - } + linkPassThrough(outId, outStructure, srcId, edge); } private void linkRename( String outId, DataStructure outStructure, String srcId, Map renameFrom) { - Map edge = Map.of("op", "rename"); + Map edge = opEdge("rename"); graph.addEdge(outId, srcId, edge); for (Component component : outStructure.values()) { String srcComponent = renameFrom.getOrDefault(component.getName(), component.getName()); @@ -304,7 +258,7 @@ private void linkRename( private void linkComponentWise( String outId, DataStructure outStructure, List operandIds, String op) { - Map edge = Map.of("op", op); + Map edge = opEdge(op); for (String operandId : operandIds) { graph.addEdge(outId, operandId, edge); } @@ -316,6 +270,13 @@ private void linkComponentWise( } } + private void linkPassThrough( + String outId, DataStructure outStructure, String srcId, Map edge) { + for (Component component : outStructure.values()) { + graph.addEdge(outId + "." + component.getName(), srcId + "." + component.getName(), edge); + } + } + private void addExpression(String exprId, String src, String datasetId, Set refs) { Map attrs = new LinkedHashMap<>(); attrs.put("kind", "expression"); @@ -326,18 +287,6 @@ private void addExpression(String exprId, String src, String datasetId, Set calcExprs, + Map renameFrom, + List conditionExprIds) { + lastOp = op; + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = calcExprs; + lastRenameFrom = renameFrom; + lastConditionExprIds = conditionExprIds; + return null; + } + + private void clearExprState() { + lastOp = null; + lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); + lastConditionExprIds = List.of(); + } + + private String nextExprId() { + exprSeq++; + return "e" + stmtIndex + "." + exprSeq; + } + + private static Map opEdge(String op) { + return Map.of("op", op); + } + /** Component names referenced in a scalar expression (not dataset bindings). */ private static Set componentRefs(VtlParser.ExprContext expr) { Set refs = new LinkedHashSet<>(); diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java index fa6c4b406..c61c52e7a 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java @@ -8,9 +8,9 @@ * Grammar-only support gate: throws {@code unsupported: …} before the structure oracle runs, so the * corpus backlog stays explicit even when the engine cannot eval the script. * - *

Supported so far: identity assign, binary dataset arithmetic (leaf operands), single {@code - * calc} / {@code filter} / {@code sub} / {@code keep}/{@code drop} / {@code rename} clause on a - * dataset varId. + *

Mirrors {@link ProvenanceVisitor} coverage: identity assign; binary dataset arithmetic (leaf + * operands); single {@code calc} / {@code filter} / {@code sub} / {@code keep}|{@code drop} / + * {@code rename} on a dataset varId. Nested clauses and other ops stay unsupported. */ class SupportCheckVisitor extends VtlBaseVisitor { @@ -54,12 +54,16 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { @Override public Void visitArithmeticExpr(VtlParser.ArithmeticExprContext ctx) { - return binaryArithmetic(ctx.left, ctx.right); + leafOperand(ctx.left); + leafOperand(ctx.right); + return null; } @Override public Void visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext ctx) { - return binaryArithmetic(ctx.left, ctx.right); + leafOperand(ctx.left); + leafOperand(ctx.right); + return null; } @Override @@ -69,22 +73,19 @@ public Void visitUnaryExpr(VtlParser.UnaryExprContext ctx) { @Override public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { - datasetVarId(ctx.expr()); + requireDatasetVarId(ctx.expr()); VtlParser.DatasetClauseContext clause = ctx.datasetClause(); if (clause.calcClause() != null) { - for (VtlParser.CalcClauseItemContext item : clause.calcClause().calcClauseItem()) { - calcRhs(item.expr()); - } + clause.calcClause().calcClauseItem().forEach(item -> calcRhs(item.expr())); return null; } if (clause.filterClause() != null) { - scalarPredicate(clause.filterClause().expr()); + requireScalarPredicate(clause.filterClause().expr()); return null; } - if (clause.subspaceClause() != null) { - return null; - } - if (clause.keepOrDropClause() != null || clause.renameClause() != null) { + if (clause.subspaceClause() != null + || clause.keepOrDropClause() != null + || clause.renameClause() != null) { return null; } throw unsupported("clause"); @@ -100,25 +101,17 @@ public Void visitConstantExpr(VtlParser.ConstantExprContext ctx) { throw unsupported("scalar"); } - private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right) { - leafOperand(left); - leafOperand(right); - return null; - } - /** Dataset name or scalar literal; nested ops deferred. */ private void leafOperand(VtlParser.ExprContext expr) { VtlParser.ExprContext current = unwrap(expr); - if (current instanceof VtlParser.VarIdExprContext) { - return; - } - if (current instanceof VtlParser.ConstantExprContext) { + if (current instanceof VtlParser.VarIdExprContext + || current instanceof VtlParser.ConstantExprContext) { return; } throw unsupported("arithmetic"); } - private void datasetVarId(VtlParser.ExprContext expr) { + private void requireDatasetVarId(VtlParser.ExprContext expr) { if (!(unwrap(expr) instanceof VtlParser.VarIdExprContext)) { throw unsupported("clause"); } @@ -145,7 +138,7 @@ private void calcRhs(VtlParser.ExprContext expr) { } /** Filter predicates may use functions/comparisons; reject nested dataset clauses only. */ - private void scalarPredicate(VtlParser.ExprContext expr) { + private void requireScalarPredicate(VtlParser.ExprContext expr) { new VtlBaseVisitor() { @Override public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) {