diff --git a/vtl-prov/specs/20260729_02_work-breakdown.md b/vtl-prov/specs/20260729_02_work-breakdown.md index 9fd476b27..653d732ae 100644 --- a/vtl-prov/specs/20260729_02_work-breakdown.md +++ b/vtl-prov/specs/20260729_02_work-breakdown.md @@ -12,10 +12,10 @@ progress paragraph. |------|----|------------|-------------| | [x] | 1 | Corpus harness (DOT import, `GraphAssert`, golden self-check) | self-check | | [x] | 2 | `VtlBaseVisitor` + structure oracle (run-once) + identity assignment; `ProvenanceExtractor` entry | 01 | -| [ ] | 3 | Component-wise dataset ops | 02, 13 | -| [ ] | 4 | Expression nodes (calc) | 03 | -| [ ] | 5 | Condition edges (filter, sub) | 04, 14 | -| [ ] | 6 | Projection & rename | 05, 06 | +| [x] | 3 | Component-wise dataset ops (`+`, `*`, …; scalar literals not nodes) | 02, 13 | +| [x] | 4 | Expression nodes (calc) | 03 | +| [x] | 5 | Condition edges (filter, sub) | 04, 14 | +| [x] | 6 | Projection & rename | 05, 06 | | [ ] | 7 | Clause chaining + anonymous intermediates | chain-filter-calc | | [ ] | 8 | Aggr | 07 | | [ ] | 9 | Join | 08 | @@ -71,8 +71,11 @@ directive parser later migrates to `vtl-test-utils` (spec 20260729_01 §6/§8). Richer RDF than today's triples is a later view, not PR-15. **PR-2** ships `ProvenanceExtractor` → `SupportCheckVisitor` / -`ProvenanceVisitor` (identity `:=` / `<-` only). Later PRs add `visit*` methods -on the same visitor. +`ProvenanceVisitor` (identity `:=` / `<-`). **PR-3** adds component-wise dataset +arithmetic (`02`, `13`). **PR-4** adds single-clause `calc` + expression nodes +(`03`). **PR-5** adds `filter` / `sub` with `role=condition` (`04`, `14`). +**PR-6** adds `keep`/`drop` / `rename` (`05`, `06`). Later PRs add more +`visit*` methods. ## Embedded decisions (flag if you disagree) diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java index 4af34b72f..cbd5a223e 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/ProvenanceVisitor.java @@ -2,23 +2,29 @@ import fr.insee.vtl.antlr.runtime.CharStream; import fr.insee.vtl.antlr.runtime.ParserRuleContext; +import fr.insee.vtl.antlr.runtime.Token; import fr.insee.vtl.antlr.runtime.misc.Interval; import fr.insee.vtl.model.Structured.Component; import fr.insee.vtl.model.Structured.DataStructure; +import fr.insee.vtl.parser.VtlBaseVisitor; import fr.insee.vtl.parser.VtlParser; import fr.insee.vtl.prov.utils.VTLTypes; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * Grammar-driven provenance walk ({@code VtlBaseVisitor}). Extends {@link * SupportCheckVisitor} for the shared {@code unsupported: …} surface; mutates a shared {@link * ProvGraph}. * - *

{@code T = Void}: the graph is the artifact; parse {@code ctx} plus run state ({@code - * versions}, oracle, {@code lastResultId}) carry what each visit needs. {@code lastResultId} is the - * dataset/expression node produced by the expression subtree just visited. + *

{@code T = Void}: the graph is the artifact. After visiting an expression, run state describes + * what the enclosing assignment materializes: identity ({@code lastOp == null}), component-wise ops + * ({@code +}, {@code *}, {@code keep}, …), or clause ops ({@code calc}, {@code filter}, {@code + * sub}, {@code rename}). */ final class ProvenanceVisitor extends SupportCheckVisitor { @@ -26,7 +32,14 @@ final class ProvenanceVisitor extends SupportCheckVisitor { private final StructureOracle oracle; private final Map versions = new LinkedHashMap<>(); private int stmtIndex; + private int exprSeq; + private String lastResultId; + private String lastOp; + private List lastOperandIds = List.of(); + private Map lastCalcExprs = Map.of(); + private Map lastRenameFrom = Map.of(); + private List lastConditionExprIds = List.of(); ProvenanceVisitor(ProvGraph graph, StructureOracle oracle, List inputs) { this.graph = graph; @@ -55,32 +68,225 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { if (id == null) { throw new IllegalStateException("unknown dataset " + name); } + clearExprState(); lastResultId = id; + lastOperandIds = List.of(id); return null; } + @Override + public Void visitArithmeticExpr(VtlParser.ArithmeticExprContext ctx) { + return binaryArithmetic(ctx.left, ctx.right, ctx.op); + } + + @Override + public Void visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext ctx) { + return binaryArithmetic(ctx.left, ctx.right, ctx.op); + } + + @Override + public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { + visit(ctx.expr()); + if (lastOp != null || lastResultId == null) { + throw unsupported("clause"); + } + String srcId = lastResultId; + VtlParser.DatasetClauseContext clause = ctx.datasetClause(); + if (clause.calcClause() != null) { + return applyCalc(srcId, clause.calcClause()); + } + if (clause.filterClause() != null) { + return applyFilter(srcId, clause.filterClause()); + } + if (clause.subspaceClause() != null) { + return applySub(srcId, clause.subspaceClause()); + } + if (clause.keepOrDropClause() != null) { + return applyKeepOrDrop(srcId, clause.keepOrDropClause()); + } + if (clause.renameClause() != null) { + return applyRename(srcId, clause.renameClause()); + } + throw unsupported("clause"); + } + + private Void applyCalc(String srcId, VtlParser.CalcClauseContext calc) { + Map calcExprs = new LinkedHashMap<>(); + for (VtlParser.CalcClauseItemContext item : calc.calcClauseItem()) { + String component = item.componentID().getText(); + VtlParser.ExprContext rhs = item.expr(); + String exprId = nextExprId(); + addExpression(exprId, text(rhs), srcId, componentRefs(rhs)); + calcExprs.put(component, exprId); + } + return finishUnaryOp("calc", srcId, Map.copyOf(calcExprs), Map.of(), List.of()); + } + + private Void applyFilter(String srcId, VtlParser.FilterClauseContext filter) { + VtlParser.ExprContext predicate = filter.expr(); + String exprId = nextExprId(); + addExpression(exprId, text(predicate), srcId, componentRefs(predicate)); + return finishUnaryOp("filter", srcId, Map.of(), Map.of(), List.of(exprId)); + } + + private Void applySub(String srcId, VtlParser.SubspaceClauseContext sub) { + List conditionIds = new ArrayList<>(); + for (VtlParser.SubspaceClauseItemContext item : sub.subspaceClauseItem()) { + String exprId = nextExprId(); + addExpression(exprId, text(item), srcId, Set.of(item.componentID().getText())); + conditionIds.add(exprId); + } + return finishUnaryOp("sub", srcId, Map.of(), Map.of(), List.copyOf(conditionIds)); + } + + private Void applyKeepOrDrop(String srcId, VtlParser.KeepOrDropClauseContext keepOrDrop) { + return finishUnaryOp(keepOrDrop.op.getText(), srcId); + } + + private Void applyRename(String srcId, VtlParser.RenameClauseContext rename) { + Map renames = new LinkedHashMap<>(); + for (VtlParser.RenameClauseItemContext item : rename.renameClauseItem()) { + renames.put(item.toName.getText(), item.fromName.getText()); + } + return finishUnaryOp("rename", srcId, Map.of(), Map.copyOf(renames), List.of()); + } + + private Void binaryArithmetic(VtlParser.ExprContext left, VtlParser.ExprContext right, Token op) { + List operands = new ArrayList<>(2); + String leftId = datasetOperand(left); + if (leftId != null) { + operands.add(leftId); + } + String rightId = datasetOperand(right); + if (rightId != null) { + operands.add(rightId); + } + if (operands.isEmpty()) { + throw unsupported("scalar"); + } + clearExprState(); + lastOp = op.getText(); + lastOperandIds = List.copyOf(operands); + return null; + } + + /** {@code null} if the operand is a scalar literal (not a provenance node). */ + private String datasetOperand(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = unwrap(expr); + if (current instanceof VtlParser.VarIdExprContext) { + visit(current); + return lastResultId; + } + if (current instanceof VtlParser.ConstantExprContext) { + return null; + } + throw unsupported("arithmetic"); + } + private Void assign(String out, VtlParser.ExprContext expr) { stmtIndex++; + exprSeq = 0; visit(expr); - String srcId = lastResultId; - if (srcId == null) { - throw new IllegalStateException("assignment RHS produced no result id"); - } String outId = out + "@" + stmtIndex; DataStructure outStructure = oracle.requireDataset(out); addDataset(outId, outStructure, text(expr)); - graph.addEdge(outId, srcId, Map.of("op", "assign")); - for (Component component : outStructure.values()) { - graph.addEdge( - outId + "." + component.getName(), - srcId + "." + component.getName(), - Map.of("op", "assign")); - } + linkAssignment(outId, outStructure); versions.put(out, outId); + clearExprState(); lastResultId = outId; + lastOperandIds = List.of(outId); return null; } + private void linkAssignment(String outId, DataStructure outStructure) { + if (lastOp == null) { + if (lastResultId == null) { + throw unsupported("scalar"); + } + linkComponentWise(outId, outStructure, List.of(lastResultId), "assign"); + return; + } + switch (lastOp) { + case "calc" -> linkCalc(outId, outStructure, lastResultId, lastCalcExprs); + case "filter", "sub" -> + linkConditionClause(outId, outStructure, lastResultId, lastConditionExprIds, lastOp); + case "rename" -> linkRename(outId, outStructure, lastResultId, lastRenameFrom); + default -> linkComponentWise(outId, outStructure, lastOperandIds, lastOp); + } + } + + private void linkCalc( + String outId, DataStructure outStructure, String srcId, Map calcExprs) { + Map edge = opEdge("calc"); + graph.addEdge(outId, srcId, edge); + for (Component component : outStructure.values()) { + String outVar = outId + "." + component.getName(); + String exprId = calcExprs.get(component.getName()); + if (exprId != null) { + graph.addEdge(outVar, exprId, edge); + } else { + graph.addEdge(outVar, srcId + "." + component.getName(), edge); + } + } + } + + private void linkConditionClause( + String outId, + DataStructure outStructure, + String srcId, + List conditionExprIds, + String op) { + Map edge = opEdge(op); + Map condition = new LinkedHashMap<>(edge); + condition.put("role", "condition"); + graph.addEdge(outId, srcId, edge); + for (String exprId : conditionExprIds) { + graph.addEdge(outId, exprId, condition); + } + linkPassThrough(outId, outStructure, srcId, edge); + } + + private void linkRename( + String outId, DataStructure outStructure, String srcId, Map renameFrom) { + Map edge = opEdge("rename"); + graph.addEdge(outId, srcId, edge); + for (Component component : outStructure.values()) { + String srcComponent = renameFrom.getOrDefault(component.getName(), component.getName()); + graph.addEdge(outId + "." + component.getName(), srcId + "." + srcComponent, edge); + } + } + + private void linkComponentWise( + String outId, DataStructure outStructure, List operandIds, String op) { + Map edge = opEdge(op); + for (String operandId : operandIds) { + graph.addEdge(outId, operandId, edge); + } + for (Component component : outStructure.values()) { + String outVar = outId + "." + component.getName(); + for (String operandId : operandIds) { + graph.addEdge(outVar, operandId + "." + component.getName(), edge); + } + } + } + + private void linkPassThrough( + String outId, DataStructure outStructure, String srcId, Map edge) { + for (Component component : outStructure.values()) { + graph.addEdge(outId + "." + component.getName(), srcId + "." + component.getName(), edge); + } + } + + private void addExpression(String exprId, String src, String datasetId, Set refs) { + Map attrs = new LinkedHashMap<>(); + attrs.put("kind", "expression"); + attrs.put("src", src); + graph.addVertex(exprId, attrs); + for (String ref : refs) { + graph.addEdge(exprId, datasetId + "." + ref, Map.of()); + } + } + private void addDataset(String id, DataStructure structure, String src) { if (graph.vertices().containsKey(id)) { return; @@ -101,6 +307,54 @@ private void addDataset(String id, DataStructure structure, String src) { } } + private Void finishUnaryOp(String op, String srcId) { + return finishUnaryOp(op, srcId, Map.of(), Map.of(), List.of()); + } + + private Void finishUnaryOp( + String op, + String srcId, + Map calcExprs, + Map renameFrom, + List conditionExprIds) { + lastOp = op; + lastResultId = srcId; + lastOperandIds = List.of(srcId); + lastCalcExprs = calcExprs; + lastRenameFrom = renameFrom; + lastConditionExprIds = conditionExprIds; + return null; + } + + private void clearExprState() { + lastOp = null; + lastCalcExprs = Map.of(); + lastRenameFrom = Map.of(); + lastConditionExprIds = List.of(); + } + + private String nextExprId() { + exprSeq++; + return "e" + stmtIndex + "." + exprSeq; + } + + private static Map opEdge(String op) { + return Map.of("op", op); + } + + /** Component names referenced in a scalar expression (not dataset bindings). */ + private static Set componentRefs(VtlParser.ExprContext expr) { + Set refs = new LinkedHashSet<>(); + new VtlBaseVisitor() { + @Override + public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { + refs.add(ctx.varID().getText()); + return null; + } + }.visit(expr); + return refs; + } + private static String text(ParserRuleContext ctx) { CharStream input = ctx.getStart().getInputStream(); return input.getText(Interval.of(ctx.getStart().getStartIndex(), ctx.getStop().getStopIndex())); diff --git a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java index 7dd06168a..c61c52e7a 100644 --- a/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java +++ b/vtl-prov/src/main/java/fr/insee/vtl/prov2/SupportCheckVisitor.java @@ -7,6 +7,10 @@ /** * Grammar-only support gate: throws {@code unsupported: …} before the structure oracle runs, so the * corpus backlog stays explicit even when the engine cannot eval the script. + * + *

Mirrors {@link ProvenanceVisitor} coverage: identity assign; binary dataset arithmetic (leaf + * operands); single {@code calc} / {@code filter} / {@code sub} / {@code keep}|{@code drop} / + * {@code rename} on a dataset varId. Nested clauses and other ops stay unsupported. */ class SupportCheckVisitor extends VtlBaseVisitor { @@ -50,12 +54,16 @@ public Void visitVarIdExpr(VtlParser.VarIdExprContext ctx) { @Override public Void visitArithmeticExpr(VtlParser.ArithmeticExprContext ctx) { - throw unsupported("arithmetic"); + leafOperand(ctx.left); + leafOperand(ctx.right); + return null; } @Override public Void visitArithmeticExprOrConcat(VtlParser.ArithmeticExprOrConcatContext ctx) { - throw unsupported("arithmetic"); + leafOperand(ctx.left); + leafOperand(ctx.right); + return null; } @Override @@ -65,6 +73,21 @@ public Void visitUnaryExpr(VtlParser.UnaryExprContext ctx) { @Override public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { + requireDatasetVarId(ctx.expr()); + VtlParser.DatasetClauseContext clause = ctx.datasetClause(); + if (clause.calcClause() != null) { + clause.calcClause().calcClauseItem().forEach(item -> calcRhs(item.expr())); + return null; + } + if (clause.filterClause() != null) { + requireScalarPredicate(clause.filterClause().expr()); + return null; + } + if (clause.subspaceClause() != null + || clause.keepOrDropClause() != null + || clause.renameClause() != null) { + return null; + } throw unsupported("clause"); } @@ -78,7 +101,61 @@ public Void visitConstantExpr(VtlParser.ConstantExprContext ctx) { throw unsupported("scalar"); } - private static UnsupportedOperationException unsupported(String what) { + /** Dataset name or scalar literal; nested ops deferred. */ + private void leafOperand(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = unwrap(expr); + if (current instanceof VtlParser.VarIdExprContext + || current instanceof VtlParser.ConstantExprContext) { + return; + } + throw unsupported("arithmetic"); + } + + private void requireDatasetVarId(VtlParser.ExprContext expr) { + if (!(unwrap(expr) instanceof VtlParser.VarIdExprContext)) { + throw unsupported("clause"); + } + } + + /** Component-level calc RHS (not dataset-level arithmetic). */ + private void calcRhs(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = unwrap(expr); + if (current instanceof VtlParser.VarIdExprContext + || current instanceof VtlParser.ConstantExprContext) { + return; + } + if (current instanceof VtlParser.ArithmeticExprContext arithmetic) { + calcRhs(arithmetic.left); + calcRhs(arithmetic.right); + return; + } + if (current instanceof VtlParser.ArithmeticExprOrConcatContext arithmetic) { + calcRhs(arithmetic.left); + calcRhs(arithmetic.right); + return; + } + throw unsupported("clause"); + } + + /** Filter predicates may use functions/comparisons; reject nested dataset clauses only. */ + private void requireScalarPredicate(VtlParser.ExprContext expr) { + new VtlBaseVisitor() { + @Override + public Void visitClauseExpr(VtlParser.ClauseExprContext ctx) { + throw unsupported("clause"); + } + }.visit(expr); + } + + static VtlParser.ExprContext unwrap(VtlParser.ExprContext expr) { + VtlParser.ExprContext current = expr; + while (current instanceof VtlParser.ParenthesisExprContext parenthesis) { + current = parenthesis.expr(); + } + return current; + } + + static UnsupportedOperationException unsupported(String what) { return new UnsupportedOperationException("unsupported: " + what); } }