From 8b0479e914b7222bd2dc0b6f7cd6f7ec506dea63 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Wed, 22 Jul 2026 10:29:05 -0400 Subject: [PATCH 01/22] Add red acceptance tests for line-aligned emission Every user statement must be emitted at its original source line so backtraces, breakpoints and debugger display are correct by construction, with no SourceMap or backtrace filtering. Four angles: - emitted text places each user statement at its source line - synthetic (loc-less) injected code never displaces user statements - raw backtrace_locations cite source lines with no filtering - the compiled iseq line table contains user statement source lines (the "can break file:N bind" proxy) All four are red today: Unparser.unparse regenerates layout from scratch. Co-authored-by: Cursor --- test/ast_transform/line_alignment_test.rb | 153 ++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 test/ast_transform/line_alignment_test.rb diff --git a/test/ast_transform/line_alignment_test.rb b/test/ast_transform/line_alignment_test.rb new file mode 100644 index 0000000..9b1fb17 --- /dev/null +++ b/test/ast_transform/line_alignment_test.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true +require 'test_helper' +require 'transformation_helper' +require 'ast_transform/instruction_sequence' +require 'ast_transform/transformation' +require 'ast_transform/transformer' + +module ASTTransform + # Acceptance tests for line-aligned emission: every user statement must be + # emitted at its original source line, so that backtraces, debuggers and + # breakpoints are correct by construction — with no mapping or filtering. + class LineAlignmentTest < Minitest::Test + extend ASTTransform::Declarative + include ASTTransform::Helpers::TransformationHelper + + # Injects a synthetic (loc-less) statement at the start of every method + # body. Represents transforms that add code: synthetic statements have no + # source-line truth, so they must never push user statements off their + # source lines. + class SetupInjectionTransformation < ASTTransform::AbstractTransformation + private + + def process_node(node) + # :defs is `def self.name`; its body sits one child later than :def's. + return method(:process).super_method.call(node) unless [:def, :defs].include?(node.type) + + *prefix, body = node.children + injected_setup = s(:send, nil, :injected_setup) + node.updated(nil, [*prefix, s(:begin, injected_setup, *Array(body))]) + end + end + + # Source layout chosen so Unparser's fresh formatting diverges from it: + # blank lines, comments and a multi-line expression shift all following + # statements when the layout is regenerated naively. The gaps are wide + # enough that no naive layout can land the statements on their source + # lines by coincidence. + FIXTURE_SOURCE = <<~HEREDOC + class LineAlignmentFixture + def self.compute + first_value = 1 + + # An explanatory comment: comments vanish from the AST, so a naive + # unparse pulls everything below this line upward. + + second_value = first_value + + 1 + + raise_helper(second_value) + end + end + HEREDOC + + FIXTURE_LINES = { + 'first_value = 1' => 3, + 'second_value = first_value' => 8, + 'raise_helper(second_value)' => 11, + }.freeze + + test "emission places each user statement at its source line" do + emitted = transform_file_source(FIXTURE_SOURCE, ASTTransform::Transformation.new) + + FIXTURE_LINES.each do |statement, source_line| + assert_equal source_line, emitted_line_number(emitted, statement), + "expected `#{statement}` at source line #{source_line} in:\n#{numbered(emitted)}" + end + end + + test "emission keeps user statements on their source lines when a transform injects synthetic code" do + emitted = transform_file_source(FIXTURE_SOURCE, SetupInjectionTransformation.new) + + assert_includes emitted, 'injected_setup' + FIXTURE_LINES.each do |statement, source_line| + assert_equal source_line, emitted_line_number(emitted, statement), + "expected `#{statement}` at source line #{source_line} in:\n#{numbered(emitted)}" + end + end + + test "raw backtrace cites the source line of the raising statement, with no filtering" do + source = <<~HEREDOC + class LineAlignmentRaiseFixture + def self.boom + value = 1 + + # Comments and blank lines force the naive unparse layout to + # diverge from the source layout. + + raise "boom" if value == 1 + end + end + HEREDOC + raise_line = 8 + + iseq = compile(source, 'line_alignment_raise_fixture.rb') + iseq.eval + + error = assert_raises(RuntimeError) { LineAlignmentRaiseFixture.boom } + + assert_equal raise_line, error.backtrace_locations.first.lineno, + "raw backtrace (no SourceMap, no filter) should cite the source line of the raise" + ensure + Object.send(:remove_const, :LineAlignmentRaiseFixture) if Object.const_defined?(:LineAlignmentRaiseFixture) + end + + test "compiled iseq line table contains the source lines of user statements" do + iseq = compile(FIXTURE_SOURCE, 'line_alignment_fixture.rb') + + lines = iseq_lines(iseq) + + FIXTURE_LINES.each do |statement, source_line| + assert_includes lines, source_line, + "a breakpoint on source line #{source_line} (`#{statement}`) should be able to bind; " \ + "line table: #{lines.sort.uniq}" + end + end + + private + + def transform_file_source(source, *transformations) + source_pathname = tmp_pathname('line_alignment_source.rb') + transformed_pathname = tmp_pathname('line_alignment_transformed.rb') + + ASTTransform::Transformer.new(*transformations) + .transform_file_source(source, source_pathname.to_s, transformed_pathname.to_s) + end + + def compile(source, file_name) + ASTTransform::InstructionSequence.source_to_transformed_iseq(source, tmp_pathname(file_name).to_s) + end + + def tmp_pathname(file_name) + Pathname.new('').join(File.expand_path(''), 'tmp', 'test', 'ast_transform', file_name) + end + + # 1-based line number of the first emitted line containing +statement+. + def emitted_line_number(emitted, statement) + index = emitted.lines.index { |line| line.include?(statement) } + index&.+(1) + end + + def numbered(emitted) + emitted.lines.map.with_index(1) { |line, number| format('%3d| %s', number, line) }.join + end + + # All line numbers the VM records for +iseq+ and its children — the lines + # a debugger can bind a `break file:N` to. + def iseq_lines(iseq) + lines = iseq.trace_points.map(&:first) + iseq.each_child { |child| lines.concat(iseq_lines(child)) } + lines + end + end +end From ccc2a44407189f4e42e0fd4a0d10652604f5993d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Wed, 22 Jul 2026 11:30:27 -0400 Subject: [PATCH 02/22] Line-aligned emission: statements emit at their source lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces bare Unparser.unparse in Transformer#transform_file_source with LineAlignedEmitter: every loc-carrying statement is emitted at its original source line (padding with blank lines), synthetic loc-less code packs onto the current line, and multi-line normalizations (e.g. modifier-if) compress back to one line when re-parse-verified safe. Backtraces, breakpoints and debugger display become correct by construction — so SourceMap and its registration are deleted. Authoring layer (TransformationHelper): s routes registered custom types (ASTTransform::Node.register) to their classes; s_at anchors fresh nodes to a source location; defer returns a Deferral placement/execution marker pair (token-linked, ControlFlowGuard-validated); run_after is the paved road for sequence-level reordering. DeferralLowering turns marker pairs into hidden-lvar lambdas and calls, reconciling tokens statically (one placement, one-or-more calls) with UnmatchedDeferralError otherwise. The emitter's postcondition rejects unlowered custom types. ast_transform/test_helpers ships assert_line_aligned and assert_backtrace_lines for transform authors' own suites. Stage 0 acceptance tests are green; full suite 70 tests, 0 failures. Co-authored-by: Cursor --- lib/ast_transform/control_flow_guard.rb | 67 +++++ lib/ast_transform/deferral.rb | 32 ++ lib/ast_transform/deferral_lowering.rb | 84 ++++++ lib/ast_transform/errors.rb | 21 ++ lib/ast_transform/kwargs_builder.rb | 7 + lib/ast_transform/line_aligned_emitter.rb | 281 ++++++++++++++++++ lib/ast_transform/node.rb | 50 ++++ lib/ast_transform/source_map.rb | 237 --------------- lib/ast_transform/test_helpers.rb | 99 ++++++ lib/ast_transform/transformation_helper.rb | 130 +++++++- lib/ast_transform/transformer.rb | 42 +-- .../line_aligned_emitter_test.rb | 124 ++++++++ test/ast_transform/source_map_test.rb | 145 --------- test/ast_transform/test_helpers_test.rb | 46 +++ .../transformation_helper_test.rb | 142 +++++++++ test/ast_transform/transformer_test.rb | 6 +- .../minitest/reporters/rake_rerun_reporter.rb | 6 +- 17 files changed, 1102 insertions(+), 417 deletions(-) create mode 100644 lib/ast_transform/control_flow_guard.rb create mode 100644 lib/ast_transform/deferral.rb create mode 100644 lib/ast_transform/deferral_lowering.rb create mode 100644 lib/ast_transform/errors.rb create mode 100644 lib/ast_transform/line_aligned_emitter.rb create mode 100644 lib/ast_transform/node.rb delete mode 100644 lib/ast_transform/source_map.rb create mode 100644 lib/ast_transform/test_helpers.rb create mode 100644 test/ast_transform/line_aligned_emitter_test.rb delete mode 100644 test/ast_transform/source_map_test.rb create mode 100644 test/ast_transform/test_helpers_test.rb create mode 100644 test/ast_transform/transformation_helper_test.rb diff --git a/lib/ast_transform/control_flow_guard.rb b/lib/ast_transform/control_flow_guard.rb new file mode 100644 index 0000000..d78462d --- /dev/null +++ b/lib/ast_transform/control_flow_guard.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true +require 'parser' +require 'ast_transform/errors' + +module ASTTransform + # Validates that statements are safe to defer. Deferral wraps statements in + # a lambda; control-flow keywords bind to the nearest enclosing scope, so + # keywords written against the original scope would silently re-bind to the + # lambda. This guard turns that semantics hazard into a transform-time error. + # + # Scope rules mirror Ruby's: + # - +break+/+next+/+redo+/+retry+ are owned by the nearest block, so blocks + # are not descended for them. + # - +return+ penetrates plain blocks (it returns from the enclosing method), + # so blocks ARE descended for it. Only defs and lambdas absorb it. + class ControlFlowGuard + BLOCK_OWNED_TYPES = [:break, :next, :redo, :retry].freeze + METHOD_OWNED_TYPES = [:return].freeze + BLOCK_TYPES = [:block, :numblock].freeze + METHOD_DEFINITION_TYPES = [:def, :defs].freeze + + # @param statements [Array] statements about to be deferred + # @return [void] + # @raise [NonDeferrableError] when a statement contains control flow that + # would re-bind to the deferral lambda + def check!(statements) + statements.each { |statement| check_node(statement, BLOCK_OWNED_TYPES + METHOD_OWNED_TYPES) } + nil + end + + private + + def check_node(node, hazardous_types) + return unless node.is_a?(::Parser::AST::Node) + + if hazardous_types.include?(node.type) + raise NonDeferrableError, + "cannot defer a statement containing `#{node.type}`: it would re-bind " \ + "to the deferral lambda and change the code's meaning" + end + + remaining = remaining_hazards(node, hazardous_types) + return if remaining.empty? + + node.children.each { |child| check_node(child, remaining) } + end + + def remaining_hazards(node, hazardous_types) + return [] if METHOD_DEFINITION_TYPES.include?(node.type) || lambda_block?(node) + return hazardous_types - BLOCK_OWNED_TYPES if BLOCK_TYPES.include?(node.type) + + hazardous_types + end + + # A literal lambda parses as (block (lambda) args body); Kernel#lambda as + # (block (send nil :lambda) args body). Unlike plain blocks, both absorb + # +return+. + def lambda_block?(node) + return false unless BLOCK_TYPES.include?(node.type) + + callee = node.children[0] + return false unless callee.is_a?(::Parser::AST::Node) + + callee.type == :lambda || (callee.type == :send && callee.children == [nil, :lambda]) + end + end +end diff --git a/lib/ast_transform/deferral.rb b/lib/ast_transform/deferral.rb new file mode 100644 index 0000000..bea1a71 --- /dev/null +++ b/lib/ast_transform/deferral.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true +require 'ast_transform/errors' + +module ASTTransform + # The handle returned by +defer+: not a node, a pair of nodes. Both markers + # are ordinary nodes in the ast_transform IR — the emitter keys on node type + # and token, never on any class — created together so the pair cannot be + # mismatched. + # + # placement:: (:ast_deferred, token, (:begin, ...)) — the deferred body, + # spliced at the statements' SOURCE position; lowered to + # +__ast_deferred___ = -> { ... }+. + # execution:: (:ast_deferred_call, token) — loc-less, spliced (or composed + # into an expression, e.g. an assert_raises block body) at the + # execution point; lowered to +__ast_deferred___.call+. + Deferral = Data.define(:placement, :execution) + + # The pairing mechanism between the two halves of a Deferral: it answers + # "which lambda does this call marker invoke?" when a scope holds several + # deferrals. The markers cannot reference each other's nodes — Processor and + # Node#updated rebuilds create new node objects, so node identity does not + # survive transformation passes. Children DO survive (carried by reference + # through every rebuild), so both markers carry this same child object and + # the emitter pairs by its object identity. No behavior needed — a named + # class over a bare Object.new only for self-documenting AST dumps and + # greppability. + class DeferralToken + def inspect + "#" + end + end +end diff --git a/lib/ast_transform/deferral_lowering.rb b/lib/ast_transform/deferral_lowering.rb new file mode 100644 index 0000000..74e48dc --- /dev/null +++ b/lib/ast_transform/deferral_lowering.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true +require 'ast_transform/node' +require 'ast_transform/errors' +require 'ast_transform/transformation_helper' + +module ASTTransform + # Lowers deferral markers into plain Ruby nodes ahead of emission: + # + # (:ast_deferred, token, (:begin, ...)) => __ast_deferred___ = -> { ... } + # (:ast_deferred_call, token) => __ast_deferred___.call + # + # Hidden lvar names are assigned per token in encounter order, so they are + # stable within a file and never collide. Pairing is by token object + # identity (see DeferralToken). + # + # Reconciliation is a static count of markers in the tree, not of runtime + # executions — a call under a conditional legitimately executes zero-or-more + # times. One placement may have many calls (multiplexing); it must have at + # least one, textually after it (the lambda must exist before it is called). + class DeferralLowering + include TransformationHelper + + def initialize + @names_by_token = {}.compare_by_identity + @called_tokens = {}.compare_by_identity + end + + # @param node [Parser::AST::Node] tree possibly containing deferral markers + # @return [Parser::AST::Node] tree with markers lowered to plain Ruby + # @raise [UnmatchedDeferralError] on missing call, orphan or premature + # call, or duplicate placement + def run(node) + lowered = lower(node) + + unexecuted = @names_by_token.keys.reject { |token| @called_tokens.key?(token) } + unless unexecuted.empty? + raise UnmatchedDeferralError, + "deferred statements were placed but never executed (#{unexecuted.size} deferral(s) " \ + "without an execution point); the deferred code would silently never run" + end + + lowered + end + + private + + def lower(node) + return node unless node.is_a?(::Parser::AST::Node) + + case node.type + when :ast_deferred then lower_placement(node) + when :ast_deferred_call then lower_call(node) + else + node.updated(nil, node.children.map { |child| lower(child) }) + end + end + + def lower_placement(node) + token, body = node.children + if @names_by_token.key?(token) + raise UnmatchedDeferralError, + "duplicate deferral placement: the hidden lambda would be assigned twice" + end + + name = :"__ast_deferred_#{@names_by_token.size + 1}__" + @names_by_token[token] = name + + s(:lvasgn, name, s(:block, s(:lambda), s(:args), lower(body))) + end + + def lower_call(node) + token = node.children[0] + name = @names_by_token[token] + unless name + raise UnmatchedDeferralError, + "deferral execution point encountered without a preceding placement; " \ + "the placement must appear textually before its execution" + end + + @called_tokens[token] = true + s(:send, s(:lvar, name), :call) + end + end +end diff --git a/lib/ast_transform/errors.rb b/lib/ast_transform/errors.rb new file mode 100644 index 0000000..9bf55f3 --- /dev/null +++ b/lib/ast_transform/errors.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +module ASTTransform + # Raised by authoring helpers (e.g. +s_at+) when a node that must carry a + # source location does not have one. + class MissingLocationError < StandardError; end + + # Raised by +defer+ when a deferred statement contains control flow that + # would re-bind to the deferral lambda (e.g. +return+), silently changing + # the meaning of the user's code. + class NonDeferrableError < StandardError; end + + # Raised at emission when deferral markers cannot be reconciled: a + # placement without any execution point, an execution point without a + # placement (or preceding it), or a duplicate placement. + class UnmatchedDeferralError < StandardError; end + + # Raised as the emitter's postcondition when a custom node type (ast_* + # markers or types registered on ASTTransform::Node) reaches the unparse + # boundary instead of being lowered by the stage that understands it. + class UnloweredNodeTypeError < StandardError; end +end diff --git a/lib/ast_transform/kwargs_builder.rb b/lib/ast_transform/kwargs_builder.rb index cc6443a..b5a2c60 100644 --- a/lib/ast_transform/kwargs_builder.rb +++ b/lib/ast_transform/kwargs_builder.rb @@ -11,6 +11,13 @@ module ASTTransform # braces: :hash gets `{}`, :kwargs does not. Since Ruby 3.0+ treats these as # semantically different (strict keyword/positional separation), we need the # AST to preserve the distinction. + # + # NOTE: parsed nodes deliberately stay plain Parser::AST::Node. Custom node + # classes exist only for registered custom types (see ASTTransform::Node), + # which are IR and never reach Unparser: AST::Node#eql? compares class, and + # Unparser verifies dynamic-string emission by re-parsing and comparing + # eql? against the freshly parsed (plain-class) node — custom-class nodes + # of standard types would fail that verification. class KwargsBuilder < Prism::Translation::Parser::Builder def associate(begin_t, pairs, end_t) node = super diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb new file mode 100644 index 0000000..9576474 --- /dev/null +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -0,0 +1,281 @@ +# frozen_string_literal: true +require 'unparser' +require 'ast_transform/node' +require 'ast_transform/errors' +require 'ast_transform/deferral_lowering' + +module ASTTransform + # Emits a transformed AST as text in which every loc-carrying statement + # occupies its original source line, so that backtraces, breakpoints and + # debugger display are correct by construction — CRuby derives line numbers + # from physical text position, so placement is our line table. + # + # Cursor algorithm over statement sequences: + # + # 1. Statement has loc and target_line > cursor: pad with newlines, emit at + # the target line. + # 2. Statement has loc and target_line <= cursor: pack (`; `) onto the + # current line. A user statement landing here means the transform moved + # it — the alignment auditor's concern, not a runtime failure. + # 3. No loc: pack onto the current line — synthetic code has no source-line + # truth to preserve. + # + # Multi-line renders advance the cursor by their height; displaced + # statements pack and emission re-anchors at the next statement that fits. + # Total: never raises on layout. + # + # Deferral markers are lowered (DeferralLowering) before layout; the + # emitter's postcondition is that no custom node type (ast_* markers or + # types registered on ASTTransform::Node) crosses the unparse boundary — + # they are IR between stages that understand them. + class LineAlignedEmitter + # Containers the emitter recurses into so nested statements align; every + # other node renders as an Unparser blob at its head line. + RECURSIVE_CONTAINER_TYPES = [:class, :module, :sclass, :def, :defs, :block, :numblock, :kwbegin].freeze + BODY_INDEXES = { + class: 2, module: 1, sclass: 1, def: 2, defs: 3, block: 2, numblock: 2 + }.freeze + # Assignments whose value is a block (e.g. the lowered deferral lambda) + # recurse into the block so its body statements align. + ASSIGNMENT_TYPES = [:lvasgn, :ivasgn, :gvasgn, :casgn].freeze + + # @param ast [Parser::AST::Node] transformed AST + # @param source_path [String] original file path (for error messages) + def initialize(ast, source_path) + @ast = ast + @source_path = source_path + @local_variables = Set.new + end + + # @return [String] transformed source, line-aligned + # @raise [UnmatchedDeferralError] if deferral markers cannot be reconciled + # @raise [UnloweredNodeTypeError] if a custom node type survived to emission + def emit + lowered = DeferralLowering.new.run(@ast) + assert_no_custom_types(lowered) + + @local_variables = collect_local_variables(lowered) + @lines = [] + emit_statements(statements_of(lowered)) + "#{@lines.join("\n")}\n" + end + + private + + def emit_statements(statements) + statements.each { |statement| emit_statement(statement) } + end + + def emit_statement(node) + if recursive_container?(node) + emit_container(node) + else + place(node.loc&.line, aligned_render(node)) + end + end + + # Unparser normalizes some single-line constructs into multi-line form + # (e.g. modifier-if into if/end), which would push following statements + # off their lines. When the render is taller than the statement's source, + # compress it back to one line — verified by re-parse so a statement that + # cannot be safely single-lined (e.g. containing a heredoc) falls back to + # its multi-line render and re-anchors after itself. + def aligned_render(node) + render = unparse(node) + loc = node.loc + return render unless loc.respond_to?(:last_line) && loc.line + + source_height = loc.last_line - loc.line + 1 + return render if render.count("\n") < source_height + + compress_to_single_line(render) || render + end + + def compress_to_single_line(render) + candidate = render.split("\n").map(&:strip).join('; ') + # Both sides parsed without scope context, so lvar/send ambiguity + # cancels out; equality means the newline join preserved structure. + Unparser.parse(candidate) == Unparser.parse(render) ? candidate : nil + rescue Parser::SyntaxError + nil + end + + # Statements are unparsed in isolation, losing the surrounding scope's + # local-variable context; without it, Unparser re-parses identifiers as + # method calls and its dstr round-trip verification fails. Feed it every + # local assigned or bound anywhere in the tree — an over-approximation + # that is safe because it only informs Unparser's re-parse verification. + def unparse(node) + Unparser.unparse(node, static_local_variables: @local_variables) + end + + LOCAL_BINDING_TYPES = [:lvasgn, :arg, :optarg, :restarg, :kwarg, :kwoptarg, :blockarg, :shadowarg].freeze + + def collect_local_variables(node, names = Set.new) + return names unless node.is_a?(::Parser::AST::Node) + + names << node.children[0] if LOCAL_BINDING_TYPES.include?(node.type) && node.children[0] + node.children.each { |child| collect_local_variables(child, names) } + names + end + + # Emits a container body that may be a bare :ensure/:rescue node (their + # begin/end context comes from the surrounding def/block/kwbegin, so the + # keywords must be emitted inline, aligned like statements). + def emit_body(body) + case body&.type + when :ensure then emit_ensure(body) + when :rescue then emit_rescue(body) + else emit_statements(statements_of(body)) + end + end + + def emit_ensure(node) + *body, ensurer = node.children + body.each { |statement| emit_body(statement) } + place_keyword(keyword_line(node), 'ensure') + emit_statements(statements_of(ensurer)) + end + + def emit_rescue(node) + body, *resbodies, else_body = node.children + emit_body(body) + resbodies.each { |resbody| emit_resbody(resbody) } + return if else_body.nil? + + place_keyword(nil, 'else') + emit_statements(statements_of(else_body)) + end + + def emit_resbody(node) + exceptions, capture, body = node.children + header = ['rescue'] + header << " #{Unparser.unparse(exceptions).delete_prefix('[').delete_suffix(']')}" if exceptions + header << " => #{capture.children[0]}" if capture + place_keyword(node.loc&.line, header.join) + emit_statements(statements_of(body)) + end + + # Keywords (rescue/ensure/else) cannot be `;`-packed after a statement; + # when their line is taken they go on a fresh line instead. + def place_keyword(target_line, keyword) + if target_line && target_line > @lines.size + place(target_line, keyword) + else + @lines << keyword + end + end + + def keyword_line(node) + loc = node.loc + loc.keyword.line if loc.respond_to?(:keyword) && loc.keyword + end + + def recursive_container?(node) + RECURSIVE_CONTAINER_TYPES.include?(node.type) || block_assignment?(node) + end + + def block_assignment?(node) + ASSIGNMENT_TYPES.include?(node.type) && node.children.last.is_a?(::Parser::AST::Node) && + [:block, :numblock].include?(node.children.last.type) + end + + # Renders a container's opener and closer from the node with its body + # emptied, then recurses into the body so nested statements align. + def emit_container(node) + opener, closer = container_delimiters(node) + place(node.loc&.line, opener) + emit_body(container_body(node)) + place(closer_line(node), closer) + end + + def container_delimiters(node) + rendered = Unparser.unparse(empty_container(node)).split("\n").reject(&:empty?) + opener = rendered[0..-2].join("\n") + closer = rendered.last + + # Unparser renders empty blocks with braces, but brace blocks cannot + # hold rescue/ensure bodies; do/end always can. + if opener.end_with?(' {') && closer == '}' + [opener.sub(/ \{\z/, ' do'), 'end'] + else + [opener, closer] + end + end + + def empty_container(node) + case node.type + when :kwbegin + node.updated(nil, []) + when *ASSIGNMENT_TYPES + block_node = node.children.last + emptied_block = block_node.updated(nil, [*block_node.children[0..-2], nil]) + node.updated(nil, [*node.children[0..-2], emptied_block]) + else + children = node.children.dup + children[BODY_INDEXES.fetch(node.type)] = nil + node.updated(nil, children) + end + end + + def container_body(node) + case node.type + when :kwbegin then node.children.size == 1 ? node.children.first : node.updated(:begin, node.children) + when *ASSIGNMENT_TYPES then node.children.last.children[2] + else node.children[BODY_INDEXES.fetch(node.type)] + end + end + + # The line the container's `end`/`}` occupies in the source, when known. + def closer_line(node) + loc = node.loc + loc.end.line if loc.respond_to?(:end) && loc.end + end + + def statements_of(body) + case body + when nil then [] + when ::Parser::AST::Node then body.type == :begin ? body.children : [body] + else [body] + end + end + + # Places +render+ at +target_line+ when the cursor hasn't passed it; + # otherwise packs onto the current line. Multi-line renders advance the + # cursor by their height. + def place(target_line, render) + first, *rest = render.split("\n") + + if target_line && target_line > @lines.size + @lines << '' while @lines.size < target_line + @lines[-1] = first + else + pack(first) + end + + @lines.concat(rest) + end + + def pack(text) + if @lines.empty? + @lines << text + elsif @lines.last.empty? + @lines[-1] = text + else + @lines[-1] = "#{@lines.last}; #{text}" + end + end + + def assert_no_custom_types(node) + return unless node.is_a?(::Parser::AST::Node) + + if node.type.start_with?('ast_') || Node.registry.key?(node.type) + raise UnloweredNodeTypeError, + "custom node type :#{node.type} reached emission in #{@source_path}; custom types are " \ + "IR between transformation stages and must be lowered by the stage that understands them" + end + + node.children.each { |child| assert_no_custom_types(child) } + end + end +end diff --git a/lib/ast_transform/node.rb b/lib/ast_transform/node.rb new file mode 100644 index 0000000..7375912 --- /dev/null +++ b/lib/ast_transform/node.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +require 'parser' + +module ASTTransform + # Base class for custom IR nodes. Transform authors subclass it and + # register a custom node type to get type-routed construction from +s+ + # with domain accessors: + # + # class InteractionNode < ASTTransform::Node + # register :rspock_interaction + # + # def cardinality = children[0] + # end + # + # s(:rspock_interaction, ...) # => InteractionNode + # + # Custom node *types* are IR between stages that understand them and must + # be lowered before emission (the emitter enforces this). Standard-typed + # nodes deliberately stay plain Parser::AST::Node everywhere — parsed and + # +s+-built alike: AST::Node#eql? compares class, and Unparser verifies + # dynamic-string emission by re-parsing and eql?-comparing, so a custom + # class on a standard type breaks emission. + class Node < ::Parser::AST::Node + class << self + # Registers +self+ as the class to construct for +type+ nodes. + # + # @param type [Symbol] the custom node type routed to this class + # @return [void] + def register(type) + Node.registry[type] = self + end + + # Builds a node of +type+: registered types construct their custom + # class, everything else a plain Parser::AST::Node. + # + # @param type [Symbol] node type + # @param children [Array] child nodes / literals + # @param properties [Hash] node properties (e.g. location:) + # @return [Parser::AST::Node] + def build(type, children, properties = {}) + klass = Node.registry.fetch(type, ::Parser::AST::Node) + klass.new(type, children, properties) + end + + def registry + @registry ||= {} + end + end + end +end diff --git a/lib/ast_transform/source_map.rb b/lib/ast_transform/source_map.rb deleted file mode 100644 index cdc67b8..0000000 --- a/lib/ast_transform/source_map.rb +++ /dev/null @@ -1,237 +0,0 @@ -# frozen_string_literal: true - -require "parser" - -module ASTTransform - class SourceMap - class << self - # Registers the given SourceMap. - # - # @param source_map [SourceMap] The source map to be registered. - # - # @return [void] - def register_source_map(source_map) - source_maps[source_map.transformed_file_path] = source_map - source_maps[source_map.source_file_path] = source_map - - nil - end - - # Retrieves the SourceMap for the given +file_path+. - # - # @param file_path [String] The transformed file path. - # - # @return [SourceMap|nil] The associated source map. - def for_file_path(file_path) - source_maps[file_path] - end - - private - - def source_maps - # Class instance var (not @@): read/written only through this method - # inside class << self, and SourceMap has no subclasses to share with. - @source_maps ||= {} - end - end - - # Constructs a new SourceMap instance. - # - # Note: +source_ranges_ast+ and +transformed_ranges_ast+ must be equivalent ASTs. - # - # @param source_file_path [String] The path to the source file. - # @param transformed_file_path [String] The path to the transformed file. - # @param source_ranges_ast [Parser::AST::Node] A transformed AST that contains the source code ranges. - # @param transformed_ranges_ast [Parser::AST::Node] A transformed AST that contains the ranges for the executed - # code. - def initialize(source_file_path, transformed_file_path, source_ranges_ast, transformed_ranges_ast) - @source_file_path = source_file_path - @transformed_file_path = transformed_file_path - @source_ranges_ast = source_ranges_ast - @transformed_ranges_ast = transformed_ranges_ast - - @lines = Hash.new { |hash, key| hash[key] = [] } - extract_source_map_data(@transformed_ranges_ast, []) - @source_map = build_source_map.freeze - end - - attr_reader :source_file_path, :transformed_file_path, :source_map - - # Retrieves the mapped line number for the given +line_number+. - # - # @param line_number [Integer] The line number in the executed code to be mapped to the source. - # - # @return [Integer|nil] The mapped line number, otherwise nil if not found. - def line(line_number) - @source_map[line_number] - end - - # Retrieves the line count for the executed code. - # - # @return [Integer] The line count. - def line_count - @transformed_ranges_ast&.loc&.expression&.last_line || 0 - end - - private - - # Extracts SourceMap data from the given node. - # - # @param node [Parser::AST::Node] The node containing ranges for the executed code. - # - # @return [void] - def extract_source_map_data(node, indexes) - return false unless node&.is_a?(Parser::AST::Node) - - range = node.loc&.expression - - if range && range.line == range.last_line - @lines[range.line] << indexes.dup - end - - node.children.each.with_index do |child, index| - extract_source_map_data(child, indexes.dup << index) - end - - nil - end - - # Builds the source map. - # - # @return [Hash] A Hash containing line numbers from executed code to source code. - def build_source_map - (1..line_count).each.with_object({}) { |it, hash| hash[it] = source_line(it) } - end - - # Retrieves the source line for the given +line_number+ in the executed code. - # - # @param line_number [Integer] The line number in the executed code. - # - # @return [Integer|nil] The line number in the source code, or nil if cannot be mapped. - def source_line(line_number) - if @lines.key?(line_number) - @lines[line_number].each do |dig_array| - source_node = approximate_dig_last_valid_node(@source_ranges_ast, dig_array) - next unless source_node - - range = search_range(source_node, 1) - return range.line if range - end - end - - nil - end - - # Recursively look for node represented by +indexes+ in +node+. If not found, goes back +depth+ nodes and search for - # the node pointed to by +indexes+. - # - # @param node [Parser::AST::Node] The node to search into. This must be a node in the +@source_ranges_ast+. - # @param indexes [Array] Child indexes pointing to the node we're looking for in +node+. - # @param depth [Integer] Number of nodes to go up to search for the node pointed to by +indexes+. - # - # @return [Parser::AST::Node|nil] The node found, nil otherwise. - def approximate_dig_last_valid_node(node, indexes, depth = 1) - return node if indexes.empty? - - result = dig_node(node, indexes) - return result if result.is_a?(Parser::AST::Node) || depth <= 0 - - queried_node = dig_last_valid_node(@transformed_ranges_ast, indexes) - - last_known_index = dig_last_valid_node_index(node, indexes[0...-depth]) - query_indexes = indexes[0...last_known_index] - - last_known_node = dig_node(node, query_indexes) - - search_node(last_known_node, queried_node) - end - - # Recursively search the children of +node+ for an equivalent +queried_node+. - # - # @param node [Parser::AST::Node] The current node to search in. - # @param queried_node [Parser::AST::Node] The equivalent node to search for. - # - # @return [Parser::AST::Node|nil] The found node from the +node+ graph, nil otherwise. - def search_node(node, queried_node) - return unless node&.is_a?(Parser::AST::Node) - return node if node == queried_node - - node.children.each do |child_node| - result = search_node(child_node, queried_node) - return result if result - end - - nil - end - - # Recursively search the given +node+ for a range. - # - # @param node [Parser::AST::Node] The current node to search in. - # @param max_range [Integer|nil] The max range to consider valid. Nil means any range is valid. If 1, only ranges - # which span one line will be considered, etc... - # - # @return [Parser::Source::Range|nil] The range, or nil if no range was found. This occurs when the tree contains - # no ranges, i.e. they're all virtually built nodes. - def search_range(node, max_range = nil) - return unless node&.is_a?(Parser::AST::Node) - - range = node.loc&.expression - if range && max_range && range.last_line - range.line < max_range || range && max_range.nil? - return range - else - node.children.each do |child_node| - result = search_range(child_node, max_range) - return result if result - end - end - - nil - end - - # Finds the index for the last valid node represented by +indexes+ in the children of +node+. - # - # @param node [Parser::AST::Node] The current node to search in. - # @param indexes [Array] The array of indexes pointing to the child node to be retrieved from +node+. - # - # @return [Integer|nil] The index of the node if found, nil otherwise. - def dig_last_valid_node_index(node, indexes) - return if indexes.empty? - - result = dig_node(node, indexes) - current_index = indexes&.size - return current_index if result.is_a?(Parser::AST::Node) - - dig_last_valid_node_index(node, indexes[0...-1]) - end - - # Recursively look for the node represented by +indexes+ in the children of +node+. If not found, returns the last - # valid node. - # - # @param node [Parser::AST::Node] The node to look into. - # @param indexes [Array] The array of indexes pointing to the child node to be retrieved from +node+. - # - # @return [Parser::AST::Node|nil] The node if found, nil otherwise. - def dig_last_valid_node(node, indexes) - return node if indexes.empty? - - result = dig_node(node, indexes) - return result if result.is_a?(Parser::AST::Node) - - dig_last_valid_node(node, indexes[0...-1]) - end - - # Recursively look for the node represented by +indexes+ in the children of +node+. - # - # @param node [Parser::AST::Node] The node to look into. - # @param indexes [Array] The array of indexes pointing to the child node to be retrieved from +node+. - # - # @return [Parser::AST::Node|nil] The node if found, nil otherwise. - def dig_node(node, indexes) - indexes.inject(node) do |node, index| - return nil unless node.is_a?(Parser::AST::Node) - - node.children[index] - end - end - end -end diff --git a/lib/ast_transform/test_helpers.rb b/lib/ast_transform/test_helpers.rb new file mode 100644 index 0000000..6bca1a5 --- /dev/null +++ b/lib/ast_transform/test_helpers.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true +require 'ast_transform/transformer' +require 'ast_transform/instruction_sequence' + +module ASTTransform + # Assertions for transform authors' own test suites — the enforcement arm + # of the authoring contract ("textual order is source order"). Never loaded + # in production; require it from test code: + # + # require "ast_transform/test_helpers" + # + # class MyTransformationTest < Minitest::Test + # include ASTTransform::TestHelpers + # end + module TestHelpers + # Transforms +source+ through the real pipeline (transform + line-aligned + # emission), re-parses both sides, matches surviving statements by + # location, and asserts each one's emitted line equals its source line. + # Statements the transform deletes (e.g. description strings) are exempt; + # statements the transform rewrites in place keep their anchor and are + # checked. + # + # @param source [String] fixture source + # @param transformations [Array] + # @param path [String] pseudo-path used for parsing and messages + # @return [void] + def assert_line_aligned(source, *transformations, path: 'fixture.rb') + transformer = Transformer.new(*transformations) + emitted = transformer.transform_file_source(source, path, path) + + source_lines_by_statement = statement_lines(transformer.build_ast(source, file_path: path)) + emitted_lines_by_statement = statement_lines(transformer.build_ast(emitted, file_path: path)) + + misaligned = source_lines_by_statement.filter_map do |render, source_line| + emitted_line = emitted_lines_by_statement[render] + next if emitted_line.nil? || emitted_line == source_line + + format(' MISALIGNED %s: source line %d, emitted line %d', render, source_line, emitted_line) + end + + assert misaligned.empty?, <<~MESSAGE + expected every surviving statement at its source line in #{path}: + #{misaligned.join("\n")} + + emitted: + #{numbered_listing(emitted)} + MESSAGE + end + + # Runtime complement of assert_line_aligned: compiles +source+ through + # the full pipeline under +path+, executes it, and asserts the raw first + # backtrace frame — no filtering of any kind — is ":". + # + # @param source [String] fixture that raises when executed + # @param path [String] pseudo source path to compile under + # @param raise_at [Integer] expected source line of the raise + # @return [void] + def assert_backtrace_lines(source, path:, raise_at:) + iseq = InstructionSequence.source_to_transformed_iseq(source, path) + + error = assert_raises(StandardError, "fixture at #{path} should raise when executed") do + iseq.eval + end + + location = error.backtrace_locations.first + assert_equal "#{location.path}:#{raise_at}", "#{location.path}:#{location.lineno}", + "raw backtrace should cite source line #{raise_at} of #{path}" + end + + private + + # Flat statement renders and their first line, keyed by unparsed text so + # source and emitted sides can be matched without location identity. + # Duplicate renders keep their first occurrence — good enough for + # fixtures, which authors control. + def statement_lines(ast, lines = {}) + return lines unless ast.is_a?(::Parser::AST::Node) + + if statement_sequence?(ast) + ast.children.each do |statement| + next unless statement.is_a?(::Parser::AST::Node) && statement.loc&.expression + + lines[Unparser.unparse(statement)] ||= statement.loc.line + end + end + + ast.children.each { |child| statement_lines(child, lines) } + lines + end + + def statement_sequence?(node) + [:begin, :kwbegin].include?(node.type) + end + + def numbered_listing(source) + source.lines.map.with_index(1) { |line, number| format('%3d| %s', number, line) }.join + end + end +end diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index df0f20f..e90f265 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -1,8 +1,23 @@ # frozen_string_literal: true - -require "parser" +require 'parser' +require 'ast_transform/node' +require 'ast_transform/deferral' +require 'ast_transform/control_flow_guard' +require 'ast_transform/errors' module ASTTransform + # The transform-authoring layer. Three shapes: + # + # - Constructors (+s+, +s_at+): type + children in, fresh node out. + # - The sequence combinator (+run_after+): sequence in, sequence out — the + # paved road for execution reordering. + # - The low-level deferral primitive (+defer+): statements in, Deferral + # pair out — for execution points inside expressions. + # + # The contract these helpers serve: textual order is source order. The + # emitter places every loc-carrying statement at its source line; when + # execution order must differ from textual order, authors express it as a + # deferral instead of moving text. module TransformationHelper class << self def included(base) @@ -12,8 +27,117 @@ def included(base) end module Methods + # Builds a loc-less node. The emitter packs loc-less nodes onto the + # current output line — the correct default for synthetic code, which + # has no source-line truth to preserve. + # + # @param type [Symbol] node type + # @param children [Array] child nodes / literals + # @param properties [Hash] node properties (e.g. location:) + # @return [ASTTransform::Node] node routed to its registered class def s(type, *children, **properties) - Parser::AST::Node.new(type, children, properties) + Node.build(type, children, properties) + end + + # Builds a fresh node anchored to another node's source location. Use + # when composing a replacement tree whose root isn't derived from the + # node it replaces (otherwise prefer +anchor.updated(...)+). The + # attached map is a clean expression-only Source::Map over + # +anchor.loc.expression+ — no stale typed sub-ranges (selector etc.). + # Anchor inheritance is shallow; children keep or lack their own locs. + # + # @param anchor [Parser::AST::Node] node whose line this code replaces + # @param type [Symbol] node type + # @param children [Array] child nodes / literals + # @return [ASTTransform::Node] node carrying anchor's expression range + # @raise [MissingLocationError] if anchor has no expression location + def s_at(anchor, type, *children) + expression = anchor.loc&.expression + raise MissingLocationError, "anchor #{anchor.type} node has no source location" unless expression + + s(type, *children, location: ::Parser::Source::Map.new(expression)) + end + + # Low-level deferral primitive. Deferral is the one reordering lever: + # text never moves and execution can only move later, so "hoist A above + # B" is expressed as "run B after A". Returns a Deferral pairing two + # plain marker nodes: splice +placement+ where the statements sit in + # the SOURCE (inner statements keep their own locs, so the emitter + # aligns the body even though execution waits) and +execution+ where + # they run — composable inside expressions, e.g. as an assert_raises + # block body. The emitter lowers the pair to a hidden-lvar lambda and + # its call (the lambda shares the enclosing method binding, so lvar + # assignments propagate out). + # + # Prefer +run_after+ when both points sit in one statement sequence. + # + # @param statements [Array] statements to defer + # @return [ASTTransform::Deferral] the placement/execution marker pair + # @raise [NonDeferrableError] if a statement contains control flow that + # would re-bind to the deferral lambda + def defer(*statements) + ControlFlowGuard.new.check!(statements) + token = DeferralToken.new + Deferral.new( + placement: s(:ast_deferred, token, s(:begin, *statements)), + execution: s(:ast_deferred_call, token) + ) + end + + # The paved road for execution reordering in flat statement sequences: + # one call, both placements handled, nothing to forget. Named for the + # constraint, not the mechanism — "run X after Y" covers hoisting and + # sinking symmetrically, because with text pinned to source lines the + # only physical lever is delaying execution. +after+ may be textually + # before or after the +run+ statements. Returns a NEW sequence in which + # the +run+ statements are replaced (in place) by one placement marker + # and its execution marker is inserted immediately after +after+. + # + # All membership checks are by identity (equal?), never ==: node + # equality ignores location, so two textually identical statements on + # different lines compare == and value matching could splice the wrong + # one. + # + # @param statements [Array] the sequence being composed + # @param run [Array] contiguous run of elements of + # +statements+ (by identity) whose execution must wait + # @param after [Parser::AST::Node] element of +statements+ (by identity, + # not inside +run+) the +run+ statements execute after + # @return [Array] new sequence with markers placed + # @raise [NonDeferrableError] per +defer+ + # @raise [ArgumentError] if +run+ is not a contiguous identity-run of + # +statements+, or +after+ is not an element (or is inside +run+) + def run_after(statements, run:, after:) + run_range = contiguous_identity_range(statements, run) + raise ArgumentError, "run: must be a contiguous run of elements of statements (by identity)" unless run_range + + after_index = statements.index { |statement| statement.equal?(after) } + raise ArgumentError, "after: must be an element of statements (by identity)" unless after_index + raise ArgumentError, "after: cannot be inside run:" if run_range.cover?(after_index) + + deferral = defer(*run) + reordered = statements.dup + reordered[run_range] = [deferral.placement] + + insertion_index = reordered.index { |statement| statement.equal?(after) } + reordered.insert(insertion_index + 1, deferral.execution) + end + + private + + # The range +members+ occupies in +sequence+, or nil unless members is + # a non-empty contiguous identity-run in order. + def contiguous_identity_range(sequence, members) + return nil if members.empty? + + start = sequence.index { |element| element.equal?(members.first) } + return nil unless start + + contiguous = members.each_with_index.all? do |member, offset| + sequence[start + offset]&.equal?(member) + end + + contiguous ? (start...(start + members.size)) : nil end end end diff --git a/lib/ast_transform/transformer.rb b/lib/ast_transform/transformer.rb index 99508e1..baab326 100644 --- a/lib/ast_transform/transformer.rb +++ b/lib/ast_transform/transformer.rb @@ -1,10 +1,9 @@ # frozen_string_literal: true - -require "prism" -require "prism/translation/parser" -require "unparser" -require "ast_transform/kwargs_builder" -require "ast_transform/source_map" +require 'prism' +require 'prism/translation/parser' +require 'unparser' +require 'ast_transform/kwargs_builder' +require 'ast_transform/line_aligned_emitter' module ASTTransform class Transformer @@ -49,7 +48,8 @@ def transform(source) # Transforms the give +file_path+. # - # @param file_path [String] The input file to be transformed. This is required for source mapping in backtraces. + # @param file_path [String] The input file to be transformed. Statement placement (and therefore + # backtrace and breakpoint line numbers) is derived from this file's source locations. # @param transformed_file_path [String] The file path to the transformed file. # # @return [String] The transformed code. @@ -61,21 +61,19 @@ def transform_file(file_path, transformed_file_path) # Transforms the given +source+ in +file_path+. # # @param source [String] The input source code to be transformed. - # @param file_path [String] The file path for the input +source+. This is required for source mapping in backtraces. - # @param transformed_file_path [String] The file path to the transformed filed. This is required to register the - # SourceMap. + # @param file_path [String] The file path for the input +source+. Statement placement (and + # therefore backtrace and breakpoint line numbers) is derived from the source locations parsed + # under this path. + # @param transformed_file_path [String] The file path the transformed file will be written to. # - # @return [String] The transformed code. - def transform_file_source(source, file_path, transformed_file_path) + # @return [String] The transformed code, line-aligned: every statement carrying a source + # location is emitted at its original source line. + def transform_file_source(source, file_path, _transformed_file_path) source_ast = build_ast(source, file_path: file_path) - # At this point, the transformed_ast contains line number mappings for the original +source+. + # At this point, the transformed_ast contains source locations for the original +source+. transformed_ast = transform_ast(source_ast) - transformed_source = Unparser.unparse(transformed_ast) - - register_source_map(file_path, transformed_file_path, transformed_ast, transformed_source) - - transformed_source + LineAlignedEmitter.new(transformed_ast, file_path).emit end # Transforms the given +ast+. @@ -102,13 +100,5 @@ def parser @parser&.reset @parser ||= Prism::Translation::Parser.new(ASTTransform::KwargsBuilder.new) end - - def register_source_map(source_file_path, transformed_file_path, transformed_ast, transformed_source) - # The transformed_source is re-parsed to get the correct line numbers for the transformed_ast, which is the code - # that will run. - rewritten_ast = build_ast(transformed_source) - source_map = ASTTransform::SourceMap.new(source_file_path, transformed_file_path, transformed_ast, rewritten_ast) - ASTTransform::SourceMap.register_source_map(source_map) - end end end diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb new file mode 100644 index 0000000..c83fe8b --- /dev/null +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true +require 'test_helper' +require 'ast_transform/line_aligned_emitter' +require 'ast_transform/transformation_helper' +require 'ast_transform/transformer' + +module ASTTransform + class LineAlignedEmitterTest < Minitest::Test + extend ASTTransform::Declarative + include ASTTransform::TransformationHelper + + class UnloweredNode < ASTTransform::Node + register :ast_transform_emitter_test_custom + end + + def parse(source) + ASTTransform::Transformer.new.build_ast(source) + end + + def emit(ast) + LineAlignedEmitter.new(ast, 'fixture.rb').emit + end + + test "emits deferral pairs as a hidden-lvar lambda and its call" do + given, when_statement, interaction = parse("given_setup\nwhen_body\ninteraction_setup\n").children + reordered = run_after([given, when_statement, interaction], run: [when_statement], after: interaction) + + emitted = emit(s(:begin, *reordered)) + + assert_includes emitted, '__ast_deferred_1__ = ->', emitted + assert_includes emitted, 'when_body', emitted + assert_includes emitted, '__ast_deferred_1__.call', emitted + # Execution order: lambda defined, interaction runs, then the call. + assert_operator emitted.index('interaction_setup'), :<, emitted.index('__ast_deferred_1__.call'), emitted + end + + test "deferred body statements stay on their source lines inside the lambda" do + source = <<~HEREDOC + given_setup + when_body_first + when_body_second + interaction_setup + HEREDOC + given, first, second, interaction = parse(source).children + reordered = run_after([given, first, second, interaction], run: [first, second], after: interaction) + + emitted = emit(s(:begin, *reordered)) + + assert_equal 2, emitted.lines.index { |line| line.include?('when_body_first') } + 1, emitted + assert_equal 3, emitted.lines.index { |line| line.include?('when_body_second') } + 1, emitted + end + + test "a placement may be executed from multiple call sites (multiplexing)" do + statement = parse("shared_body\n") + deferral = defer(statement) + + emitted = emit(s(:begin, deferral.placement, deferral.execution, deferral.execution)) + + assert_equal 2, emitted.scan('__ast_deferred_1__.call').size, emitted + end + + test "a placement with no execution point raises UnmatchedDeferralError" do + deferral = defer(parse("orphan_body\n")) + + error = assert_raises(UnmatchedDeferralError) { emit(s(:begin, deferral.placement)) } + + assert_includes error.message, 'never executed' + end + + test "an execution point before its placement raises UnmatchedDeferralError" do + deferral = defer(parse("body\n")) + + error = assert_raises(UnmatchedDeferralError) do + emit(s(:begin, deferral.execution, deferral.placement)) + end + + assert_includes error.message, 'before' + end + + test "a duplicate placement raises UnmatchedDeferralError" do + deferral = defer(parse("body\n")) + + error = assert_raises(UnmatchedDeferralError) do + emit(s(:begin, deferral.placement, deferral.placement, deferral.execution)) + end + + assert_includes error.message, 'duplicate' + end + + test "distinct deferrals get distinct hidden lvar names" do + first_deferral = defer(parse("first_body\n")) + second_deferral = defer(parse("second_body\n")) + + emitted = emit(s(:begin, + first_deferral.placement, second_deferral.placement, + first_deferral.execution, second_deferral.execution)) + + assert_includes emitted, '__ast_deferred_1__ = ->', emitted + assert_includes emitted, '__ast_deferred_2__ = ->', emitted + end + + test "an unlowered custom node type raises UnloweredNodeTypeError" do + error = assert_raises(UnloweredNodeTypeError) do + emit(s(:begin, s(:ast_transform_emitter_test_custom))) + end + + assert_includes error.message, 'ast_transform_emitter_test_custom' + end + + test "execution markers compose inside expressions" do + when_body = parse("raise_helper\n") + deferral = defer(when_body) + assert_raises_call = s(:block, + s(:send, nil, :assert_raises, s(:const, nil, :RuntimeError)), + s(:args), + deferral.execution) + + emitted = emit(s(:begin, deferral.placement, assert_raises_call)) + + assert_includes emitted, 'assert_raises(RuntimeError)', emitted + assert_includes emitted, '__ast_deferred_1__.call', emitted + end + end +end diff --git a/test/ast_transform/source_map_test.rb b/test/ast_transform/source_map_test.rb deleted file mode 100644 index ae7ba57..0000000 --- a/test/ast_transform/source_map_test.rb +++ /dev/null @@ -1,145 +0,0 @@ -# frozen_string_literal: true - -require "test_helper" -require "transformation_helper" -require "ast_transform/abstract_transformation" -require "ast_transform/transformer" - -module ASTTransform - class SourceMapTest < Minitest::Test - extend ASTTransform::Declarative - include ASTTransform::Helpers::TransformationHelper - - test "#line returns the correct line number when transformation wraps node in a virtual node" do - transformation = Class.new(ASTTransform::AbstractTransformation) do - def run(node) - s(:send, node, :+, s(:int, 1)) - end - end.new - - transformer = ASTTransform::Transformer.new(transformation) - - source = <<~HEREDOC - method_call - HEREDOC - - actual_transformed_source = transformer.transform_file_source(source, "src", "transformed") - - assert_equal "method_call + 1", actual_transformed_source - - source_map = ASTTransform::SourceMap.for_file_path("transformed") - - assert_equal 1, source_map.line(1) - end - - test "#line returns the correct line number when transformation updates node" do - transformation = Class.new(ASTTransform::AbstractTransformation) do - def run(node) - node.updated(:send, [node, :+, s(:int, 1)]) - end - end.new - - transformer = ASTTransform::Transformer.new(transformation) - - source = <<~HEREDOC - method_call - HEREDOC - - actual_transformed_source = transformer.transform_file_source(source, "src", "transformed") - - assert_equal "method_call + 1", actual_transformed_source - - transformer.transform_file_source(source, "src", "transformed") - source_map = ASTTransform::SourceMap.for_file_path("transformed") - - assert_equal 1, source_map.line(1) - end - - test "#line returns the correct line number when transformation makes code collapse on the same line" do - transformation = Class.new(ASTTransform::AbstractTransformation) do - def run(node) - s(:send, node.children[0], :+, node.children[1]) - end - end.new - - transformer = ASTTransform::Transformer.new(transformation) - - source = <<~HEREDOC - method_call1 - method_call2 - HEREDOC - - actual_transformed_source = transformer.transform_file_source(source, "src", "transformed") - - assert_equal "method_call1 + method_call2", actual_transformed_source - - source_map = ASTTransform::SourceMap.for_file_path("transformed") - - assert_equal 1, source_map.line(1) - end - - test "#line returns the correct line number when transformation makes code expand on multiple lines" do - transformation = Class.new(ASTTransform::AbstractTransformation) do - def run(node) - node.updated(:begin, [node.children[0], node.children[2]]) - end - end.new - - transformer = ASTTransform::Transformer.new(transformation) - - source = <<~HEREDOC - method_call1 + method_call2 - HEREDOC - - expected_transformed_source = <<~HEREDOC - method_call1 - method_call2 - HEREDOC - - actual_transformed_source = transformer.transform_file_source(source, "src", "transformed") - - assert_equal expected_transformed_source, actual_transformed_source - - source_map = ASTTransform::SourceMap.for_file_path("transformed") - - assert_equal 1, source_map.line(1) - assert_equal 1, source_map.line(2) - end - - test "source map is retrievable by source file path" do - transformer = ASTTransform::Transformer.new - - source = <<~HEREDOC - method_call - HEREDOC - - transformer.transform_file_source(source, "/original/path.rb", "/transformed/path.rb") - - source_map = ASTTransform::SourceMap.for_file_path("/original/path.rb") - refute_nil source_map, "source map should be retrievable by source file path" - assert_equal "/original/path.rb", source_map.source_file_path - end - - test "#line returns nil when transformation creates nodes that don't contain previous nodes" do - transformation = Class.new(ASTTransform::AbstractTransformation) do - def run(node) - s(:send, s(:int, 1), :+, s(:int, 2)) - end - end.new - - transformer = ASTTransform::Transformer.new(transformation) - - source = <<~HEREDOC - method_call1 + method_call2 - HEREDOC - - actual_transformed_source = transformer.transform_file_source(source, "src", "transformed") - - assert_equal "1 + 2", actual_transformed_source - - source_map = ASTTransform::SourceMap.for_file_path("transformed") - - assert_nil source_map.line(1) - end - end -end diff --git a/test/ast_transform/test_helpers_test.rb b/test/ast_transform/test_helpers_test.rb new file mode 100644 index 0000000..f0cad8d --- /dev/null +++ b/test/ast_transform/test_helpers_test.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true +require 'test_helper' +require 'ast_transform/test_helpers' +require 'ast_transform/abstract_transformation' + +module ASTTransform + class TestHelpersTest < Minitest::Test + extend ASTTransform::Declarative + include ASTTransform::TestHelpers + + # Rewrites statements in place (keeps anchors) — always aligned. + class InPlaceTransformation < ASTTransform::AbstractTransformation + private + + def process_node(node) + return method(:process).super_method.call(node) unless node.type == :send && node.children[0].nil? + + node.updated(nil, [nil, :"renamed_#{node.children[1]}"]) + end + end + + ALIGNED_SOURCE = <<~HEREDOC + first_call + + second_call + HEREDOC + + test "assert_line_aligned passes for an in-place transform" do + assert_line_aligned(ALIGNED_SOURCE, InPlaceTransformation.new) + end + + test "assert_line_aligned passes with no transformations" do + assert_line_aligned(ALIGNED_SOURCE) + end + + test "assert_backtrace_lines passes when the raise cites its source line" do + source = <<~HEREDOC + value = 1 + + raise "expected boom" if value == 1 + HEREDOC + + assert_backtrace_lines(source, path: File.expand_path('tmp/test/helpers_fixture.rb'), raise_at: 3) + end + end +end diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb new file mode 100644 index 0000000..4ae7cf2 --- /dev/null +++ b/test/ast_transform/transformation_helper_test.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true +require 'test_helper' +require 'ast_transform/transformation_helper' +require 'ast_transform/transformer' + +module ASTTransform + class TransformationHelperTest < Minitest::Test + extend ASTTransform::Declarative + include ASTTransform::TransformationHelper + + class RegisteredNode < ASTTransform::Node + register :ast_transform_test_registered + + def payload = children[0] + end + + def parse(source) + ASTTransform::Transformer.new.build_ast(source) + end + + test "s builds a plain loc-less node" do + node = s(:send, nil, :foo) + + assert_instance_of ::Parser::AST::Node, node + assert_nil node.loc + end + + test "s routes registered custom types to their class" do + node = s(:ast_transform_test_registered, 42) + + assert_instance_of RegisteredNode, node + assert_equal 42, node.payload + end + + test "s_at anchors a fresh node to another node's source location" do + anchor = parse("foo(1)\n") + + node = s_at(anchor, :send, nil, :bar) + + assert_equal :bar, node.children[1] + assert_equal anchor.loc.line, node.loc.line + assert_equal anchor.loc.expression, node.loc.expression + end + + test "s_at raises MissingLocationError for loc-less anchors" do + error = assert_raises(MissingLocationError) { s_at(s(:send, nil, :foo), :send, nil, :bar) } + + assert_includes error.message, 'send' + end + + test "defer returns a Deferral whose markers share a token" do + statements = parse("foo\nbar\n").children + + deferral = defer(*statements) + + assert_equal :ast_deferred, deferral.placement.type + assert_equal :ast_deferred_call, deferral.execution.type + assert_same deferral.placement.children[0], deferral.execution.children[0] + assert_instance_of DeferralToken, deferral.placement.children[0] + end + + test "defer rejects statements containing return" do + statement = parse("return 1 if early\n") + + assert_raises(NonDeferrableError) { defer(statement) } + end + + test "defer rejects break and next at the deferred scope's level" do + assert_raises(NonDeferrableError) { defer(parse("break\n")) } + assert_raises(NonDeferrableError) { defer(parse("next\n")) } + end + + test "defer allows break and next owned by a nested block" do + statement = parse("items.each { |item| next if item.nil? }\n") + + deferral = defer(statement) + + assert_equal :ast_deferred, deferral.placement.type + end + + test "defer rejects return inside a nested block (it penetrates to the method)" do + statement = parse("items.each { |item| return item }\n") + + assert_raises(NonDeferrableError) { defer(statement) } + end + + test "defer allows return absorbed by a nested def or lambda" do + assert_equal :ast_deferred, defer(parse("def helper = (return 1)\n")).placement.type + assert_equal :ast_deferred, defer(parse("callback = -> { return 1 }\n")).placement.type + end + + test "run_after replaces the run with a placement and inserts the execution after the anchor" do + setup_statement, when_statement, interaction = parse("given\nwhen_body\ninteraction\n").children + + reordered = run_after([setup_statement, when_statement, interaction], run: [when_statement], after: interaction) + + assert_equal [:send, :ast_deferred, :send, :ast_deferred_call], reordered.map(&:type) + assert_same setup_statement, reordered[0] + assert_same interaction, reordered[2] + assert_same reordered[1].children[0], reordered[3].children[0] + end + + test "run_after supports after: textually before the run (pure sink)" do + first, second, third = parse("first\nsecond\nthird\n").children + + reordered = run_after([first, second, third], run: [third], after: first) + + assert_equal [:send, :ast_deferred_call, :send, :ast_deferred], reordered.map(&:type) + end + + test "run_after matches statements by identity, not equality" do + # Two textually identical statements: value matching would be ambiguous. + first, duplicate_of_first, last = parse("foo\nfoo\nbar\n").children + + reordered = run_after([first, duplicate_of_first, last], run: [duplicate_of_first], after: last) + + assert_same first, reordered[0] + assert_equal :ast_deferred, reordered[1].type + assert_same last, reordered[2] + end + + test "run_after rejects a non-contiguous run" do + first, second, third = parse("first\nsecond\nthird\n").children + + assert_raises(ArgumentError) { run_after([first, second, third], run: [first, third], after: second) } + end + + test "run_after rejects after: inside the run" do + first, second, third = parse("first\nsecond\nthird\n").children + + assert_raises(ArgumentError) { run_after([first, second, third], run: [first, second], after: second) } + end + + test "run_after rejects statements not in the sequence" do + first, second = parse("first\nsecond\n").children + foreign = s(:send, nil, :foreign) + + assert_raises(ArgumentError) { run_after([first, second], run: [foreign], after: second) } + assert_raises(ArgumentError) { run_after([first, second], run: [first], after: foreign) } + end + end +end diff --git a/test/ast_transform/transformer_test.rb b/test/ast_transform/transformer_test.rb index db76fc8..b2c6642 100644 --- a/test/ast_transform/transformer_test.rb +++ b/test/ast_transform/transformer_test.rb @@ -93,7 +93,8 @@ def setup transformed_source = @multi_transformer.transform_file(pathname.to_s, transformed_pathname.to_s) - assert_equal("foo_bar", transformed_source) + # Line-aligned emission always ends files with a newline. + assert_equal "foo_bar\n", transformed_source ensure File.delete(pathname.to_s) if File.exist?(pathname.to_s) end @@ -109,7 +110,8 @@ def setup transformed_source = @multi_transformer.transform_file_source(@source, pathname.to_s, transformed_pathname.to_s) - assert_equal("foo_bar", transformed_source) + # Line-aligned emission always ends files with a newline. + assert_equal "foo_bar\n", transformed_source ensure File.delete(pathname.to_s) if File.exist?(pathname.to_s) end diff --git a/test/minitest/reporters/rake_rerun_reporter.rb b/test/minitest/reporters/rake_rerun_reporter.rb index 85a5c43..c515dd5 100644 --- a/test/minitest/reporters/rake_rerun_reporter.rb +++ b/test/minitest/reporters/rake_rerun_reporter.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true - -require "minitest/reporters" -require "ast_transform/source_map" +require 'minitest/reporters' module Minitest module Reporters @@ -33,8 +31,8 @@ def print_rerun_command(test) end def rerun_message_for(test) + # Line-aligned emission makes backtrace paths the source paths; no mapping needed. file_path = location(test.failure).gsub(/(\:\d*)\z/, "") - file_path = ASTTransform::SourceMap.for_file_path(file_path)&.source_file_path || file_path "Rerun:\n#{@rerun_user_prefix} rake test TEST=#{file_path} TESTOPTS=\"--name=#{test.name} -v\"" end From a7b6c01559c519cf51e8bc31b6d8f81975a9c0b4 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Wed, 22 Jul 2026 12:01:46 -0400 Subject: [PATCH 03/22] Route Transformer#transform through LineAlignedEmitter transform(source) now emits line-aligned output like transform_file_source, so in-memory callers (and rspock's transformation tests) observe the exact text the loader would compile. Expectations in TransformationTest update from Unparser's normalized re-indentation to source-anchored layout: statements keep their source lines, consumed transform! annotations leave blank lines, and nested bodies are emitted flush-left (indentation is not semantic; lines are). Co-authored-by: Cursor --- lib/ast_transform/transformer.rb | 4 +- test/ast_transform/transformation_test.rb | 55 ++++++++++++++--------- test/ast_transform/transformer_test.rb | 4 +- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/lib/ast_transform/transformer.rb b/lib/ast_transform/transformer.rb index baab326..2b63173 100644 --- a/lib/ast_transform/transformer.rb +++ b/lib/ast_transform/transformer.rb @@ -39,11 +39,11 @@ def build_ast_from_file(file_path) # # @param source [String] The input source code to be transformed. # - # @return [String] The transformed code. + # @return [String] The transformed code, line-aligned (see #transform_file_source). def transform(source) ast = build_ast(source) transformed_ast = transform_ast(ast) - Unparser.unparse(transformed_ast) + LineAlignedEmitter.new(transformed_ast, 'tmp').emit end # Transforms the give +file_path+. diff --git a/test/ast_transform/transformation_test.rb b/test/ast_transform/transformation_test.rb index 43c4ba6..7110266 100644 --- a/test/ast_transform/transformation_test.rb +++ b/test/ast_transform/transformation_test.rb @@ -43,9 +43,7 @@ def setup transform!(FooTransformation) HEREDOC - expected = "transform!(FooTransformation)" - - assert_equal expected, transform(source, @transformation) + assert_equal source, transform(source, @transformation) end test "transform! is not considered an annotation if it does not annotate anything" do @@ -71,14 +69,7 @@ class Potato end HEREDOC - expected = <<~HEREDOC - transform! - - class Potato - end - HEREDOC - - assert_equal expected, transform(source, @transformation) + assert_equal source, transform(source, @transformation) end test "transform! runs the transformation if annotating a Class node" do @@ -88,7 +79,12 @@ class Potato end HEREDOC - expected = "foo" + # The consumed transform! annotation leaves line 1 blank; the class node's + # replacement stays anchored at the class's source line. + expected = <<~HEREDOC + + foo + HEREDOC assert_equal expected, transform(source, @transformation) end @@ -105,7 +101,11 @@ class Potato HEREDOC expected = <<~HEREDOC + foo + + + foo HEREDOC @@ -123,8 +123,11 @@ class Bar HEREDOC expected = <<~HEREDOC + class PrefixFoo - foo + + foo + end HEREDOC @@ -141,9 +144,10 @@ class Bar HEREDOC expected = <<~HEREDOC + class PrefixFoo - class Bar - end + class Bar + end end HEREDOC @@ -181,8 +185,10 @@ class Bar HEREDOC expected = <<~HEREDOC + foo + class Bar end HEREDOC @@ -201,10 +207,11 @@ def setup HEREDOC expected = <<~HEREDOC + class PrefixFoo - def setup - @obj = MyClass.new(bar: 1, baz: 2) - end + def setup + @obj = MyClass.new(bar: 1, baz: 2) + end end HEREDOC @@ -222,10 +229,11 @@ def call HEREDOC expected = <<~HEREDOC + class PrefixFoo - def call - method("hello", bar: 1) - end + def call + method("hello", bar: 1) + end end HEREDOC @@ -239,7 +247,10 @@ def call end HEREDOC - expected = "foo" + expected = <<~HEREDOC + + foo + HEREDOC assert_equal expected, transform(source, @transformation) end diff --git a/test/ast_transform/transformer_test.rb b/test/ast_transform/transformer_test.rb index b2c6642..3a8c8c5 100644 --- a/test/ast_transform/transformer_test.rb +++ b/test/ast_transform/transformer_test.rb @@ -75,11 +75,11 @@ def setup end test "#transform with no transformation" do - assert_equal strip_end_line(@source), @transformer.transform(@source) + assert_equal @source, @transformer.transform(@source) end test "#transform with multiple transformations" do - assert_equal "foo_bar", @multi_transformer.transform(@source) + assert_equal "foo_bar\n", @multi_transformer.transform(@source) end test "#transform_file returns the expected transformed code" do From 4dc755fd6a0038a2d5d9bf4340bca65019264ed7 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Wed, 22 Jul 2026 12:02:27 -0400 Subject: [PATCH 04/22] Bump version to 3.0.0 Co-authored-by: Cursor --- lib/ast_transform/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ast_transform/version.rb b/lib/ast_transform/version.rb index b2aa222..2cfdbce 100644 --- a/lib/ast_transform/version.rb +++ b/lib/ast_transform/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ASTTransform - VERSION = "2.1.4" + VERSION = "3.0.0" end From 80f086653d9e0c5d02ca67c9170de4a56243f6ea Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Wed, 22 Jul 2026 12:24:16 -0400 Subject: [PATCH 05/22] Unparse container delimiters with scope locals Container openers (e.g. a test("name #{row} line #{line}") block header) can contain dstr interpolations referencing scope locals; without the local-variable context Unparser's round-trip verification fails. Route container_delimiters through the same scoped unparse as statements. Co-authored-by: Cursor --- Gemfile.lock | 2 +- lib/ast_transform/line_aligned_emitter.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index d307024..89b7538 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ast_transform (2.1.4) + ast_transform (3.0.0) parser (>= 3.0) prism (>= 1.5) unparser (>= 0.6) diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index 9576474..d1739b7 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -190,7 +190,7 @@ def emit_container(node) end def container_delimiters(node) - rendered = Unparser.unparse(empty_container(node)).split("\n").reject(&:empty?) + rendered = unparse(empty_container(node)).split("\n").reject(&:empty?) opener = rendered[0..-2].join("\n") closer = rendered.last From 00b5fdd2a678f6a384124a937d3e9a85ffd94837 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Wed, 22 Jul 2026 12:37:18 -0400 Subject: [PATCH 06/22] Document line-aligned emission and the authoring contract README gains the authoring contract section (loc'd nodes emit at their source line; synthetic nodes pack; textual order is source order), the TransformationHelper toolkit (s/s_at/defer/run_after), custom IR node registration, and the test helpers. CHANGELOG for 3.0.0. Co-authored-by: Cursor --- CHANGELOG.md | 14 ++++++++++++++ README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e603b39..6dadc48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.0.0] - Unreleased +### Added +- Line-aligned emission: transformed code is emitted with every loc-carrying statement on its original source line, making backtraces, breakpoints, and debugger display correct by construction (`LineAlignedEmitter`). +- Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `defer` (deferred-execution marker pairs with a control-flow guard), and `run_after` (sequence-level execution reordering that preserves textual/source order). +- `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. +- `ast_transform/test_helpers` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. +- Error types: `MissingLocationError`, `NonDeferrableError`, `UnmatchedDeferralError`, `UnloweredNodeTypeError`. + +### Removed +- **Breaking:** `ASTTransform::SourceMap` and source-map registration. Line-aligned emission makes raw VM line numbers the source line numbers, so there is nothing left to map at display time. + +### Changed +- **Breaking:** `Transformer#transform` and `#transform_file_source` emit line-aligned output (source-anchored layout, always newline-terminated) instead of Unparser's re-normalized formatting. + ## [0.1.4] 2019-06-20 ### Fixed - Source mapping for transformations wrapping source nodes into virtual nodes now work. diff --git a/README.md b/README.md index 9b1f55d..7e10efb 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ ASTTransform is an Abstract Syntax Tree (AST) transformation framework. It hooks into the compilation process and allows to perform AST transformations using an annotation: `transform!`. +Transformed code is emitted **line-aligned**: every statement carrying a source location is placed on its original source line. Backtraces, failure messages, `break file:line` breakpoints, and debugger display are therefore correct by construction — no source maps, no backtrace filtering, no debugger integration required. + ## Installation Add this line to your application's Gemfile: @@ -160,6 +162,44 @@ In the above, `node#updated` allows updating the node, either its type or its ch The [ast gem](https://github.com/whitequark/ast) uses a pattern in which a Transformation may implement a method matching a node type, i.e. `on_class`, `on_send`, `on_lvar`, etc... This is very useful when transformations should process all nodes of this type. +### Line-aligned emission and the authoring contract + +ASTTransform owns text and lines; transform authors own semantics and execution order. The contract: + +* A node **with** a source location is emitted at that location's line (the emitter pads with blank lines to reach it, and packs with `;` when a line is already occupied). +* A node **without** a source location is synthetic: it packs onto the current line and inherits its neighbors' line number. +* Textual order is source order. If your transform needs code to *execute* in a different order than it *appears*, use the deferral facility below instead of moving nodes. + +`ASTTransform::TransformationHelper` (included by `AbstractTransformation`) provides the authoring toolkit: + +* `s(type, *children)` — builds a loc-less (synthetic) node. Registered custom types (see below) construct their registered class. +* `s_at(anchor, type, *children)` — builds a node anchored at `anchor`'s source location, so it is emitted at `anchor`'s line. Raises `MissingLocationError` if the anchor has no location. +* `defer(*statements)` — wraps statements for deferred execution. Returns a `Deferral` with two marker nodes: `placement` (splice where the statements *appear* — lowered to a hidden lambda) and `execution` (splice or compose where they must *run* — lowered to the lambda call). Raises `NonDeferrableError` if a statement contains control flow (`return`, `break`, ...) that would re-bind to the lambda. Unmatched markers fail emission with `UnmatchedDeferralError`. +* `run_after(statements, run:, after:)` — the paved road over `defer`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. + +#### Custom node types (intermediate representation) + +Transformations that parse a DSL can build their own IR by registering node classes: + +```ruby +class InteractionNode < ASTTransform::Node + register :my_interaction + + def cardinality = children[0] +end + +s(:my_interaction, ...) # => InteractionNode, with domain accessors +``` + +Custom node types are IR **between stages that understand them** — the stage that owns a type must lower it to plain Ruby nodes before emission. The emitter enforces this: any registered or `ast_`-prefixed type reaching emission raises `UnloweredNodeTypeError`. + +#### Testing your transformation + +`require 'ast_transform/test_helpers'` (test-only) provides: + +* `assert_line_aligned(source, *transformations)` — transforms `source` through the real pipeline and asserts every surviving statement is emitted at its source line. +* `assert_backtrace_lines(source, path:, raise_at:)` — compiles and executes `source`, asserting the raw first backtrace frame cites `path:raise_at` with no filtering. + ### Parameterizable transformations If you want your transformation to be customizable, accept the parameters in the constructor. The annotation can the be changed accordingly: From d9ef5b63f32659ee25602178ce9bea93af636b61 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 08:31:48 -0400 Subject: [PATCH 07/22] Require Ruby >= 3.3; honest dependency floors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruby 3.2 is EOL (March 2026) — required_ruby_version and the CI matrix move to 3.3+. unparser floor raised to >= 0.8: the emitter passes static_local_variables: (an 0.7 interface — 0.6 would ArgumentError) and relies on 0.8's prism-based round-trip verification for Ruby >= 3.4 syntax. parser floor raised to >= 3.3 to match unparser's own floor (the declared >= 3.0 could never actually resolve lower). Lockfile now on unparser 0.9.0. Co-authored-by: Cursor --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 2 ++ Gemfile.lock | 26 ++++++++++---------------- ast_transform.gemspec | 12 ++++++++---- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35965b9..24aeb72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: ['3.2', '3.3', '4.0'] + ruby: ['3.3', '4.0'] steps: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6dadc48..e5ed601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Breaking:** `Transformer#transform` and `#transform_file_source` emit line-aligned output (source-anchored layout, always newline-terminated) instead of Unparser's re-normalized formatting. +- **Breaking:** requires Ruby >= 3.3 (3.2 is EOL since March 2026). +- Dependency floors now reflect reality: `unparser >= 0.8` (the emitter uses `static_local_variables:`, a 0.7 interface, and 0.8's prism-based round-trip verification is required for Ruby >= 3.4 syntax) and `parser >= 3.3` (unparser's own floor; the declared `>= 3.0` could never resolve lower). ## [0.1.4] 2019-06-20 ### Fixed diff --git a/Gemfile.lock b/Gemfile.lock index 89b7538..c998a18 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,18 +2,18 @@ PATH remote: . specs: ast_transform (3.0.0) - parser (>= 3.0) + parser (>= 3.3) prism (>= 1.5) - unparser (>= 0.6) + unparser (>= 0.8) GEM remote: https://rubygems.org/ specs: - ansi (1.5.0) + ansi (1.6.0) ast (2.4.3) builder (3.3.0) coderay (1.1.3) - diff-lcs (1.6.2) + diff-lcs (2.0.0) docile (1.4.1) io-console (0.8.2) json (2.21.1) @@ -21,13 +21,12 @@ GEM lint_roller (1.1.0) method_source (1.1.0) minitest (5.27.0) - minitest-reporters (1.7.1) + minitest-reporters (1.8.0) ansi builder - minitest (>= 5.0) + minitest (>= 5.0, < 7) ruby-progressbar - parallel (1.28.0) - parser (3.3.10.2) + parser (3.3.12.0) ast (~> 2.4.1) racc prism (1.9.0) @@ -36,9 +35,7 @@ GEM method_source (~> 1.0) reline (>= 0.6.0) racc (1.8.1) - rainbow (3.1.1) - rake (13.3.1) - regexp_parser (2.12.0) + rake (13.4.2) reline (0.6.3) io-console (~> 0.5) rubocop (1.88.2) @@ -64,11 +61,8 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) - unicode-display_width (3.2.0) - unicode-emoji (~> 4.1) - unicode-emoji (4.2.0) - unparser (0.8.1) - diff-lcs (~> 1.6) + unparser (0.9.0) + diff-lcs (>= 1.6, < 3) parser (>= 3.3.0) prism (>= 1.5.1) diff --git a/ast_transform.gemspec b/ast_transform.gemspec index dd6d03e..4f7ff1e 100644 --- a/ast_transform.gemspec +++ b/ast_transform.gemspec @@ -20,7 +20,7 @@ Gem::Specification.new do |spec| spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] - spec.required_ruby_version = ">= 3.2" + spec.required_ruby_version = '>= 3.3' # Development dependencies spec.add_development_dependency("bundler", ">= 2.1") @@ -33,7 +33,11 @@ Gem::Specification.new do |spec| spec.add_development_dependency("simplecov", "~> 0.22") # Runtime dependencies - spec.add_runtime_dependency("parser", ">= 3.0") - spec.add_runtime_dependency("prism", ">= 1.5") - spec.add_runtime_dependency("unparser", ">= 0.6") + # parser provides the runtime AST vocabulary (Parser::AST::Node/Processor, + # Source::Buffer/Map); parsing itself goes through prism's translation layer. + spec.add_runtime_dependency "parser", ">= 3.3" + spec.add_runtime_dependency "prism", ">= 1.5" + # unparser >= 0.8: static_local_variables: (0.7 interface) + the prism-based + # round-trip verification parser required for Ruby >= 3.4 syntax. + spec.add_runtime_dependency "unparser", ">= 0.8" end From 5f6f6dd037bdb6c1752514cbc7177bfad39235f2 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 08:48:51 -0400 Subject: [PATCH 08/22] Cover the emitter's rescue/ensure/kwbegin paths; delete two dead branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codecov flagged the patch gaps. Real fixtures now exercise rescue headers (exception list + capture), else, ensure, and standalone begin/end containers at their source lines; assert_line_aligned's failure path is pinned by a statement-swapping transform (the contract violation it exists to catch — note AST::Node#updated compares children by ==, which ignores locations, so a loc-only rewrite is invisible); DeferralToken#inspect and compress_to_single_line's parse-failure fallback (a totality guard with no natural trigger while Unparser normalizes heredocs to inline strings) get direct tests. Two branches were genuinely unreachable and are deleted rather than covered: statements_of only ever receives nodes or nil, and pack never sees a blank last line because place overwrites padded lines immediately. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- lib/ast_transform/line_aligned_emitter.rb | 12 +++--- .../line_aligned_emitter_test.rb | 42 +++++++++++++++++++ test/ast_transform/test_helpers_test.rb | 21 ++++++++++ .../transformation_helper_test.rb | 6 +++ 5 files changed, 75 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e5ed601..00ee9af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - **Breaking:** `Transformer#transform` and `#transform_file_source` emit line-aligned output (source-anchored layout, always newline-terminated) instead of Unparser's re-normalized formatting. -- **Breaking:** requires Ruby >= 3.3 (3.2 is EOL since March 2026). +- **Breaking:** dropped Ruby 3.2 support (EOL since March 2026); `required_ruby_version` is now `>= 3.3`. - Dependency floors now reflect reality: `unparser >= 0.8` (the emitter uses `static_local_variables:`, a 0.7 interface, and 0.8's prism-based round-trip verification is required for Ruby >= 3.4 syntax) and `parser >= 3.3` (unparser's own floor; the declared `>= 3.0` could never resolve lower). ## [0.1.4] 2019-06-20 diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index d1739b7..6abe58a 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -233,11 +233,9 @@ def closer_line(node) end def statements_of(body) - case body - when nil then [] - when ::Parser::AST::Node then body.type == :begin ? body.children : [body] - else [body] - end + return [] if body.nil? + + body.type == :begin ? body.children : [body] end # Places +render+ at +target_line+ when the cursor hasn't passed it; @@ -256,11 +254,11 @@ def place(target_line, render) @lines.concat(rest) end + # The last line is never blank here: padding blanks are only created + # inside +place+, which immediately overwrites the padded line. def pack(text) if @lines.empty? @lines << text - elsif @lines.last.empty? - @lines[-1] = text else @lines[-1] = "#{@lines.last}; #{text}" end diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index c83fe8b..fc10c5a 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -107,6 +107,48 @@ def emit(ast) assert_includes error.message, 'ast_transform_emitter_test_custom' end + test "rescue/else/ensure keywords and their statements emit at their source lines" do + source = <<~HEREDOC + def risky + compute + rescue ArgumentError, TypeError => error + handle(error) + else + celebrate + ensure + cleanup + end + HEREDOC + + emitted_lines = emit(parse(source)).lines.map(&:strip) + + assert_equal ['def risky', 'compute', 'rescue ArgumentError, TypeError => error', 'handle(error)', + 'else', 'celebrate', 'ensure', 'cleanup', 'end'], emitted_lines + end + + test "a standalone begin/end block emits its statements at their source lines" do + source = <<~HEREDOC + begin + first_call + second_call + end + HEREDOC + + emitted_lines = emit(parse(source)).lines.map(&:strip) + + assert_equal ['begin', 'first_call', 'second_call', 'end'], emitted_lines + end + + test "compress_to_single_line declines renders whose single-line join does not parse" do + emitter = LineAlignedEmitter.new(parse("noop\n"), 'fixture.rb') + + # No current Unparser render joins into invalid syntax (heredocs are + # normalized to inline strings), so exercise the totality guard + # directly: layout must fall back, never raise, whatever future + # Unparser output looks like. + assert_nil emitter.send(:compress_to_single_line, "value = <<~TXT\n hi\nTXT") + end + test "execution markers compose inside expressions" do when_body = parse("raise_helper\n") deferral = defer(when_body) diff --git a/test/ast_transform/test_helpers_test.rb b/test/ast_transform/test_helpers_test.rb index f0cad8d..f76f919 100644 --- a/test/ast_transform/test_helpers_test.rb +++ b/test/ast_transform/test_helpers_test.rb @@ -33,6 +33,27 @@ def process_node(node) assert_line_aligned(ALIGNED_SOURCE) end + # Reorders statements textually instead of deferring execution — exactly + # the contract violation assert_line_aligned exists to catch: the moved + # statement can no longer be emitted at its source line. + class StatementSwappingTransformation < ASTTransform::AbstractTransformation + private + + def process_node(node) + return method(:process).super_method.call(node) unless node.type == :begin + + node.updated(nil, node.children.reverse) + end + end + + test "assert_line_aligned reports each misaligned statement" do + error = assert_raises(Minitest::Assertion) do + assert_line_aligned(ALIGNED_SOURCE, StatementSwappingTransformation.new) + end + + assert_includes error.message, 'MISALIGNED first_call: source line 1, emitted line 3' + end + test "assert_backtrace_lines passes when the raise cites its source line" do source = <<~HEREDOC value = 1 diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index 4ae7cf2..fb52b9f 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -59,6 +59,12 @@ def parse(source) assert_instance_of DeferralToken, deferral.placement.children[0] end + test "DeferralToken#inspect names the class so AST dumps are self-documenting" do + token = defer(parse("foo\n")).placement.children[0] + + assert_match(/\A#\z/, token.inspect) + end + test "defer rejects statements containing return" do statement = parse("return 1 if early\n") From b25916550168b0bf212092b89dd10f9ff0e8e577 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 09:32:12 -0400 Subject: [PATCH 09/22] Adopt dev + shadowenv for the development environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dev.yml pins Ruby 4.0.6 (latest; converges with the org toolchain) and declares up/test; dev up provisions the per-machine .shadowenv.d (now gitignored) via rbenv, so the right Ruby activates without manual PATH surgery. .ruby-version gives plain rbenv users the same pin. Inert for external contributors — plain Bundler still works, README says so. Co-authored-by: Cursor --- .gitignore | 5 ++++- .ruby-version | 1 + README.md | 2 ++ dev.yml | 4 +--- 4 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 .ruby-version diff --git a/.gitignore b/.gitignore index 19257f0..a86f13e 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,7 @@ build-iPhoneSimulator/ .rvmrc # RubyMine -/.idea \ No newline at end of file +/.idea + +# Generated per-machine by `dev up` (d3mlabs dev tool); never committed +/.shadowenv.d/ \ No newline at end of file diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..d13e837 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +4.0.6 diff --git a/README.md b/README.md index 7e10efb..6d292fe 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,8 @@ end After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. +If you use the [d3mlabs dev tool](https://github.com/d3mlabs/dev), `dev up` provisions the pinned Ruby (see `.ruby-version`) with a per-project shadowenv, and `dev test` runs the suite — plain Bundler as above works just as well. + To install this gem onto your local machine, run `bundle exec rake install`. ## Releasing a New Version diff --git a/dev.yml b/dev.yml index 96a944d..ea3414f 100644 --- a/dev.yml +++ b/dev.yml @@ -1,4 +1,5 @@ name: ast_transform +ruby: "4.0.6" commands: up: desc: Install gems @@ -6,6 +7,3 @@ commands: test: desc: Run this repo's tests run: bundle exec rake test - style: - desc: Run RuboCop - run: bundle exec rubocop From 9637d0b5d56287b114999dc32211a63a9eeede8d Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 10:04:06 -0400 Subject: [PATCH 10/22] Document the location-only-rewrite gotcha in the authoring contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parser::AST::Node#updated returns self when children compare == — and AST::Node#== ignores source locations, so a Processor pass replacing a node with an equal-valued one (different loc) silently no-ops through every updated() up the tree. Surfaced while writing the misalignment test for assert_line_aligned. Co-authored-by: Cursor --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 6d292fe..1320cfb 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,10 @@ ASTTransform owns text and lines; transform authors own semantics and execution * `defer(*statements)` — wraps statements for deferred execution. Returns a `Deferral` with two marker nodes: `placement` (splice where the statements *appear* — lowered to a hidden lambda) and `execution` (splice or compose where they must *run* — lowered to the lambda call). Raises `NonDeferrableError` if a statement contains control flow (`return`, `break`, ...) that would re-bind to the lambda. Unmatched markers fail emission with `UnmatchedDeferralError`. * `run_after(statements, run:, after:)` — the paved road over `defer`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. +#### Gotcha: location-only rewrites are silently dropped + +`Parser::AST::Node#updated` returns `self` when the new children compare `==` to the old ones — and `AST::Node#==` ignores source locations. A `Processor` pass that replaces a node with an equal-valued one (e.g. the same call rebuilt loc-less, hoping to change its emitted line) is a no-op: every `node.updated(nil, process_all(node))` up the tree discards the replacement. Location is part of a node's *emission*, not its *value* — to change where a node emits, change what it is (`s_at` an anchored rebuild with different children), or restructure the parent explicitly rather than relying on `updated`. + #### Custom node types (intermediate representation) Transformations that parse a DSL can build their own IR by registering node classes: From 88b345f666b920d9a10eb97793331e01638b402e Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 12:58:35 -0400 Subject: [PATCH 11/22] Declare the Ruby toolchain in dependencies.rb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dev now single-sources the project Ruby from the dependencies.rb manifest; dev.yml's ruby: key is removed. Toolchain-only manifest — gems stay bundler-managed via the hand-written gemspec/Gemfile. Co-authored-by: Cursor --- dependencies.rb | 11 +++++++++++ dev.yml | 1 - 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 dependencies.rb diff --git a/dependencies.rb b/dependencies.rb new file mode 100644 index 0000000..7d83308 --- /dev/null +++ b/dependencies.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# Toolchain-only manifest for d3mlabs' dev tool: it provisions this exact +# Ruby (rbenv + shadowenv) for `dev` commands. Gems stay bundler-managed +# through the hand-written gemspec/Gemfile; contributors without dev can +# ignore this file and use .ruby-version. +require "dev/deps" + +Dev::Deps.define do + ruby "4.0.6" +end diff --git a/dev.yml b/dev.yml index ea3414f..a2a8f20 100644 --- a/dev.yml +++ b/dev.yml @@ -1,5 +1,4 @@ name: ast_transform -ruby: "4.0.6" commands: up: desc: Install gems From cc94d3e010eb6c641878a4e4ca57a8b6e4608cd2 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 16:21:44 -0400 Subject: [PATCH 12/22] Lower deferrals to procs and drop the control-flow guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deferral now has near-transparent semantics instead of transform-time policing: - The hidden closure is a non-lambda proc, so a deferred return still returns from the enclosing method (placement and execution always share one method activation). - Locals assigned by deferred statements are pre-declared (x = x) before the proc, keeping them method-scope so statements after the execution point can read them. - ControlFlowGuard and NonDeferrableError are gone. Jumps severed from their owner keep Ruby's native behavior (break/retry fail loudly, next/redo silently alter flow) — what a surface allows users to defer is the transform author's call, documented on defer. Also: the emitter recurses into :itblock containers (parity with :block/:numblock), flattens loc-less :begin statements (the lowered placement carries its pre-declarations in one), and CI adds Ruby 3.4. Co-authored-by: Cursor --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 4 +- README.md | 8 +- lib/ast_transform/control_flow_guard.rb | 67 ---------- lib/ast_transform/deferral.rb | 5 +- lib/ast_transform/deferral_lowering.rb | 53 +++++++- lib/ast_transform/errors.rb | 7 +- lib/ast_transform/line_aligned_emitter.rb | 17 ++- lib/ast_transform/transformation_helper.rb | 18 +-- .../line_aligned_emitter_test.rb | 124 +++++++++++++++++- .../transformation_helper_test.rb | 33 +---- 11 files changed, 208 insertions(+), 130 deletions(-) delete mode 100644 lib/ast_transform/control_flow_guard.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24aeb72..86af1b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: ['3.3', '4.0'] + ruby: ['3.3', '3.4', '4.0'] steps: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 00ee9af..cc9518e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [3.0.0] - Unreleased ### Added - Line-aligned emission: transformed code is emitted with every loc-carrying statement on its original source line, making backtraces, breakpoints, and debugger display correct by construction (`LineAlignedEmitter`). -- Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `defer` (deferred-execution marker pairs with a control-flow guard), and `run_after` (sequence-level execution reordering that preserves textual/source order). +- Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `defer` (deferred-execution marker pairs), and `run_after` (sequence-level execution reordering that preserves textual/source order). Deferral lowers to a non-lambda proc, so `return` still returns from the enclosing method, and locals assigned by deferred statements are pre-declared to stay method-scope. - `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. - `ast_transform/test_helpers` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. -- Error types: `MissingLocationError`, `NonDeferrableError`, `UnmatchedDeferralError`, `UnloweredNodeTypeError`. +- Error types: `MissingLocationError`, `UnmatchedDeferralError`, `UnloweredNodeTypeError`. ### Removed - **Breaking:** `ASTTransform::SourceMap` and source-map registration. Line-aligned emission makes raw VM line numbers the source line numbers, so there is nothing left to map at display time. diff --git a/README.md b/README.md index 1320cfb..3a64d94 100644 --- a/README.md +++ b/README.md @@ -174,9 +174,15 @@ ASTTransform owns text and lines; transform authors own semantics and execution * `s(type, *children)` — builds a loc-less (synthetic) node. Registered custom types (see below) construct their registered class. * `s_at(anchor, type, *children)` — builds a node anchored at `anchor`'s source location, so it is emitted at `anchor`'s line. Raises `MissingLocationError` if the anchor has no location. -* `defer(*statements)` — wraps statements for deferred execution. Returns a `Deferral` with two marker nodes: `placement` (splice where the statements *appear* — lowered to a hidden lambda) and `execution` (splice or compose where they must *run* — lowered to the lambda call). Raises `NonDeferrableError` if a statement contains control flow (`return`, `break`, ...) that would re-bind to the lambda. Unmatched markers fail emission with `UnmatchedDeferralError`. +* `defer(*statements)` — wraps statements for deferred execution. Returns a `Deferral` with two marker nodes: `placement` (splice where the statements *appear* — lowered to a hidden proc) and `execution` (splice or compose where they must *run* — lowered to the proc's call). Unmatched markers fail emission with `UnmatchedDeferralError`. * `run_after(statements, run:, after:)` — the paved road over `defer`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. +Deferred statements keep their original meaning as far as Ruby's closure semantics allow: + +* `return` still returns from the enclosing method — the hidden closure is a non-lambda proc, and placement and execution always share one method activation. +* Locals assigned by the deferred statements stay method-scope: the lowering pre-declares each one (`result = result`) before the proc, so code after the execution point can read them. Before the deferred code runs they are `nil` — exactly what an unexecuted assignment yields. +* Jump keywords whose owner lies *outside* the deferred statements keep Ruby's native behavior: `break`/`retry` raise `LocalJumpError` at the jump's own source line, while `next`/`redo` silently end or restart the deferred body. ASTTransform does not validate this — what a transform surface allows users to defer is the transform author's call. + #### Gotcha: location-only rewrites are silently dropped `Parser::AST::Node#updated` returns `self` when the new children compare `==` to the old ones — and `AST::Node#==` ignores source locations. A `Processor` pass that replaces a node with an equal-valued one (e.g. the same call rebuilt loc-less, hoping to change its emitted line) is a no-op: every `node.updated(nil, process_all(node))` up the tree discards the replacement. Location is part of a node's *emission*, not its *value* — to change where a node emits, change what it is (`s_at` an anchored rebuild with different children), or restructure the parent explicitly rather than relying on `updated`. diff --git a/lib/ast_transform/control_flow_guard.rb b/lib/ast_transform/control_flow_guard.rb deleted file mode 100644 index d78462d..0000000 --- a/lib/ast_transform/control_flow_guard.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true -require 'parser' -require 'ast_transform/errors' - -module ASTTransform - # Validates that statements are safe to defer. Deferral wraps statements in - # a lambda; control-flow keywords bind to the nearest enclosing scope, so - # keywords written against the original scope would silently re-bind to the - # lambda. This guard turns that semantics hazard into a transform-time error. - # - # Scope rules mirror Ruby's: - # - +break+/+next+/+redo+/+retry+ are owned by the nearest block, so blocks - # are not descended for them. - # - +return+ penetrates plain blocks (it returns from the enclosing method), - # so blocks ARE descended for it. Only defs and lambdas absorb it. - class ControlFlowGuard - BLOCK_OWNED_TYPES = [:break, :next, :redo, :retry].freeze - METHOD_OWNED_TYPES = [:return].freeze - BLOCK_TYPES = [:block, :numblock].freeze - METHOD_DEFINITION_TYPES = [:def, :defs].freeze - - # @param statements [Array] statements about to be deferred - # @return [void] - # @raise [NonDeferrableError] when a statement contains control flow that - # would re-bind to the deferral lambda - def check!(statements) - statements.each { |statement| check_node(statement, BLOCK_OWNED_TYPES + METHOD_OWNED_TYPES) } - nil - end - - private - - def check_node(node, hazardous_types) - return unless node.is_a?(::Parser::AST::Node) - - if hazardous_types.include?(node.type) - raise NonDeferrableError, - "cannot defer a statement containing `#{node.type}`: it would re-bind " \ - "to the deferral lambda and change the code's meaning" - end - - remaining = remaining_hazards(node, hazardous_types) - return if remaining.empty? - - node.children.each { |child| check_node(child, remaining) } - end - - def remaining_hazards(node, hazardous_types) - return [] if METHOD_DEFINITION_TYPES.include?(node.type) || lambda_block?(node) - return hazardous_types - BLOCK_OWNED_TYPES if BLOCK_TYPES.include?(node.type) - - hazardous_types - end - - # A literal lambda parses as (block (lambda) args body); Kernel#lambda as - # (block (send nil :lambda) args body). Unlike plain blocks, both absorb - # +return+. - def lambda_block?(node) - return false unless BLOCK_TYPES.include?(node.type) - - callee = node.children[0] - return false unless callee.is_a?(::Parser::AST::Node) - - callee.type == :lambda || (callee.type == :send && callee.children == [nil, :lambda]) - end - end -end diff --git a/lib/ast_transform/deferral.rb b/lib/ast_transform/deferral.rb index bea1a71..4855dd1 100644 --- a/lib/ast_transform/deferral.rb +++ b/lib/ast_transform/deferral.rb @@ -9,14 +9,15 @@ module ASTTransform # # placement:: (:ast_deferred, token, (:begin, ...)) — the deferred body, # spliced at the statements' SOURCE position; lowered to - # +__ast_deferred___ = -> { ... }+. + # +__ast_deferred___ = proc { ... }+ (plus pre-declarations + # for the locals the body assigns — see DeferralLowering). # execution:: (:ast_deferred_call, token) — loc-less, spliced (or composed # into an expression, e.g. an assert_raises block body) at the # execution point; lowered to +__ast_deferred___.call+. Deferral = Data.define(:placement, :execution) # The pairing mechanism between the two halves of a Deferral: it answers - # "which lambda does this call marker invoke?" when a scope holds several + # "which proc does this call marker invoke?" when a scope holds several # deferrals. The markers cannot reference each other's nodes — Processor and # Node#updated rebuilds create new node objects, so node identity does not # survive transformation passes. Children DO survive (carried by reference diff --git a/lib/ast_transform/deferral_lowering.rb b/lib/ast_transform/deferral_lowering.rb index 74e48dc..725ebc0 100644 --- a/lib/ast_transform/deferral_lowering.rb +++ b/lib/ast_transform/deferral_lowering.rb @@ -6,9 +6,25 @@ module ASTTransform # Lowers deferral markers into plain Ruby nodes ahead of emission: # - # (:ast_deferred, token, (:begin, ...)) => __ast_deferred___ = -> { ... } + # (:ast_deferred, token, (:begin, ...)) => x = x; __ast_deferred___ = proc { ... } # (:ast_deferred_call, token) => __ast_deferred___.call # + # The closure is a non-lambda proc on purpose: `return` inside a proc + # returns from the method where the proc was defined, and placement and + # execution always share one method activation (the hidden lvar cannot be + # referenced across a def boundary), so a deferred `return` keeps its + # original meaning. Jump keywords whose owner lies outside the deferred + # statements keep Ruby's native behavior — no transform-time validation: + # what a transform chooses to defer is the transform author's call. + # + # The `x = x` pre-declarations cover every local the deferred statements + # assign at method scope. A local first assigned inside a block literal is + # block-local, so without a textual method-scope assignment before the + # proc, deferred assignments would be invisible to the statements that + # read them after the execution point. Self-assignment registers the name + # (nil until the deferred code runs — exactly what an unexecuted + # assignment yields) without clobbering an already-assigned value. + # # Hidden lvar names are assigned per token in encounter order, so they are # stable within a file and never collide. Pairing is by token object # identity (see DeferralToken). @@ -16,7 +32,7 @@ module ASTTransform # Reconciliation is a static count of markers in the tree, not of runtime # executions — a call under a conditional legitimately executes zero-or-more # times. One placement may have many calls (multiplexing); it must have at - # least one, textually after it (the lambda must exist before it is called). + # least one, textually after it (the proc must exist before it is called). class DeferralLowering include TransformationHelper @@ -59,13 +75,19 @@ def lower_placement(node) token, body = node.children if @names_by_token.key?(token) raise UnmatchedDeferralError, - "duplicate deferral placement: the hidden lambda would be assigned twice" + "duplicate deferral placement: the hidden proc would be assigned twice" end name = :"__ast_deferred_#{@names_by_token.size + 1}__" @names_by_token[token] = name - s(:lvasgn, name, s(:block, s(:lambda), s(:args), lower(body))) + deferred_assignment = s(:lvasgn, name, s(:block, s(:send, nil, :proc), s(:args), lower(body))) + pre_declarations = method_scope_assignments(body).map { |local| s(:lvasgn, local, s(:lvar, local)) } + return deferred_assignment if pre_declarations.empty? + + # A loc-less :begin in statement position; the emitter flattens it into + # the surrounding statement stream so the proc body still aligns. + s(:begin, *pre_declarations, deferred_assignment) end def lower_call(node) @@ -80,5 +102,28 @@ def lower_call(node) @called_tokens[token] = true s(:send, s(:lvar, name), :call) end + + # Node types opening a new local-variable scope: assignments inside them + # were invisible to the method scope in the original source too, so they + # get no pre-declaration. + NEW_SCOPE_TYPES = [:def, :defs, :class, :module, :sclass].freeze + # Block literals: locals first assigned inside them are block-local (the + # same lexical rule the pre-declarations exist to work around), but their + # callee/arguments evaluate at method scope and are still descended. + BLOCK_TYPES = [:block, :numblock, :itblock].freeze + + # Locals the deferred statements assign at method scope, in + # first-assignment order (covers masgn/op_asgn targets — they all carry + # :lvasgn nodes). + def method_scope_assignments(node, names = []) + return names unless node.is_a?(::Parser::AST::Node) + return names if NEW_SCOPE_TYPES.include?(node.type) + + names << node.children[0] if node.type == :lvasgn && !names.include?(node.children[0]) + + children = BLOCK_TYPES.include?(node.type) ? [node.children[0]] : node.children + children.each { |child| method_scope_assignments(child, names) } + names + end end end diff --git a/lib/ast_transform/errors.rb b/lib/ast_transform/errors.rb index 9bf55f3..6b22ad4 100644 --- a/lib/ast_transform/errors.rb +++ b/lib/ast_transform/errors.rb @@ -4,12 +4,7 @@ module ASTTransform # source location does not have one. class MissingLocationError < StandardError; end - # Raised by +defer+ when a deferred statement contains control flow that - # would re-bind to the deferral lambda (e.g. +return+), silently changing - # the meaning of the user's code. - class NonDeferrableError < StandardError; end - - # Raised at emission when deferral markers cannot be reconciled: a + # Raised at emission when deferral markers cannot be reconciled: a # placement without any execution point, an execution point without a # placement (or preceding it), or a duplicate placement. class UnmatchedDeferralError < StandardError; end diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index 6abe58a..86ef304 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -31,13 +31,14 @@ module ASTTransform class LineAlignedEmitter # Containers the emitter recurses into so nested statements align; every # other node renders as an Unparser blob at its head line. - RECURSIVE_CONTAINER_TYPES = [:class, :module, :sclass, :def, :defs, :block, :numblock, :kwbegin].freeze + RECURSIVE_CONTAINER_TYPES = [:class, :module, :sclass, :def, :defs, :block, :numblock, :itblock, :kwbegin].freeze BODY_INDEXES = { - class: 2, module: 1, sclass: 1, def: 2, defs: 3, block: 2, numblock: 2 + class: 2, module: 1, sclass: 1, def: 2, defs: 3, block: 2, numblock: 2, itblock: 2 }.freeze - # Assignments whose value is a block (e.g. the lowered deferral lambda) + # Assignments whose value is a block (e.g. the lowered deferral proc) # recurse into the block so its body statements align. ASSIGNMENT_TYPES = [:lvasgn, :ivasgn, :gvasgn, :casgn].freeze + BLOCK_VALUE_TYPES = [:block, :numblock, :itblock].freeze # @param ast [Parser::AST::Node] transformed AST # @param source_path [String] original file path (for error messages) @@ -177,7 +178,7 @@ def recursive_container?(node) def block_assignment?(node) ASSIGNMENT_TYPES.include?(node.type) && node.children.last.is_a?(::Parser::AST::Node) && - [:block, :numblock].include?(node.children.last.type) + BLOCK_VALUE_TYPES.include?(node.children.last.type) end # Renders a container's opener and closer from the node with its body @@ -232,10 +233,16 @@ def closer_line(node) loc.end.line if loc.respond_to?(:end) && loc.end end + # Loc-less :begin nodes in statement position (e.g. a lowered deferral + # placement carrying its pre-declarations) are grouping, not structure: + # flatten them so each inner statement is laid out independently. def statements_of(body) return [] if body.nil? + return [body] unless body.type == :begin - body.type == :begin ? body.children : [body] + body.children.flat_map do |child| + child.is_a?(::Parser::AST::Node) && child.type == :begin && child.loc.nil? ? statements_of(child) : [child] + end end # Places +render+ at +target_line+ when the cursor hasn't passed it; diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index e90f265..9758cb6 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -2,7 +2,6 @@ require 'parser' require 'ast_transform/node' require 'ast_transform/deferral' -require 'ast_transform/control_flow_guard' require 'ast_transform/errors' module ASTTransform @@ -65,18 +64,22 @@ def s_at(anchor, type, *children) # the SOURCE (inner statements keep their own locs, so the emitter # aligns the body even though execution waits) and +execution+ where # they run — composable inside expressions, e.g. as an assert_raises - # block body. The emitter lowers the pair to a hidden-lvar lambda and - # its call (the lambda shares the enclosing method binding, so lvar - # assignments propagate out). + # block body. + # + # The emitter lowers the pair to a hidden-lvar proc and its call, with + # near-transparent semantics (see DeferralLowering): +return+ still + # returns from the enclosing method (non-lambda proc), and locals the + # deferred statements assign stay method-scope (pre-declared before the + # proc). Jump keywords whose owner lies outside the deferred statements + # keep Ruby's native behavior — +break+/+retry+ fail loudly at the + # jump's own source line, +next+/+redo+ silently end or restart the + # deferred body. Weigh that when choosing what your surface defers. # # Prefer +run_after+ when both points sit in one statement sequence. # # @param statements [Array] statements to defer # @return [ASTTransform::Deferral] the placement/execution marker pair - # @raise [NonDeferrableError] if a statement contains control flow that - # would re-bind to the deferral lambda def defer(*statements) - ControlFlowGuard.new.check!(statements) token = DeferralToken.new Deferral.new( placement: s(:ast_deferred, token, s(:begin, *statements)), @@ -104,7 +107,6 @@ def defer(*statements) # @param after [Parser::AST::Node] element of +statements+ (by identity, # not inside +run+) the +run+ statements execute after # @return [Array] new sequence with markers placed - # @raise [NonDeferrableError] per +defer+ # @raise [ArgumentError] if +run+ is not a contiguous identity-run of # +statements+, or +after+ is not an element (or is inside +run+) def run_after(statements, run:, after:) diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index fc10c5a..9295f2f 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -21,20 +21,28 @@ def emit(ast) LineAlignedEmitter.new(ast, 'fixture.rb').emit end - test "emits deferral pairs as a hidden-lvar lambda and its call" do + # Runs emitted code with real method semantics (return target, method + # scope for locals) — exactly the environment deferred code lives in. + def run_as_method(emitted) + harness = Module.new + harness.module_eval("def self.run_case\n#{emitted}\nend", 'fixture.rb', 0) + harness.run_case + end + + test "emits deferral pairs as a hidden-lvar proc and its call" do given, when_statement, interaction = parse("given_setup\nwhen_body\ninteraction_setup\n").children reordered = run_after([given, when_statement, interaction], run: [when_statement], after: interaction) emitted = emit(s(:begin, *reordered)) - assert_includes emitted, '__ast_deferred_1__ = ->', emitted + assert_includes emitted, '__ast_deferred_1__ = proc', emitted assert_includes emitted, 'when_body', emitted assert_includes emitted, '__ast_deferred_1__.call', emitted - # Execution order: lambda defined, interaction runs, then the call. + # Execution order: proc defined, interaction runs, then the call. assert_operator emitted.index('interaction_setup'), :<, emitted.index('__ast_deferred_1__.call'), emitted end - test "deferred body statements stay on their source lines inside the lambda" do + test "deferred body statements stay on their source lines inside the proc" do source = <<~HEREDOC given_setup when_body_first @@ -95,8 +103,8 @@ def emit(ast) first_deferral.placement, second_deferral.placement, first_deferral.execution, second_deferral.execution)) - assert_includes emitted, '__ast_deferred_1__ = ->', emitted - assert_includes emitted, '__ast_deferred_2__ = ->', emitted + assert_includes emitted, '__ast_deferred_1__ = proc', emitted + assert_includes emitted, '__ast_deferred_2__ = proc', emitted end test "an unlowered custom node type raises UnloweredNodeTypeError" do @@ -149,6 +157,110 @@ def risky assert_nil emitter.send(:compress_to_single_line, "value = <<~TXT\n hi\nTXT") end + test "pre-declares locals the deferred statements assign at method scope" do + first, second, third = parse("given_setup\nresult = compute\ninteraction_setup\n").children + reordered = run_after([first, second, third], run: [second], after: third) + + emitted = emit(s(:begin, *reordered)) + + assert_includes emitted, 'result = result; __ast_deferred_1__ = proc', emitted + end + + test "pre-declarations skip block-local assignments but cover the block call's arguments" do + source = <<~HEREDOC + outer = items.map { |item| inner = item } + buffer.take(width = limit) { |line| sink(line) } + HEREDOC + deferral = defer(*parse(source).children) + + emitted = emit(s(:begin, deferral.placement, deferral.execution)) + + assert_includes emitted, 'outer = outer', emitted + # width is assigned in the block call's ARGUMENTS, which evaluate at + # method scope; inner is first assigned inside the block, so it is + # block-local in the original source too. + assert_includes emitted, 'width = width', emitted + refute_includes emitted, 'inner = inner', emitted + end + + test "pre-declarations skip assignments inside nested defs" do + deferral = defer(parse("def helper = (scoped = 1)\n")) + + emitted = emit(s(:begin, deferral.placement, deferral.execution)) + + refute_includes emitted, 'scoped = scoped', emitted + end + + test "a deferred assignment runs late but propagates to the enclosing method scope" do + source = <<~HEREDOC + log = [] + result = log.size + log << :setup + [log, result] + HEREDOC + first, second, third, fourth = parse(source).children + reordered = run_after([first, second, third, fourth], run: [second], after: third) + + # Deferred `result = log.size` runs after `log << :setup`, so result is + # 1 (textual order would give 0) — proving both the reordering and that + # the assignment escaped the proc into the method scope. + assert_equal [[:setup], 1], run_as_method(emit(s(:begin, *reordered))) + end + + test "a pre-declaration does not clobber an already-assigned local" do + source = <<~HEREDOC + value = :given + value = :reassigned + snapshot = value + [snapshot, value] + HEREDOC + first, second, third, fourth = parse(source).children + reordered = run_after([first, second, third, fourth], run: [second], after: third) + + # snapshot reads value between the proc's definition and its call: the + # pre-declaration must preserve :given, and the deferred reassignment + # must land afterwards. + assert_equal [:given, :reassigned], run_as_method(emit(s(:begin, *reordered))) + end + + test "a deferred return exits the enclosing method (non-lambda proc semantics)" do + source = <<~HEREDOC + log = [] + return [:early, log] unless log.empty? + log << :setup + :late + HEREDOC + first, second, third, fourth = parse(source).children + reordered = run_after([first, second, third, fourth], run: [second], after: third) + + # The deferred return fires from inside the proc but returns from the + # method — and only after the setup it was deferred past has run. + assert_equal [:early, [:setup]], run_as_method(emit(s(:begin, *reordered))) + end + + test "a deferred break severed from its loop keeps Ruby's native LocalJumpError" do + deferral = defer(parse("break\n")) + + emitted = emit(s(:begin, deferral.placement, deferral.execution)) + + assert_raises(LocalJumpError) { run_as_method(emitted) } + end + + test "statements inside an it-block container stay on their source lines" do + source = <<~HEREDOC + items.each do + first_call(it) + + second_call(it) + end + HEREDOC + + emitted_lines = emit(parse(source)).lines.map(&:strip) + + assert_equal 2, emitted_lines.index { |line| line.include?('first_call') } + 1, emitted_lines.join + assert_equal 4, emitted_lines.index { |line| line.include?('second_call') } + 1, emitted_lines.join + end + test "execution markers compose inside expressions" do when_body = parse("raise_helper\n") deferral = defer(when_body) diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index fb52b9f..4c44831 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -65,34 +65,11 @@ def parse(source) assert_match(/\A#\z/, token.inspect) end - test "defer rejects statements containing return" do - statement = parse("return 1 if early\n") - - assert_raises(NonDeferrableError) { defer(statement) } - end - - test "defer rejects break and next at the deferred scope's level" do - assert_raises(NonDeferrableError) { defer(parse("break\n")) } - assert_raises(NonDeferrableError) { defer(parse("next\n")) } - end - - test "defer allows break and next owned by a nested block" do - statement = parse("items.each { |item| next if item.nil? }\n") - - deferral = defer(statement) - - assert_equal :ast_deferred, deferral.placement.type - end - - test "defer rejects return inside a nested block (it penetrates to the method)" do - statement = parse("items.each { |item| return item }\n") - - assert_raises(NonDeferrableError) { defer(statement) } - end - - test "defer allows return absorbed by a nested def or lambda" do - assert_equal :ast_deferred, defer(parse("def helper = (return 1)\n")).placement.type - assert_equal :ast_deferred, defer(parse("callback = -> { return 1 }\n")).placement.type + test "defer imposes no control-flow validation (semantics are the proc lowering's contract)" do + # return is transparent through the non-lambda proc; severed jumps keep + # Ruby's native behavior (see DeferralLowering / LineAlignedEmitterTest). + assert_equal :ast_deferred, defer(parse("return 1 if early\n")).placement.type + assert_equal :ast_deferred, defer(parse("break\n")).placement.type end test "run_after replaces the run with a placement and inserts the execution after the anchor" do From 49bf60cbb46ab417727c5a8601f4d80a0494ddc9 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 21:24:00 -0400 Subject: [PATCH 13/22] Replace the Deferral marker pair with a single Thunk node MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two free-floating markers let authors build programs that only failed two stages later (run_after's pure-sink arrangement pinned behavior the lowering rejected). Thunk makes those states unrepresentable: - Thunk < ASTTransform::Node, children [token, *body], invariants enforced in initialize — every construction path shares it, including Processor rebuilds (MalformedThunkError). Built via thunk(*statements); the token is internal. - One node, placed where the body must EXECUTE; ThunkLowering derives the proc's textual placement from the body's source locations and inserts it into the enclosing statement sequence by line order. Placements never cross a def boundary (the hidden lvar must share the call's activation) but do escape block literals (closures). - Multiplexing is reusing the node: one proc, N calls. Diverging bodies on one token and bodies whose lines fall after the execution point raise ThunkPlacementError; the three UnmatchedDeferralError modes (orphan/premature/duplicate) no longer exist. - run_after keeps its surface, now removing the run and inserting a thunk; its impossible pure-sink support is gone. - AbstractTransformation#on_ast_thunk descends thunk bodies by default, so later passes process wrapped statements instead of skipping them via handler_missing. - Hidden lvars renamed __ast_deferred_N__ -> __ast_thunk_N__. Co-authored-by: Cursor --- CHANGELOG.md | 4 +- README.md | 14 +- lib/ast_transform/abstract_transformation.rb | 9 + lib/ast_transform/deferral.rb | 33 --- lib/ast_transform/deferral_lowering.rb | 129 ---------- lib/ast_transform/errors.rb | 15 +- lib/ast_transform/line_aligned_emitter.rb | 23 +- lib/ast_transform/thunk.rb | 55 +++++ lib/ast_transform/thunk_lowering.rb | 222 ++++++++++++++++++ lib/ast_transform/transformation_helper.rb | 70 +++--- .../line_aligned_emitter_test.rb | 124 +++++----- .../transformation_helper_test.rb | 85 ++++--- 12 files changed, 475 insertions(+), 308 deletions(-) delete mode 100644 lib/ast_transform/deferral.rb delete mode 100644 lib/ast_transform/deferral_lowering.rb create mode 100644 lib/ast_transform/thunk.rb create mode 100644 lib/ast_transform/thunk_lowering.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index cc9518e..3ad1ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [3.0.0] - Unreleased ### Added - Line-aligned emission: transformed code is emitted with every loc-carrying statement on its original source line, making backtraces, breakpoints, and debugger display correct by construction (`LineAlignedEmitter`). -- Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `defer` (deferred-execution marker pairs), and `run_after` (sequence-level execution reordering that preserves textual/source order). Deferral lowers to a non-lambda proc, so `return` still returns from the enclosing method, and locals assigned by deferred statements are pre-declared to stay method-scope. +- Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `thunk` (a single invariant-checked `Thunk` node spliced at the execution point; the lowering derives the hidden proc's textual placement from the body's source locations), and `run_after` (sequence-level execution reordering that preserves textual/source order). Thunks lower to a non-lambda proc, so `return` still returns from the enclosing method, and locals assigned by thunked statements are pre-declared to stay method-scope. Reusing one thunk node executes its body from several points. - `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. - `ast_transform/test_helpers` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. -- Error types: `MissingLocationError`, `UnmatchedDeferralError`, `UnloweredNodeTypeError`. +- Error types: `MissingLocationError`, `MalformedThunkError`, `ThunkPlacementError`, `UnloweredNodeTypeError`. ### Removed - **Breaking:** `ASTTransform::SourceMap` and source-map registration. Line-aligned emission makes raw VM line numbers the source line numbers, so there is nothing left to map at display time. diff --git a/README.md b/README.md index 3a64d94..6be47d6 100644 --- a/README.md +++ b/README.md @@ -168,20 +168,20 @@ ASTTransform owns text and lines; transform authors own semantics and execution * A node **with** a source location is emitted at that location's line (the emitter pads with blank lines to reach it, and packs with `;` when a line is already occupied). * A node **without** a source location is synthetic: it packs onto the current line and inherits its neighbors' line number. -* Textual order is source order. If your transform needs code to *execute* in a different order than it *appears*, use the deferral facility below instead of moving nodes. +* Textual order is source order. If your transform needs code to *execute* in a different order than it *appears*, use a thunk (below) instead of moving nodes. `ASTTransform::TransformationHelper` (included by `AbstractTransformation`) provides the authoring toolkit: * `s(type, *children)` — builds a loc-less (synthetic) node. Registered custom types (see below) construct their registered class. * `s_at(anchor, type, *children)` — builds a node anchored at `anchor`'s source location, so it is emitted at `anchor`'s line. Raises `MissingLocationError` if the anchor has no location. -* `defer(*statements)` — wraps statements for deferred execution. Returns a `Deferral` with two marker nodes: `placement` (splice where the statements *appear* — lowered to a hidden proc) and `execution` (splice or compose where they must *run* — lowered to the proc's call). Unmatched markers fail emission with `UnmatchedDeferralError`. -* `run_after(statements, run:, after:)` — the paved road over `defer`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. +* `thunk(*statements)` — wraps statements in a single `Thunk` node: splice it wherever the statements must *run*, in statement position or composed inside an expression (e.g. an `assert_raises` block body). The wrapped statements keep their own locations, and the lowering derives the hidden proc's textual placement from them — the body still emits on its source lines even though execution waits. Reuse the same node to execute one body from several points. Thunk construction is invariant-checked (`MalformedThunkError`); a body whose source lines fall after its execution point fails lowering with `ThunkPlacementError` (a thunk can only delay execution, never text). +* `run_after(statements, run:, after:)` — the paved road over `thunk`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. -Deferred statements keep their original meaning as far as Ruby's closure semantics allow: +Thunked statements keep their original meaning as far as Ruby's closure semantics allow: -* `return` still returns from the enclosing method — the hidden closure is a non-lambda proc, and placement and execution always share one method activation. -* Locals assigned by the deferred statements stay method-scope: the lowering pre-declares each one (`result = result`) before the proc, so code after the execution point can read them. Before the deferred code runs they are `nil` — exactly what an unexecuted assignment yields. -* Jump keywords whose owner lies *outside* the deferred statements keep Ruby's native behavior: `break`/`retry` raise `LocalJumpError` at the jump's own source line, while `next`/`redo` silently end or restart the deferred body. ASTTransform does not validate this — what a transform surface allows users to defer is the transform author's call. +* `return` still returns from the enclosing method — the hidden closure is a non-lambda proc, and the proc and its call always share one method activation (placements never cross a `def` boundary). +* Locals assigned by the thunked statements stay method-scope: the lowering pre-declares each one (`result = result`) before the proc, so code after the execution point can read them. Before the thunk runs they are `nil` — exactly what an unexecuted assignment yields. +* Jump keywords whose owner lies *outside* the thunked statements keep Ruby's native behavior: `break`/`retry` raise `LocalJumpError` at the jump's own source line, while `next`/`redo` silently end or restart the thunk body. ASTTransform does not validate this — what a transform surface allows users to thunk is the transform author's call. #### Gotcha: location-only rewrites are silently dropped diff --git a/lib/ast_transform/abstract_transformation.rb b/lib/ast_transform/abstract_transformation.rb index 782cfe9..8104496 100644 --- a/lib/ast_transform/abstract_transformation.rb +++ b/lib/ast_transform/abstract_transformation.rb @@ -23,6 +23,15 @@ def process(node) process_node(node) end + # Thunks are framework-owned IR: descend into the body so passes that + # don't know about thunks still process the wrapped statements. Without + # this, Processor's handler_missing default would pass the node through + # opaquely, hiding the body from every later transformation. The token + # (first child) is not a node and passes through untouched. + def on_ast_thunk(node) + node.updated(nil, [node.children[0], *process_all(node.children.drop(1))]) + end + private # Processes the given +node+. diff --git a/lib/ast_transform/deferral.rb b/lib/ast_transform/deferral.rb deleted file mode 100644 index 4855dd1..0000000 --- a/lib/ast_transform/deferral.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true -require 'ast_transform/errors' - -module ASTTransform - # The handle returned by +defer+: not a node, a pair of nodes. Both markers - # are ordinary nodes in the ast_transform IR — the emitter keys on node type - # and token, never on any class — created together so the pair cannot be - # mismatched. - # - # placement:: (:ast_deferred, token, (:begin, ...)) — the deferred body, - # spliced at the statements' SOURCE position; lowered to - # +__ast_deferred___ = proc { ... }+ (plus pre-declarations - # for the locals the body assigns — see DeferralLowering). - # execution:: (:ast_deferred_call, token) — loc-less, spliced (or composed - # into an expression, e.g. an assert_raises block body) at the - # execution point; lowered to +__ast_deferred___.call+. - Deferral = Data.define(:placement, :execution) - - # The pairing mechanism between the two halves of a Deferral: it answers - # "which proc does this call marker invoke?" when a scope holds several - # deferrals. The markers cannot reference each other's nodes — Processor and - # Node#updated rebuilds create new node objects, so node identity does not - # survive transformation passes. Children DO survive (carried by reference - # through every rebuild), so both markers carry this same child object and - # the emitter pairs by its object identity. No behavior needed — a named - # class over a bare Object.new only for self-documenting AST dumps and - # greppability. - class DeferralToken - def inspect - "#" - end - end -end diff --git a/lib/ast_transform/deferral_lowering.rb b/lib/ast_transform/deferral_lowering.rb deleted file mode 100644 index 725ebc0..0000000 --- a/lib/ast_transform/deferral_lowering.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true -require 'ast_transform/node' -require 'ast_transform/errors' -require 'ast_transform/transformation_helper' - -module ASTTransform - # Lowers deferral markers into plain Ruby nodes ahead of emission: - # - # (:ast_deferred, token, (:begin, ...)) => x = x; __ast_deferred___ = proc { ... } - # (:ast_deferred_call, token) => __ast_deferred___.call - # - # The closure is a non-lambda proc on purpose: `return` inside a proc - # returns from the method where the proc was defined, and placement and - # execution always share one method activation (the hidden lvar cannot be - # referenced across a def boundary), so a deferred `return` keeps its - # original meaning. Jump keywords whose owner lies outside the deferred - # statements keep Ruby's native behavior — no transform-time validation: - # what a transform chooses to defer is the transform author's call. - # - # The `x = x` pre-declarations cover every local the deferred statements - # assign at method scope. A local first assigned inside a block literal is - # block-local, so without a textual method-scope assignment before the - # proc, deferred assignments would be invisible to the statements that - # read them after the execution point. Self-assignment registers the name - # (nil until the deferred code runs — exactly what an unexecuted - # assignment yields) without clobbering an already-assigned value. - # - # Hidden lvar names are assigned per token in encounter order, so they are - # stable within a file and never collide. Pairing is by token object - # identity (see DeferralToken). - # - # Reconciliation is a static count of markers in the tree, not of runtime - # executions — a call under a conditional legitimately executes zero-or-more - # times. One placement may have many calls (multiplexing); it must have at - # least one, textually after it (the proc must exist before it is called). - class DeferralLowering - include TransformationHelper - - def initialize - @names_by_token = {}.compare_by_identity - @called_tokens = {}.compare_by_identity - end - - # @param node [Parser::AST::Node] tree possibly containing deferral markers - # @return [Parser::AST::Node] tree with markers lowered to plain Ruby - # @raise [UnmatchedDeferralError] on missing call, orphan or premature - # call, or duplicate placement - def run(node) - lowered = lower(node) - - unexecuted = @names_by_token.keys.reject { |token| @called_tokens.key?(token) } - unless unexecuted.empty? - raise UnmatchedDeferralError, - "deferred statements were placed but never executed (#{unexecuted.size} deferral(s) " \ - "without an execution point); the deferred code would silently never run" - end - - lowered - end - - private - - def lower(node) - return node unless node.is_a?(::Parser::AST::Node) - - case node.type - when :ast_deferred then lower_placement(node) - when :ast_deferred_call then lower_call(node) - else - node.updated(nil, node.children.map { |child| lower(child) }) - end - end - - def lower_placement(node) - token, body = node.children - if @names_by_token.key?(token) - raise UnmatchedDeferralError, - "duplicate deferral placement: the hidden proc would be assigned twice" - end - - name = :"__ast_deferred_#{@names_by_token.size + 1}__" - @names_by_token[token] = name - - deferred_assignment = s(:lvasgn, name, s(:block, s(:send, nil, :proc), s(:args), lower(body))) - pre_declarations = method_scope_assignments(body).map { |local| s(:lvasgn, local, s(:lvar, local)) } - return deferred_assignment if pre_declarations.empty? - - # A loc-less :begin in statement position; the emitter flattens it into - # the surrounding statement stream so the proc body still aligns. - s(:begin, *pre_declarations, deferred_assignment) - end - - def lower_call(node) - token = node.children[0] - name = @names_by_token[token] - unless name - raise UnmatchedDeferralError, - "deferral execution point encountered without a preceding placement; " \ - "the placement must appear textually before its execution" - end - - @called_tokens[token] = true - s(:send, s(:lvar, name), :call) - end - - # Node types opening a new local-variable scope: assignments inside them - # were invisible to the method scope in the original source too, so they - # get no pre-declaration. - NEW_SCOPE_TYPES = [:def, :defs, :class, :module, :sclass].freeze - # Block literals: locals first assigned inside them are block-local (the - # same lexical rule the pre-declarations exist to work around), but their - # callee/arguments evaluate at method scope and are still descended. - BLOCK_TYPES = [:block, :numblock, :itblock].freeze - - # Locals the deferred statements assign at method scope, in - # first-assignment order (covers masgn/op_asgn targets — they all carry - # :lvasgn nodes). - def method_scope_assignments(node, names = []) - return names unless node.is_a?(::Parser::AST::Node) - return names if NEW_SCOPE_TYPES.include?(node.type) - - names << node.children[0] if node.type == :lvasgn && !names.include?(node.children[0]) - - children = BLOCK_TYPES.include?(node.type) ? [node.children[0]] : node.children - children.each { |child| method_scope_assignments(child, names) } - names - end - end -end diff --git a/lib/ast_transform/errors.rb b/lib/ast_transform/errors.rb index 6b22ad4..5f57e0e 100644 --- a/lib/ast_transform/errors.rb +++ b/lib/ast_transform/errors.rb @@ -4,10 +4,17 @@ module ASTTransform # source location does not have one. class MissingLocationError < StandardError; end - # Raised at emission when deferral markers cannot be reconciled: a - # placement without any execution point, an execution point without a - # placement (or preceding it), or a duplicate placement. - class UnmatchedDeferralError < StandardError; end + # Raised at construction when a Thunk node's children violate its + # invariants (missing token, empty body). Every construction path funnels + # through Thunk#initialize — including Processor rebuilds — so a malformed + # thunk cannot exist in a tree. + class MalformedThunkError < StandardError; end + + # Raised at lowering when a thunk cannot be placed: its body's source + # lines fall after the execution point (the hidden proc's text IS its + # assignment, so a call can never textually precede the body), or two + # occurrences of the same thunk carry diverging bodies. + class ThunkPlacementError < StandardError; end # Raised as the emitter's postcondition when a custom node type (ast_* # markers or types registered on ASTTransform::Node) reaches the unparse diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index 86ef304..107acdd 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -2,7 +2,7 @@ require 'unparser' require 'ast_transform/node' require 'ast_transform/errors' -require 'ast_transform/deferral_lowering' +require 'ast_transform/thunk_lowering' module ASTTransform # Emits a transformed AST as text in which every loc-carrying statement @@ -24,10 +24,10 @@ module ASTTransform # statements pack and emission re-anchors at the next statement that fits. # Total: never raises on layout. # - # Deferral markers are lowered (DeferralLowering) before layout; the - # emitter's postcondition is that no custom node type (ast_* markers or - # types registered on ASTTransform::Node) crosses the unparse boundary — - # they are IR between stages that understand them. + # Thunk nodes are lowered (ThunkLowering) before layout; the emitter's + # postcondition is that no custom node type (ast_* markers or types + # registered on ASTTransform::Node) crosses the unparse boundary — they + # are IR between stages that understand them. class LineAlignedEmitter # Containers the emitter recurses into so nested statements align; every # other node renders as an Unparser blob at its head line. @@ -35,7 +35,7 @@ class LineAlignedEmitter BODY_INDEXES = { class: 2, module: 1, sclass: 1, def: 2, defs: 3, block: 2, numblock: 2, itblock: 2 }.freeze - # Assignments whose value is a block (e.g. the lowered deferral proc) + # Assignments whose value is a block (e.g. the lowered thunk proc) # recurse into the block so its body statements align. ASSIGNMENT_TYPES = [:lvasgn, :ivasgn, :gvasgn, :casgn].freeze BLOCK_VALUE_TYPES = [:block, :numblock, :itblock].freeze @@ -49,10 +49,10 @@ def initialize(ast, source_path) end # @return [String] transformed source, line-aligned - # @raise [UnmatchedDeferralError] if deferral markers cannot be reconciled + # @raise [ThunkPlacementError] if a thunk cannot be textually placed # @raise [UnloweredNodeTypeError] if a custom node type survived to emission def emit - lowered = DeferralLowering.new.run(@ast) + lowered = ThunkLowering.new.run(@ast) assert_no_custom_types(lowered) @local_variables = collect_local_variables(lowered) @@ -233,9 +233,10 @@ def closer_line(node) loc.end.line if loc.respond_to?(:end) && loc.end end - # Loc-less :begin nodes in statement position (e.g. a lowered deferral - # placement carrying its pre-declarations) are grouping, not structure: - # flatten them so each inner statement is laid out independently. + # Loc-less :begin nodes in statement position (e.g. a lowered thunk in a + # single-statement container body, carried with its placement) are + # grouping, not structure: flatten them so each inner statement is laid + # out independently. def statements_of(body) return [] if body.nil? return [body] unless body.type == :begin diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb new file mode 100644 index 0000000..2ba4cac --- /dev/null +++ b/lib/ast_transform/thunk.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true +require 'ast_transform/node' +require 'ast_transform/errors' + +module ASTTransform + # The reordering primitive: an eagerly built wrapper node, spliced wherever + # the wrapped statements must EXECUTE — statement position or composed + # inside an expression (e.g. an assert_raises block body). Its body keeps + # its own source locations, and the lowering derives the wrapper's textual + # placement from them (see ThunkLowering), so the statements still emit on + # their original lines even though execution waits. + # + # Children are +[token, *body_statements]+ and the invariants are enforced + # here in +initialize+, which every construction path shares — +s+ routing, + # the +thunk+ helper, and Processor rebuilds (+updated+ re-initializes). + # Build thunks with +TransformationHelper#thunk+; reuse the same node to + # execute one body from several points (multiplexing). + # + # Runtime semantics are near-transparent (proc lowering): +return+ still + # returns from the enclosing method, and locals the body assigns stay + # method-scope. See ThunkLowering for the full contract. + class Thunk < Node + register :ast_thunk + + def initialize(type, children, properties = {}) + token, *body = children + unless token.is_a?(ThunkToken) + raise MalformedThunkError, + "a Thunk's first child must be its ThunkToken (got #{token.class}); " \ + "build thunks with the thunk(*statements) helper" + end + raise MalformedThunkError, "a Thunk must wrap at least one statement" if body.empty? + + super + end + + def token = children[0] + + def body = children.drop(1) + end + + # The identity of a Thunk across transformation passes: Processor and + # Node#updated rebuilds create new node objects, so node identity does not + # survive — but children DO (carried by reference through every rebuild). + # Every rebuild of a thunk therefore carries this same token object, and + # the lowering groups occurrences by its object identity: one proc, one + # call per occurrence. Minted internally by the +thunk+ helper, never + # handled by authors. No behavior — a named class over a bare Object.new + # only for self-documenting AST dumps and greppability. + class ThunkToken + def inspect + "#" + end + end +end diff --git a/lib/ast_transform/thunk_lowering.rb b/lib/ast_transform/thunk_lowering.rb new file mode 100644 index 0000000..703bb2f --- /dev/null +++ b/lib/ast_transform/thunk_lowering.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true +require 'ast_transform/node' +require 'ast_transform/thunk' +require 'ast_transform/errors' +require 'ast_transform/transformation_helper' + +module ASTTransform + # Lowers Thunk nodes into plain Ruby ahead of emission. Each unique thunk + # (grouped by token identity) becomes a hidden proc; each occurrence + # becomes the proc's call: + # + # thunk placed at the execution point + # => x = x; __ast_thunk___ = proc { body } (at the body's source lines) + # ... + # __ast_thunk___.call (at the occurrence) + # + # Placement is inferred, not authored: the proc's text is inserted into + # the statement sequence enclosing the occurrence, positioned among its + # siblings by the body's first source line — the lines the author removed + # the statements from. A loc-less body has no textual home and packs + # immediately before its call. Placements never escape a scope boundary + # (def/class/module bodies absorb their own), because the hidden lvar must + # share the call's method activation; they DO escape block literals, which + # close over the defining scope. + # + # The closure is a non-lambda proc on purpose: `return` inside a proc + # returns from the method where the proc was defined, and placement and + # execution always share one method activation, so a thunked `return` + # keeps its original meaning. Jump keywords whose owner lies outside the + # body keep Ruby's native behavior (`break`/`retry` fail loudly, + # `next`/`redo` silently alter flow) — what a transform chooses to thunk + # is the transform author's call. + # + # The `x = x` pre-declarations cover every local the body assigns at + # method scope. A local first assigned inside a block literal is + # block-local, so without a textual method-scope assignment before the + # proc, thunked assignments would be invisible to the statements that + # read them after the execution point. Self-assignment registers the name + # (nil until the thunk runs — exactly what an unexecuted assignment + # yields) without clobbering an already-assigned value. + class ThunkLowering + include TransformationHelper + + # A pending proc definition: +line+ is the body's first source line + # (nil for fully synthetic bodies), +statements+ the pre-declarations + # plus the proc assignment. + Placement = Struct.new(:line, :statements) + + SEQUENCE_TYPES = [:begin, :kwbegin].freeze + # Scope-opening containers: the hidden lvar cannot be referenced across + # these boundaries, so placements arising inside must land inside. + SCOPE_BODY_INDEXES = { def: 2, defs: 3, class: 2, module: 1, sclass: 1 }.freeze + + def initialize + @names_by_token = {}.compare_by_identity + @bodies_by_token = {}.compare_by_identity + end + + # @param node [Parser::AST::Node] tree possibly containing Thunk nodes + # @return [Parser::AST::Node] tree with thunks lowered to plain Ruby + # @raise [ThunkPlacementError] when a thunk body's source lines fall + # after its execution point, or occurrences of one thunk diverge + def run(node) + lower_body(node) + end + + private + + # Lowers a node standing in statement-body position (a container's body + # or the root), absorbing any placements that arise within it. + def lower_body(node) + return node unless node.is_a?(::Parser::AST::Node) + return lower_sequence(node) if SEQUENCE_TYPES.include?(node.type) + + lowered, placements = lower_expression(node) + return lowered if placements.empty? + + # A loc-less :begin in statement position; the emitter flattens it + # into the surrounding statement stream. + s(:begin, *placements.flat_map(&:statements), lowered) + end + + # Lowers a statement sequence, inserting each placement among the + # statements by the body's source line. + def lower_sequence(node) + statements = [] + + node.children.each_with_index do |child, index| + lowered, placements = lower_expression(child) + placements.each do |placement| + check_placement_precedes_execution!(placement, child, node.children[(index + 1)..]) + statements.insert(insertion_index(statements, placement), *placement.statements) + end + statements << lowered + end + + node.updated(nil, statements) + end + + # Lowers a node in expression position. Returns the lowered node and the + # placements that must be inserted into the enclosing statement + # sequence. + # + # @return [Array(Parser::AST::Node, Array)] + def lower_expression(node) + return [node, []] unless node.is_a?(::Parser::AST::Node) + + case node.type + when :ast_thunk + lower_thunk(node) + when *SEQUENCE_TYPES + [lower_sequence(node), []] + when :ensure, :rescue + [node.updated(nil, node.children.map { |child| lower_body(child) }), []] + when :resbody + exceptions, capture, body = node.children + [node.updated(nil, [exceptions, capture, lower_body(body)]), []] + else + lower_generic(node) + end + end + + def lower_generic(node) + scope_body_index = SCOPE_BODY_INDEXES[node.type] + pending = [] + + children = node.children.each_with_index.map do |child, index| + if index == scope_body_index + lower_body(child) + else + lowered, placements = lower_expression(child) + pending.concat(placements) + lowered + end + end + + [node.updated(nil, children), pending] + end + + # An occurrence of a thunk: the first occurrence of its token yields the + # placement; every occurrence yields the call. + def lower_thunk(node) + token = node.token + + if @names_by_token.key?(token) + unless @bodies_by_token[token] == node.body + raise ThunkPlacementError, + "occurrences of one thunk carry diverging bodies; reuse the same thunk node to multiplex" + end + return [call_node(token), []] + end + + name = :"__ast_thunk_#{@names_by_token.size + 1}__" + @names_by_token[token] = name + @bodies_by_token[token] = node.body + + lowered_body = lower_sequence(s(:begin, *node.body)) + placement = Placement.new(body_first_line(node.body), placement_statements(name, lowered_body)) + [call_node(token), [placement]] + end + + def call_node(token) + s(:send, s(:lvar, @names_by_token.fetch(token)), :call) + end + + def placement_statements(name, lowered_body) + assignment = s(:lvasgn, name, s(:block, s(:send, nil, :proc), s(:args), lowered_body)) + hidden_names = @names_by_token.values + pre_declared = method_scope_assignments(lowered_body).reject { |local| hidden_names.include?(local) } + pre_declared.map { |local| s(:lvasgn, local, s(:lvar, local)) } << assignment + end + + def body_first_line(body) + body.filter_map { |statement| statement.loc&.line }.min + end + + # The proc's text must precede its call: a placement whose body lines + # fall at or after the executing statement (or any statement after it) + # cannot be laid out — the assignment would complete after the call. + def check_placement_precedes_execution!(placement, executing_statement, following_statements) + return if placement.line.nil? + + conflicting = [executing_statement, *following_statements].find do |statement| + line = statement.is_a?(::Parser::AST::Node) ? statement.loc&.line : nil + line && line < placement.line + end + return if conflicting.nil? + + raise ThunkPlacementError, + "thunk body's source lines (from line #{placement.line}) fall after its execution point " \ + "(statement at line #{conflicting.loc.line}); a thunk can only delay execution, never text" + end + + def insertion_index(statements, placement) + return statements.size if placement.line.nil? + + statements.index { |statement| statement.loc&.line && statement.loc.line > placement.line } || statements.size + end + + # Node types opening a new local-variable scope: assignments inside them + # were invisible to the method scope in the original source too, so they + # get no pre-declaration. + NEW_SCOPE_TYPES = [:def, :defs, :class, :module, :sclass].freeze + # Block literals: locals first assigned inside them are block-local (the + # same lexical rule the pre-declarations exist to work around), but their + # callee/arguments evaluate at method scope and are still descended. + BLOCK_TYPES = [:block, :numblock, :itblock].freeze + + # Locals the thunk body assigns at method scope, in first-assignment + # order (covers masgn/op_asgn targets — they all carry :lvasgn nodes). + def method_scope_assignments(node, names = []) + return names unless node.is_a?(::Parser::AST::Node) + return names if NEW_SCOPE_TYPES.include?(node.type) + + names << node.children[0] if node.type == :lvasgn && !names.include?(node.children[0]) + + children = BLOCK_TYPES.include?(node.type) ? [node.children[0]] : node.children + children.each { |child| method_scope_assignments(child, names) } + names + end + end +end diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index 9758cb6..52d5a95 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'parser' require 'ast_transform/node' -require 'ast_transform/deferral' +require 'ast_transform/thunk' require 'ast_transform/errors' module ASTTransform @@ -10,13 +10,13 @@ module ASTTransform # - Constructors (+s+, +s_at+): type + children in, fresh node out. # - The sequence combinator (+run_after+): sequence in, sequence out — the # paved road for execution reordering. - # - The low-level deferral primitive (+defer+): statements in, Deferral - # pair out — for execution points inside expressions. + # - The low-level reordering primitive (+thunk+): statements in, Thunk + # node out — for execution points inside expressions. # # The contract these helpers serve: textual order is source order. The # emitter places every loc-carrying statement at its source line; when # execution order must differ from textual order, authors express it as a - # deferral instead of moving text. + # thunk instead of moving text. module TransformationHelper class << self def included(base) @@ -57,44 +57,39 @@ def s_at(anchor, type, *children) s(type, *children, location: ::Parser::Source::Map.new(expression)) end - # Low-level deferral primitive. Deferral is the one reordering lever: - # text never moves and execution can only move later, so "hoist A above - # B" is expressed as "run B after A". Returns a Deferral pairing two - # plain marker nodes: splice +placement+ where the statements sit in - # the SOURCE (inner statements keep their own locs, so the emitter - # aligns the body even though execution waits) and +execution+ where - # they run — composable inside expressions, e.g. as an assert_raises - # block body. + # The low-level reordering primitive. Thunking is the one reordering + # lever: text never moves and execution can only move later, so "hoist + # A above B" is expressed as "run B after A". Returns a single Thunk + # node: splice it where the statements must RUN — statement position + # or composed inside an expression, e.g. as an assert_raises block + # body. The wrapped statements keep their own locs, and the lowering + # derives the hidden proc's textual placement from them, so the body + # still emits on its source lines even though execution waits. Reuse + # the same node to execute one body from several points. # - # The emitter lowers the pair to a hidden-lvar proc and its call, with - # near-transparent semantics (see DeferralLowering): +return+ still + # Semantics are near-transparent (see ThunkLowering): +return+ still # returns from the enclosing method (non-lambda proc), and locals the - # deferred statements assign stay method-scope (pre-declared before the - # proc). Jump keywords whose owner lies outside the deferred statements + # wrapped statements assign stay method-scope (pre-declared before the + # proc). Jump keywords whose owner lies outside the wrapped statements # keep Ruby's native behavior — +break+/+retry+ fail loudly at the # jump's own source line, +next+/+redo+ silently end or restart the - # deferred body. Weigh that when choosing what your surface defers. + # thunk body. Weigh that when choosing what your surface thunks. # - # Prefer +run_after+ when both points sit in one statement sequence. + # Prefer +run_after+ when the execution point sits in the same + # statement sequence as the statements. # - # @param statements [Array] statements to defer - # @return [ASTTransform::Deferral] the placement/execution marker pair - def defer(*statements) - token = DeferralToken.new - Deferral.new( - placement: s(:ast_deferred, token, s(:begin, *statements)), - execution: s(:ast_deferred_call, token) - ) + # @param statements [Array] statements to wrap + # @return [ASTTransform::Thunk] the thunk node + def thunk(*statements) + s(:ast_thunk, ThunkToken.new, *statements) end - # The paved road for execution reordering in flat statement sequences: - # one call, both placements handled, nothing to forget. Named for the - # constraint, not the mechanism — "run X after Y" covers hoisting and - # sinking symmetrically, because with text pinned to source lines the - # only physical lever is delaying execution. +after+ may be textually - # before or after the +run+ statements. Returns a NEW sequence in which - # the +run+ statements are replaced (in place) by one placement marker - # and its execution marker is inserted immediately after +after+. + # The paved road for execution reordering in flat statement sequences. + # Named for the constraint, not the mechanism — with text pinned to + # source lines the only physical lever is delaying execution, so "run + # X after Y" is the constraint an author states. Returns a NEW + # sequence in which the +run+ statements are removed and a thunk + # wrapping them is inserted immediately after +after+. # # All membership checks are by identity (equal?), never ==: node # equality ignores location, so two textually identical statements on @@ -106,7 +101,7 @@ def defer(*statements) # +statements+ (by identity) whose execution must wait # @param after [Parser::AST::Node] element of +statements+ (by identity, # not inside +run+) the +run+ statements execute after - # @return [Array] new sequence with markers placed + # @return [Array] new sequence with the thunk placed # @raise [ArgumentError] if +run+ is not a contiguous identity-run of # +statements+, or +after+ is not an element (or is inside +run+) def run_after(statements, run:, after:) @@ -117,12 +112,11 @@ def run_after(statements, run:, after:) raise ArgumentError, "after: must be an element of statements (by identity)" unless after_index raise ArgumentError, "after: cannot be inside run:" if run_range.cover?(after_index) - deferral = defer(*run) reordered = statements.dup - reordered[run_range] = [deferral.placement] + reordered[run_range] = [] insertion_index = reordered.index { |statement| statement.equal?(after) } - reordered.insert(insertion_index + 1, deferral.execution) + reordered.insert(insertion_index + 1, thunk(*run)) end private diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index 9295f2f..ef696c6 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -22,27 +22,27 @@ def emit(ast) end # Runs emitted code with real method semantics (return target, method - # scope for locals) — exactly the environment deferred code lives in. + # scope for locals) — exactly the environment thunked code lives in. def run_as_method(emitted) harness = Module.new harness.module_eval("def self.run_case\n#{emitted}\nend", 'fixture.rb', 0) harness.run_case end - test "emits deferral pairs as a hidden-lvar proc and its call" do + test "lowers a thunk to a hidden-lvar proc at the body's source lines and its call" do given, when_statement, interaction = parse("given_setup\nwhen_body\ninteraction_setup\n").children reordered = run_after([given, when_statement, interaction], run: [when_statement], after: interaction) emitted = emit(s(:begin, *reordered)) - assert_includes emitted, '__ast_deferred_1__ = proc', emitted + assert_includes emitted, '__ast_thunk_1__ = proc', emitted assert_includes emitted, 'when_body', emitted - assert_includes emitted, '__ast_deferred_1__.call', emitted + assert_includes emitted, '__ast_thunk_1__.call', emitted # Execution order: proc defined, interaction runs, then the call. - assert_operator emitted.index('interaction_setup'), :<, emitted.index('__ast_deferred_1__.call'), emitted + assert_operator emitted.index('interaction_setup'), :<, emitted.index('__ast_thunk_1__.call'), emitted end - test "deferred body statements stay on their source lines inside the proc" do + test "thunk body statements stay on their source lines inside the proc" do source = <<~HEREDOC given_setup when_body_first @@ -58,53 +58,52 @@ def run_as_method(emitted) assert_equal 3, emitted.lines.index { |line| line.include?('when_body_second') } + 1, emitted end - test "a placement may be executed from multiple call sites (multiplexing)" do - statement = parse("shared_body\n") - deferral = defer(statement) + test "reusing one thunk node executes its body from multiple call sites (multiplexing)" do + shared = thunk(parse("shared_body\n")) - emitted = emit(s(:begin, deferral.placement, deferral.execution, deferral.execution)) + emitted = emit(s(:begin, shared, shared)) - assert_equal 2, emitted.scan('__ast_deferred_1__.call').size, emitted + assert_equal 1, emitted.scan('__ast_thunk_1__ = proc').size, emitted + assert_equal 2, emitted.scan('__ast_thunk_1__.call').size, emitted end - test "a placement with no execution point raises UnmatchedDeferralError" do - deferral = defer(parse("orphan_body\n")) + test "a thunk whose body lines fall after its execution point raises ThunkPlacementError" do + first, second, third = parse("first\nsecond\nthird\n").children + # third's text (line 3) cannot execute after first (line 1) yet before + # second (line 2): the proc's text IS its assignment. + reordered = run_after([first, second, third], run: [third], after: first) - error = assert_raises(UnmatchedDeferralError) { emit(s(:begin, deferral.placement)) } + error = assert_raises(ThunkPlacementError) { emit(s(:begin, *reordered)) } - assert_includes error.message, 'never executed' + assert_includes error.message, 'fall after its execution point' end - test "an execution point before its placement raises UnmatchedDeferralError" do - deferral = defer(parse("body\n")) + test "occurrences of one thunk with diverging bodies raise ThunkPlacementError" do + original = thunk(parse("foo\n")) + diverged = original.updated(nil, [original.token, parse("bar\n")]) - error = assert_raises(UnmatchedDeferralError) do - emit(s(:begin, deferral.execution, deferral.placement)) - end + error = assert_raises(ThunkPlacementError) { emit(s(:begin, original, diverged)) } - assert_includes error.message, 'before' + assert_includes error.message, 'diverging' end - test "a duplicate placement raises UnmatchedDeferralError" do - deferral = defer(parse("body\n")) + test "a loc-less thunk body packs immediately before its call" do + synthetic = thunk(s(:send, nil, :synthetic_body)) - error = assert_raises(UnmatchedDeferralError) do - emit(s(:begin, deferral.placement, deferral.placement, deferral.execution)) - end + emitted = emit(s(:begin, parse("real_statement\n"), synthetic)) - assert_includes error.message, 'duplicate' + assert_includes emitted, 'real_statement; __ast_thunk_1__ = proc', emitted + assert_operator emitted.index('synthetic_body'), :<, emitted.index('__ast_thunk_1__.call'), emitted end - test "distinct deferrals get distinct hidden lvar names" do - first_deferral = defer(parse("first_body\n")) - second_deferral = defer(parse("second_body\n")) + test "distinct thunks get distinct hidden lvar names" do + first_thunk = thunk(parse("first_body\n")) + second_thunk = thunk(parse("second_body\n")) - emitted = emit(s(:begin, - first_deferral.placement, second_deferral.placement, - first_deferral.execution, second_deferral.execution)) + emitted = emit(s(:begin, first_thunk, second_thunk)) - assert_includes emitted, '__ast_deferred_1__ = proc', emitted - assert_includes emitted, '__ast_deferred_2__ = proc', emitted + assert_includes emitted, '__ast_thunk_1__ = proc', emitted + assert_includes emitted, '__ast_thunk_2__ = proc', emitted end test "an unlowered custom node type raises UnloweredNodeTypeError" do @@ -157,13 +156,13 @@ def risky assert_nil emitter.send(:compress_to_single_line, "value = <<~TXT\n hi\nTXT") end - test "pre-declares locals the deferred statements assign at method scope" do + test "pre-declares locals the thunk body assigns at method scope" do first, second, third = parse("given_setup\nresult = compute\ninteraction_setup\n").children reordered = run_after([first, second, third], run: [second], after: third) emitted = emit(s(:begin, *reordered)) - assert_includes emitted, 'result = result; __ast_deferred_1__ = proc', emitted + assert_includes emitted, 'result = result; __ast_thunk_1__ = proc', emitted end test "pre-declarations skip block-local assignments but cover the block call's arguments" do @@ -171,9 +170,8 @@ def risky outer = items.map { |item| inner = item } buffer.take(width = limit) { |line| sink(line) } HEREDOC - deferral = defer(*parse(source).children) - emitted = emit(s(:begin, deferral.placement, deferral.execution)) + emitted = emit(s(:begin, thunk(*parse(source).children))) assert_includes emitted, 'outer = outer', emitted # width is assigned in the block call's ARGUMENTS, which evaluate at @@ -184,14 +182,12 @@ def risky end test "pre-declarations skip assignments inside nested defs" do - deferral = defer(parse("def helper = (scoped = 1)\n")) - - emitted = emit(s(:begin, deferral.placement, deferral.execution)) + emitted = emit(s(:begin, thunk(parse("def helper = (scoped = 1)\n")))) refute_includes emitted, 'scoped = scoped', emitted end - test "a deferred assignment runs late but propagates to the enclosing method scope" do + test "a thunked assignment runs late but propagates to the enclosing method scope" do source = <<~HEREDOC log = [] result = log.size @@ -201,7 +197,7 @@ def risky first, second, third, fourth = parse(source).children reordered = run_after([first, second, third, fourth], run: [second], after: third) - # Deferred `result = log.size` runs after `log << :setup`, so result is + # Thunked `result = log.size` runs after `log << :setup`, so result is # 1 (textual order would give 0) — proving both the reordering and that # the assignment escaped the proc into the method scope. assert_equal [[:setup], 1], run_as_method(emit(s(:begin, *reordered))) @@ -218,12 +214,12 @@ def risky reordered = run_after([first, second, third, fourth], run: [second], after: third) # snapshot reads value between the proc's definition and its call: the - # pre-declaration must preserve :given, and the deferred reassignment + # pre-declaration must preserve :given, and the thunked reassignment # must land afterwards. assert_equal [:given, :reassigned], run_as_method(emit(s(:begin, *reordered))) end - test "a deferred return exits the enclosing method (non-lambda proc semantics)" do + test "a thunked return exits the enclosing method (non-lambda proc semantics)" do source = <<~HEREDOC log = [] return [:early, log] unless log.empty? @@ -233,15 +229,13 @@ def risky first, second, third, fourth = parse(source).children reordered = run_after([first, second, third, fourth], run: [second], after: third) - # The deferred return fires from inside the proc but returns from the - # method — and only after the setup it was deferred past has run. + # The thunked return fires from inside the proc but returns from the + # method — and only after the setup it was thunked past has run. assert_equal [:early, [:setup]], run_as_method(emit(s(:begin, *reordered))) end - test "a deferred break severed from its loop keeps Ruby's native LocalJumpError" do - deferral = defer(parse("break\n")) - - emitted = emit(s(:begin, deferral.placement, deferral.execution)) + test "a thunked break severed from its loop keeps Ruby's native LocalJumpError" do + emitted = emit(s(:begin, thunk(parse("break\n")))) assert_raises(LocalJumpError) { run_as_method(emitted) } end @@ -261,18 +255,34 @@ def risky assert_equal 4, emitted_lines.index { |line| line.include?('second_call') } + 1, emitted_lines.join end - test "execution markers compose inside expressions" do + test "a thunk composes inside expressions; its placement hoists to the enclosing sequence" do when_body = parse("raise_helper\n") - deferral = defer(when_body) assert_raises_call = s(:block, s(:send, nil, :assert_raises, s(:const, nil, :RuntimeError)), s(:args), - deferral.execution) + thunk(when_body)) - emitted = emit(s(:begin, deferral.placement, assert_raises_call)) + emitted = emit(s(:begin, assert_raises_call)) + assert_includes emitted, '__ast_thunk_1__ = proc', emitted assert_includes emitted, 'assert_raises(RuntimeError)', emitted - assert_includes emitted, '__ast_deferred_1__.call', emitted + assert_includes emitted, '__ast_thunk_1__.call', emitted + # The proc's definition precedes the assert_raises call that runs it. + assert_operator emitted.index('__ast_thunk_1__ = proc'), :<, emitted.index('assert_raises'), emitted + end + + test "a thunk inside a def stays inside the def (scope boundary)" do + def_node = parse("def run\n helper\nend\n") + name, args, body = def_node.children + thunked_def = def_node.updated(nil, [name, args, thunk(body)]) + + emitted = emit(thunked_def) + + # The proc and its call both sit between the def opener and its end; + # the body statement keeps its source line. + assert_operator emitted.index('def run'), :<, emitted.index('__ast_thunk_1__ = proc'), emitted + assert_operator emitted.index('__ast_thunk_1__.call'), :<, emitted.rindex('end'), emitted + assert_equal 2, emitted.lines.index { |line| line.include?('helper') } + 1, emitted end end end diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index 4c44831..05ea457 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'test_helper' require 'ast_transform/transformation_helper' +require 'ast_transform/abstract_transformation' require 'ast_transform/transformer' module ASTTransform @@ -48,47 +49,76 @@ def parse(source) assert_includes error.message, 'send' end - test "defer returns a Deferral whose markers share a token" do + test "thunk builds a Thunk node carrying an internal token and the body" do statements = parse("foo\nbar\n").children - deferral = defer(*statements) + node = thunk(*statements) - assert_equal :ast_deferred, deferral.placement.type - assert_equal :ast_deferred_call, deferral.execution.type - assert_same deferral.placement.children[0], deferral.execution.children[0] - assert_instance_of DeferralToken, deferral.placement.children[0] + assert_instance_of Thunk, node + assert_equal :ast_thunk, node.type + assert_instance_of ThunkToken, node.token + assert_equal statements, node.body end - test "DeferralToken#inspect names the class so AST dumps are self-documenting" do - token = defer(parse("foo\n")).placement.children[0] + test "a Thunk without a token cannot be constructed" do + error = assert_raises(MalformedThunkError) { s(:ast_thunk, parse("foo\n")) } - assert_match(/\A#\z/, token.inspect) + assert_includes error.message, 'ThunkToken' end - test "defer imposes no control-flow validation (semantics are the proc lowering's contract)" do - # return is transparent through the non-lambda proc; severed jumps keep - # Ruby's native behavior (see DeferralLowering / LineAlignedEmitterTest). - assert_equal :ast_deferred, defer(parse("return 1 if early\n")).placement.type - assert_equal :ast_deferred, defer(parse("break\n")).placement.type + test "a Thunk with an empty body cannot be constructed" do + error = assert_raises(MalformedThunkError) { thunk } + + assert_includes error.message, 'at least one statement' end - test "run_after replaces the run with a placement and inserts the execution after the anchor" do - setup_statement, when_statement, interaction = parse("given\nwhen_body\ninteraction\n").children + test "a Processor rebuild preserves the Thunk class, token, and invariants" do + node = thunk(parse("foo\n")) - reordered = run_after([setup_statement, when_statement, interaction], run: [when_statement], after: interaction) + rebuilt = node.updated(nil, [node.token, parse("bar\n")]) - assert_equal [:send, :ast_deferred, :send, :ast_deferred_call], reordered.map(&:type) - assert_same setup_statement, reordered[0] - assert_same interaction, reordered[2] - assert_same reordered[1].children[0], reordered[3].children[0] + assert_instance_of Thunk, rebuilt + assert_same node.token, rebuilt.token + assert_raises(MalformedThunkError) { node.updated(nil, [node.token]) } end - test "run_after supports after: textually before the run (pure sink)" do - first, second, third = parse("first\nsecond\nthird\n").children + test "AbstractTransformation descends thunk bodies by default" do + swap_foo_for_bar = Class.new(AbstractTransformation) do + def on_send(node) + node.children[1] == :foo ? node.updated(nil, [node.children[0], :bar]) : node + end + end + node = thunk(parse("foo\n")) + + processed = swap_foo_for_bar.new.run(node) - reordered = run_after([first, second, third], run: [third], after: first) + assert_instance_of Thunk, processed + assert_same node.token, processed.token + assert_equal :bar, processed.body[0].children[1] + end - assert_equal [:send, :ast_deferred_call, :send, :ast_deferred], reordered.map(&:type) + test "ThunkToken#inspect names the class so AST dumps are self-documenting" do + token = thunk(parse("foo\n")).token + + assert_match(/\A#\z/, token.inspect) + end + + test "thunk imposes no control-flow validation (semantics are the proc lowering's contract)" do + # return is transparent through the non-lambda proc; severed jumps keep + # Ruby's native behavior (see ThunkLowering / LineAlignedEmitterTest). + assert_instance_of Thunk, thunk(parse("return 1 if early\n")) + assert_instance_of Thunk, thunk(parse("break\n")) + end + + test "run_after removes the run and inserts a thunk after the anchor" do + setup_statement, when_statement, interaction = parse("given\nwhen_body\ninteraction\n").children + + reordered = run_after([setup_statement, when_statement, interaction], run: [when_statement], after: interaction) + + assert_equal [:send, :send, :ast_thunk], reordered.map(&:type) + assert_same setup_statement, reordered[0] + assert_same interaction, reordered[1] + assert_equal [when_statement], reordered[2].body end test "run_after matches statements by identity, not equality" do @@ -98,8 +128,9 @@ def parse(source) reordered = run_after([first, duplicate_of_first, last], run: [duplicate_of_first], after: last) assert_same first, reordered[0] - assert_equal :ast_deferred, reordered[1].type - assert_same last, reordered[2] + assert_same last, reordered[1] + assert_equal :ast_thunk, reordered[2].type + assert_same duplicate_of_first, reordered[2].body[0] end test "run_after rejects a non-contiguous run" do From b4d14048051d51ce54b7e951b8e48f3565135f75 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Thu, 23 Jul 2026 23:02:36 -0400 Subject: [PATCH 14/22] Indent emitted statements to their source column MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Line-aligned emission previously emitted everything flush-left, which kept line fidelity but made emitted artifacts and test expectations hard to read against the source. Statements opening a fresh line now carry their original loc column as leading whitespace — cosmetic only, packed statements and Unparser continuation lines are unaffected. Co-authored-by: Cursor --- lib/ast_transform/line_aligned_emitter.rb | 45 +++++++++++++++++------ test/ast_transform/transformation_test.rb | 18 ++++----- 2 files changed, 42 insertions(+), 21 deletions(-) diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index 107acdd..dbcc828 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -13,7 +13,7 @@ module ASTTransform # Cursor algorithm over statement sequences: # # 1. Statement has loc and target_line > cursor: pad with newlines, emit at - # the target line. + # the target line, indented to the statement's source column. # 2. Statement has loc and target_line <= cursor: pack (`; `) onto the # current line. A user statement landing here means the transform moved # it — the alignment auditor's concern, not a runtime failure. @@ -71,7 +71,7 @@ def emit_statement(node) if recursive_container?(node) emit_container(node) else - place(node.loc&.line, aligned_render(node)) + place(node.loc&.line, aligned_render(node), column: node.loc&.column) end end @@ -134,7 +134,7 @@ def emit_body(body) def emit_ensure(node) *body, ensurer = node.children body.each { |statement| emit_body(statement) } - place_keyword(keyword_line(node), 'ensure') + place_keyword(keyword_line(node), 'ensure', column: keyword_column(node)) emit_statements(statements_of(ensurer)) end @@ -144,7 +144,8 @@ def emit_rescue(node) resbodies.each { |resbody| emit_resbody(resbody) } return if else_body.nil? - place_keyword(nil, 'else') + else_range = node.loc.else if node.loc.respond_to?(:else) + place_keyword(else_range&.line, 'else', column: else_range&.column) emit_statements(statements_of(else_body)) end @@ -153,15 +154,15 @@ def emit_resbody(node) header = ['rescue'] header << " #{Unparser.unparse(exceptions).delete_prefix('[').delete_suffix(']')}" if exceptions header << " => #{capture.children[0]}" if capture - place_keyword(node.loc&.line, header.join) + place_keyword(node.loc&.line, header.join, column: node.loc&.column) emit_statements(statements_of(body)) end # Keywords (rescue/ensure/else) cannot be `;`-packed after a statement; # when their line is taken they go on a fresh line instead. - def place_keyword(target_line, keyword) + def place_keyword(target_line, keyword, column: nil) if target_line && target_line > @lines.size - place(target_line, keyword) + place(target_line, keyword, column: column) else @lines << keyword end @@ -172,6 +173,11 @@ def keyword_line(node) loc.keyword.line if loc.respond_to?(:keyword) && loc.keyword end + def keyword_column(node) + loc = node.loc + loc.keyword.column if loc.respond_to?(:keyword) && loc.keyword + end + def recursive_container?(node) RECURSIVE_CONTAINER_TYPES.include?(node.type) || block_assignment?(node) end @@ -185,9 +191,9 @@ def block_assignment?(node) # emptied, then recurses into the body so nested statements align. def emit_container(node) opener, closer = container_delimiters(node) - place(node.loc&.line, opener) + place(node.loc&.line, opener, column: node.loc&.column) emit_body(container_body(node)) - place(closer_line(node), closer) + place(closer_line(node), closer, column: closer_column(node)) end def container_delimiters(node) @@ -233,6 +239,11 @@ def closer_line(node) loc.end.line if loc.respond_to?(:end) && loc.end end + def closer_column(node) + loc = node.loc + loc.end.column if loc.respond_to?(:end) && loc.end + end + # Loc-less :begin nodes in statement position (e.g. a lowered thunk in a # single-statement container body, carried with its placement) are # grouping, not structure: flatten them so each inner statement is laid @@ -248,13 +259,19 @@ def statements_of(body) # Places +render+ at +target_line+ when the cursor hasn't passed it; # otherwise packs onto the current line. Multi-line renders advance the - # cursor by their height. - def place(target_line, render) + # cursor by their height. When opening a fresh line, the render is + # indented to the statement's source +column+ — cosmetic only (leading + # whitespace is never significant in emitted code; heredocs are + # normalized to inline strings), but it keeps the artifact and test + # expectations visually close to the source. Packed statements ignore + # the column, as do an Unparser render's continuation lines (they keep + # Unparser's own relative indentation). + def place(target_line, render, column: nil) first, *rest = render.split("\n") if target_line && target_line > @lines.size @lines << '' while @lines.size < target_line - @lines[-1] = first + @lines[-1] = indented(first, column) else pack(first) end @@ -262,6 +279,10 @@ def place(target_line, render) @lines.concat(rest) end + def indented(text, column) + column && column.positive? ? "#{' ' * column}#{text}" : text + end + # The last line is never blank here: padding blanks are only created # inside +place+, which immediately overwrites the padded line. def pack(text) diff --git a/test/ast_transform/transformation_test.rb b/test/ast_transform/transformation_test.rb index 7110266..be55fd7 100644 --- a/test/ast_transform/transformation_test.rb +++ b/test/ast_transform/transformation_test.rb @@ -126,7 +126,7 @@ class Bar class PrefixFoo - foo + foo end HEREDOC @@ -146,8 +146,8 @@ class Bar expected = <<~HEREDOC class PrefixFoo - class Bar - end + class Bar + end end HEREDOC @@ -209,9 +209,9 @@ def setup expected = <<~HEREDOC class PrefixFoo - def setup - @obj = MyClass.new(bar: 1, baz: 2) - end + def setup + @obj = MyClass.new(bar: 1, baz: 2) + end end HEREDOC @@ -231,9 +231,9 @@ def call expected = <<~HEREDOC class PrefixFoo - def call - method("hello", bar: 1) - end + def call + method("hello", bar: 1) + end end HEREDOC From f7e29740ed1a7574d775aecf5a15ce9794f78417 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 00:12:21 -0400 Subject: [PATCH 15/22] Apply Shopify style across the rebased branch Style the files this branch adds or rewrites, bump rubocop-shopify to ~> 3.0 now that the Ruby floor is 3.3 (dropping the parallel pin), and move development dependencies to the Gemfile per Gemspec/DevelopmentDependencies, which 3.0 enables. Co-authored-by: Cursor --- Gemfile | 11 ++++++++--- Gemfile.lock | 14 ++++++++++---- ast_transform.gemspec | 10 +--------- dev.yml | 3 +++ lib/ast_transform/errors.rb | 1 + lib/ast_transform/line_aligned_emitter.rb | 1 + lib/ast_transform/node.rb | 1 + lib/ast_transform/test_helpers.rb | 1 + lib/ast_transform/thunk.rb | 1 + lib/ast_transform/thunk_lowering.rb | 1 + lib/ast_transform/transformation_helper.rb | 1 + lib/ast_transform/transformer.rb | 1 + test/ast_transform/line_aligned_emitter_test.rb | 1 + test/ast_transform/line_alignment_test.rb | 1 + test/ast_transform/test_helpers_test.rb | 1 + test/ast_transform/transformation_helper_test.rb | 1 + test/minitest/reporters/rake_rerun_reporter.rb | 1 + 17 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Gemfile b/Gemfile index 0e2a8f0..4719249 100644 --- a/Gemfile +++ b/Gemfile @@ -7,6 +7,11 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } # Specify your gem's dependencies in ast_transform.gemspec gemspec -# Transitive dependency of rubocop: parallel >= 2 requires Ruby >= 3.3, but we -# still support 3.2. Drop this pin when our Ruby floor moves to 3.3. -gem "parallel", "< 2", require: false +# Development dependencies +gem "bundler", ">= 2.1" +gem "minitest", "~> 5.14" +gem "minitest-reporters", "~> 1.4" +gem "pry", ">= 0.14" +gem "rake", "~> 13.0" +gem "rubocop-shopify", "~> 3.0", require: false +gem "simplecov", "~> 0.22" diff --git a/Gemfile.lock b/Gemfile.lock index c998a18..6a40419 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -26,6 +26,7 @@ GEM builder minitest (>= 5.0, < 7) ruby-progressbar + parallel (2.1.0) parser (3.3.12.0) ast (~> 2.4.1) racc @@ -35,7 +36,9 @@ GEM method_source (~> 1.0) reline (>= 0.6.0) racc (1.8.1) + rainbow (3.1.1) rake (13.4.2) + regexp_parser (2.12.0) reline (0.6.3) io-console (~> 0.5) rubocop (1.88.2) @@ -52,8 +55,9 @@ GEM rubocop-ast (1.50.0) parser (>= 3.3.7.2) prism (~> 1.7) - rubocop-shopify (2.18.0) - rubocop (~> 1.62) + rubocop-shopify (3.0.1) + lint_roller + rubocop (~> 1.72, >= 1.72.1) ruby-progressbar (1.13.0) simplecov (0.22.0) docile (~> 1.1) @@ -61,6 +65,9 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) + unicode-display_width (3.2.0) + unicode-emoji (~> 4.1) + unicode-emoji (4.2.0) unparser (0.9.0) diff-lcs (>= 1.6, < 3) parser (>= 3.3.0) @@ -75,10 +82,9 @@ DEPENDENCIES bundler (>= 2.1) minitest (~> 5.14) minitest-reporters (~> 1.4) - parallel (< 2) pry (>= 0.14) rake (~> 13.0) - rubocop-shopify (~> 2.18) + rubocop-shopify (~> 3.0) simplecov (~> 0.22) BUNDLED WITH diff --git a/ast_transform.gemspec b/ast_transform.gemspec index 4f7ff1e..ffb0067 100644 --- a/ast_transform.gemspec +++ b/ast_transform.gemspec @@ -22,15 +22,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = '>= 3.3' - # Development dependencies - spec.add_development_dependency("bundler", ">= 2.1") - spec.add_development_dependency("minitest", "~> 5.14") - spec.add_development_dependency("minitest-reporters", "~> 1.4") - spec.add_development_dependency("pry", ">= 0.14") - spec.add_development_dependency("rake", "~> 13.0") - # rubocop-shopify >= 3.0 requires Ruby >= 3.3; bump alongside our own floor. - spec.add_development_dependency("rubocop-shopify", "~> 2.18") - spec.add_development_dependency("simplecov", "~> 0.22") + # Development dependencies live in the Gemfile (Gemspec/DevelopmentDependencies). # Runtime dependencies # parser provides the runtime AST vocabulary (Parser::AST::Node/Processor, diff --git a/dev.yml b/dev.yml index a2a8f20..96a944d 100644 --- a/dev.yml +++ b/dev.yml @@ -6,3 +6,6 @@ commands: test: desc: Run this repo's tests run: bundle exec rake test + style: + desc: Run RuboCop + run: bundle exec rubocop diff --git a/lib/ast_transform/errors.rb b/lib/ast_transform/errors.rb index 5f57e0e..cab67d6 100644 --- a/lib/ast_transform/errors.rb +++ b/lib/ast_transform/errors.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module ASTTransform # Raised by authoring helpers (e.g. +s_at+) when a node that must carry a # source location does not have one. diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index dbcc828..e861ac7 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'unparser' require 'ast_transform/node' require 'ast_transform/errors' diff --git a/lib/ast_transform/node.rb b/lib/ast_transform/node.rb index 7375912..cd35615 100644 --- a/lib/ast_transform/node.rb +++ b/lib/ast_transform/node.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'parser' module ASTTransform diff --git a/lib/ast_transform/test_helpers.rb b/lib/ast_transform/test_helpers.rb index 6bca1a5..789bcb0 100644 --- a/lib/ast_transform/test_helpers.rb +++ b/lib/ast_transform/test_helpers.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'ast_transform/transformer' require 'ast_transform/instruction_sequence' diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb index 2ba4cac..77fdb74 100644 --- a/lib/ast_transform/thunk.rb +++ b/lib/ast_transform/thunk.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'ast_transform/node' require 'ast_transform/errors' diff --git a/lib/ast_transform/thunk_lowering.rb b/lib/ast_transform/thunk_lowering.rb index 703bb2f..ffe5daf 100644 --- a/lib/ast_transform/thunk_lowering.rb +++ b/lib/ast_transform/thunk_lowering.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'ast_transform/node' require 'ast_transform/thunk' require 'ast_transform/errors' diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index 52d5a95..f617e83 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'parser' require 'ast_transform/node' require 'ast_transform/thunk' diff --git a/lib/ast_transform/transformer.rb b/lib/ast_transform/transformer.rb index 2b63173..12b5b0a 100644 --- a/lib/ast_transform/transformer.rb +++ b/lib/ast_transform/transformer.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'prism' require 'prism/translation/parser' require 'unparser' diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index ef696c6..2bba508 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'test_helper' require 'ast_transform/line_aligned_emitter' require 'ast_transform/transformation_helper' diff --git a/test/ast_transform/line_alignment_test.rb b/test/ast_transform/line_alignment_test.rb index 9b1fb17..7163ad0 100644 --- a/test/ast_transform/line_alignment_test.rb +++ b/test/ast_transform/line_alignment_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'test_helper' require 'transformation_helper' require 'ast_transform/instruction_sequence' diff --git a/test/ast_transform/test_helpers_test.rb b/test/ast_transform/test_helpers_test.rb index f76f919..a411016 100644 --- a/test/ast_transform/test_helpers_test.rb +++ b/test/ast_transform/test_helpers_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'test_helper' require 'ast_transform/test_helpers' require 'ast_transform/abstract_transformation' diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index 05ea457..cca1c81 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'test_helper' require 'ast_transform/transformation_helper' require 'ast_transform/abstract_transformation' diff --git a/test/minitest/reporters/rake_rerun_reporter.rb b/test/minitest/reporters/rake_rerun_reporter.rb index c515dd5..778e6db 100644 --- a/test/minitest/reporters/rake_rerun_reporter.rb +++ b/test/minitest/reporters/rake_rerun_reporter.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'minitest/reporters' module Minitest From e587836dd3caebe9d6e221227cb35d77647d4793 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 00:17:04 -0400 Subject: [PATCH 16/22] Rewrap comment blocks to the 120-column line length The narrative comments were wrapped at ~80 columns, wasting vertical space now that the style guide's line length is 120. Code examples and diagrams keep their layout. Co-authored-by: Cursor --- lib/ast_transform/abstract_transformation.rb | 9 +- lib/ast_transform/errors.rb | 22 ++-- lib/ast_transform/kwargs_builder.rb | 21 ++-- lib/ast_transform/line_aligned_emitter.rb | 102 ++++++++----------- lib/ast_transform/node.rb | 19 ++-- lib/ast_transform/test_helpers.rb | 27 ++--- lib/ast_transform/thunk.rb | 37 +++---- lib/ast_transform/thunk_lowering.rb | 92 +++++++---------- lib/ast_transform/transformation_helper.rb | 89 +++++++--------- 9 files changed, 174 insertions(+), 244 deletions(-) diff --git a/lib/ast_transform/abstract_transformation.rb b/lib/ast_transform/abstract_transformation.rb index 8104496..56fd0c7 100644 --- a/lib/ast_transform/abstract_transformation.rb +++ b/lib/ast_transform/abstract_transformation.rb @@ -23,11 +23,10 @@ def process(node) process_node(node) end - # Thunks are framework-owned IR: descend into the body so passes that - # don't know about thunks still process the wrapped statements. Without - # this, Processor's handler_missing default would pass the node through - # opaquely, hiding the body from every later transformation. The token - # (first child) is not a node and passes through untouched. + # Thunks are framework-owned IR: descend into the body so passes that don't know about thunks still process the + # wrapped statements. Without this, Processor's handler_missing default would pass the node through opaquely, + # hiding the body from every later transformation. The token (first child) is not a node and passes through + # untouched. def on_ast_thunk(node) node.updated(nil, [node.children[0], *process_all(node.children.drop(1))]) end diff --git a/lib/ast_transform/errors.rb b/lib/ast_transform/errors.rb index cab67d6..ed5c9c7 100644 --- a/lib/ast_transform/errors.rb +++ b/lib/ast_transform/errors.rb @@ -1,24 +1,20 @@ # frozen_string_literal: true module ASTTransform - # Raised by authoring helpers (e.g. +s_at+) when a node that must carry a - # source location does not have one. + # Raised by authoring helpers (e.g. +s_at+) when a node that must carry a source location does not have one. class MissingLocationError < StandardError; end - # Raised at construction when a Thunk node's children violate its - # invariants (missing token, empty body). Every construction path funnels - # through Thunk#initialize — including Processor rebuilds — so a malformed - # thunk cannot exist in a tree. + # Raised at construction when a Thunk node's children violate its invariants (missing token, empty body). Every + # construction path funnels through Thunk#initialize — including Processor rebuilds — so a malformed thunk cannot + # exist in a tree. class MalformedThunkError < StandardError; end - # Raised at lowering when a thunk cannot be placed: its body's source - # lines fall after the execution point (the hidden proc's text IS its - # assignment, so a call can never textually precede the body), or two - # occurrences of the same thunk carry diverging bodies. + # Raised at lowering when a thunk cannot be placed: its body's source lines fall after the execution point (the + # hidden proc's text IS its assignment, so a call can never textually precede the body), or two occurrences of the + # same thunk carry diverging bodies. class ThunkPlacementError < StandardError; end - # Raised as the emitter's postcondition when a custom node type (ast_* - # markers or types registered on ASTTransform::Node) reaches the unparse - # boundary instead of being lowered by the stage that understands it. + # Raised as the emitter's postcondition when a custom node type (ast_* markers or types registered on + # ASTTransform::Node) reaches the unparse boundary instead of being lowered by the stage that understands it. class UnloweredNodeTypeError < StandardError; end end diff --git a/lib/ast_transform/kwargs_builder.rb b/lib/ast_transform/kwargs_builder.rb index b5a2c60..a0a02be 100644 --- a/lib/ast_transform/kwargs_builder.rb +++ b/lib/ast_transform/kwargs_builder.rb @@ -3,21 +3,16 @@ require "prism/translation/parser" module ASTTransform - # Extends the default Prism parser builder to distinguish keyword arguments - # from hash literals in the AST. + # Extends the default Prism parser builder to distinguish keyword arguments from hash literals in the AST. # - # The upstream builder always emits :hash nodes for both `foo(bar: 1)` and - # `foo({ bar: 1 })`. Unparser uses the node type to decide whether to emit - # braces: :hash gets `{}`, :kwargs does not. Since Ruby 3.0+ treats these as - # semantically different (strict keyword/positional separation), we need the - # AST to preserve the distinction. + # The upstream builder always emits :hash nodes for both `foo(bar: 1)` and `foo({ bar: 1 })`. Unparser uses the + # node type to decide whether to emit braces: :hash gets `{}`, :kwargs does not. Since Ruby 3.0+ treats these as + # semantically different (strict keyword/positional separation), we need the AST to preserve the distinction. # - # NOTE: parsed nodes deliberately stay plain Parser::AST::Node. Custom node - # classes exist only for registered custom types (see ASTTransform::Node), - # which are IR and never reach Unparser: AST::Node#eql? compares class, and - # Unparser verifies dynamic-string emission by re-parsing and comparing - # eql? against the freshly parsed (plain-class) node — custom-class nodes - # of standard types would fail that verification. + # NOTE: parsed nodes deliberately stay plain Parser::AST::Node. Custom node classes exist only for registered + # custom types (see ASTTransform::Node), which are IR and never reach Unparser: AST::Node#eql? compares class, and + # Unparser verifies dynamic-string emission by re-parsing and comparing eql? against the freshly parsed + # (plain-class) node — custom-class nodes of standard types would fail that verification. class KwargsBuilder < Prism::Translation::Parser::Builder def associate(begin_t, pairs, end_t) node = super diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index e861ac7..2d1d0f7 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -6,38 +6,33 @@ require 'ast_transform/thunk_lowering' module ASTTransform - # Emits a transformed AST as text in which every loc-carrying statement - # occupies its original source line, so that backtraces, breakpoints and - # debugger display are correct by construction — CRuby derives line numbers - # from physical text position, so placement is our line table. + # Emits a transformed AST as text in which every loc-carrying statement occupies its original source line, so that + # backtraces, breakpoints and debugger display are correct by construction — CRuby derives line numbers from + # physical text position, so placement is our line table. # # Cursor algorithm over statement sequences: # - # 1. Statement has loc and target_line > cursor: pad with newlines, emit at - # the target line, indented to the statement's source column. - # 2. Statement has loc and target_line <= cursor: pack (`; `) onto the - # current line. A user statement landing here means the transform moved - # it — the alignment auditor's concern, not a runtime failure. - # 3. No loc: pack onto the current line — synthetic code has no source-line - # truth to preserve. + # 1. Statement has loc and target_line > cursor: pad with newlines, emit at the target line, indented to the + # statement's source column. + # 2. Statement has loc and target_line <= cursor: pack (`; `) onto the current line. A user statement landing here + # means the transform moved it — the alignment auditor's concern, not a runtime failure. + # 3. No loc: pack onto the current line — synthetic code has no source-line truth to preserve. # - # Multi-line renders advance the cursor by their height; displaced - # statements pack and emission re-anchors at the next statement that fits. - # Total: never raises on layout. + # Multi-line renders advance the cursor by their height; displaced statements pack and emission re-anchors at the + # next statement that fits. Total: never raises on layout. # - # Thunk nodes are lowered (ThunkLowering) before layout; the emitter's - # postcondition is that no custom node type (ast_* markers or types - # registered on ASTTransform::Node) crosses the unparse boundary — they - # are IR between stages that understand them. + # Thunk nodes are lowered (ThunkLowering) before layout; the emitter's postcondition is that no custom node type + # (ast_* markers or types registered on ASTTransform::Node) crosses the unparse boundary — they are IR between + # stages that understand them. class LineAlignedEmitter - # Containers the emitter recurses into so nested statements align; every - # other node renders as an Unparser blob at its head line. + # Containers the emitter recurses into so nested statements align; every other node renders as an Unparser blob + # at its head line. RECURSIVE_CONTAINER_TYPES = [:class, :module, :sclass, :def, :defs, :block, :numblock, :itblock, :kwbegin].freeze BODY_INDEXES = { class: 2, module: 1, sclass: 1, def: 2, defs: 3, block: 2, numblock: 2, itblock: 2 }.freeze - # Assignments whose value is a block (e.g. the lowered thunk proc) - # recurse into the block so its body statements align. + # Assignments whose value is a block (e.g. the lowered thunk proc) recurse into the block so its body statements + # align. ASSIGNMENT_TYPES = [:lvasgn, :ivasgn, :gvasgn, :casgn].freeze BLOCK_VALUE_TYPES = [:block, :numblock, :itblock].freeze @@ -76,12 +71,10 @@ def emit_statement(node) end end - # Unparser normalizes some single-line constructs into multi-line form - # (e.g. modifier-if into if/end), which would push following statements - # off their lines. When the render is taller than the statement's source, - # compress it back to one line — verified by re-parse so a statement that - # cannot be safely single-lined (e.g. containing a heredoc) falls back to - # its multi-line render and re-anchors after itself. + # Unparser normalizes some single-line constructs into multi-line form (e.g. modifier-if into if/end), which + # would push following statements off their lines. When the render is taller than the statement's source, + # compress it back to one line — verified by re-parse so a statement that cannot be safely single-lined + # (e.g. containing a heredoc) falls back to its multi-line render and re-anchors after itself. def aligned_render(node) render = unparse(node) loc = node.loc @@ -95,18 +88,17 @@ def aligned_render(node) def compress_to_single_line(render) candidate = render.split("\n").map(&:strip).join('; ') - # Both sides parsed without scope context, so lvar/send ambiguity - # cancels out; equality means the newline join preserved structure. + # Both sides parsed without scope context, so lvar/send ambiguity cancels out; equality means the newline join + # preserved structure. Unparser.parse(candidate) == Unparser.parse(render) ? candidate : nil rescue Parser::SyntaxError nil end - # Statements are unparsed in isolation, losing the surrounding scope's - # local-variable context; without it, Unparser re-parses identifiers as - # method calls and its dstr round-trip verification fails. Feed it every - # local assigned or bound anywhere in the tree — an over-approximation - # that is safe because it only informs Unparser's re-parse verification. + # Statements are unparsed in isolation, losing the surrounding scope's local-variable context; without it, + # Unparser re-parses identifiers as method calls and its dstr round-trip verification fails. Feed it every local + # assigned or bound anywhere in the tree — an over-approximation that is safe because it only informs Unparser's + # re-parse verification. def unparse(node) Unparser.unparse(node, static_local_variables: @local_variables) end @@ -121,9 +113,8 @@ def collect_local_variables(node, names = Set.new) names end - # Emits a container body that may be a bare :ensure/:rescue node (their - # begin/end context comes from the surrounding def/block/kwbegin, so the - # keywords must be emitted inline, aligned like statements). + # Emits a container body that may be a bare :ensure/:rescue node (their begin/end context comes from the + # surrounding def/block/kwbegin, so the keywords must be emitted inline, aligned like statements). def emit_body(body) case body&.type when :ensure then emit_ensure(body) @@ -159,8 +150,8 @@ def emit_resbody(node) emit_statements(statements_of(body)) end - # Keywords (rescue/ensure/else) cannot be `;`-packed after a statement; - # when their line is taken they go on a fresh line instead. + # Keywords (rescue/ensure/else) cannot be `;`-packed after a statement; when their line is taken they go on a + # fresh line instead. def place_keyword(target_line, keyword, column: nil) if target_line && target_line > @lines.size place(target_line, keyword, column: column) @@ -188,8 +179,8 @@ def block_assignment?(node) BLOCK_VALUE_TYPES.include?(node.children.last.type) end - # Renders a container's opener and closer from the node with its body - # emptied, then recurses into the body so nested statements align. + # Renders a container's opener and closer from the node with its body emptied, then recurses into the body so + # nested statements align. def emit_container(node) opener, closer = container_delimiters(node) place(node.loc&.line, opener, column: node.loc&.column) @@ -202,8 +193,8 @@ def container_delimiters(node) opener = rendered[0..-2].join("\n") closer = rendered.last - # Unparser renders empty blocks with braces, but brace blocks cannot - # hold rescue/ensure bodies; do/end always can. + # Unparser renders empty blocks with braces, but brace blocks cannot hold rescue/ensure bodies; do/end always + # can. if opener.end_with?(' {') && closer == '}' [opener.sub(/ \{\z/, ' do'), 'end'] else @@ -245,10 +236,8 @@ def closer_column(node) loc.end.column if loc.respond_to?(:end) && loc.end end - # Loc-less :begin nodes in statement position (e.g. a lowered thunk in a - # single-statement container body, carried with its placement) are - # grouping, not structure: flatten them so each inner statement is laid - # out independently. + # Loc-less :begin nodes in statement position (e.g. a lowered thunk in a single-statement container body, carried + # with its placement) are grouping, not structure: flatten them so each inner statement is laid out independently. def statements_of(body) return [] if body.nil? return [body] unless body.type == :begin @@ -258,14 +247,11 @@ def statements_of(body) end end - # Places +render+ at +target_line+ when the cursor hasn't passed it; - # otherwise packs onto the current line. Multi-line renders advance the - # cursor by their height. When opening a fresh line, the render is - # indented to the statement's source +column+ — cosmetic only (leading - # whitespace is never significant in emitted code; heredocs are - # normalized to inline strings), but it keeps the artifact and test - # expectations visually close to the source. Packed statements ignore - # the column, as do an Unparser render's continuation lines (they keep + # Places +render+ at +target_line+ when the cursor hasn't passed it; otherwise packs onto the current line. + # Multi-line renders advance the cursor by their height. When opening a fresh line, the render is indented to the + # statement's source +column+ — cosmetic only (leading whitespace is never significant in emitted code; heredocs + # are normalized to inline strings), but it keeps the artifact and test expectations visually close to the + # source. Packed statements ignore the column, as do an Unparser render's continuation lines (they keep # Unparser's own relative indentation). def place(target_line, render, column: nil) first, *rest = render.split("\n") @@ -284,8 +270,8 @@ def indented(text, column) column && column.positive? ? "#{' ' * column}#{text}" : text end - # The last line is never blank here: padding blanks are only created - # inside +place+, which immediately overwrites the padded line. + # The last line is never blank here: padding blanks are only created inside +place+, which immediately overwrites + # the padded line. def pack(text) if @lines.empty? @lines << text diff --git a/lib/ast_transform/node.rb b/lib/ast_transform/node.rb index cd35615..c2a4f67 100644 --- a/lib/ast_transform/node.rb +++ b/lib/ast_transform/node.rb @@ -3,9 +3,8 @@ require 'parser' module ASTTransform - # Base class for custom IR nodes. Transform authors subclass it and - # register a custom node type to get type-routed construction from +s+ - # with domain accessors: + # Base class for custom IR nodes. Transform authors subclass it and register a custom node type to get type-routed + # construction from +s+ with domain accessors: # # class InteractionNode < ASTTransform::Node # register :rspock_interaction @@ -15,12 +14,10 @@ module ASTTransform # # s(:rspock_interaction, ...) # => InteractionNode # - # Custom node *types* are IR between stages that understand them and must - # be lowered before emission (the emitter enforces this). Standard-typed - # nodes deliberately stay plain Parser::AST::Node everywhere — parsed and - # +s+-built alike: AST::Node#eql? compares class, and Unparser verifies - # dynamic-string emission by re-parsing and eql?-comparing, so a custom - # class on a standard type breaks emission. + # Custom node *types* are IR between stages that understand them and must be lowered before emission (the emitter + # enforces this). Standard-typed nodes deliberately stay plain Parser::AST::Node everywhere — parsed and +s+-built + # alike: AST::Node#eql? compares class, and Unparser verifies dynamic-string emission by re-parsing and + # eql?-comparing, so a custom class on a standard type breaks emission. class Node < ::Parser::AST::Node class << self # Registers +self+ as the class to construct for +type+ nodes. @@ -31,8 +28,8 @@ def register(type) Node.registry[type] = self end - # Builds a node of +type+: registered types construct their custom - # class, everything else a plain Parser::AST::Node. + # Builds a node of +type+: registered types construct their custom class, everything else a plain + # Parser::AST::Node. # # @param type [Symbol] node type # @param children [Array] child nodes / literals diff --git a/lib/ast_transform/test_helpers.rb b/lib/ast_transform/test_helpers.rb index 789bcb0..470dab6 100644 --- a/lib/ast_transform/test_helpers.rb +++ b/lib/ast_transform/test_helpers.rb @@ -4,9 +4,8 @@ require 'ast_transform/instruction_sequence' module ASTTransform - # Assertions for transform authors' own test suites — the enforcement arm - # of the authoring contract ("textual order is source order"). Never loaded - # in production; require it from test code: + # Assertions for transform authors' own test suites — the enforcement arm of the authoring contract ("textual + # order is source order"). Never loaded in production; require it from test code: # # require "ast_transform/test_helpers" # @@ -14,12 +13,10 @@ module ASTTransform # include ASTTransform::TestHelpers # end module TestHelpers - # Transforms +source+ through the real pipeline (transform + line-aligned - # emission), re-parses both sides, matches surviving statements by - # location, and asserts each one's emitted line equals its source line. - # Statements the transform deletes (e.g. description strings) are exempt; - # statements the transform rewrites in place keep their anchor and are - # checked. + # Transforms +source+ through the real pipeline (transform + line-aligned emission), re-parses both sides, + # matches surviving statements by location, and asserts each one's emitted line equals its source line. + # Statements the transform deletes (e.g. description strings) are exempt; statements the transform rewrites in + # place keep their anchor and are checked. # # @param source [String] fixture source # @param transformations [Array] @@ -48,9 +45,8 @@ def assert_line_aligned(source, *transformations, path: 'fixture.rb') MESSAGE end - # Runtime complement of assert_line_aligned: compiles +source+ through - # the full pipeline under +path+, executes it, and asserts the raw first - # backtrace frame — no filtering of any kind — is ":". + # Runtime complement of assert_line_aligned: compiles +source+ through the full pipeline under +path+, executes + # it, and asserts the raw first backtrace frame — no filtering of any kind — is ":". # # @param source [String] fixture that raises when executed # @param path [String] pseudo source path to compile under @@ -70,10 +66,9 @@ def assert_backtrace_lines(source, path:, raise_at:) private - # Flat statement renders and their first line, keyed by unparsed text so - # source and emitted sides can be matched without location identity. - # Duplicate renders keep their first occurrence — good enough for - # fixtures, which authors control. + # Flat statement renders and their first line, keyed by unparsed text so source and emitted sides can be matched + # without location identity. Duplicate renders keep their first occurrence — good enough for fixtures, which + # authors control. def statement_lines(ast, lines = {}) return lines unless ast.is_a?(::Parser::AST::Node) diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb index 77fdb74..067299b 100644 --- a/lib/ast_transform/thunk.rb +++ b/lib/ast_transform/thunk.rb @@ -4,22 +4,18 @@ require 'ast_transform/errors' module ASTTransform - # The reordering primitive: an eagerly built wrapper node, spliced wherever - # the wrapped statements must EXECUTE — statement position or composed - # inside an expression (e.g. an assert_raises block body). Its body keeps - # its own source locations, and the lowering derives the wrapper's textual - # placement from them (see ThunkLowering), so the statements still emit on - # their original lines even though execution waits. + # The reordering primitive: an eagerly built wrapper node, spliced wherever the wrapped statements must EXECUTE — + # statement position or composed inside an expression (e.g. an assert_raises block body). Its body keeps its own + # source locations, and the lowering derives the wrapper's textual placement from them (see ThunkLowering), so the + # statements still emit on their original lines even though execution waits. # - # Children are +[token, *body_statements]+ and the invariants are enforced - # here in +initialize+, which every construction path shares — +s+ routing, - # the +thunk+ helper, and Processor rebuilds (+updated+ re-initializes). - # Build thunks with +TransformationHelper#thunk+; reuse the same node to - # execute one body from several points (multiplexing). + # Children are +[token, *body_statements]+ and the invariants are enforced here in +initialize+, which every + # construction path shares — +s+ routing, the +thunk+ helper, and Processor rebuilds (+updated+ re-initializes). + # Build thunks with +TransformationHelper#thunk+; reuse the same node to execute one body from several points + # (multiplexing). # - # Runtime semantics are near-transparent (proc lowering): +return+ still - # returns from the enclosing method, and locals the body assigns stay - # method-scope. See ThunkLowering for the full contract. + # Runtime semantics are near-transparent (proc lowering): +return+ still returns from the enclosing method, and + # locals the body assigns stay method-scope. See ThunkLowering for the full contract. class Thunk < Node register :ast_thunk @@ -40,14 +36,11 @@ def token = children[0] def body = children.drop(1) end - # The identity of a Thunk across transformation passes: Processor and - # Node#updated rebuilds create new node objects, so node identity does not - # survive — but children DO (carried by reference through every rebuild). - # Every rebuild of a thunk therefore carries this same token object, and - # the lowering groups occurrences by its object identity: one proc, one - # call per occurrence. Minted internally by the +thunk+ helper, never - # handled by authors. No behavior — a named class over a bare Object.new - # only for self-documenting AST dumps and greppability. + # The identity of a Thunk across transformation passes: Processor and Node#updated rebuilds create new node + # objects, so node identity does not survive — but children DO (carried by reference through every rebuild). Every + # rebuild of a thunk therefore carries this same token object, and the lowering groups occurrences by its object + # identity: one proc, one call per occurrence. Minted internally by the +thunk+ helper, never handled by authors. + # No behavior — a named class over a bare Object.new only for self-documenting AST dumps and greppability. class ThunkToken def inspect "#" diff --git a/lib/ast_transform/thunk_lowering.rb b/lib/ast_transform/thunk_lowering.rb index ffe5daf..56a224f 100644 --- a/lib/ast_transform/thunk_lowering.rb +++ b/lib/ast_transform/thunk_lowering.rb @@ -6,50 +6,40 @@ require 'ast_transform/transformation_helper' module ASTTransform - # Lowers Thunk nodes into plain Ruby ahead of emission. Each unique thunk - # (grouped by token identity) becomes a hidden proc; each occurrence - # becomes the proc's call: + # Lowers Thunk nodes into plain Ruby ahead of emission. Each unique thunk (grouped by token identity) becomes a + # hidden proc; each occurrence becomes the proc's call: # # thunk placed at the execution point # => x = x; __ast_thunk___ = proc { body } (at the body's source lines) # ... # __ast_thunk___.call (at the occurrence) # - # Placement is inferred, not authored: the proc's text is inserted into - # the statement sequence enclosing the occurrence, positioned among its - # siblings by the body's first source line — the lines the author removed - # the statements from. A loc-less body has no textual home and packs - # immediately before its call. Placements never escape a scope boundary - # (def/class/module bodies absorb their own), because the hidden lvar must - # share the call's method activation; they DO escape block literals, which - # close over the defining scope. + # Placement is inferred, not authored: the proc's text is inserted into the statement sequence enclosing the + # occurrence, positioned among its siblings by the body's first source line — the lines the author removed the + # statements from. A loc-less body has no textual home and packs immediately before its call. Placements never + # escape a scope boundary (def/class/module bodies absorb their own), because the hidden lvar must share the call's + # method activation; they DO escape block literals, which close over the defining scope. # - # The closure is a non-lambda proc on purpose: `return` inside a proc - # returns from the method where the proc was defined, and placement and - # execution always share one method activation, so a thunked `return` - # keeps its original meaning. Jump keywords whose owner lies outside the - # body keep Ruby's native behavior (`break`/`retry` fail loudly, - # `next`/`redo` silently alter flow) — what a transform chooses to thunk - # is the transform author's call. + # The closure is a non-lambda proc on purpose: `return` inside a proc returns from the method where the proc was + # defined, and placement and execution always share one method activation, so a thunked `return` keeps its original + # meaning. Jump keywords whose owner lies outside the body keep Ruby's native behavior (`break`/`retry` fail loudly, + # `next`/`redo` silently alter flow) — what a transform chooses to thunk is the transform author's call. # - # The `x = x` pre-declarations cover every local the body assigns at - # method scope. A local first assigned inside a block literal is - # block-local, so without a textual method-scope assignment before the - # proc, thunked assignments would be invisible to the statements that - # read them after the execution point. Self-assignment registers the name - # (nil until the thunk runs — exactly what an unexecuted assignment - # yields) without clobbering an already-assigned value. + # The `x = x` pre-declarations cover every local the body assigns at method scope. A local first assigned inside a + # block literal is block-local, so without a textual method-scope assignment before the proc, thunked assignments + # would be invisible to the statements that read them after the execution point. Self-assignment registers the name + # (nil until the thunk runs — exactly what an unexecuted assignment yields) without clobbering an already-assigned + # value. class ThunkLowering include TransformationHelper - # A pending proc definition: +line+ is the body's first source line - # (nil for fully synthetic bodies), +statements+ the pre-declarations - # plus the proc assignment. + # A pending proc definition: +line+ is the body's first source line (nil for fully synthetic bodies), + # +statements+ the pre-declarations plus the proc assignment. Placement = Struct.new(:line, :statements) SEQUENCE_TYPES = [:begin, :kwbegin].freeze - # Scope-opening containers: the hidden lvar cannot be referenced across - # these boundaries, so placements arising inside must land inside. + # Scope-opening containers: the hidden lvar cannot be referenced across these boundaries, so placements arising + # inside must land inside. SCOPE_BODY_INDEXES = { def: 2, defs: 3, class: 2, module: 1, sclass: 1 }.freeze def initialize @@ -59,16 +49,16 @@ def initialize # @param node [Parser::AST::Node] tree possibly containing Thunk nodes # @return [Parser::AST::Node] tree with thunks lowered to plain Ruby - # @raise [ThunkPlacementError] when a thunk body's source lines fall - # after its execution point, or occurrences of one thunk diverge + # @raise [ThunkPlacementError] when a thunk body's source lines fall after its execution point, or occurrences of + # one thunk diverge def run(node) lower_body(node) end private - # Lowers a node standing in statement-body position (a container's body - # or the root), absorbing any placements that arise within it. + # Lowers a node standing in statement-body position (a container's body or the root), absorbing any placements + # that arise within it. def lower_body(node) return node unless node.is_a?(::Parser::AST::Node) return lower_sequence(node) if SEQUENCE_TYPES.include?(node.type) @@ -76,13 +66,11 @@ def lower_body(node) lowered, placements = lower_expression(node) return lowered if placements.empty? - # A loc-less :begin in statement position; the emitter flattens it - # into the surrounding statement stream. + # A loc-less :begin in statement position; the emitter flattens it into the surrounding statement stream. s(:begin, *placements.flat_map(&:statements), lowered) end - # Lowers a statement sequence, inserting each placement among the - # statements by the body's source line. + # Lowers a statement sequence, inserting each placement among the statements by the body's source line. def lower_sequence(node) statements = [] @@ -98,9 +86,8 @@ def lower_sequence(node) node.updated(nil, statements) end - # Lowers a node in expression position. Returns the lowered node and the - # placements that must be inserted into the enclosing statement - # sequence. + # Lowers a node in expression position. Returns the lowered node and the placements that must be inserted into + # the enclosing statement sequence. # # @return [Array(Parser::AST::Node, Array)] def lower_expression(node) @@ -138,8 +125,8 @@ def lower_generic(node) [node.updated(nil, children), pending] end - # An occurrence of a thunk: the first occurrence of its token yields the - # placement; every occurrence yields the call. + # An occurrence of a thunk: the first occurrence of its token yields the placement; every occurrence yields the + # call. def lower_thunk(node) token = node.token @@ -175,9 +162,8 @@ def body_first_line(body) body.filter_map { |statement| statement.loc&.line }.min end - # The proc's text must precede its call: a placement whose body lines - # fall at or after the executing statement (or any statement after it) - # cannot be laid out — the assignment would complete after the call. + # The proc's text must precede its call: a placement whose body lines fall at or after the executing statement + # (or any statement after it) cannot be laid out — the assignment would complete after the call. def check_placement_precedes_execution!(placement, executing_statement, following_statements) return if placement.line.nil? @@ -198,17 +184,15 @@ def insertion_index(statements, placement) statements.index { |statement| statement.loc&.line && statement.loc.line > placement.line } || statements.size end - # Node types opening a new local-variable scope: assignments inside them - # were invisible to the method scope in the original source too, so they - # get no pre-declaration. + # Node types opening a new local-variable scope: assignments inside them were invisible to the method scope in + # the original source too, so they get no pre-declaration. NEW_SCOPE_TYPES = [:def, :defs, :class, :module, :sclass].freeze - # Block literals: locals first assigned inside them are block-local (the - # same lexical rule the pre-declarations exist to work around), but their - # callee/arguments evaluate at method scope and are still descended. + # Block literals: locals first assigned inside them are block-local (the same lexical rule the pre-declarations + # exist to work around), but their callee/arguments evaluate at method scope and are still descended. BLOCK_TYPES = [:block, :numblock, :itblock].freeze - # Locals the thunk body assigns at method scope, in first-assignment - # order (covers masgn/op_asgn targets — they all carry :lvasgn nodes). + # Locals the thunk body assigns at method scope, in first-assignment order (covers masgn/op_asgn targets — they + # all carry :lvasgn nodes). def method_scope_assignments(node, names = []) return names unless node.is_a?(::Parser::AST::Node) return names if NEW_SCOPE_TYPES.include?(node.type) diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index f617e83..bf7c6ff 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -9,15 +9,13 @@ module ASTTransform # The transform-authoring layer. Three shapes: # # - Constructors (+s+, +s_at+): type + children in, fresh node out. - # - The sequence combinator (+run_after+): sequence in, sequence out — the - # paved road for execution reordering. - # - The low-level reordering primitive (+thunk+): statements in, Thunk - # node out — for execution points inside expressions. + # - The sequence combinator (+run_after+): sequence in, sequence out — the paved road for execution reordering. + # - The low-level reordering primitive (+thunk+): statements in, Thunk node out — for execution points inside + # expressions. # - # The contract these helpers serve: textual order is source order. The - # emitter places every loc-carrying statement at its source line; when - # execution order must differ from textual order, authors express it as a - # thunk instead of moving text. + # The contract these helpers serve: textual order is source order. The emitter places every loc-carrying statement + # at its source line; when execution order must differ from textual order, authors express it as a thunk instead of + # moving text. module TransformationHelper class << self def included(base) @@ -27,9 +25,8 @@ def included(base) end module Methods - # Builds a loc-less node. The emitter packs loc-less nodes onto the - # current output line — the correct default for synthetic code, which - # has no source-line truth to preserve. + # Builds a loc-less node. The emitter packs loc-less nodes onto the current output line — the correct default + # for synthetic code, which has no source-line truth to preserve. # # @param type [Symbol] node type # @param children [Array] child nodes / literals @@ -39,11 +36,9 @@ def s(type, *children, **properties) Node.build(type, children, properties) end - # Builds a fresh node anchored to another node's source location. Use - # when composing a replacement tree whose root isn't derived from the - # node it replaces (otherwise prefer +anchor.updated(...)+). The - # attached map is a clean expression-only Source::Map over - # +anchor.loc.expression+ — no stale typed sub-ranges (selector etc.). + # Builds a fresh node anchored to another node's source location. Use when composing a replacement tree whose + # root isn't derived from the node it replaces (otherwise prefer +anchor.updated(...)+). The attached map is a + # clean expression-only Source::Map over +anchor.loc.expression+ — no stale typed sub-ranges (selector etc.). # Anchor inheritance is shallow; children keep or lack their own locs. # # @param anchor [Parser::AST::Node] node whose line this code replaces @@ -58,26 +53,20 @@ def s_at(anchor, type, *children) s(type, *children, location: ::Parser::Source::Map.new(expression)) end - # The low-level reordering primitive. Thunking is the one reordering - # lever: text never moves and execution can only move later, so "hoist - # A above B" is expressed as "run B after A". Returns a single Thunk - # node: splice it where the statements must RUN — statement position - # or composed inside an expression, e.g. as an assert_raises block - # body. The wrapped statements keep their own locs, and the lowering - # derives the hidden proc's textual placement from them, so the body - # still emits on its source lines even though execution waits. Reuse - # the same node to execute one body from several points. + # The low-level reordering primitive. Thunking is the one reordering lever: text never moves and execution can + # only move later, so "hoist A above B" is expressed as "run B after A". Returns a single Thunk node: splice it + # where the statements must RUN — statement position or composed inside an expression, e.g. as an assert_raises + # block body. The wrapped statements keep their own locs, and the lowering derives the hidden proc's textual + # placement from them, so the body still emits on its source lines even though execution waits. Reuse the same + # node to execute one body from several points. # - # Semantics are near-transparent (see ThunkLowering): +return+ still - # returns from the enclosing method (non-lambda proc), and locals the - # wrapped statements assign stay method-scope (pre-declared before the - # proc). Jump keywords whose owner lies outside the wrapped statements - # keep Ruby's native behavior — +break+/+retry+ fail loudly at the - # jump's own source line, +next+/+redo+ silently end or restart the - # thunk body. Weigh that when choosing what your surface thunks. + # Semantics are near-transparent (see ThunkLowering): +return+ still returns from the enclosing method + # (non-lambda proc), and locals the wrapped statements assign stay method-scope (pre-declared before the proc). + # Jump keywords whose owner lies outside the wrapped statements keep Ruby's native behavior — +break+/+retry+ + # fail loudly at the jump's own source line, +next+/+redo+ silently end or restart the thunk body. Weigh that + # when choosing what your surface thunks. # - # Prefer +run_after+ when the execution point sits in the same - # statement sequence as the statements. + # Prefer +run_after+ when the execution point sits in the same statement sequence as the statements. # # @param statements [Array] statements to wrap # @return [ASTTransform::Thunk] the thunk node @@ -85,26 +74,22 @@ def thunk(*statements) s(:ast_thunk, ThunkToken.new, *statements) end - # The paved road for execution reordering in flat statement sequences. - # Named for the constraint, not the mechanism — with text pinned to - # source lines the only physical lever is delaying execution, so "run - # X after Y" is the constraint an author states. Returns a NEW - # sequence in which the +run+ statements are removed and a thunk - # wrapping them is inserted immediately after +after+. + # The paved road for execution reordering in flat statement sequences. Named for the constraint, not the + # mechanism — with text pinned to source lines the only physical lever is delaying execution, so "run X after + # Y" is the constraint an author states. Returns a NEW sequence in which the +run+ statements are removed and a + # thunk wrapping them is inserted immediately after +after+. # - # All membership checks are by identity (equal?), never ==: node - # equality ignores location, so two textually identical statements on - # different lines compare == and value matching could splice the wrong - # one. + # All membership checks are by identity (equal?), never ==: node equality ignores location, so two textually + # identical statements on different lines compare == and value matching could splice the wrong one. # # @param statements [Array] the sequence being composed - # @param run [Array] contiguous run of elements of - # +statements+ (by identity) whose execution must wait - # @param after [Parser::AST::Node] element of +statements+ (by identity, - # not inside +run+) the +run+ statements execute after + # @param run [Array] contiguous run of elements of +statements+ (by identity) whose + # execution must wait + # @param after [Parser::AST::Node] element of +statements+ (by identity, not inside +run+) the +run+ statements + # execute after # @return [Array] new sequence with the thunk placed - # @raise [ArgumentError] if +run+ is not a contiguous identity-run of - # +statements+, or +after+ is not an element (or is inside +run+) + # @raise [ArgumentError] if +run+ is not a contiguous identity-run of +statements+, or +after+ is not an + # element (or is inside +run+) def run_after(statements, run:, after:) run_range = contiguous_identity_range(statements, run) raise ArgumentError, "run: must be a contiguous run of elements of statements (by identity)" unless run_range @@ -122,8 +107,8 @@ def run_after(statements, run:, after:) private - # The range +members+ occupies in +sequence+, or nil unless members is - # a non-empty contiguous identity-run in order. + # The range +members+ occupies in +sequence+, or nil unless members is a non-empty contiguous identity-run in + # order. def contiguous_identity_range(sequence, members) return nil if members.empty? From adcc36990e7ff86c22dd1fa871c3988f31090461 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 01:33:41 -0400 Subject: [PATCH 17/22] Extract Layout and StatementRenderer; make emitter and lowering stateless collaborators LineAlignedEmitter.new(ast, source_path).emit was the narrow-responsibility smell: operation parameters in the constructor, a sole no-param public method, single-use instances. Gathering what changes together uncovers two concepts that were smeared across the emitter's ivars: - Layout: line-addressed output (pad-or-pack cursor mechanics), AST-free. - StatementRenderer: the Unparser adapter, built once per tree with the collected locals to work around statement-in-isolation unparsing. The emitter keeps only the Ruby knowledge (structure walking, line targeting, keyword non-packing) and goes stateless; ThunkLowering gets the same treatment with its token hashes extracted into a private Registry and run renamed to the domain verb lower. Stateless services are constructor-time collaborators (Transformer wires the emitter, the emitter wires the lowering, kwarg defaults as the DI seam); operation-scoped objects (Layout, StatementRenderer, Registry) are created at entry and die with the call. Co-authored-by: Cursor --- lib/ast_transform/layout.rb | 64 ++++++ lib/ast_transform/line_aligned_emitter.rb | 192 ++++++------------ lib/ast_transform/statement_renderer.rb | 76 +++++++ lib/ast_transform/thunk_lowering.rb | 107 ++++++---- lib/ast_transform/transformer.rb | 8 +- test/ast_transform/layout_test.rb | 91 +++++++++ .../line_aligned_emitter_test.rb | 12 +- test/ast_transform/statement_renderer_test.rb | 66 ++++++ 8 files changed, 430 insertions(+), 186 deletions(-) create mode 100644 lib/ast_transform/layout.rb create mode 100644 lib/ast_transform/statement_renderer.rb create mode 100644 test/ast_transform/layout_test.rb create mode 100644 test/ast_transform/statement_renderer_test.rb diff --git a/lib/ast_transform/layout.rb b/lib/ast_transform/layout.rb new file mode 100644 index 0000000..f6e85d7 --- /dev/null +++ b/lib/ast_transform/layout.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module ASTTransform + # Line-addressed output: text is placed at absolute line numbers, top to bottom, and the cursor never rewinds. + # When a placement's target line is already behind the cursor, the text is packed (`; `) onto the current line + # instead — Ruby lets statements share a physical line, so alignment degrades locally and the next placement + # whose target is still ahead re-anchors. Knows nothing about Ruby structure or ASTs; callers decide WHAT goes + # on WHICH line, the layout owns the pad-or-pack mechanics. + class Layout + def initialize + @lines = [] + end + + # The line number currently being written; the next fresh line would be +cursor + 1+. + def cursor + @lines.size + end + + # Places +text+ at +target_line+ when the cursor hasn't passed it; otherwise packs onto the current line. + # Multi-line text advances the cursor by its height. When opening a fresh line, the first line is indented to + # +column+ — cosmetic only (leading whitespace is never significant in emitted code), but it keeps the artifact + # visually close to the source. Packed text ignores the column, as do continuation lines (they keep their own + # relative indentation). + def place(target_line, text, column: nil) + first, *rest = text.split("\n") + + if target_line && target_line > @lines.size + @lines << '' while @lines.size < target_line + @lines[-1] = indented(first, column) + else + pack(first) + end + + @lines.concat(rest) + end + + # Appends +text+ on a new line unconditionally — for text that must never be `;`-packed after a statement + # (e.g. keywords). + def place_on_fresh_line(text) + @lines << text + end + + # Appends +text+ to the current line with a `; ` separator. The last line is never blank here: padding blanks + # are only created inside +place+, which immediately overwrites the padded line. + def pack(text) + if @lines.empty? + @lines << text + else + @lines[-1] = "#{@lines.last}; #{text}" + end + end + + # @return [String] the laid-out text, with a trailing newline. + def to_source + "#{@lines.join("\n")}\n" + end + + private + + def indented(text, column) + column && column.positive? ? "#{' ' * column}#{text}" : text + end + end +end diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index 2d1d0f7..744ef36 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -1,8 +1,9 @@ # frozen_string_literal: true -require 'unparser' require 'ast_transform/node' require 'ast_transform/errors' +require 'ast_transform/layout' +require 'ast_transform/statement_renderer' require 'ast_transform/thunk_lowering' module ASTTransform @@ -10,16 +11,17 @@ module ASTTransform # backtraces, breakpoints and debugger display are correct by construction — CRuby derives line numbers from # physical text position, so placement is our line table. # - # Cursor algorithm over statement sequences: + # Placement policy over statement sequences: # - # 1. Statement has loc and target_line > cursor: pad with newlines, emit at the target line, indented to the - # statement's source column. - # 2. Statement has loc and target_line <= cursor: pack (`; `) onto the current line. A user statement landing here - # means the transform moved it — the alignment auditor's concern, not a runtime failure. - # 3. No loc: pack onto the current line — synthetic code has no source-line truth to preserve. + # 1. Statement has loc: target its source line — the Layout pads to reach it, or packs (`; `) when the cursor has + # already passed it. A user statement packing means the transform moved it — the alignment auditor's concern, + # not a runtime failure. + # 2. No loc: pack onto the current line — synthetic code has no source-line truth to preserve. # - # Multi-line renders advance the cursor by their height; displaced statements pack and emission re-anchors at the - # next statement that fits. Total: never raises on layout. + # The emitter owns the Ruby knowledge: walking statement structure, deciding which line each node targets, and + # that keywords can never be `;`-packed. The pad-or-pack mechanics live in Layout; statement-to-text rendering + # (and its isolation workarounds) in StatementRenderer — both created per emission, so the emitter itself is + # stateless and an instance is a reusable collaborator. # # Thunk nodes are lowered (ThunkLowering) before layout; the emitter's postcondition is that no custom node type # (ast_* markers or types registered on ASTTransform::Node) crosses the unparse boundary — they are IR between @@ -36,127 +38,84 @@ class LineAlignedEmitter ASSIGNMENT_TYPES = [:lvasgn, :ivasgn, :gvasgn, :casgn].freeze BLOCK_VALUE_TYPES = [:block, :numblock, :itblock].freeze - # @param ast [Parser::AST::Node] transformed AST - # @param source_path [String] original file path (for error messages) - def initialize(ast, source_path) - @ast = ast - @source_path = source_path - @local_variables = Set.new + # @param thunk_lowering [ThunkLowering] the lowering run ahead of emission. + def initialize(thunk_lowering: ThunkLowering.new) + @thunk_lowering = thunk_lowering end + # @param ast [Parser::AST::Node] transformed AST + # @param source_path [String] original file path (for error messages) # @return [String] transformed source, line-aligned # @raise [ThunkPlacementError] if a thunk cannot be textually placed # @raise [UnloweredNodeTypeError] if a custom node type survived to emission - def emit - lowered = ThunkLowering.new.run(@ast) - assert_no_custom_types(lowered) + def emit(ast, source_path) + lowered = @thunk_lowering.lower(ast) + assert_no_custom_types(lowered, source_path) - @local_variables = collect_local_variables(lowered) - @lines = [] - emit_statements(statements_of(lowered)) - "#{@lines.join("\n")}\n" + layout = Layout.new + renderer = StatementRenderer.for_tree(lowered) + emit_statements(statements_of(lowered), layout, renderer) + layout.to_source end private - def emit_statements(statements) - statements.each { |statement| emit_statement(statement) } + def emit_statements(statements, layout, renderer) + statements.each { |statement| emit_statement(statement, layout, renderer) } end - def emit_statement(node) + def emit_statement(node, layout, renderer) if recursive_container?(node) - emit_container(node) + emit_container(node, layout, renderer) else - place(node.loc&.line, aligned_render(node), column: node.loc&.column) + layout.place(node.loc&.line, renderer.aligned_render(node), column: node.loc&.column) end end - # Unparser normalizes some single-line constructs into multi-line form (e.g. modifier-if into if/end), which - # would push following statements off their lines. When the render is taller than the statement's source, - # compress it back to one line — verified by re-parse so a statement that cannot be safely single-lined - # (e.g. containing a heredoc) falls back to its multi-line render and re-anchors after itself. - def aligned_render(node) - render = unparse(node) - loc = node.loc - return render unless loc.respond_to?(:last_line) && loc.line - - source_height = loc.last_line - loc.line + 1 - return render if render.count("\n") < source_height - - compress_to_single_line(render) || render - end - - def compress_to_single_line(render) - candidate = render.split("\n").map(&:strip).join('; ') - # Both sides parsed without scope context, so lvar/send ambiguity cancels out; equality means the newline join - # preserved structure. - Unparser.parse(candidate) == Unparser.parse(render) ? candidate : nil - rescue Parser::SyntaxError - nil - end - - # Statements are unparsed in isolation, losing the surrounding scope's local-variable context; without it, - # Unparser re-parses identifiers as method calls and its dstr round-trip verification fails. Feed it every local - # assigned or bound anywhere in the tree — an over-approximation that is safe because it only informs Unparser's - # re-parse verification. - def unparse(node) - Unparser.unparse(node, static_local_variables: @local_variables) - end - - LOCAL_BINDING_TYPES = [:lvasgn, :arg, :optarg, :restarg, :kwarg, :kwoptarg, :blockarg, :shadowarg].freeze - - def collect_local_variables(node, names = Set.new) - return names unless node.is_a?(::Parser::AST::Node) - - names << node.children[0] if LOCAL_BINDING_TYPES.include?(node.type) && node.children[0] - node.children.each { |child| collect_local_variables(child, names) } - names - end - # Emits a container body that may be a bare :ensure/:rescue node (their begin/end context comes from the # surrounding def/block/kwbegin, so the keywords must be emitted inline, aligned like statements). - def emit_body(body) + def emit_body(body, layout, renderer) case body&.type - when :ensure then emit_ensure(body) - when :rescue then emit_rescue(body) - else emit_statements(statements_of(body)) + when :ensure then emit_ensure(body, layout, renderer) + when :rescue then emit_rescue(body, layout, renderer) + else emit_statements(statements_of(body), layout, renderer) end end - def emit_ensure(node) + def emit_ensure(node, layout, renderer) *body, ensurer = node.children - body.each { |statement| emit_body(statement) } - place_keyword(keyword_line(node), 'ensure', column: keyword_column(node)) - emit_statements(statements_of(ensurer)) + body.each { |statement| emit_body(statement, layout, renderer) } + place_keyword(layout, keyword_line(node), 'ensure', column: keyword_column(node)) + emit_statements(statements_of(ensurer), layout, renderer) end - def emit_rescue(node) + def emit_rescue(node, layout, renderer) body, *resbodies, else_body = node.children - emit_body(body) - resbodies.each { |resbody| emit_resbody(resbody) } + emit_body(body, layout, renderer) + resbodies.each { |resbody| emit_resbody(resbody, layout, renderer) } return if else_body.nil? else_range = node.loc.else if node.loc.respond_to?(:else) - place_keyword(else_range&.line, 'else', column: else_range&.column) - emit_statements(statements_of(else_body)) + place_keyword(layout, else_range&.line, 'else', column: else_range&.column) + emit_statements(statements_of(else_body), layout, renderer) end - def emit_resbody(node) + def emit_resbody(node, layout, renderer) exceptions, capture, body = node.children header = ['rescue'] - header << " #{Unparser.unparse(exceptions).delete_prefix('[').delete_suffix(']')}" if exceptions + header << " #{renderer.unparse(exceptions).delete_prefix('[').delete_suffix(']')}" if exceptions header << " => #{capture.children[0]}" if capture - place_keyword(node.loc&.line, header.join, column: node.loc&.column) - emit_statements(statements_of(body)) + place_keyword(layout, node.loc&.line, header.join, column: node.loc&.column) + emit_statements(statements_of(body), layout, renderer) end # Keywords (rescue/ensure/else) cannot be `;`-packed after a statement; when their line is taken they go on a # fresh line instead. - def place_keyword(target_line, keyword, column: nil) - if target_line && target_line > @lines.size - place(target_line, keyword, column: column) + def place_keyword(layout, target_line, keyword, column: nil) + if target_line && target_line > layout.cursor + layout.place(target_line, keyword, column: column) else - @lines << keyword + layout.place_on_fresh_line(keyword) end end @@ -181,15 +140,15 @@ def block_assignment?(node) # Renders a container's opener and closer from the node with its body emptied, then recurses into the body so # nested statements align. - def emit_container(node) - opener, closer = container_delimiters(node) - place(node.loc&.line, opener, column: node.loc&.column) - emit_body(container_body(node)) - place(closer_line(node), closer, column: closer_column(node)) + def emit_container(node, layout, renderer) + opener, closer = container_delimiters(node, renderer) + layout.place(node.loc&.line, opener, column: node.loc&.column) + emit_body(container_body(node), layout, renderer) + layout.place(closer_line(node), closer, column: closer_column(node)) end - def container_delimiters(node) - rendered = unparse(empty_container(node)).split("\n").reject(&:empty?) + def container_delimiters(node, renderer) + rendered = renderer.unparse(empty_container(node)).split("\n").reject(&:empty?) opener = rendered[0..-2].join("\n") closer = rendered.last @@ -247,49 +206,16 @@ def statements_of(body) end end - # Places +render+ at +target_line+ when the cursor hasn't passed it; otherwise packs onto the current line. - # Multi-line renders advance the cursor by their height. When opening a fresh line, the render is indented to the - # statement's source +column+ — cosmetic only (leading whitespace is never significant in emitted code; heredocs - # are normalized to inline strings), but it keeps the artifact and test expectations visually close to the - # source. Packed statements ignore the column, as do an Unparser render's continuation lines (they keep - # Unparser's own relative indentation). - def place(target_line, render, column: nil) - first, *rest = render.split("\n") - - if target_line && target_line > @lines.size - @lines << '' while @lines.size < target_line - @lines[-1] = indented(first, column) - else - pack(first) - end - - @lines.concat(rest) - end - - def indented(text, column) - column && column.positive? ? "#{' ' * column}#{text}" : text - end - - # The last line is never blank here: padding blanks are only created inside +place+, which immediately overwrites - # the padded line. - def pack(text) - if @lines.empty? - @lines << text - else - @lines[-1] = "#{@lines.last}; #{text}" - end - end - - def assert_no_custom_types(node) + def assert_no_custom_types(node, source_path) return unless node.is_a?(::Parser::AST::Node) if node.type.start_with?('ast_') || Node.registry.key?(node.type) raise UnloweredNodeTypeError, - "custom node type :#{node.type} reached emission in #{@source_path}; custom types are " \ + "custom node type :#{node.type} reached emission in #{source_path}; custom types are " \ "IR between transformation stages and must be lowered by the stage that understands them" end - node.children.each { |child| assert_no_custom_types(child) } + node.children.each { |child| assert_no_custom_types(child, source_path) } end end end diff --git a/lib/ast_transform/statement_renderer.rb b/lib/ast_transform/statement_renderer.rb new file mode 100644 index 0000000..a5df222 --- /dev/null +++ b/lib/ast_transform/statement_renderer.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'unparser' + +module ASTTransform + # Renders individual statements to text via Unparser, working around the two consequences of unparsing them in + # isolation (line-aligned emission places each statement independently, so each is ripped out of its context): + # + # * Isolation loses the surrounding scope's local variables, making Unparser re-parse identifiers as method calls + # and fail its dstr round-trip verification — so a renderer is built once per tree with every local bound + # anywhere in it, and feeds Unparser that set on each render. + # * Unparser normalizes some single-line constructs into multi-line form, which would push following statements + # off their lines — so renders taller than their source are compressed back to one line when safely possible. + # + # Immutable: configured with the tree's locals at construction, no per-render state. + class StatementRenderer + # Node types that bind a local variable name: assignments plus every method/block parameter flavor. + LOCAL_BINDING_TYPES = [:lvasgn, :arg, :optarg, :restarg, :kwarg, :kwoptarg, :blockarg, :shadowarg].freeze + + class << self + # Builds a renderer for statements of +node+'s tree, holding every local bound anywhere in it — an + # over-approximation that is safe because the set only informs Unparser's re-parse verification, never the + # rendered text. + def for_tree(node) + new(local_variables: collect_local_variables(node)) + end + + private + + def collect_local_variables(node, names = Set.new) + return names unless node.is_a?(::Parser::AST::Node) + + names << node.children[0] if LOCAL_BINDING_TYPES.include?(node.type) && node.children[0] + node.children.each { |child| collect_local_variables(child, names) } + names + end + end + + # @param local_variables [Set] every local bound in the tree the statements come from. + def initialize(local_variables:) + @local_variables = local_variables + end + + # @param node [Parser::AST::Node] the statement to render. + # @return [String] Unparser's render, informed of the tree's locals. + def unparse(node) + Unparser.unparse(node, static_local_variables: @local_variables) + end + + # Renders +node+ no taller than its source when safely possible. Unparser normalizes some single-line + # constructs into multi-line form (e.g. modifier-if into if/end); when the render is taller than the + # statement's source, compress it back to one line — verified by re-parse so a statement that cannot be safely + # single-lined (e.g. containing a heredoc) falls back to its multi-line render. + def aligned_render(node) + render = unparse(node) + loc = node.loc + return render unless loc.respond_to?(:last_line) && loc.line + + source_height = loc.last_line - loc.line + 1 + return render if render.count("\n") < source_height + + compress_to_single_line(render) || render + end + + private + + def compress_to_single_line(render) + candidate = render.split("\n").map(&:strip).join('; ') + # Both sides parsed without scope context, so lvar/send ambiguity cancels out; equality means the newline + # join preserved structure. + Unparser.parse(candidate) == Unparser.parse(render) ? candidate : nil + rescue Parser::SyntaxError + nil + end + end +end diff --git a/lib/ast_transform/thunk_lowering.rb b/lib/ast_transform/thunk_lowering.rb index 56a224f..01fe747 100644 --- a/lib/ast_transform/thunk_lowering.rb +++ b/lib/ast_transform/thunk_lowering.rb @@ -30,6 +30,9 @@ module ASTTransform # would be invisible to the statements that read them after the execution point. Self-assignment registers the name # (nil until the thunk runs — exactly what an unexecuted assignment yields) without clobbering an already-assigned # value. + # + # Stateless: the thunks encountered during one lowering are tracked in a Registry created at +lower+ entry, so an + # instance is a reusable collaborator. class ThunkLowering include TransformationHelper @@ -37,33 +40,65 @@ class ThunkLowering # +statements+ the pre-declarations plus the proc assignment. Placement = Struct.new(:line, :statements) + # The thunks encountered during one lowering, keyed by token identity: allocates each thunk's hidden lvar name + # on first occurrence and verifies later occurrences carry the same body. + class Registry + def initialize + @names_by_token = {}.compare_by_identity + @bodies_by_token = {}.compare_by_identity + end + + def known?(token) + @names_by_token.key?(token) + end + + # @return [Symbol] the hidden lvar name allocated for +token+. + def register(token, body) + name = :"__ast_thunk_#{@names_by_token.size + 1}__" + @names_by_token[token] = name + @bodies_by_token[token] = body + name + end + + def name_for(token) + @names_by_token.fetch(token) + end + + def verify_same_body!(token, body) + return if @bodies_by_token[token] == body + + raise ThunkPlacementError, + 'occurrences of one thunk carry diverging bodies; reuse the same thunk node to multiplex' + end + + def hidden_names + @names_by_token.values + end + end + private_constant :Registry + SEQUENCE_TYPES = [:begin, :kwbegin].freeze # Scope-opening containers: the hidden lvar cannot be referenced across these boundaries, so placements arising # inside must land inside. SCOPE_BODY_INDEXES = { def: 2, defs: 3, class: 2, module: 1, sclass: 1 }.freeze - def initialize - @names_by_token = {}.compare_by_identity - @bodies_by_token = {}.compare_by_identity - end - # @param node [Parser::AST::Node] tree possibly containing Thunk nodes # @return [Parser::AST::Node] tree with thunks lowered to plain Ruby # @raise [ThunkPlacementError] when a thunk body's source lines fall after its execution point, or occurrences of # one thunk diverge - def run(node) - lower_body(node) + def lower(node) + lower_body(node, Registry.new) end private # Lowers a node standing in statement-body position (a container's body or the root), absorbing any placements # that arise within it. - def lower_body(node) + def lower_body(node, registry) return node unless node.is_a?(::Parser::AST::Node) - return lower_sequence(node) if SEQUENCE_TYPES.include?(node.type) + return lower_sequence(node, registry) if SEQUENCE_TYPES.include?(node.type) - lowered, placements = lower_expression(node) + lowered, placements = lower_expression(node, registry) return lowered if placements.empty? # A loc-less :begin in statement position; the emitter flattens it into the surrounding statement stream. @@ -71,11 +106,11 @@ def lower_body(node) end # Lowers a statement sequence, inserting each placement among the statements by the body's source line. - def lower_sequence(node) + def lower_sequence(node, registry) statements = [] node.children.each_with_index do |child, index| - lowered, placements = lower_expression(child) + lowered, placements = lower_expression(child, registry) placements.each do |placement| check_placement_precedes_execution!(placement, child, node.children[(index + 1)..]) statements.insert(insertion_index(statements, placement), *placement.statements) @@ -90,33 +125,33 @@ def lower_sequence(node) # the enclosing statement sequence. # # @return [Array(Parser::AST::Node, Array)] - def lower_expression(node) + def lower_expression(node, registry) return [node, []] unless node.is_a?(::Parser::AST::Node) case node.type when :ast_thunk - lower_thunk(node) + lower_thunk(node, registry) when *SEQUENCE_TYPES - [lower_sequence(node), []] + [lower_sequence(node, registry), []] when :ensure, :rescue - [node.updated(nil, node.children.map { |child| lower_body(child) }), []] + [node.updated(nil, node.children.map { |child| lower_body(child, registry) }), []] when :resbody exceptions, capture, body = node.children - [node.updated(nil, [exceptions, capture, lower_body(body)]), []] + [node.updated(nil, [exceptions, capture, lower_body(body, registry)]), []] else - lower_generic(node) + lower_generic(node, registry) end end - def lower_generic(node) + def lower_generic(node, registry) scope_body_index = SCOPE_BODY_INDEXES[node.type] pending = [] children = node.children.each_with_index.map do |child, index| if index == scope_body_index - lower_body(child) + lower_body(child, registry) else - lowered, placements = lower_expression(child) + lowered, placements = lower_expression(child, registry) pending.concat(placements) lowered end @@ -127,33 +162,27 @@ def lower_generic(node) # An occurrence of a thunk: the first occurrence of its token yields the placement; every occurrence yields the # call. - def lower_thunk(node) + def lower_thunk(node, registry) token = node.token - if @names_by_token.key?(token) - unless @bodies_by_token[token] == node.body - raise ThunkPlacementError, - "occurrences of one thunk carry diverging bodies; reuse the same thunk node to multiplex" - end - return [call_node(token), []] + if registry.known?(token) + registry.verify_same_body!(token, node.body) + return [call_node(token, registry), []] end - name = :"__ast_thunk_#{@names_by_token.size + 1}__" - @names_by_token[token] = name - @bodies_by_token[token] = node.body - - lowered_body = lower_sequence(s(:begin, *node.body)) - placement = Placement.new(body_first_line(node.body), placement_statements(name, lowered_body)) - [call_node(token), [placement]] + name = registry.register(token, node.body) + lowered_body = lower_sequence(s(:begin, *node.body), registry) + placement = Placement.new(body_first_line(node.body), placement_statements(name, lowered_body, registry)) + [call_node(token, registry), [placement]] end - def call_node(token) - s(:send, s(:lvar, @names_by_token.fetch(token)), :call) + def call_node(token, registry) + s(:send, s(:lvar, registry.name_for(token)), :call) end - def placement_statements(name, lowered_body) + def placement_statements(name, lowered_body, registry) assignment = s(:lvasgn, name, s(:block, s(:send, nil, :proc), s(:args), lowered_body)) - hidden_names = @names_by_token.values + hidden_names = registry.hidden_names pre_declared = method_scope_assignments(lowered_body).reject { |local| hidden_names.include?(local) } pre_declared.map { |local| s(:lvasgn, local, s(:lvar, local)) } << assignment end diff --git a/lib/ast_transform/transformer.rb b/lib/ast_transform/transformer.rb index 12b5b0a..567aaf4 100644 --- a/lib/ast_transform/transformer.rb +++ b/lib/ast_transform/transformer.rb @@ -11,8 +11,10 @@ class Transformer # Constructs a new Transformer instance. # # @param transformations [Array] The transformations to be run. - def initialize(*transformations) + # @param emitter [ASTTransform::LineAlignedEmitter] The emitter rendering transformed ASTs back to source. + def initialize(*transformations, emitter: LineAlignedEmitter.new) @transformations = transformations + @emitter = emitter end # Builds the AST for the given +source+. @@ -44,7 +46,7 @@ def build_ast_from_file(file_path) def transform(source) ast = build_ast(source) transformed_ast = transform_ast(ast) - LineAlignedEmitter.new(transformed_ast, 'tmp').emit + @emitter.emit(transformed_ast, 'tmp') end # Transforms the give +file_path+. @@ -74,7 +76,7 @@ def transform_file_source(source, file_path, _transformed_file_path) # At this point, the transformed_ast contains source locations for the original +source+. transformed_ast = transform_ast(source_ast) - LineAlignedEmitter.new(transformed_ast, file_path).emit + @emitter.emit(transformed_ast, file_path) end # Transforms the given +ast+. diff --git a/test/ast_transform/layout_test.rb b/test/ast_transform/layout_test.rb new file mode 100644 index 0000000..011c1f5 --- /dev/null +++ b/test/ast_transform/layout_test.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'ast_transform/layout' + +module ASTTransform + class LayoutTest < Minitest::Test + extend ASTTransform::Declarative + + test "place pads with blank lines to reach a target line ahead of the cursor" do + layout = Layout.new + layout.place(3, 'statement') + + assert_equal "\n\nstatement\n", layout.to_source + end + + test "place indents a fresh line to the requested column" do + layout = Layout.new + layout.place(1, 'statement', column: 2) + + assert_equal " statement\n", layout.to_source + end + + test "place packs onto the current line when the cursor has passed the target" do + layout = Layout.new + layout.place(1, 'first') + layout.place(1, 'displaced') + + assert_equal "first; displaced\n", layout.to_source + end + + test "packed text ignores the column" do + layout = Layout.new + layout.place(1, 'first') + layout.place(1, 'displaced', column: 4) + + assert_equal "first; displaced\n", layout.to_source + end + + test "loc-less text (nil target) packs onto the current line" do + layout = Layout.new + layout.place(1, 'first') + layout.place(nil, 'synthetic') + + assert_equal "first; synthetic\n", layout.to_source + end + + test "multi-line text advances the cursor by its height" do + layout = Layout.new + layout.place(1, "opener\n continuation") + + assert_equal 2, layout.cursor + # Continuation lines keep their own relative indentation. + assert_equal "opener\n continuation\n", layout.to_source + end + + test "emission re-anchors at the next target still ahead of the cursor" do + layout = Layout.new + layout.place(1, "tall\ntall\ntall") + layout.place(2, 'displaced') + layout.place(5, 'aligned') + + assert_equal "tall\ntall\ntall; displaced\n\naligned\n", layout.to_source + end + + test "place_on_fresh_line never packs" do + layout = Layout.new + layout.place(1, 'statement') + layout.place_on_fresh_line('ensure') + + assert_equal "statement\nensure\n", layout.to_source + end + + test "pack onto an empty layout opens the first line" do + layout = Layout.new + layout.pack('lonely') + + assert_equal "lonely\n", layout.to_source + end + + test "cursor reports the line currently being written" do + layout = Layout.new + + assert_equal 0, layout.cursor + + layout.place(2, 'statement') + + assert_equal 2, layout.cursor + end + end +end diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index 2bba508..847e97c 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -19,7 +19,7 @@ def parse(source) end def emit(ast) - LineAlignedEmitter.new(ast, 'fixture.rb').emit + LineAlignedEmitter.new.emit(ast, 'fixture.rb') end # Runs emitted code with real method semantics (return target, method @@ -147,16 +147,6 @@ def risky assert_equal ['begin', 'first_call', 'second_call', 'end'], emitted_lines end - test "compress_to_single_line declines renders whose single-line join does not parse" do - emitter = LineAlignedEmitter.new(parse("noop\n"), 'fixture.rb') - - # No current Unparser render joins into invalid syntax (heredocs are - # normalized to inline strings), so exercise the totality guard - # directly: layout must fall back, never raise, whatever future - # Unparser output looks like. - assert_nil emitter.send(:compress_to_single_line, "value = <<~TXT\n hi\nTXT") - end - test "pre-declares locals the thunk body assigns at method scope" do first, second, third = parse("given_setup\nresult = compute\ninteraction_setup\n").children reordered = run_after([first, second, third], run: [second], after: third) diff --git a/test/ast_transform/statement_renderer_test.rb b/test/ast_transform/statement_renderer_test.rb new file mode 100644 index 0000000..0ce7f04 --- /dev/null +++ b/test/ast_transform/statement_renderer_test.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'ast_transform/statement_renderer' +require 'ast_transform/transformer' + +module ASTTransform + class StatementRendererTest < Minitest::Test + extend ASTTransform::Declarative + + def parse(source) + ASTTransform::Transformer.new.build_ast(source) + end + + test "unparse renders an isolated statement using locals collected from the whole tree" do + tree = parse("name = fetch\nmessage = \"hi \#{name}\"\n") + isolated_dstr_statement = tree.children[1] + + # Without the tree's locals, Unparser's dstr round-trip verification re-parses `name` as a method call and + # raises; for_tree restores the context the statement was ripped out of. + rendered = StatementRenderer.for_tree(tree).unparse(isolated_dstr_statement) + + assert_equal "message = \"hi \#{name}\"", rendered + end + + test "aligned_render compresses a render taller than its source back to one line" do + statement = parse("raise ArgumentError if strict\n") + + rendered = StatementRenderer.for_tree(statement).aligned_render(statement) + + assert_equal 1, rendered.lines.size, rendered + end + + test "aligned_render keeps a render that already fits its source height" do + statement = parse("def risky\n compute\nend\n") + + rendered = StatementRenderer.for_tree(statement).aligned_render(statement) + + assert_operator rendered.lines.size, :>, 1, rendered + end + + test "compress_to_single_line declines renders whose single-line join does not parse" do + renderer = StatementRenderer.for_tree(parse("noop\n")) + + # No current Unparser render joins into invalid syntax (heredocs are normalized to inline strings), so + # exercise the totality guard directly: rendering must fall back, never raise, whatever future Unparser + # output looks like. + assert_nil renderer.send(:compress_to_single_line, "value = <<~TXT\n hi\nTXT") + end + + test "for_tree collects assignments and every parameter flavor" do + tree = parse(<<~HEREDOC) + assigned = 1 + def a_method(plain, optional = 1, *splat, keyword:, optional_keyword: 2, &block_arg) + end + items.map { |item; shadow| item } + HEREDOC + + renderer = StatementRenderer.for_tree(tree) + locals = renderer.instance_variable_get(:@local_variables) + + assert_equal Set[:assigned, :plain, :optional, :splat, :keyword, :optional_keyword, :block_arg, :item, :shadow], + locals + end + end +end From 55a1fd8a6c774ccf7912044314ee2329f781cb27 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 02:26:37 -0400 Subject: [PATCH 18/22] Capture Thunk token/body in initialize instead of re-deriving per call initialize already destructures children for invariant checks; capture the pieces there (before super freezes the node) rather than allocating a fresh body array on every accessor call. The body array is frozen so the shared reference cannot be mutated out from under children, and updated re-runs initialize so the capture cannot go stale. Co-authored-by: Cursor --- lib/ast_transform/thunk.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb index 067299b..2863a32 100644 --- a/lib/ast_transform/thunk.rb +++ b/lib/ast_transform/thunk.rb @@ -19,6 +19,8 @@ module ASTTransform class Thunk < Node register :ast_thunk + attr_reader :token, :body + def initialize(type, children, properties = {}) token, *body = children unless token.is_a?(ThunkToken) @@ -28,12 +30,13 @@ def initialize(type, children, properties = {}) end raise MalformedThunkError, "a Thunk must wrap at least one statement" if body.empty? + # Captured before super (which freezes the node); frozen so the shared array cannot be mutated out from under + # +children+. Rebuilds via +updated+ re-run initialize, so the capture can never go stale. + @token = token + @body = body.freeze + super end - - def token = children[0] - - def body = children.drop(1) end # The identity of a Thunk across transformation passes: Processor and Node#updated rebuilds create new node From 0440490827ba3e14b4717d976449d79004883ea1 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 10:58:08 -0400 Subject: [PATCH 19/22] Renamed Thunk#token to Thunk#id --- lib/ast_transform/thunk.rb | 36 +++++++------ lib/ast_transform/thunk_lowering.rb | 50 +++++++++---------- lib/ast_transform/transformation_helper.rb | 2 +- .../line_aligned_emitter_test.rb | 2 +- .../transformation_helper_test.rb | 24 ++++----- 5 files changed, 56 insertions(+), 58 deletions(-) diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb index 2863a32..086bc6c 100644 --- a/lib/ast_transform/thunk.rb +++ b/lib/ast_transform/thunk.rb @@ -19,34 +19,38 @@ module ASTTransform class Thunk < Node register :ast_thunk - attr_reader :token, :body + # The identity of a Thunk across transformation passes: Processor and Node#updated rebuilds create new node + # objects, so node identity does not survive — but children DO (carried by reference through every rebuild). Every + # rebuild of a thunk therefore carries this same id object, and the lowering groups occurrences by its object + # identity: one proc, one call per occurrence. Minted internally by the +thunk+ helper, never handled by authors. + # No behavior — a named class over a bare Object.new only for self-documenting AST dumps and greppability. + class Id; end def initialize(type, children, properties = {}) - token, *body = children - unless token.is_a?(ThunkToken) + id, *body = children + unless id.is_a?(ASTTransform::Thunk::Id) raise MalformedThunkError, - "a Thunk's first child must be its ThunkToken (got #{token.class}); " \ - "build thunks with the thunk(*statements) helper" + "a Thunk's first child must be its #{Thunk::Id} (got #{id.class}); build thunks with the " \ + "thunk(*statements) helper" end raise MalformedThunkError, "a Thunk must wrap at least one statement" if body.empty? # Captured before super (which freezes the node); frozen so the shared array cannot be mutated out from under # +children+. Rebuilds via +updated+ re-run initialize, so the capture can never go stale. - @token = token + @id = id @body = body.freeze super end - end - # The identity of a Thunk across transformation passes: Processor and Node#updated rebuilds create new node - # objects, so node identity does not survive — but children DO (carried by reference through every rebuild). Every - # rebuild of a thunk therefore carries this same token object, and the lowering groups occurrences by its object - # identity: one proc, one call per occurrence. Minted internally by the +thunk+ helper, never handled by authors. - # No behavior — a named class over a bare Object.new only for self-documenting AST dumps and greppability. - class ThunkToken - def inspect - "#" - end + # Retrieves the Thunk's id. + # @return [ASTTransform::Thunk::Id] The Id. + attr_reader(:id) + + # Retrieves the Thunk's body. + # + # @note Same as the +Parser::AST::Node#children+ + # @return [Array] The nodes forming the body of the Thunk. + attr_reader(:body) end end diff --git a/lib/ast_transform/thunk_lowering.rb b/lib/ast_transform/thunk_lowering.rb index 01fe747..865741f 100644 --- a/lib/ast_transform/thunk_lowering.rb +++ b/lib/ast_transform/thunk_lowering.rb @@ -6,7 +6,7 @@ require 'ast_transform/transformation_helper' module ASTTransform - # Lowers Thunk nodes into plain Ruby ahead of emission. Each unique thunk (grouped by token identity) becomes a + # Lowers Thunk nodes into plain Ruby ahead of emission. Each unique thunk (grouped by id identity) becomes a # hidden proc; each occurrence becomes the proc's call: # # thunk placed at the execution point @@ -40,39 +40,39 @@ class ThunkLowering # +statements+ the pre-declarations plus the proc assignment. Placement = Struct.new(:line, :statements) - # The thunks encountered during one lowering, keyed by token identity: allocates each thunk's hidden lvar name + # The thunks encountered during one lowering, keyed by identity: allocates each thunk's hidden lvar name # on first occurrence and verifies later occurrences carry the same body. class Registry def initialize - @names_by_token = {}.compare_by_identity - @bodies_by_token = {}.compare_by_identity + @names_by_id = {}.compare_by_identity + @bodies_by_id = {}.compare_by_identity end - def known?(token) - @names_by_token.key?(token) + def known?(id) + @names_by_id.key?(id) end - # @return [Symbol] the hidden lvar name allocated for +token+. - def register(token, body) - name = :"__ast_thunk_#{@names_by_token.size + 1}__" - @names_by_token[token] = name - @bodies_by_token[token] = body + # @return [Symbol] the hidden lvar name allocated for +id+. + def register(id, body) + name = :"__ast_thunk_#{@names_by_id.size + 1}__" + @names_by_id[id] = name + @bodies_by_id[id] = body name end - def name_for(token) - @names_by_token.fetch(token) + def name_for(id) + @names_by_id.fetch(id) end - def verify_same_body!(token, body) - return if @bodies_by_token[token] == body + def verify_same_body!(id, body) + return if @bodies_by_id[id] == body raise ThunkPlacementError, 'occurrences of one thunk carry diverging bodies; reuse the same thunk node to multiplex' end def hidden_names - @names_by_token.values + @names_by_id.values end end private_constant :Registry @@ -160,24 +160,24 @@ def lower_generic(node, registry) [node.updated(nil, children), pending] end - # An occurrence of a thunk: the first occurrence of its token yields the placement; every occurrence yields the + # An occurrence of a thunk: the first occurrence of its id yields the placement; every occurrence yields the # call. def lower_thunk(node, registry) - token = node.token + id = node.id - if registry.known?(token) - registry.verify_same_body!(token, node.body) - return [call_node(token, registry), []] + if registry.known?(id) + registry.verify_same_body!(id, node.body) + return [call_node(id, registry), []] end - name = registry.register(token, node.body) + name = registry.register(id, node.body) lowered_body = lower_sequence(s(:begin, *node.body), registry) placement = Placement.new(body_first_line(node.body), placement_statements(name, lowered_body, registry)) - [call_node(token, registry), [placement]] + [call_node(id, registry), [placement]] end - def call_node(token, registry) - s(:send, s(:lvar, registry.name_for(token)), :call) + def call_node(id, registry) + s(:send, s(:lvar, registry.name_for(id)), :call) end def placement_statements(name, lowered_body, registry) diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index bf7c6ff..96656dd 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -71,7 +71,7 @@ def s_at(anchor, type, *children) # @param statements [Array] statements to wrap # @return [ASTTransform::Thunk] the thunk node def thunk(*statements) - s(:ast_thunk, ThunkToken.new, *statements) + s(:ast_thunk, Thunk::Id.new, *statements) end # The paved road for execution reordering in flat statement sequences. Named for the constraint, not the diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index 847e97c..6df40f3 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -81,7 +81,7 @@ def run_as_method(emitted) test "occurrences of one thunk with diverging bodies raise ThunkPlacementError" do original = thunk(parse("foo\n")) - diverged = original.updated(nil, [original.token, parse("bar\n")]) + diverged = original.updated(nil, [original.id, parse("bar\n")]) error = assert_raises(ThunkPlacementError) { emit(s(:begin, original, diverged)) } diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index cca1c81..0b94eda 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -50,21 +50,21 @@ def parse(source) assert_includes error.message, 'send' end - test "thunk builds a Thunk node carrying an internal token and the body" do + test "thunk builds a Thunk node carrying an internal id and the body" do statements = parse("foo\nbar\n").children node = thunk(*statements) assert_instance_of Thunk, node assert_equal :ast_thunk, node.type - assert_instance_of ThunkToken, node.token + assert_instance_of Thunk::Id, node.id assert_equal statements, node.body end - test "a Thunk without a token cannot be constructed" do + test "a Thunk without an id cannot be constructed" do error = assert_raises(MalformedThunkError) { s(:ast_thunk, parse("foo\n")) } - assert_includes error.message, 'ThunkToken' + assert_includes error.message, 'Thunk::Id' end test "a Thunk with an empty body cannot be constructed" do @@ -73,14 +73,14 @@ def parse(source) assert_includes error.message, 'at least one statement' end - test "a Processor rebuild preserves the Thunk class, token, and invariants" do + test "a Processor rebuild preserves the Thunk class, id, and invariants" do node = thunk(parse("foo\n")) - rebuilt = node.updated(nil, [node.token, parse("bar\n")]) + rebuilt = node.updated(nil, [node.id, parse("bar\n")]) assert_instance_of Thunk, rebuilt - assert_same node.token, rebuilt.token - assert_raises(MalformedThunkError) { node.updated(nil, [node.token]) } + assert_same node.id, rebuilt.id + assert_raises(MalformedThunkError) { node.updated(nil, [node.id]) } end test "AbstractTransformation descends thunk bodies by default" do @@ -94,16 +94,10 @@ def on_send(node) processed = swap_foo_for_bar.new.run(node) assert_instance_of Thunk, processed - assert_same node.token, processed.token + assert_same node.id, processed.id assert_equal :bar, processed.body[0].children[1] end - test "ThunkToken#inspect names the class so AST dumps are self-documenting" do - token = thunk(parse("foo\n")).token - - assert_match(/\A#\z/, token.inspect) - end - test "thunk imposes no control-flow validation (semantics are the proc lowering's contract)" do # return is transparent through the non-lambda proc; severed jumps keep # Ruby's native behavior (see ThunkLowering / LineAlignedEmitterTest). From a1a7e6b78c96cef89053891900820d6c00c93bfb Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 11:06:58 -0400 Subject: [PATCH 20/22] Move each error into its producing class; delete errors.rb An agglomerated errors.rb obscures ownership. Each error is defined by the class that raises it, following the Thunk::Id nesting pattern (de-stuttered where the namespace already says thunk): - TransformationHelper::MissingLocationError (raised by s_at) - Thunk::MalformedError (raised by Thunk#initialize) - ThunkLowering::PlacementError (raised during lowering) - LineAlignedEmitter::UnloweredNodeTypeError (the emitter postcondition) Co-authored-by: Cursor --- CHANGELOG.md | 4 ++-- README.md | 6 +++--- lib/ast_transform/errors.rb | 20 ------------------- lib/ast_transform/line_aligned_emitter.rb | 7 +++++-- lib/ast_transform/thunk.rb | 12 +++++++---- lib/ast_transform/thunk_lowering.rb | 12 +++++++---- lib/ast_transform/transformation_helper.rb | 4 +++- .../line_aligned_emitter_test.rb | 12 +++++------ .../transformation_helper_test.rb | 10 +++++----- 9 files changed, 40 insertions(+), 47 deletions(-) delete mode 100644 lib/ast_transform/errors.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ad1ab7..1ded249 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Line-aligned emission: transformed code is emitted with every loc-carrying statement on its original source line, making backtraces, breakpoints, and debugger display correct by construction (`LineAlignedEmitter`). - Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `thunk` (a single invariant-checked `Thunk` node spliced at the execution point; the lowering derives the hidden proc's textual placement from the body's source locations), and `run_after` (sequence-level execution reordering that preserves textual/source order). Thunks lower to a non-lambda proc, so `return` still returns from the enclosing method, and locals assigned by thunked statements are pre-declared to stay method-scope. Reusing one thunk node executes its body from several points. -- `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. +- `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`LineAlignedEmitter::UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. - `ast_transform/test_helpers` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. -- Error types: `MissingLocationError`, `MalformedThunkError`, `ThunkPlacementError`, `UnloweredNodeTypeError`. +- Error types, each owned by its producer: `TransformationHelper::MissingLocationError`, `Thunk::MalformedError`, `ThunkLowering::PlacementError`, `LineAlignedEmitter::UnloweredNodeTypeError`. ### Removed - **Breaking:** `ASTTransform::SourceMap` and source-map registration. Line-aligned emission makes raw VM line numbers the source line numbers, so there is nothing left to map at display time. diff --git a/README.md b/README.md index 6be47d6..77884a3 100644 --- a/README.md +++ b/README.md @@ -173,8 +173,8 @@ ASTTransform owns text and lines; transform authors own semantics and execution `ASTTransform::TransformationHelper` (included by `AbstractTransformation`) provides the authoring toolkit: * `s(type, *children)` — builds a loc-less (synthetic) node. Registered custom types (see below) construct their registered class. -* `s_at(anchor, type, *children)` — builds a node anchored at `anchor`'s source location, so it is emitted at `anchor`'s line. Raises `MissingLocationError` if the anchor has no location. -* `thunk(*statements)` — wraps statements in a single `Thunk` node: splice it wherever the statements must *run*, in statement position or composed inside an expression (e.g. an `assert_raises` block body). The wrapped statements keep their own locations, and the lowering derives the hidden proc's textual placement from them — the body still emits on its source lines even though execution waits. Reuse the same node to execute one body from several points. Thunk construction is invariant-checked (`MalformedThunkError`); a body whose source lines fall after its execution point fails lowering with `ThunkPlacementError` (a thunk can only delay execution, never text). +* `s_at(anchor, type, *children)` — builds a node anchored at `anchor`'s source location, so it is emitted at `anchor`'s line. Raises `TransformationHelper::MissingLocationError` if the anchor has no location. +* `thunk(*statements)` — wraps statements in a single `Thunk` node: splice it wherever the statements must *run*, in statement position or composed inside an expression (e.g. an `assert_raises` block body). The wrapped statements keep their own locations, and the lowering derives the hidden proc's textual placement from them — the body still emits on its source lines even though execution waits. Reuse the same node to execute one body from several points. Thunk construction is invariant-checked (`Thunk::MalformedError`); a body whose source lines fall after its execution point fails lowering with `ThunkLowering::PlacementError` (a thunk can only delay execution, never text). * `run_after(statements, run:, after:)` — the paved road over `thunk`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. Thunked statements keep their original meaning as far as Ruby's closure semantics allow: @@ -201,7 +201,7 @@ end s(:my_interaction, ...) # => InteractionNode, with domain accessors ``` -Custom node types are IR **between stages that understand them** — the stage that owns a type must lower it to plain Ruby nodes before emission. The emitter enforces this: any registered or `ast_`-prefixed type reaching emission raises `UnloweredNodeTypeError`. +Custom node types are IR **between stages that understand them** — the stage that owns a type must lower it to plain Ruby nodes before emission. The emitter enforces this: any registered or `ast_`-prefixed type reaching emission raises `LineAlignedEmitter::UnloweredNodeTypeError`. #### Testing your transformation diff --git a/lib/ast_transform/errors.rb b/lib/ast_transform/errors.rb deleted file mode 100644 index ed5c9c7..0000000 --- a/lib/ast_transform/errors.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -module ASTTransform - # Raised by authoring helpers (e.g. +s_at+) when a node that must carry a source location does not have one. - class MissingLocationError < StandardError; end - - # Raised at construction when a Thunk node's children violate its invariants (missing token, empty body). Every - # construction path funnels through Thunk#initialize — including Processor rebuilds — so a malformed thunk cannot - # exist in a tree. - class MalformedThunkError < StandardError; end - - # Raised at lowering when a thunk cannot be placed: its body's source lines fall after the execution point (the - # hidden proc's text IS its assignment, so a call can never textually precede the body), or two occurrences of the - # same thunk carry diverging bodies. - class ThunkPlacementError < StandardError; end - - # Raised as the emitter's postcondition when a custom node type (ast_* markers or types registered on - # ASTTransform::Node) reaches the unparse boundary instead of being lowered by the stage that understands it. - class UnloweredNodeTypeError < StandardError; end -end diff --git a/lib/ast_transform/line_aligned_emitter.rb b/lib/ast_transform/line_aligned_emitter.rb index 744ef36..f7c46db 100644 --- a/lib/ast_transform/line_aligned_emitter.rb +++ b/lib/ast_transform/line_aligned_emitter.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'ast_transform/node' -require 'ast_transform/errors' require 'ast_transform/layout' require 'ast_transform/statement_renderer' require 'ast_transform/thunk_lowering' @@ -27,6 +26,10 @@ module ASTTransform # (ast_* markers or types registered on ASTTransform::Node) crosses the unparse boundary — they are IR between # stages that understand them. class LineAlignedEmitter + # Raised as the emitter's postcondition when a custom node type (ast_* markers or types registered on + # ASTTransform::Node) reaches the unparse boundary instead of being lowered by the stage that understands it. + class UnloweredNodeTypeError < StandardError; end + # Containers the emitter recurses into so nested statements align; every other node renders as an Unparser blob # at its head line. RECURSIVE_CONTAINER_TYPES = [:class, :module, :sclass, :def, :defs, :block, :numblock, :itblock, :kwbegin].freeze @@ -46,7 +49,7 @@ def initialize(thunk_lowering: ThunkLowering.new) # @param ast [Parser::AST::Node] transformed AST # @param source_path [String] original file path (for error messages) # @return [String] transformed source, line-aligned - # @raise [ThunkPlacementError] if a thunk cannot be textually placed + # @raise [ThunkLowering::PlacementError] if a thunk cannot be textually placed # @raise [UnloweredNodeTypeError] if a custom node type survived to emission def emit(ast, source_path) lowered = @thunk_lowering.lower(ast) diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb index 086bc6c..1a7756e 100644 --- a/lib/ast_transform/thunk.rb +++ b/lib/ast_transform/thunk.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'ast_transform/node' -require 'ast_transform/errors' module ASTTransform # The reordering primitive: an eagerly built wrapper node, spliced wherever the wrapped statements must EXECUTE — @@ -9,7 +8,7 @@ module ASTTransform # source locations, and the lowering derives the wrapper's textual placement from them (see ThunkLowering), so the # statements still emit on their original lines even though execution waits. # - # Children are +[token, *body_statements]+ and the invariants are enforced here in +initialize+, which every + # Children are +[id, *body_statements]+ and the invariants are enforced here in +initialize+, which every # construction path shares — +s+ routing, the +thunk+ helper, and Processor rebuilds (+updated+ re-initializes). # Build thunks with +TransformationHelper#thunk+; reuse the same node to execute one body from several points # (multiplexing). @@ -26,14 +25,19 @@ class Thunk < Node # No behavior — a named class over a bare Object.new only for self-documenting AST dumps and greppability. class Id; end + # Raised at construction when a Thunk node's children violate its invariants (missing id, empty body). Every + # construction path funnels through Thunk#initialize — including Processor rebuilds — so a malformed thunk + # cannot exist in a tree. + class MalformedError < StandardError; end + def initialize(type, children, properties = {}) id, *body = children unless id.is_a?(ASTTransform::Thunk::Id) - raise MalformedThunkError, + raise MalformedError, "a Thunk's first child must be its #{Thunk::Id} (got #{id.class}); build thunks with the " \ "thunk(*statements) helper" end - raise MalformedThunkError, "a Thunk must wrap at least one statement" if body.empty? + raise MalformedError, "a Thunk must wrap at least one statement" if body.empty? # Captured before super (which freezes the node); frozen so the shared array cannot be mutated out from under # +children+. Rebuilds via +updated+ re-run initialize, so the capture can never go stale. diff --git a/lib/ast_transform/thunk_lowering.rb b/lib/ast_transform/thunk_lowering.rb index 865741f..2081605 100644 --- a/lib/ast_transform/thunk_lowering.rb +++ b/lib/ast_transform/thunk_lowering.rb @@ -2,7 +2,6 @@ require 'ast_transform/node' require 'ast_transform/thunk' -require 'ast_transform/errors' require 'ast_transform/transformation_helper' module ASTTransform @@ -40,6 +39,11 @@ class ThunkLowering # +statements+ the pre-declarations plus the proc assignment. Placement = Struct.new(:line, :statements) + # Raised when a thunk cannot be placed: its body's source lines fall after the execution point (the hidden + # proc's text IS its assignment, so a call can never textually precede the body), or two occurrences of the + # same thunk carry diverging bodies. + class PlacementError < StandardError; end + # The thunks encountered during one lowering, keyed by identity: allocates each thunk's hidden lvar name # on first occurrence and verifies later occurrences carry the same body. class Registry @@ -67,7 +71,7 @@ def name_for(id) def verify_same_body!(id, body) return if @bodies_by_id[id] == body - raise ThunkPlacementError, + raise PlacementError, 'occurrences of one thunk carry diverging bodies; reuse the same thunk node to multiplex' end @@ -84,7 +88,7 @@ def hidden_names # @param node [Parser::AST::Node] tree possibly containing Thunk nodes # @return [Parser::AST::Node] tree with thunks lowered to plain Ruby - # @raise [ThunkPlacementError] when a thunk body's source lines fall after its execution point, or occurrences of + # @raise [PlacementError] when a thunk body's source lines fall after its execution point, or occurrences of # one thunk diverge def lower(node) lower_body(node, Registry.new) @@ -202,7 +206,7 @@ def check_placement_precedes_execution!(placement, executing_statement, followin end return if conflicting.nil? - raise ThunkPlacementError, + raise PlacementError, "thunk body's source lines (from line #{placement.line}) fall after its execution point " \ "(statement at line #{conflicting.loc.line}); a thunk can only delay execution, never text" end diff --git a/lib/ast_transform/transformation_helper.rb b/lib/ast_transform/transformation_helper.rb index 96656dd..d1f3b85 100644 --- a/lib/ast_transform/transformation_helper.rb +++ b/lib/ast_transform/transformation_helper.rb @@ -3,7 +3,6 @@ require 'parser' require 'ast_transform/node' require 'ast_transform/thunk' -require 'ast_transform/errors' module ASTTransform # The transform-authoring layer. Three shapes: @@ -17,6 +16,9 @@ module ASTTransform # at its source line; when execution order must differ from textual order, authors express it as a thunk instead of # moving text. module TransformationHelper + # Raised by authoring helpers (e.g. +s_at+) when a node that must carry a source location does not have one. + class MissingLocationError < StandardError; end + class << self def included(base) base.extend(Methods) diff --git a/test/ast_transform/line_aligned_emitter_test.rb b/test/ast_transform/line_aligned_emitter_test.rb index 6df40f3..47bfb98 100644 --- a/test/ast_transform/line_aligned_emitter_test.rb +++ b/test/ast_transform/line_aligned_emitter_test.rb @@ -68,22 +68,22 @@ def run_as_method(emitted) assert_equal 2, emitted.scan('__ast_thunk_1__.call').size, emitted end - test "a thunk whose body lines fall after its execution point raises ThunkPlacementError" do + test "a thunk whose body lines fall after its execution point raises ThunkLowering::PlacementError" do first, second, third = parse("first\nsecond\nthird\n").children # third's text (line 3) cannot execute after first (line 1) yet before # second (line 2): the proc's text IS its assignment. reordered = run_after([first, second, third], run: [third], after: first) - error = assert_raises(ThunkPlacementError) { emit(s(:begin, *reordered)) } + error = assert_raises(ThunkLowering::PlacementError) { emit(s(:begin, *reordered)) } assert_includes error.message, 'fall after its execution point' end - test "occurrences of one thunk with diverging bodies raise ThunkPlacementError" do + test "occurrences of one thunk with diverging bodies raise ThunkLowering::PlacementError" do original = thunk(parse("foo\n")) diverged = original.updated(nil, [original.id, parse("bar\n")]) - error = assert_raises(ThunkPlacementError) { emit(s(:begin, original, diverged)) } + error = assert_raises(ThunkLowering::PlacementError) { emit(s(:begin, original, diverged)) } assert_includes error.message, 'diverging' end @@ -107,8 +107,8 @@ def run_as_method(emitted) assert_includes emitted, '__ast_thunk_2__ = proc', emitted end - test "an unlowered custom node type raises UnloweredNodeTypeError" do - error = assert_raises(UnloweredNodeTypeError) do + test "an unlowered custom node type raises LineAlignedEmitter::UnloweredNodeTypeError" do + error = assert_raises(LineAlignedEmitter::UnloweredNodeTypeError) do emit(s(:begin, s(:ast_transform_emitter_test_custom))) end diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index 0b94eda..06b8684 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -44,8 +44,8 @@ def parse(source) assert_equal anchor.loc.expression, node.loc.expression end - test "s_at raises MissingLocationError for loc-less anchors" do - error = assert_raises(MissingLocationError) { s_at(s(:send, nil, :foo), :send, nil, :bar) } + test "s_at raises TransformationHelper::MissingLocationError for loc-less anchors" do + error = assert_raises(TransformationHelper::MissingLocationError) { s_at(s(:send, nil, :foo), :send, nil, :bar) } assert_includes error.message, 'send' end @@ -62,13 +62,13 @@ def parse(source) end test "a Thunk without an id cannot be constructed" do - error = assert_raises(MalformedThunkError) { s(:ast_thunk, parse("foo\n")) } + error = assert_raises(Thunk::MalformedError) { s(:ast_thunk, parse("foo\n")) } assert_includes error.message, 'Thunk::Id' end test "a Thunk with an empty body cannot be constructed" do - error = assert_raises(MalformedThunkError) { thunk } + error = assert_raises(Thunk::MalformedError) { thunk } assert_includes error.message, 'at least one statement' end @@ -80,7 +80,7 @@ def parse(source) assert_instance_of Thunk, rebuilt assert_same node.id, rebuilt.id - assert_raises(MalformedThunkError) { node.updated(nil, [node.id]) } + assert_raises(Thunk::MalformedError) { node.updated(nil, [node.id]) } end test "AbstractTransformation descends thunk bodies by default" do From 4da21d2f0be1c986b96f1201f0883f67c54e5682 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 11:09:18 -0400 Subject: [PATCH 21/22] Raise ArgumentError for Thunk construction invariants Constructing a Thunk with wrong arguments is exactly what ArgumentError is for; a custom MalformedError class added ceremony without meaning. Matches run_after, which already raises ArgumentError for bad arguments. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- README.md | 2 +- lib/ast_transform/thunk.rb | 9 ++------- test/ast_transform/transformation_helper_test.rb | 6 +++--- 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ded249..91e5e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `thunk` (a single invariant-checked `Thunk` node spliced at the execution point; the lowering derives the hidden proc's textual placement from the body's source locations), and `run_after` (sequence-level execution reordering that preserves textual/source order). Thunks lower to a non-lambda proc, so `return` still returns from the enclosing method, and locals assigned by thunked statements are pre-declared to stay method-scope. Reusing one thunk node executes its body from several points. - `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`LineAlignedEmitter::UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. - `ast_transform/test_helpers` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. -- Error types, each owned by its producer: `TransformationHelper::MissingLocationError`, `Thunk::MalformedError`, `ThunkLowering::PlacementError`, `LineAlignedEmitter::UnloweredNodeTypeError`. +- Error types, each owned by its producer: `TransformationHelper::MissingLocationError`, `ThunkLowering::PlacementError`, `LineAlignedEmitter::UnloweredNodeTypeError`. Thunk construction invariants raise plain `ArgumentError`. ### Removed - **Breaking:** `ASTTransform::SourceMap` and source-map registration. Line-aligned emission makes raw VM line numbers the source line numbers, so there is nothing left to map at display time. diff --git a/README.md b/README.md index 77884a3..55bc5cd 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,7 @@ ASTTransform owns text and lines; transform authors own semantics and execution * `s(type, *children)` — builds a loc-less (synthetic) node. Registered custom types (see below) construct their registered class. * `s_at(anchor, type, *children)` — builds a node anchored at `anchor`'s source location, so it is emitted at `anchor`'s line. Raises `TransformationHelper::MissingLocationError` if the anchor has no location. -* `thunk(*statements)` — wraps statements in a single `Thunk` node: splice it wherever the statements must *run*, in statement position or composed inside an expression (e.g. an `assert_raises` block body). The wrapped statements keep their own locations, and the lowering derives the hidden proc's textual placement from them — the body still emits on its source lines even though execution waits. Reuse the same node to execute one body from several points. Thunk construction is invariant-checked (`Thunk::MalformedError`); a body whose source lines fall after its execution point fails lowering with `ThunkLowering::PlacementError` (a thunk can only delay execution, never text). +* `thunk(*statements)` — wraps statements in a single `Thunk` node: splice it wherever the statements must *run*, in statement position or composed inside an expression (e.g. an `assert_raises` block body). The wrapped statements keep their own locations, and the lowering derives the hidden proc's textual placement from them — the body still emits on its source lines even though execution waits. Reuse the same node to execute one body from several points. Thunk construction is invariant-checked (`ArgumentError` on a missing id or empty body); a body whose source lines fall after its execution point fails lowering with `ThunkLowering::PlacementError` (a thunk can only delay execution, never text). * `run_after(statements, run:, after:)` — the paved road over `thunk`: returns a reordered copy of `statements` where the contiguous `run` executes after `after`, while remaining at its source position textually. Elements are matched by object identity. Thunked statements keep their original meaning as far as Ruby's closure semantics allow: diff --git a/lib/ast_transform/thunk.rb b/lib/ast_transform/thunk.rb index 1a7756e..0dff280 100644 --- a/lib/ast_transform/thunk.rb +++ b/lib/ast_transform/thunk.rb @@ -25,19 +25,14 @@ class Thunk < Node # No behavior — a named class over a bare Object.new only for self-documenting AST dumps and greppability. class Id; end - # Raised at construction when a Thunk node's children violate its invariants (missing id, empty body). Every - # construction path funnels through Thunk#initialize — including Processor rebuilds — so a malformed thunk - # cannot exist in a tree. - class MalformedError < StandardError; end - def initialize(type, children, properties = {}) id, *body = children unless id.is_a?(ASTTransform::Thunk::Id) - raise MalformedError, + raise ArgumentError, "a Thunk's first child must be its #{Thunk::Id} (got #{id.class}); build thunks with the " \ "thunk(*statements) helper" end - raise MalformedError, "a Thunk must wrap at least one statement" if body.empty? + raise ArgumentError, "a Thunk must wrap at least one statement" if body.empty? # Captured before super (which freezes the node); frozen so the shared array cannot be mutated out from under # +children+. Rebuilds via +updated+ re-run initialize, so the capture can never go stale. diff --git a/test/ast_transform/transformation_helper_test.rb b/test/ast_transform/transformation_helper_test.rb index 06b8684..b9c4a74 100644 --- a/test/ast_transform/transformation_helper_test.rb +++ b/test/ast_transform/transformation_helper_test.rb @@ -62,13 +62,13 @@ def parse(source) end test "a Thunk without an id cannot be constructed" do - error = assert_raises(Thunk::MalformedError) { s(:ast_thunk, parse("foo\n")) } + error = assert_raises(ArgumentError) { s(:ast_thunk, parse("foo\n")) } assert_includes error.message, 'Thunk::Id' end test "a Thunk with an empty body cannot be constructed" do - error = assert_raises(Thunk::MalformedError) { thunk } + error = assert_raises(ArgumentError) { thunk } assert_includes error.message, 'at least one statement' end @@ -80,7 +80,7 @@ def parse(source) assert_instance_of Thunk, rebuilt assert_same node.id, rebuilt.id - assert_raises(Thunk::MalformedError) { node.updated(nil, [node.id]) } + assert_raises(ArgumentError) { node.updated(nil, [node.id]) } end test "AbstractTransformation descends thunk bodies by default" do From 0609d420d528e18c413aeac3ce52501a79c29a30 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Date: Fri, 24 Jul 2026 11:35:32 -0400 Subject: [PATCH 22/22] Namespace consumer test assertions under ast_transform/testing TestHelpers becomes ASTTransform::Testing::Assertions (ast_transform/testing/assertions), following the active_support/testing shape: the testing namespace marks the consumer-facing test-only surface and leaves room for future framework-specific entries. Co-authored-by: Cursor --- CHANGELOG.md | 2 +- README.md | 2 +- lib/ast_transform/test_helpers.rb | 95 ------------------ lib/ast_transform/testing/assertions.rb | 97 +++++++++++++++++++ .../assertions_test.rb} | 8 +- 5 files changed, 103 insertions(+), 101 deletions(-) delete mode 100644 lib/ast_transform/test_helpers.rb create mode 100644 lib/ast_transform/testing/assertions.rb rename test/ast_transform/{test_helpers_test.rb => testing/assertions_test.rb} (91%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91e5e28..afbacba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Line-aligned emission: transformed code is emitted with every loc-carrying statement on its original source line, making backtraces, breakpoints, and debugger display correct by construction (`LineAlignedEmitter`). - Authoring toolkit in `TransformationHelper`: `s_at` (loc-anchored node construction), `thunk` (a single invariant-checked `Thunk` node spliced at the execution point; the lowering derives the hidden proc's textual placement from the body's source locations), and `run_after` (sequence-level execution reordering that preserves textual/source order). Thunks lower to a non-lambda proc, so `return` still returns from the enclosing method, and locals assigned by thunked statements are pre-declared to stay method-scope. Reusing one thunk node executes its body from several points. - `ASTTransform::Node.register`: type-routed construction of custom IR node classes through `s`, with an emitter postcondition (`LineAlignedEmitter::UnloweredNodeTypeError`) rejecting custom types that were not lowered before emission. -- `ast_transform/test_helpers` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. +- `ast_transform/testing/assertions` (test-only): `assert_line_aligned` and `assert_backtrace_lines` for transform authors' suites. - Error types, each owned by its producer: `TransformationHelper::MissingLocationError`, `ThunkLowering::PlacementError`, `LineAlignedEmitter::UnloweredNodeTypeError`. Thunk construction invariants raise plain `ArgumentError`. ### Removed diff --git a/README.md b/README.md index 55bc5cd..a505849 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ Custom node types are IR **between stages that understand them** — the stage t #### Testing your transformation -`require 'ast_transform/test_helpers'` (test-only) provides: +`require 'ast_transform/testing/assertions'` (test-only) provides `ASTTransform::Testing::Assertions`, a Minitest-flavored module to include in your test class: * `assert_line_aligned(source, *transformations)` — transforms `source` through the real pipeline and asserts every surviving statement is emitted at its source line. * `assert_backtrace_lines(source, path:, raise_at:)` — compiles and executes `source`, asserting the raw first backtrace frame cites `path:raise_at` with no filtering. diff --git a/lib/ast_transform/test_helpers.rb b/lib/ast_transform/test_helpers.rb deleted file mode 100644 index 470dab6..0000000 --- a/lib/ast_transform/test_helpers.rb +++ /dev/null @@ -1,95 +0,0 @@ -# frozen_string_literal: true - -require 'ast_transform/transformer' -require 'ast_transform/instruction_sequence' - -module ASTTransform - # Assertions for transform authors' own test suites — the enforcement arm of the authoring contract ("textual - # order is source order"). Never loaded in production; require it from test code: - # - # require "ast_transform/test_helpers" - # - # class MyTransformationTest < Minitest::Test - # include ASTTransform::TestHelpers - # end - module TestHelpers - # Transforms +source+ through the real pipeline (transform + line-aligned emission), re-parses both sides, - # matches surviving statements by location, and asserts each one's emitted line equals its source line. - # Statements the transform deletes (e.g. description strings) are exempt; statements the transform rewrites in - # place keep their anchor and are checked. - # - # @param source [String] fixture source - # @param transformations [Array] - # @param path [String] pseudo-path used for parsing and messages - # @return [void] - def assert_line_aligned(source, *transformations, path: 'fixture.rb') - transformer = Transformer.new(*transformations) - emitted = transformer.transform_file_source(source, path, path) - - source_lines_by_statement = statement_lines(transformer.build_ast(source, file_path: path)) - emitted_lines_by_statement = statement_lines(transformer.build_ast(emitted, file_path: path)) - - misaligned = source_lines_by_statement.filter_map do |render, source_line| - emitted_line = emitted_lines_by_statement[render] - next if emitted_line.nil? || emitted_line == source_line - - format(' MISALIGNED %s: source line %d, emitted line %d', render, source_line, emitted_line) - end - - assert misaligned.empty?, <<~MESSAGE - expected every surviving statement at its source line in #{path}: - #{misaligned.join("\n")} - - emitted: - #{numbered_listing(emitted)} - MESSAGE - end - - # Runtime complement of assert_line_aligned: compiles +source+ through the full pipeline under +path+, executes - # it, and asserts the raw first backtrace frame — no filtering of any kind — is ":". - # - # @param source [String] fixture that raises when executed - # @param path [String] pseudo source path to compile under - # @param raise_at [Integer] expected source line of the raise - # @return [void] - def assert_backtrace_lines(source, path:, raise_at:) - iseq = InstructionSequence.source_to_transformed_iseq(source, path) - - error = assert_raises(StandardError, "fixture at #{path} should raise when executed") do - iseq.eval - end - - location = error.backtrace_locations.first - assert_equal "#{location.path}:#{raise_at}", "#{location.path}:#{location.lineno}", - "raw backtrace should cite source line #{raise_at} of #{path}" - end - - private - - # Flat statement renders and their first line, keyed by unparsed text so source and emitted sides can be matched - # without location identity. Duplicate renders keep their first occurrence — good enough for fixtures, which - # authors control. - def statement_lines(ast, lines = {}) - return lines unless ast.is_a?(::Parser::AST::Node) - - if statement_sequence?(ast) - ast.children.each do |statement| - next unless statement.is_a?(::Parser::AST::Node) && statement.loc&.expression - - lines[Unparser.unparse(statement)] ||= statement.loc.line - end - end - - ast.children.each { |child| statement_lines(child, lines) } - lines - end - - def statement_sequence?(node) - [:begin, :kwbegin].include?(node.type) - end - - def numbered_listing(source) - source.lines.map.with_index(1) { |line, number| format('%3d| %s', number, line) }.join - end - end -end diff --git a/lib/ast_transform/testing/assertions.rb b/lib/ast_transform/testing/assertions.rb new file mode 100644 index 0000000..adefdf9 --- /dev/null +++ b/lib/ast_transform/testing/assertions.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'ast_transform/transformer' +require 'ast_transform/instruction_sequence' + +module ASTTransform + module Testing + # Assertions for transform authors' own test suites — the enforcement arm of the authoring contract ("textual + # order is source order"). Minitest-flavored. Never loaded in production; require it from test code: + # + # require "ast_transform/testing/assertions" + # + # class MyTransformationTest < Minitest::Test + # include ASTTransform::Testing::Assertions + # end + module Assertions + # Transforms +source+ through the real pipeline (transform + line-aligned emission), re-parses both sides, + # matches surviving statements by location, and asserts each one's emitted line equals its source line. + # Statements the transform deletes (e.g. description strings) are exempt; statements the transform rewrites in + # place keep their anchor and are checked. + # + # @param source [String] fixture source + # @param transformations [Array] + # @param path [String] pseudo-path used for parsing and messages + # @return [void] + def assert_line_aligned(source, *transformations, path: 'fixture.rb') + transformer = Transformer.new(*transformations) + emitted = transformer.transform_file_source(source, path, path) + + source_lines_by_statement = statement_lines(transformer.build_ast(source, file_path: path)) + emitted_lines_by_statement = statement_lines(transformer.build_ast(emitted, file_path: path)) + + misaligned = source_lines_by_statement.filter_map do |render, source_line| + emitted_line = emitted_lines_by_statement[render] + next if emitted_line.nil? || emitted_line == source_line + + format(' MISALIGNED %s: source line %d, emitted line %d', render, source_line, emitted_line) + end + + assert misaligned.empty?, <<~MESSAGE + expected every surviving statement at its source line in #{path}: + #{misaligned.join("\n")} + + emitted: + #{numbered_listing(emitted)} + MESSAGE + end + + # Runtime complement of assert_line_aligned: compiles +source+ through the full pipeline under +path+, executes + # it, and asserts the raw first backtrace frame — no filtering of any kind — is ":". + # + # @param source [String] fixture that raises when executed + # @param path [String] pseudo source path to compile under + # @param raise_at [Integer] expected source line of the raise + # @return [void] + def assert_backtrace_lines(source, path:, raise_at:) + iseq = InstructionSequence.source_to_transformed_iseq(source, path) + + error = assert_raises(StandardError, "fixture at #{path} should raise when executed") do + iseq.eval + end + + location = error.backtrace_locations.first + assert_equal "#{location.path}:#{raise_at}", "#{location.path}:#{location.lineno}", + "raw backtrace should cite source line #{raise_at} of #{path}" + end + + private + + # Flat statement renders and their first line, keyed by unparsed text so source and emitted sides can be matched + # without location identity. Duplicate renders keep their first occurrence — good enough for fixtures, which + # authors control. + def statement_lines(ast, lines = {}) + return lines unless ast.is_a?(::Parser::AST::Node) + + if statement_sequence?(ast) + ast.children.each do |statement| + next unless statement.is_a?(::Parser::AST::Node) && statement.loc&.expression + + lines[Unparser.unparse(statement)] ||= statement.loc.line + end + end + + ast.children.each { |child| statement_lines(child, lines) } + lines + end + + def statement_sequence?(node) + [:begin, :kwbegin].include?(node.type) + end + + def numbered_listing(source) + source.lines.map.with_index(1) { |line, number| format('%3d| %s', number, line) }.join + end + end + end +end diff --git a/test/ast_transform/test_helpers_test.rb b/test/ast_transform/testing/assertions_test.rb similarity index 91% rename from test/ast_transform/test_helpers_test.rb rename to test/ast_transform/testing/assertions_test.rb index a411016..f94a15b 100644 --- a/test/ast_transform/test_helpers_test.rb +++ b/test/ast_transform/testing/assertions_test.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true require 'test_helper' -require 'ast_transform/test_helpers' +require 'ast_transform/testing/assertions' require 'ast_transform/abstract_transformation' module ASTTransform - class TestHelpersTest < Minitest::Test + class AssertionsTest < Minitest::Test extend ASTTransform::Declarative - include ASTTransform::TestHelpers + include ASTTransform::Testing::Assertions # Rewrites statements in place (keeps anchors) — always aligned. class InPlaceTransformation < ASTTransform::AbstractTransformation @@ -62,7 +62,7 @@ def process_node(node) raise "expected boom" if value == 1 HEREDOC - assert_backtrace_lines(source, path: File.expand_path('tmp/test/helpers_fixture.rb'), raise_at: 3) + assert_backtrace_lines(source, path: File.expand_path('tmp/test/assertions_fixture.rb'), raise_at: 3) end end end