Transpose/Test: Add support for compact transposition output matrices and a working verilator flow

.github/workflows/deploy.yml

@@ -15,6 +15,11 @@ on:
         required: false
         type: string
         default: "rw_axi r_obi_w_axi r_axi_w_obi rw_axi_rw_axis rw_obi r_obi_rw_init_w_axi r_axi_rw_init_rw_obi rw_axi_rw_init_rw_obi"
+      IDMA_ADD_FE_IDS:
+        description: Register frontend variants to generate.
+        required: false
+        type: string
+        default: "reg32_4d reg64_4d"
       IDMA_REG_CPUIF:
         description: PeakRDL CPU interface used for generated register frontends.
         required: false
@@ -27,6 +32,11 @@ on:
         required: false
         type: string
         default: "rw_axi r_obi_w_axi r_axi_w_obi rw_axi_rw_axis rw_obi r_obi_rw_init_w_axi r_axi_rw_init_rw_obi rw_axi_rw_init_rw_obi"
+      IDMA_ADD_FE_IDS:
+        description: Register frontend variants to generate.
+        required: false
+        type: string
+        default: "reg32_4d reg64_4d"
       IDMA_REG_CPUIF:
         description: PeakRDL CPU interface used for generated register frontends.
         required: false
@@ -63,7 +73,8 @@ jobs:
       run: |
         uv run --locked make -B idma_hw_all idma_sw_all \
           IDMA_BACKEND_IDS="${{ inputs.IDMA_BACKEND_IDS }}" \
-          IDMA_REG_CPUIF="${{ inputs.IDMA_REG_CPUIF }}"
+          IDMA_REG_CPUIF="${{ inputs.IDMA_REG_CPUIF }}" \
+          IDMA_ADD_FE_IDS="${{ inputs.IDMA_ADD_FE_IDS }}"
     -
       name: Deploy generated files
       run: |

.gitignore

@@ -11,4 +11,6 @@ working_dir
 bender
 todo
 .venv
-modelsim.ini
+modelsim.ini
+# for emacs enjoyers
+**/*.~undo-tree~

Bender.yml

@@ -68,6 +68,7 @@ sources:
       - src/midend/idma_nd_midend.sv
       - src/midend/idma_rt_midend.sv
       - src/midend/idma_transpose_midend.sv
+      - src/midend/idma_transpose_req_replay.sv
 
   # RISC-V opcode package for ooc use of inst64
   - target: all(rtl,snitch_cluster)
@@ -133,8 +134,10 @@ sources:
   - target: idma_test
     files:
       - target/rtl/tb_idma_generated.sv
+      - test/tb_idma_otf_transpose.sv
       - test/tb_idma_transpose_nd.sv
       - test/tb_idma_transpose_b2b.sv
+      - test/idma_transpose_dpi.c
 
   # Multi-head directed backend testbenches
   - target: multihead

idma.mk

@@ -500,31 +500,8 @@ idma_vcs_clean:
 # Verilator
 # --------------
 
-.PHONY: idma_verilator_clean
-
-IDMA_VLT_DIR   := $(IDMA_ROOT)/target/sim/verilator
-IDMA_VLT_ARGS  := --cc \
-				  --Wall \
-				  --Wno-fatal \
-				  +1800-2017ext+ \
-				  --assert \
-				  --error-limit 1000 \
-				  --hierarchical \
-				  --no-skip-identical
-
-IDMA_VLT_TOP     ?=
-IDMA_VLT_PARAMS  ?=
-
-.PRECIOUS: $(IDMA_VLT_DIR)/%_elab.log
-
-$(IDMA_VLT_DIR)/%_elab.log: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL)
-	mkdir -p $(IDMA_VLT_DIR)
-	# We need a dedicated pickle here to set the defines
-	$(BENDER) pickle $(IDMA_PICKLE_TARGETS) --top $(IDMA_VLT_TOP) -D VERILATOR --expand-macros -o $(IDMA_VLT_DIR)/$(IDMA_VLT_TOP).sv
-	cd $(IDMA_VLT_DIR); $(VERILATOR) $(IDMA_VLT_ARGS) $(IDMA_VLT_PARAMS) -Mdir obj_$* $(IDMA_VLT_TOP).sv --top-module $(IDMA_VLT_TOP) 2> $*_elab.log
-
-idma_verilator_clean:
-	rm -rf $(IDMA_VLT_DIR)
+include $(IDMA_ROOT)/target/sim/vlt/testbenches.mk
+include $(IDMA_ROOT)/target/sim/vlt/vlt.mk
 
 
 # ---------------

src/backend/idma_otf_transpose.sv

@@ -27,7 +27,7 @@ module idma_otf_transpose #(
   input  logic rst_ni,
   input  logic clear_i,
 
-  /// Element size select: 0->1B, 1->2B, 2->4B (E = 1<<transp_mode_i)
+  /// Element size select: 0->1B, 1->2B, 2->4B, 3->8B (E = 1<<transp_mode_i)
   input  logic [1:0]  transp_mode_i,
   /// Matrix dimensions in elements
   input  logic [DimWidth-1:0] tensor_size_m_i,

src/frontend/reg/idma_reg.rdl

@@ -151,6 +151,31 @@ addrmap idma_reg #(
         } reps [31:0] = 0;
     };
 
+    reg compute_cfg {
+        name = "compute_cfg";
+        desc = "Per-transfer on-the-fly compute configuration. The fields are sampled when next_id is read to launch a transfer.";
+        default sw = rw;
+        default hw = r;
+        field {
+            desc = "Enable on-the-fly compute for the launched transfer.";
+        } compute_enable [0:0] = 0;
+        field {
+            desc = "Compute operation selector. Currently, 1 selects transpose.";
+        } compute_op [4:1] = 0;
+        field {
+            desc = "Transpose element-size mode. The element size is 1 << transpose_mode bytes, encoding 8b, 16b, 32b, and 64b elements.";
+        } transpose_mode [6:5] = 0;
+        field {
+            desc = "Transpose tensor M dimension in elements. Must be non-zero when transpose is enabled.";
+        } transpose_tensor_m [18:7] = 0;
+        field {
+            desc = "Transpose tensor N dimension in elements. Must be non-zero when transpose is enabled.";
+        } transpose_tensor_n [30:19] = 0;
+        field {
+            desc = "Use compact row-major output storage. When clear, transpose uses the legacy tile-padded layout.";
+        } transpose_compact [31:31] = 0;
+    };
+
     regfile dim {
         dst_stride dst_stride[SysAddrWidth/32];
         src_stride src_stride[SysAddrWidth/32];
@@ -166,6 +191,7 @@ addrmap idma_reg #(
              src_addr src_addr[SysAddrWidth/32];
              length   length[SysAddrWidth/32];
              dim      dim[NumDims-1 | NumDims == 1];
+             compute_cfg compute_cfg;
 };
 
 `endif // IDMA_REG_RDL

src/frontend/reg/tpl/idma_reg.sv.tpl

@@ -242,6 +242,22 @@ module idma_${identifier} #(
       nxt_dma_req${sep}opt.beo.src_reduce_len = dma_reg2hw[i].conf.src_reduce_len.value;
       nxt_dma_req${sep}opt.beo.dst_reduce_len = dma_reg2hw[i].conf.dst_reduce_len.value;
 
+      // Optional on-the-fly compute settings are part of the transfer descriptor and
+      // are captured together with the address/stride fields when next_id is read.
+      nxt_dma_req${sep}opt.compute.enable                    =
+          dma_reg2hw[i].compute_cfg.compute_enable.value;
+      nxt_dma_req${sep}opt.compute.op                        =
+          idma_pkg::compute_op_e'(dma_reg2hw[i].compute_cfg.compute_op.value);
+      nxt_dma_req${sep}opt.compute.params.transpose.mode     =
+          dma_reg2hw[i].compute_cfg.transpose_mode.value;
+      nxt_dma_req${sep}opt.compute.params.transpose.tensor_m =
+          dma_reg2hw[i].compute_cfg.transpose_tensor_m.value;
+      nxt_dma_req${sep}opt.compute.params.transpose.tensor_n =
+          dma_reg2hw[i].compute_cfg.transpose_tensor_n.value;
+      // Compact mode removes tile padding from destination rows.
+      nxt_dma_req${sep}opt.compute.params.transpose.compact  =
+          dma_reg2hw[i].compute_cfg.transpose_compact.value;
+
 % if num_dim != 1:
       // ND connections
 % for nd in range(0, num_dim-1):

src/idma_pkg.sv

@@ -90,10 +90,21 @@ package idma_pkg;
     /// Transpose tensor dimension width (elements)
     localparam int unsigned TransposeDimWidth = 32'd12;
 
-    /// Transpose options (E = 1<<mode: 1/2/4 B)
+    /// Transpose options.
+    ///
+    /// `E` denotes the width of one matrix element in bytes. The encoded
+    /// element width is `E = 1 << mode`, selecting 1, 2, 4, or 8 byte
+    /// elements. A transpose tile therefore contains `StrbWidth / E`
+    /// elements along each side, where `StrbWidth` is the datapath width in
+    /// bytes.
     typedef struct packed {
+        /// Store output rows without tile padding.
+        logic                         compact;
+        /// Base-2 logarithm of the element width in bytes (`E = 1 << mode`).
         logic [1:0]                   mode;
+        /// Number of rows in the source matrix, measured in elements.
         logic [TransposeDimWidth-1:0] tensor_m;
+        /// Number of columns in the source matrix, measured in elements.
         logic [TransposeDimWidth-1:0] tensor_n;
     } transpose_options_t;
 

src/midend/idma_transpose_midend.sv

@@ -49,7 +49,7 @@ module idma_transpose_midend #(
     always_comb begin : proc_expand
         logic [ModeW-1:0]        mode;
         logic [TensorW-1:0]      tm, tn;
-        logic signed [WorkW-1:0] m, n, log2ne, ne, yt, nt, nxe, mpe;
+        logic signed [WorkW-1:0] m, n, log2ne, ne, yt, nt, nxe, me, mpe, dst_row_bytes;
         logic signed [WorkW-1:0] strb_c;   // NE*E == StrbWidth (mode cancels)
 
         nd_req_o = nd_req_i;   // passthrough
@@ -66,24 +66,30 @@ module idma_transpose_midend #(
             yt     = (m + ne - 1) >>> log2ne;                  // ceil(M/NE)
             nt     = (n + ne - 1) >>> log2ne;                  // ceil(N/NE)
             nxe    = n  <<< mode;                              // N*E  (E = 1<<mode)
-            mpe    = yt <<< Log2Strb;                          // MP*E = YT*NE*E = YT*StrbWidth
+            me     = m  <<< mode;                              // M*E: compact destination row size
+            mpe    = yt <<< Log2Strb;                          // padded destination row size
+            // Compact rows can be unaligned to the datapath and may require split AXI beats.
+            // Legacy mode retains one complete tile-row slot per row tile.
+            dst_row_bytes = nd_req_i.burst_req.opt.compute.params.transpose.compact ? me : mpe;
             strb_c = $signed(WorkW'(StrbWidth));               // NE*E (one tile-row = StrbWidth B)
 
             nd_req_o.burst_req.length     = LenW'(StrbWidth);
 
             // d_req[0] = local row within tile (reps NE)
             nd_req_o.d_req[0].reps        = ne[RepW-1:0];
             nd_req_o.d_req[0].src_strides = addr_t'(nxe);
-            nd_req_o.d_req[0].dst_strides = addr_t'(mpe);
-            // d_req[1] = row-tile (reps YT). (NE-1)*MPE = (MPE<<log2ne) - MPE.
+            nd_req_o.d_req[0].dst_strides = addr_t'(dst_row_bytes);
+            // d_req[1] advances to the next row tile and rewinds the local output-row walk.
             nd_req_o.d_req[1].reps        = yt[RepW-1:0];
             nd_req_o.d_req[1].src_strides = addr_t'(nxe);
-            nd_req_o.d_req[1].dst_strides = addr_t'(strb_c - (mpe <<< log2ne) + mpe);
+            nd_req_o.d_req[1].dst_strides =
+                addr_t'(strb_c - (dst_row_bytes <<< log2ne) + dst_row_bytes);
             // d_req[2] = col-tile (reps NT). (YT*NE-1)*NXE = ((YT*N)<<Log2Strb) - NXE;
-            //            the dst rewind MPE-(YT-1)*StrbWidth collapses to StrbWidth.
+            //            destination output rows are dst_row_bytes apart.
             nd_req_o.d_req[2].reps        = nt[RepW-1:0];
             nd_req_o.d_req[2].src_strides = addr_t'(strb_c - ((yt * n) <<< Log2Strb) + nxe);
-            nd_req_o.d_req[2].dst_strides = addr_t'(strb_c);
+            nd_req_o.d_req[2].dst_strides =
+                addr_t'(dst_row_bytes - (yt <<< Log2Strb) + strb_c);
             // the walk is exactly 4-D: neutralize any higher dims
             for (int unsigned d = 3; d < NumDim-1; d++) begin
                 nd_req_o.d_req[d].reps        = RepW'(1);
@@ -105,11 +111,10 @@ module idma_transpose_midend #(
                RepW, (TensorW > Log2Strb+1) ? TensorW : Log2Strb+1);
     initial assert (LenW > Log2Strb) else
         $fatal(1, "idma_transpose_midend: length field %0d b cannot hold StrbWidth", LenW);
-    // reserved mode 3 (EB=8) and zero-size tensors are out of contract
+    // Zero-size tensors are out of contract.  Modes 0..3 select 1, 2, 4,
+    // and 8 byte elements; mode 3 degenerates to NE=1 for a 64-bit datapath.
     always_comb begin : check_domain
         if (is_transpose) begin
-            assert (nd_req_i.burst_req.opt.compute.params.transpose.mode != 2'd3) else
-                $error("idma_transpose_midend: reserved element mode 3 (EB=8)");
             assert (nd_req_i.burst_req.opt.compute.params.transpose.tensor_m != '0 &&
                     nd_req_i.burst_req.opt.compute.params.transpose.tensor_n != '0) else
                 $error("idma_transpose_midend: zero-size tensor (M or N == 0)");

src/midend/idma_transpose_req_replay.sv

@@ -0,0 +1,143 @@
+// Copyright 2026 Mosaic SoC AG
+// Solderpad Hardware License, Version 0.51, see LICENSE for details.
+// SPDX-License-Identifier: SHL-0.51
+
+// Authors:
+// - Georg Rutishauser <georg@mosaic-soc.com>
+
+/// Redirect requests for invalid transpose edge rows to known-mapped addresses.
+///
+/// The current transport architecture requires one paired read/write descriptor
+/// for every row of a complete square tile. Partial bottom and right tiles
+/// therefore still issue descriptors for nonexistent source and destination
+/// rows. This stage keeps the descriptor count unchanged, but replays the
+/// first source/destination address for those requests. The transpose engine's
+/// output strobe suppresses all data belonging to replayed requests.
+module idma_transpose_req_replay #(
+    /// Write datapath width in bytes.
+    parameter int unsigned StrbWidth = 32'd64,
+    /// 1D iDMA request type.
+    parameter type idma_req_t = logic
+) (
+    input  logic      clk_i,
+    input  logic      rst_ni,
+
+    input  idma_req_t req_i,
+    input  logic      valid_i,
+    output logic      ready_o,
+
+    output idma_req_t req_o,
+    output logic      valid_o,
+    input  logic      ready_i
+);
+
+    localparam int unsigned Log2Strb = $clog2(StrbWidth);
+    localparam int unsigned LocalW   = Log2Strb;
+    localparam int unsigned ModeW    = $bits(req_i.opt.compute.params.transpose.mode);
+    localparam int unsigned TensorW  = $bits(req_i.opt.compute.params.transpose.tensor_m);
+    localparam int unsigned AddrW    = $bits(req_i.src_addr);
+    localparam int unsigned WorkW    = (TensorW > LocalW) ? TensorW + 1 : LocalW + 1;
+
+    typedef logic [AddrW-1:0] addr_t;
+
+    logic [LocalW-1:0]  local_idx_q;
+    logic [TensorW-1:0] row_tile_idx_q, col_tile_idx_q;
+    addr_t safe_src_addr_q, safe_dst_addr_q;
+
+    logic is_transpose;
+    logic req_handshake;
+    logic first_request, last_request;
+    logic invalid_read, invalid_write;
+    logic [WorkW-1:0] ne, ne_m1, row_tiles, col_tiles;
+    logic [WorkW-1:0] remaining_rows, remaining_cols;
+
+    assign is_transpose = req_i.opt.compute.enable &
+                          (req_i.opt.compute.op == idma_pkg::COMPUTE_TRANSPOSE);
+    assign req_handshake = valid_o & ready_i;
+    assign first_request = (local_idx_q == '0) && (row_tile_idx_q == '0) &&
+                           (col_tile_idx_q == '0);
+
+    // Tile dimensions are powers of two, so edge classification only needs
+    // shifts and masks. The request carries the original tensor dimensions.
+    always_comb begin : proc_geometry
+        logic [ModeW-1:0] mode;
+        logic [WorkW-1:0] rows, cols;
+        logic [WorkW-1:0] log2_ne;
+
+        mode    = req_i.opt.compute.params.transpose.mode;
+        rows    = WorkW'(req_i.opt.compute.params.transpose.tensor_m);
+        cols    = WorkW'(req_i.opt.compute.params.transpose.tensor_n);
+        log2_ne = WorkW'(Log2Strb) - WorkW'(mode);
+        ne      = WorkW'(1) << log2_ne;
+        ne_m1   = ne - 1'b1;
+        row_tiles = (rows + ne_m1) >> log2_ne;
+        col_tiles = (cols + ne_m1) >> log2_ne;
+        remaining_rows = rows & ne_m1;
+        remaining_cols = cols & ne_m1;
+    end
+
+    // Invalid edge requests do not contribute transpose data in either output
+    // layout. Replaying them to known mapped addresses avoids unsafe accesses
+    // without changing the generated data or strobes.
+    assign invalid_read = is_transpose && (remaining_rows != '0) &&
+                          (WorkW'(row_tile_idx_q) == row_tiles - 1'b1) &&
+                          (WorkW'(local_idx_q) >= remaining_rows);
+    assign invalid_write = is_transpose && (remaining_cols != '0) &&
+                           (WorkW'(col_tile_idx_q) == col_tiles - 1'b1) &&
+                           (WorkW'(local_idx_q) >= remaining_cols);
+    assign last_request = is_transpose &&
+                          (WorkW'(local_idx_q) == ne - 1'b1) &&
+                          (WorkW'(row_tile_idx_q) == row_tiles - 1'b1) &&
+                          (WorkW'(col_tile_idx_q) == col_tiles - 1'b1);
+
+    // The stage is transparent to handshaking and all non-address payload.
+    always_comb begin : proc_replay
+        req_o = req_i;
+        if (invalid_read) begin
+            req_o.src_addr = safe_src_addr_q;
+        end
+        if (invalid_write) begin
+            req_o.dst_addr = safe_dst_addr_q;
+        end
+    end
+    assign valid_o = valid_i;
+    assign ready_o = ready_i;
+
+    // Mirror the transpose ND walk: local row, row tile, then column tile.
+    always_ff @(posedge clk_i or negedge rst_ni) begin : proc_position
+        if (!rst_ni) begin
+            local_idx_q     <= '0;
+            row_tile_idx_q  <= '0;
+            col_tile_idx_q  <= '0;
+            safe_src_addr_q <= '0;
+            safe_dst_addr_q <= '0;
+        end else if (req_handshake && is_transpose) begin
+            if (first_request) begin
+                safe_src_addr_q <= req_i.src_addr;
+                safe_dst_addr_q <= req_i.dst_addr;
+            end
+
+            if (last_request) begin
+                local_idx_q    <= '0;
+                row_tile_idx_q <= '0;
+                col_tile_idx_q <= '0;
+            end else if (WorkW'(local_idx_q) == ne - 1'b1) begin
+                local_idx_q <= '0;
+                if (WorkW'(row_tile_idx_q) == row_tiles - 1'b1) begin
+                    row_tile_idx_q <= '0;
+                    col_tile_idx_q <= col_tile_idx_q + 1'b1;
+                end else begin
+                    row_tile_idx_q <= row_tile_idx_q + 1'b1;
+                end
+            end else begin
+                local_idx_q <= local_idx_q + 1'b1;
+            end
+        end
+    end
+
+`ifndef SYNTHESIS
+    initial assert (StrbWidth >= 2 && (StrbWidth & (StrbWidth - 1)) == 0) else
+        $fatal(1, "idma_transpose_req_replay: StrbWidth must be a power of two >= 2");
+`endif
+
+endmodule

target/sim/vlt/.gitignore

@@ -0,0 +1 @@
+build-*