diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 47b8d32f..2b9ed075 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,6 +15,11 @@ on: required: false type: string default: "rw_axi r_obi_w_axi r_axi_w_obi rw_axi_rw_axis rw_obi r_obi_rw_init_w_axi r_axi_rw_init_rw_obi rw_axi_rw_init_rw_obi" + IDMA_ADD_FE_IDS: + description: Register frontend variants to generate. + required: false + type: string + default: "reg32_4d reg64_4d" IDMA_REG_CPUIF: description: PeakRDL CPU interface used for generated register frontends. required: false @@ -27,6 +32,11 @@ on: required: false type: string default: "rw_axi r_obi_w_axi r_axi_w_obi rw_axi_rw_axis rw_obi r_obi_rw_init_w_axi r_axi_rw_init_rw_obi rw_axi_rw_init_rw_obi" + IDMA_ADD_FE_IDS: + description: Register frontend variants to generate. + required: false + type: string + default: "reg32_4d reg64_4d" IDMA_REG_CPUIF: description: PeakRDL CPU interface used for generated register frontends. required: false @@ -63,7 +73,8 @@ jobs: run: | uv run --locked make -B idma_hw_all idma_sw_all \ IDMA_BACKEND_IDS="${{ inputs.IDMA_BACKEND_IDS }}" \ - IDMA_REG_CPUIF="${{ inputs.IDMA_REG_CPUIF }}" + IDMA_REG_CPUIF="${{ inputs.IDMA_REG_CPUIF }}" \ + IDMA_ADD_FE_IDS="${{ inputs.IDMA_ADD_FE_IDS }}" - name: Deploy generated files run: | diff --git a/.gitignore b/.gitignore index cc9cb323..a85e4a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,6 @@ working_dir bender todo .venv -modelsim.ini \ No newline at end of file +modelsim.ini +# for emacs enjoyers +**/*.~undo-tree~ diff --git a/Bender.yml b/Bender.yml index a29c1eff..f3e20e9f 100644 --- a/Bender.yml +++ b/Bender.yml @@ -68,6 +68,7 @@ sources: - src/midend/idma_nd_midend.sv - src/midend/idma_rt_midend.sv - src/midend/idma_transpose_midend.sv + - src/midend/idma_transpose_req_replay.sv # RISC-V opcode package for ooc use of inst64 - target: all(rtl,snitch_cluster) @@ -133,8 +134,10 @@ sources: - target: idma_test files: - target/rtl/tb_idma_generated.sv + - test/tb_idma_otf_transpose.sv - test/tb_idma_transpose_nd.sv - test/tb_idma_transpose_b2b.sv + - test/idma_transpose_dpi.c # Multi-head directed backend testbenches - target: multihead diff --git a/idma.mk b/idma.mk index 7c763f21..7de96836 100644 --- a/idma.mk +++ b/idma.mk @@ -500,31 +500,8 @@ idma_vcs_clean: # Verilator # -------------- -.PHONY: idma_verilator_clean - -IDMA_VLT_DIR := $(IDMA_ROOT)/target/sim/verilator -IDMA_VLT_ARGS := --cc \ - --Wall \ - --Wno-fatal \ - +1800-2017ext+ \ - --assert \ - --error-limit 1000 \ - --hierarchical \ - --no-skip-identical - -IDMA_VLT_TOP ?= -IDMA_VLT_PARAMS ?= - -.PRECIOUS: $(IDMA_VLT_DIR)/%_elab.log - -$(IDMA_VLT_DIR)/%_elab.log: $(IDMA_BENDER_FILES) $(IDMA_FULL_TB) $(IDMA_FULL_RTL) $(IDMA_INCLUDE_ALL) - mkdir -p $(IDMA_VLT_DIR) - # We need a dedicated pickle here to set the defines - $(BENDER) pickle $(IDMA_PICKLE_TARGETS) --top $(IDMA_VLT_TOP) -D VERILATOR --expand-macros -o $(IDMA_VLT_DIR)/$(IDMA_VLT_TOP).sv - cd $(IDMA_VLT_DIR); $(VERILATOR) $(IDMA_VLT_ARGS) $(IDMA_VLT_PARAMS) -Mdir obj_$* $(IDMA_VLT_TOP).sv --top-module $(IDMA_VLT_TOP) 2> $*_elab.log - -idma_verilator_clean: - rm -rf $(IDMA_VLT_DIR) +include $(IDMA_ROOT)/target/sim/vlt/testbenches.mk +include $(IDMA_ROOT)/target/sim/vlt/vlt.mk # --------------- diff --git a/src/backend/idma_otf_transpose.sv b/src/backend/idma_otf_transpose.sv index 2eccb48e..4452a3ce 100644 --- a/src/backend/idma_otf_transpose.sv +++ b/src/backend/idma_otf_transpose.sv @@ -27,7 +27,7 @@ module idma_otf_transpose #( input logic rst_ni, input logic clear_i, - /// Element size select: 0->1B, 1->2B, 2->4B (E = 1<1B, 1->2B, 2->4B, 3->8B (E = 1<>> log2ne; // ceil(M/NE) nt = (n + ne - 1) >>> log2ne; // ceil(N/NE) nxe = n <<< mode; // N*E (E = 1< Log2Strb+1) ? TensorW : Log2Strb+1); initial assert (LenW > Log2Strb) else $fatal(1, "idma_transpose_midend: length field %0d b cannot hold StrbWidth", LenW); - // reserved mode 3 (EB=8) and zero-size tensors are out of contract + // Zero-size tensors are out of contract. Modes 0..3 select 1, 2, 4, + // and 8 byte elements; mode 3 degenerates to NE=1 for a 64-bit datapath. always_comb begin : check_domain if (is_transpose) begin - assert (nd_req_i.burst_req.opt.compute.params.transpose.mode != 2'd3) else - $error("idma_transpose_midend: reserved element mode 3 (EB=8)"); assert (nd_req_i.burst_req.opt.compute.params.transpose.tensor_m != '0 && nd_req_i.burst_req.opt.compute.params.transpose.tensor_n != '0) else $error("idma_transpose_midend: zero-size tensor (M or N == 0)"); diff --git a/src/midend/idma_transpose_req_replay.sv b/src/midend/idma_transpose_req_replay.sv new file mode 100644 index 00000000..73a0fd1b --- /dev/null +++ b/src/midend/idma_transpose_req_replay.sv @@ -0,0 +1,143 @@ +// Copyright 2026 Mosaic SoC AG +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 + +// Authors: +// - Georg Rutishauser + +/// Redirect requests for invalid transpose edge rows to known-mapped addresses. +/// +/// The current transport architecture requires one paired read/write descriptor +/// for every row of a complete square tile. Partial bottom and right tiles +/// therefore still issue descriptors for nonexistent source and destination +/// rows. This stage keeps the descriptor count unchanged, but replays the +/// first source/destination address for those requests. The transpose engine's +/// output strobe suppresses all data belonging to replayed requests. +module idma_transpose_req_replay #( + /// Write datapath width in bytes. + parameter int unsigned StrbWidth = 32'd64, + /// 1D iDMA request type. + parameter type idma_req_t = logic +) ( + input logic clk_i, + input logic rst_ni, + + input idma_req_t req_i, + input logic valid_i, + output logic ready_o, + + output idma_req_t req_o, + output logic valid_o, + input logic ready_i +); + + localparam int unsigned Log2Strb = $clog2(StrbWidth); + localparam int unsigned LocalW = Log2Strb; + localparam int unsigned ModeW = $bits(req_i.opt.compute.params.transpose.mode); + localparam int unsigned TensorW = $bits(req_i.opt.compute.params.transpose.tensor_m); + localparam int unsigned AddrW = $bits(req_i.src_addr); + localparam int unsigned WorkW = (TensorW > LocalW) ? TensorW + 1 : LocalW + 1; + + typedef logic [AddrW-1:0] addr_t; + + logic [LocalW-1:0] local_idx_q; + logic [TensorW-1:0] row_tile_idx_q, col_tile_idx_q; + addr_t safe_src_addr_q, safe_dst_addr_q; + + logic is_transpose; + logic req_handshake; + logic first_request, last_request; + logic invalid_read, invalid_write; + logic [WorkW-1:0] ne, ne_m1, row_tiles, col_tiles; + logic [WorkW-1:0] remaining_rows, remaining_cols; + + assign is_transpose = req_i.opt.compute.enable & + (req_i.opt.compute.op == idma_pkg::COMPUTE_TRANSPOSE); + assign req_handshake = valid_o & ready_i; + assign first_request = (local_idx_q == '0) && (row_tile_idx_q == '0) && + (col_tile_idx_q == '0); + + // Tile dimensions are powers of two, so edge classification only needs + // shifts and masks. The request carries the original tensor dimensions. + always_comb begin : proc_geometry + logic [ModeW-1:0] mode; + logic [WorkW-1:0] rows, cols; + logic [WorkW-1:0] log2_ne; + + mode = req_i.opt.compute.params.transpose.mode; + rows = WorkW'(req_i.opt.compute.params.transpose.tensor_m); + cols = WorkW'(req_i.opt.compute.params.transpose.tensor_n); + log2_ne = WorkW'(Log2Strb) - WorkW'(mode); + ne = WorkW'(1) << log2_ne; + ne_m1 = ne - 1'b1; + row_tiles = (rows + ne_m1) >> log2_ne; + col_tiles = (cols + ne_m1) >> log2_ne; + remaining_rows = rows & ne_m1; + remaining_cols = cols & ne_m1; + end + + // Invalid edge requests do not contribute transpose data in either output + // layout. Replaying them to known mapped addresses avoids unsafe accesses + // without changing the generated data or strobes. + assign invalid_read = is_transpose && (remaining_rows != '0) && + (WorkW'(row_tile_idx_q) == row_tiles - 1'b1) && + (WorkW'(local_idx_q) >= remaining_rows); + assign invalid_write = is_transpose && (remaining_cols != '0) && + (WorkW'(col_tile_idx_q) == col_tiles - 1'b1) && + (WorkW'(local_idx_q) >= remaining_cols); + assign last_request = is_transpose && + (WorkW'(local_idx_q) == ne - 1'b1) && + (WorkW'(row_tile_idx_q) == row_tiles - 1'b1) && + (WorkW'(col_tile_idx_q) == col_tiles - 1'b1); + + // The stage is transparent to handshaking and all non-address payload. + always_comb begin : proc_replay + req_o = req_i; + if (invalid_read) begin + req_o.src_addr = safe_src_addr_q; + end + if (invalid_write) begin + req_o.dst_addr = safe_dst_addr_q; + end + end + assign valid_o = valid_i; + assign ready_o = ready_i; + + // Mirror the transpose ND walk: local row, row tile, then column tile. + always_ff @(posedge clk_i or negedge rst_ni) begin : proc_position + if (!rst_ni) begin + local_idx_q <= '0; + row_tile_idx_q <= '0; + col_tile_idx_q <= '0; + safe_src_addr_q <= '0; + safe_dst_addr_q <= '0; + end else if (req_handshake && is_transpose) begin + if (first_request) begin + safe_src_addr_q <= req_i.src_addr; + safe_dst_addr_q <= req_i.dst_addr; + end + + if (last_request) begin + local_idx_q <= '0; + row_tile_idx_q <= '0; + col_tile_idx_q <= '0; + end else if (WorkW'(local_idx_q) == ne - 1'b1) begin + local_idx_q <= '0; + if (WorkW'(row_tile_idx_q) == row_tiles - 1'b1) begin + row_tile_idx_q <= '0; + col_tile_idx_q <= col_tile_idx_q + 1'b1; + end else begin + row_tile_idx_q <= row_tile_idx_q + 1'b1; + end + end else begin + local_idx_q <= local_idx_q + 1'b1; + end + end + end + +`ifndef SYNTHESIS + initial assert (StrbWidth >= 2 && (StrbWidth & (StrbWidth - 1)) == 0) else + $fatal(1, "idma_transpose_req_replay: StrbWidth must be a power of two >= 2"); +`endif + +endmodule diff --git a/target/sim/vlt/.gitignore b/target/sim/vlt/.gitignore new file mode 100644 index 00000000..dd07bff2 --- /dev/null +++ b/target/sim/vlt/.gitignore @@ -0,0 +1 @@ +build-* diff --git a/target/sim/vlt/testbenches.mk b/target/sim/vlt/testbenches.mk new file mode 100644 index 00000000..e64c7a91 --- /dev/null +++ b/target/sim/vlt/testbenches.mk @@ -0,0 +1,52 @@ +# Copyright 2026 Mosaic SoC AG +# Solderpad Hardware License, Version 0.51, see LICENSE for details. +# SPDX-License-Identifier: SHL-0.51 + +# Each testbench declares its configuration suffixes here. The generic +# plumbing combines them into "__" build and run targets. +IDMA_VLT_TESTBENCHES := \ + tb_idma_otf_transpose \ + tb_idma_transpose_nd \ + tb_idma_transpose_b2b \ + tb_idma_transpose_midend \ + tb_idma_nd_midend_b2b \ + tb_idma_reg_frontend \ + tb_idma_rt_midend + +IDMA_VLT_SUFFIXES_tb_idma_otf_transpose := sw8_fd0 sw8_fd1 sw64_fd0 sw64_fd1 +IDMA_VLT_SUFFIXES_tb_idma_transpose_nd := dw32 dw64 +IDMA_VLT_SUFFIXES_tb_idma_transpose_b2b := dw32 dw64 +IDMA_VLT_SUFFIXES_tb_idma_transpose_midend := dw64 dw512 +IDMA_VLT_SUFFIXES_tb_idma_nd_midend_b2b := default +IDMA_VLT_SUFFIXES_tb_idma_reg_frontend := ns1_nr1 ns2_nr1 ns2_nr2 +IDMA_VLT_SUFFIXES_tb_idma_rt_midend := default + +# Standalone transpose engine: explicitly enumerate the orthogonal +# StrbWidth x FullDuplex matrix. +IDMA_VLT_ELAB_ARGS_tb_idma_otf_transpose__sw8_fd0 := \ + -GStrbWidth=8 -GFullDuplex=0 +IDMA_VLT_ELAB_ARGS_tb_idma_otf_transpose__sw8_fd1 := \ + -GStrbWidth=8 -GFullDuplex=1 +IDMA_VLT_ELAB_ARGS_tb_idma_otf_transpose__sw64_fd0 := \ + -GStrbWidth=64 -GFullDuplex=0 +IDMA_VLT_ELAB_ARGS_tb_idma_otf_transpose__sw64_fd1 := \ + -GStrbWidth=64 -GFullDuplex=1 + +IDMA_VLT_RUN_ARGS_tb_idma_otf_transpose__sw8_fd0 := +BP +IDMA_VLT_RUN_ARGS_tb_idma_otf_transpose__sw8_fd1 := +BP +IDMA_VLT_RUN_ARGS_tb_idma_otf_transpose__sw64_fd0 := +BP +IDMA_VLT_RUN_ARGS_tb_idma_otf_transpose__sw64_fd1 := +BP + +# End-to-end transpose and transpose-midend bus-width configurations. +IDMA_VLT_ELAB_ARGS_tb_idma_transpose_nd__dw32 := -GDataWidth=32 +IDMA_VLT_ELAB_ARGS_tb_idma_transpose_nd__dw64 := -GDataWidth=64 +IDMA_VLT_ELAB_ARGS_tb_idma_transpose_b2b__dw32 := -GDataWidth=32 +IDMA_VLT_ELAB_ARGS_tb_idma_transpose_b2b__dw64 := -GDataWidth=64 +IDMA_VLT_ELAB_ARGS_tb_idma_transpose_midend__dw64 := -GDataWidth=64 +IDMA_VLT_ELAB_ARGS_tb_idma_transpose_midend__dw512 := -GDataWidth=512 + +# Register frontend configurations cover one and two streams, plus arbitration +# between two independent register ports. +IDMA_VLT_ELAB_ARGS_tb_idma_reg_frontend__ns1_nr1 := -GNumStreams=1 -GNumRegs=1 +IDMA_VLT_ELAB_ARGS_tb_idma_reg_frontend__ns2_nr1 := -GNumStreams=2 -GNumRegs=1 +IDMA_VLT_ELAB_ARGS_tb_idma_reg_frontend__ns2_nr2 := -GNumStreams=2 -GNumRegs=2 diff --git a/target/sim/vlt/vlt.mk b/target/sim/vlt/vlt.mk new file mode 100644 index 00000000..f4661194 --- /dev/null +++ b/target/sim/vlt/vlt.mk @@ -0,0 +1,128 @@ +# Copyright 2026 Mosaic SoC AG +# Solderpad Hardware License, Version 0.51, see LICENSE for details. +# SPDX-License-Identifier: SHL-0.51 + +# Generic Verilator build and run plumbing. Testbench-specific variants, +# elaboration parameters, and runtime arguments belong in separate +# configuration fragments. + +IDMA_VLT_DIR := $(abspath $(IDMA_ROOT)/target/sim/vlt) +IDMA_VLT_FILELIST_TMP_DIR := $(IDMA_VLT_DIR)/.tmp + +IDMA_TRACE ?= $(TRACE) + +IDMA_VLT_BENDER_ARGS := \ + -t test \ + -t idma_test \ + -t rtl \ + -t synth \ + -t simulation \ + -t snitch_cluster + +IDMA_VLT_ARGS ?= \ + --assert \ + --binary \ + --error-limit 1000 \ + --timing \ + --timescale 1ns/1ps \ + -Wno-fatal + +# Optional flags for Verilator's generated C++ compilation and final link. +IDMA_VLT_CFLAGS ?= +IDMA_VLT_LDFLAGS ?= + +ifeq ($(IDMA_TRACE), 1) +IDMA_VLT_ARGS += --trace-fst +endif + +ifeq ($(IDMA_TRACE), 2) +IDMA_VLT_ARGS += --trace-fst --trace-structs +endif +# Configuration names use "__". The explicit separator keeps +# top extraction independent of the underscores commonly used in module names. +idma_vlt_top = $(word 1,$(subst __, ,$(1))) +idma_vlt_configs = $(addprefix $(1)__,$(IDMA_VLT_SUFFIXES_$(1))) + +IDMA_VLT_CONFIGS := $(strip $(foreach testbench,$(IDMA_VLT_TESTBENCHES),\ + $(call idma_vlt_configs,$(testbench)))) + +# Make cannot inspect the sources named by the Verilator file list. Use +# Bender's plain file list for source dependencies and conservatively track all +# Verilog headers in this package and its checked-out dependencies. +IDMA_VLT_SOURCES := $(shell \ + $(BENDER) script flist $(IDMA_VLT_BENDER_ARGS)) +IDMA_VLT_HEADERS := $(shell \ + find $(IDMA_ROOT)/src $(IDMA_ROOT)/target/rtl/include $(IDMA_ROOT)/test \ + $(IDMA_ROOT)/.bender/git/checkouts -type f \ + \( -name '*.svh' -o -name '*.vh' \) 2>/dev/null) + +# Bender records the DPI model but omits C sources from both its Verilator +# script and plain file list, so pass this shared source to Verilator directly. +IDMA_VLT_DPI_SOURCES := $(IDMA_ROOT)/test/idma_transpose_dpi.c + +.PHONY: idma_vlt_force +idma_vlt_force: + +.PRECIOUS: $(IDMA_VLT_DIR)/build-%/vlt-sources.flist + +# Regenerate a configuration-specific candidate filelist on every invocation. +# The temporary files share one directory but retain the configuration name so +# parallel builds cannot overwrite each other. rsync preserves the stable +# build-local file's timestamp when its contents did not change. +$(IDMA_VLT_DIR)/build-%/vlt-sources.flist: \ + idma_vlt_force \ + $(IDMA_FULL_RTL) \ + $(IDMA_FULL_TB) \ + $(IDMA_INCLUDE_ALL) + mkdir -p $(@D) + mkdir -p $(IDMA_VLT_FILELIST_TMP_DIR) + $(BENDER) script verilator \ + $(IDMA_VLT_BENDER_ARGS) \ + > $(IDMA_VLT_FILELIST_TMP_DIR)/$*.flist + rsync -c $(IDMA_VLT_FILELIST_TMP_DIR)/$*.flist $@ + rm -f $(IDMA_VLT_FILELIST_TMP_DIR)/$*.flist + +$(IDMA_VLT_DIR)/build-%/vlt.bin: \ + $(IDMA_VLT_DIR)/build-%/vlt-sources.flist \ + $(IDMA_VLT_SOURCES) \ + $(IDMA_VLT_HEADERS) \ + $(IDMA_VLT_DPI_SOURCES) \ + $(IDMA_INCLUDE_ALL) + $(VERILATOR) \ + -f $(@D)/vlt-sources.flist \ + $(IDMA_VLT_DPI_SOURCES) \ + $(IDMA_VLT_ARGS) \ + $(if $(IDMA_VLT_CFLAGS),-CFLAGS "$(IDMA_VLT_CFLAGS)") \ + $(if $(IDMA_VLT_LDFLAGS),-LDFLAGS "$(IDMA_VLT_LDFLAGS)") \ + $(IDMA_VLT_ELAB_ARGS_$*) \ + --top-module $(call idma_vlt_top,$*) \ + --Mdir $(@D)/obj_dir \ + -o $(abspath $@) + +define idma_vlt_define_run_target +.PHONY: idma_vlt_run_$(1) +idma_vlt_run_$(1): $(IDMA_VLT_DIR)/build-$(1)/vlt.bin + $$< $$(IDMA_VLT_RUN_ARGS_$(1)) +endef + +$(foreach config,$(IDMA_VLT_CONFIGS),\ + $(eval $(call idma_vlt_define_run_target,$(config)))) + +# Friendly per-testbench targets run every suffix declared for that top. +define idma_vlt_define_testbench_target +.PHONY: idma_sim_vlt_$(1) +idma_sim_vlt_$(1): $$(addprefix idma_vlt_run_,$$(call idma_vlt_configs,$(1))) +endef + +$(foreach testbench,$(IDMA_VLT_TESTBENCHES),\ + $(eval $(call idma_vlt_define_testbench_target,$(testbench)))) + +IDMA_VLT_TB_TARGETS := $(addprefix idma_sim_vlt_,$(IDMA_VLT_TESTBENCHES)) + +.PHONY: idma_sim_vlt_all +idma_sim_vlt_all: $(IDMA_VLT_TB_TARGETS) + +.PHONY: idma_vlt_clean idma_verilator_clean +idma_vlt_clean idma_verilator_clean: + rm -rf $(IDMA_VLT_DIR)/build-* + rm -rf $(IDMA_VLT_FILELIST_TMP_DIR) diff --git a/test/frontend/tb_idma_reg_frontend.sv b/test/frontend/tb_idma_reg_frontend.sv index 7132d1b3..40706a5a 100644 --- a/test/frontend/tb_idma_reg_frontend.sv +++ b/test/frontend/tb_idma_reg_frontend.sv @@ -125,6 +125,16 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( cnt_width_t [NumStreams-1:0] done_id; idma_busy_t [NumStreams-1:0] busy; logic [NumStreams-1:0] midend_busy; + logic issue; + logic retire; + + // Sample the backend request immediately before the active edge and apply + // backpressure after it, avoiding races with the DUT's arbitration state. + clocking backend_cb @(posedge clk); + default input #1step output #0; + input req_valid, dma_req, stream_idx, issue; + output req_ready; + endclocking // -------------------------------------------------------------------------- // APB DV interfaces + drivers: one per config port. Each interface is bridged @@ -154,9 +164,6 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( // Transfer-id generator (owns the next/completed counters). Reset next=2. // issue on an accepted launch, retire on a modeled backend completion. // -------------------------------------------------------------------------- - logic issue; - logic retire; - idma_transfer_id_gen #( .IdWidth ( IdCounterWidth ) ) i_id_gen ( @@ -231,8 +238,8 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( end // Unified backend model: capture launches, count outstanding, and retire FIFO. - always @(posedge clk) begin - automatic bit accept = rst_n && req_valid && req_ready; + always @(backend_cb) begin + automatic bit accept = rst_n && backend_cb.issue; automatic bit do_retire = 1'b0; retire <= 1'b0; @@ -242,7 +249,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( end else begin // 1) capture an accepted launch if (accept) - captured_q.push_back(dma_req); + captured_q.push_back(backend_cb.dma_req); // 2) advance / fire the retire timer if (backend_auto_retire && retire_timer == 0) begin @@ -298,9 +305,9 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( // -------------------------------------------------------------------------- int unsigned launch_accept_count; int unsigned launch_acc_base; // accept-count snapshot taken at a launch read - always @(posedge clk) begin + always @(backend_cb) begin if (!rst_n) launch_accept_count <= 0; - else if (issue) launch_accept_count <= launch_accept_count + 1; + else if (backend_cb.issue) launch_accept_count <= launch_accept_count + 1; end // Test 5 (multi-port arbitration) scoreboard state. Each config port programs a @@ -367,13 +374,19 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( int unsigned tries; tries = 0; while (launch_accept_count == launch_acc_base) begin - @(posedge clk); + @(backend_cb); tries++; if (tries > 1000) $fatal(1, "wait_launch_accepted: launch never accepted"); end endtask + // Drive backend backpressure through the clocking block after the active edge. + task automatic set_req_ready(input logic ready); + @(backend_cb); + backend_cb.req_ready <= ready; + endtask + task automatic read_done(output logic [31:0] id, input int unsigned s = 0); logic err; drv[0].read(reg_done_id(s), id, err); @@ -441,7 +454,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( // ------------------------------------------------------------------ $display("\n--- Test 1: basic launch ---"); backend_auto_retire = 1'b1; - req_ready = 1'b1; + set_req_ready(1'b1); captured_q.delete(); program_transfer(32'h1000_0000, 32'h2000_0000, 32'h0000_0040); // id gen resets next=2, so the very first launch must return id 2 @@ -473,7 +486,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( backend_auto_retire = 1'b0; // no auto retire while we hold the stall captured_q.delete(); // model a busy backend: hold req_ready LOW so the arbiter cannot grant - req_ready = 1'b0; + set_req_ready(1'b0); program_transfer(32'h3000_0000, 32'h4000_0000, 32'h0000_0080); exp_id = next_id; // id that this launch returns // the read completes despite req_ready low — the non-blocking property @@ -485,7 +498,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( // the launch is held pending (not yet granted): id must not have advanced yet check_eq(next_id, exp_id, "Test2 id held (no issue) while req_ready low"); // release backpressure — the held launch now completes exactly once - req_ready = 1'b1; + set_req_ready(1'b1); wait_launch_accepted(); check_eq(captured_q.size(), 32'd1, "Test2 launch accepted exactly once"); if (captured_q.size() > 0) begin @@ -505,7 +518,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( $display("\n--- Test 2b: launch integrity (late grant, no drop) ---"); backend_auto_retire = 1'b0; captured_q.delete(); - req_ready = 1'b0; + set_req_ready(1'b0); program_transfer(32'h7000_0000, 32'h8000_0000, 32'h0000_00C0); exp_id = next_id; begin @@ -516,17 +529,17 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( check_eq(rcyc <= MaxReadLatency, 1'b1, "Test2b read within bounded latency (BP)"); check_eq(id3, exp_id, "Test2b launch id == next_id"); // hold the grant off for several cycles: the launch must stay pending, not drop - repeat (12) @(posedge clk); + repeat (12) @(backend_cb); check_eq(launch_accept_count, acc_before, "Test2b no accept while req_ready low"); - check_eq(req_valid, 1'b1, "Test2b req_valid held high across late grant"); + check_eq(backend_cb.req_valid, 1'b1, "Test2b req_valid held high across late grant"); check_eq(captured_q.size(), 32'd0, "Test2b nothing captured before grant"); // release: exactly one accept, exactly one captured launch - req_ready = 1'b1; + set_req_ready(1'b1); wait_launch_accepted(); check_eq(launch_accept_count, acc_before + 32'd1, "Test2b launch fired exactly once"); end // give the arbiter a settle cycle, then confirm no second spurious launch - repeat (4) @(posedge clk); + repeat (4) @(backend_cb); check_eq(captured_q.size(), 32'd1, "Test2b exactly one launch captured"); if (captured_q.size() > 0) begin check_eq(captured_q[0].burst_req.src_addr, 32'h7000_0000, "Test2b src_addr held"); @@ -549,7 +562,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( $display("\n--- Test 3: multi-stream stream_idx held until grant ---"); backend_auto_retire = 1'b0; captured_q.delete(); - req_ready = 1'b0; + set_req_ready(1'b0); program_transfer(32'h5000_0000, 32'h6000_0000, 32'h0000_0100); exp_id = next_id; held1_cnt = 0; @@ -559,16 +572,16 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( check_eq(rcyc <= MaxReadLatency, 1'b1, "Test3 read within bounded latency (BP)"); // while the launch is pending (req_valid high, grant withheld) stream_idx==1 repeat (12) begin - @(posedge clk); - if (req_valid && !req_ready) begin - if (stream_idx == 1) held1_cnt++; - else bad_idx = 1'b1; // wrong / dropped stream index + @(backend_cb); + if (backend_cb.req_valid) begin + if (backend_cb.stream_idx == 1) held1_cnt++; + else bad_idx = 1'b1; // wrong / dropped stream index end end check_eq(held1_cnt >= 32'd8, 1'b1, "Test3 stream_idx held == 1 across stall"); check_eq(bad_idx, 1'b0, "Test3 stream_idx never pointed at wrong stream"); // release: exactly one accept on stream 1 - req_ready = 1'b1; + set_req_ready(1'b1); wait_launch_accepted(); check_eq(launch_accept_count, acc_before + 32'd1, "Test3 stream1 accepted once"); check_eq(id2, exp_id, "Test3 stream1 launch id == next_id"); @@ -589,7 +602,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( // ------------------------------------------------------------------ $display("\n--- Test 4: back-to-back launches ---"); backend_auto_retire = 1'b1; - req_ready = 1'b1; + set_req_ready(1'b1); captured_q.delete(); begin logic [31:0] ids[4]; @@ -629,7 +642,7 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( $display("\n--- Test 5: concurrent multi-port arbitration (stream_idx) ---"); backend_auto_retire = 1'b0; captured_q.delete(); - req_ready = 1'b0; + set_req_ready(1'b0); // port 0 -> stream 0, port 1 -> stream 1, each with a unique src_addr sb_addr_stream0 = 32'hAAAA_0000; sb_addr_stream1 = 32'hBBBB_0000; @@ -652,18 +665,17 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( int unsigned held_checks; held_checks = 0; repeat (12) begin - @(negedge clk); - #(TCK/10); // let combinational DUT outputs settle - if (req_valid && !req_ready) begin + @(backend_cb); + if (backend_cb.req_valid) begin automatic int unsigned won_stream = 32'hFFFF_FFFF; - if (dma_req.burst_req.src_addr == sb_addr_stream0) won_stream = 0; - else if (dma_req.burst_req.src_addr == sb_addr_stream1) won_stream = 1; + if (backend_cb.dma_req.burst_req.src_addr == sb_addr_stream0) won_stream = 0; + else if (backend_cb.dma_req.burst_req.src_addr == sb_addr_stream1) won_stream = 1; if (won_stream != 32'hFFFF_FFFF) begin held_checks++; - if (stream_idx != won_stream[$bits(stream_idx)-1:0]) begin + if (backend_cb.stream_idx != won_stream[$bits(stream_idx)-1:0]) begin sb_mismatch++; $display("[Test5] MISMATCH: dma_req_o=port for stream %0d but stream_idx=%0d", - won_stream, stream_idx); + won_stream, backend_cb.stream_idx); end end end @@ -672,13 +684,13 @@ module tb_idma_reg_frontend import idma_pkg::*; import apb_test::apb_driver; #( check_eq(sb_mismatch, 32'd0, "Test5 stream_idx matches arbitrated port (winner)"); end // now let both launches drain and confirm both transfers are captured correctly - req_ready = 1'b1; + set_req_ready(1'b1); backend_auto_retire = 1'b1; begin int unsigned tries; tries = 0; while (captured_q.size() < 2) begin - @(posedge clk); + @(backend_cb); tries++; if (tries > 1000) $fatal(1, "Test5: both launches never drained (got %0d)", captured_q.size()); diff --git a/test/idma_transpose_dpi.c b/test/idma_transpose_dpi.c index 4082b21d..f9035338 100644 --- a/test/idma_transpose_dpi.c +++ b/test/idma_transpose_dpi.c @@ -15,6 +15,10 @@ static uint8_t gm_in[GM_MAX_BYTES]; static uint8_t gm_out[GM_MAX_BYTES]; +#ifdef __cplusplus +extern "C" { +#endif + // Load one input byte at flat byte index. void gm_load(int idx, int val) { if (idx >= 0 && idx < GM_MAX_BYTES) gm_in[idx] = (uint8_t)val; @@ -33,3 +37,7 @@ int gm_get(int idx) { if (idx >= 0 && idx < GM_MAX_BYTES) return (int)gm_out[idx]; return -1; } + +#ifdef __cplusplus +} +#endif diff --git a/test/midend/tb_idma_nd_midend_b2b.sv b/test/midend/tb_idma_nd_midend_b2b.sv index e849a1e0..d9106ab8 100644 --- a/test/midend/tb_idma_nd_midend_b2b.sv +++ b/test/midend/tb_idma_nd_midend_b2b.sv @@ -47,6 +47,12 @@ module tb_idma_nd_midend_b2b; clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + clocking req_cb @(posedge clk); + default input #1step output #0; + input nd_req_ready; + output nd_req, nd_req_valid; + endclocking + idma_nd_midend #( .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) @@ -101,23 +107,24 @@ module tb_idma_nd_midend_b2b; repeat (3) @(posedge clk); // ── transfer 1 ── - nd_req = mk_req(S1, D1); nd_req_valid = 1'b1; - @(posedge clk); - while (!nd_req_ready) @(posedge clk); - // ── transfer 2 : BACK-TO-BACK (keep valid high, swap payload the cycle after accept) ── - nd_req = mk_req(S2, D2); - @(posedge clk); - while (!nd_req_ready) @(posedge clk); - nd_req_valid = 1'b0; - nd_req = '0; + @(req_cb); + req_cb.nd_req <= mk_req(S1, D1); + req_cb.nd_req_valid <= 1'b1; + do @(req_cb); while (!req_cb.nd_req_ready); + // ── transfer 2: keep valid high and replace the payload after T1's accept ── + req_cb.nd_req <= mk_req(S2, D2); + do @(req_cb); while (!req_cb.nd_req_ready); + req_cb.nd_req_valid <= 1'b0; + req_cb.nd_req <= '0; // ── idle gap ── - repeat (5) @(posedge clk); + repeat (5) @(req_cb); // ── transfer 3 : after the gap ── - nd_req = mk_req(S3, D3); nd_req_valid = 1'b1; - @(posedge clk); - while (!nd_req_ready) @(posedge clk); - nd_req_valid = 1'b0; - repeat (3) @(posedge clk); + req_cb.nd_req <= mk_req(S3, D3); + req_cb.nd_req_valid <= 1'b1; + do @(req_cb); while (!req_cb.nd_req_ready); + req_cb.nd_req_valid <= 1'b0; + req_cb.nd_req <= '0; + repeat (3) @(req_cb); // ── checks ── if (cap_src.size() != 3*NB) diff --git a/test/midend/tb_idma_rt_midend.sv b/test/midend/tb_idma_rt_midend.sv index 3b11c1da..e22d623d 100644 --- a/test/midend/tb_idma_rt_midend.sv +++ b/test/midend/tb_idma_rt_midend.sv @@ -107,6 +107,12 @@ module tb_idma_rt_midend; logic out_req_handshake; assign out_req_handshake = out_req_valid & out_req_ready; + clocking bypass_cb @(posedge clk); + default input #1step output #0; + input byp_req_ready; + output byp_req, byp_req_valid; + endclocking + always_ff @(posedge clk or negedge rst_n) begin if (!rst_n) begin out_rsp_valid <= 1'b0; @@ -141,23 +147,23 @@ module tb_idma_rt_midend; // -- Bypass stimulus ----------------------------------------------- initial begin : drive_bypass + automatic idma_nd_req_t request = '0; byp_req_valid = 1'b0; byp_req = '0; - byp_req.burst_req.length = 32'h0000_1000; - byp_req.burst_req.src_addr = 32'hC000_0000; - byp_req.burst_req.dst_addr = 32'hD000_0000; + request.burst_req.src_addr = 32'hC000_0000; + request.burst_req.dst_addr = 32'hD000_0000; wait (rst_n === 1'b1); @(posedge clk); // Issue 8 bypass requests interleaved with the counter traffic. for (int i = 0; i < 8; i++) begin - repeat (3 + (i % 4)) @(posedge clk); - byp_req_valid = 1'b1; - byp_req.burst_req.length = 32'h0000_1000 + i; - @(posedge clk); - while (!byp_req_ready) @(posedge clk); - byp_req_valid = 1'b0; + repeat (3 + (i % 4)) @(bypass_cb); + request.burst_req.length = 32'h0000_1000 + i; + bypass_cb.byp_req <= request; + bypass_cb.byp_req_valid <= 1'b1; + do @(bypass_cb); while (!bypass_cb.byp_req_ready); + bypass_cb.byp_req_valid <= 1'b0; end end diff --git a/test/midend/tb_idma_transpose_midend.sv b/test/midend/tb_idma_transpose_midend.sv index 755b8ec5..39255325 100644 --- a/test/midend/tb_idma_transpose_midend.sv +++ b/test/midend/tb_idma_transpose_midend.sv @@ -8,9 +8,9 @@ `include "idma/typedef.svh" /// Unit check for idma_transpose_midend: the expanded NumDim=4 ND request must -/// match the golden geometry that tb_idma_transpose_nd hand-builds, and a -/// non-transpose request must pass through unchanged. Sweeps a geometry list -/// internally (one elaboration per bus width). +/// match the golden compact and tile-padded geometries, and a non-transpose +/// request must pass through unchanged. Sweeps a geometry list internally +/// (one elaboration per bus width). module tb_idma_transpose_midend #( parameter int unsigned DataWidth = 512, parameter int unsigned AddrWidth = 64 @@ -54,18 +54,21 @@ module tb_idma_transpose_midend #( end endtask - // Check the expanded geometry for one m x n transpose of eb-byte elements. - task automatic chk_geom(input int unsigned m, input int unsigned n, input int unsigned eb); + // Check both compact and tile-padded expansion for one transpose geometry. + task automatic chk_geom(input int unsigned m, input int unsigned n, input int unsigned eb, + input bit compact); automatic int unsigned ne = StrbWidth/eb; automatic int unsigned mode = (eb==4) ? 2 : (eb==2) ? 1 : 0; automatic int unsigned yt = (m + ne - 1)/ne; automatic int unsigned nt = (n + ne - 1)/ne; automatic int unsigned mp = yt*ne; + automatic int unsigned dp = compact ? m : mp; nd_in = '0; nd_in.burst_req.src_addr = 64'h1000; nd_in.burst_req.dst_addr = 64'h2000; nd_in.burst_req.opt.compute.enable = 1'b1; nd_in.burst_req.opt.compute.op = COMPUTE_TRANSPOSE; + nd_in.burst_req.opt.compute.params.transpose.compact = compact; nd_in.burst_req.opt.compute.params.transpose.mode = 2'(mode); nd_in.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); nd_in.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); @@ -74,19 +77,20 @@ module tb_idma_transpose_midend #( chk("length", nd_out.burst_req.length, ne*eb); chk("d0.reps", nd_out.d_req[0].reps, ne); chk("d0.src", nd_out.d_req[0].src_strides, addr_t'(n*eb)); - chk("d0.dst", nd_out.d_req[0].dst_strides, addr_t'(mp*eb)); + chk("d0.dst", nd_out.d_req[0].dst_strides, addr_t'(dp*eb)); chk("d1.reps", nd_out.d_req[1].reps, yt); chk("d1.src", nd_out.d_req[1].src_strides, addr_t'(n*eb)); - chk("d1.dst", nd_out.d_req[1].dst_strides, addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb))); + chk("d1.dst", nd_out.d_req[1].dst_strides, addr_t'(int'(ne*eb) - int'((ne-1)*dp*eb))); chk("d2.reps", nd_out.d_req[2].reps, nt); chk("d2.src", nd_out.d_req[2].src_strides, addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb))); - chk("d2.dst", nd_out.d_req[2].dst_strides, addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb))); + chk("d2.dst", nd_out.d_req[2].dst_strides, addr_t'(int'(dp*eb) - int'((yt-1)*ne*eb))); // addresses + compute must survive untouched chk("src_addr", nd_out.burst_req.src_addr, 64'h1000); chk("dst_addr", nd_out.burst_req.dst_addr, 64'h2000); chk("cmp_en", nd_out.burst_req.opt.compute.enable, 1); - if (errs == 0) $display("[MID] PASS: %0dx%0d EB=%0d golden (NE=%0d YT=%0d NT=%0d MP=%0d)", - m, n, eb, ne, yt, nt, mp); + if (errs == 0) + $display("[MID] PASS: %0dx%0d EB=%0d compact=%0d (NE=%0d YT=%0d NT=%0d DP=%0d)", + m, n, eb, compact, ne, yt, nt, dp); endtask initial begin @@ -104,10 +108,12 @@ module tb_idma_transpose_midend #( // --- transpose cases --- for (int unsigned k = 0; k < NCases; k++) begin if (Cases[k][2] > StrbWidth) continue; // element must fit the bus - chk_geom(Cases[k][0], Cases[k][1], Cases[k][2]); + chk_geom(Cases[k][0], Cases[k][1], Cases[k][2], 1'b0); + chk_geom(Cases[k][0], Cases[k][1], Cases[k][2], 1'b1); end - if (errs == 0) $display("[MID] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + if (errs == 0) $display("[MID] ALL PASS (%0d geometries x 2 layouts, StrbWidth=%0d)", + NCases, StrbWidth); else $fatal(1, "[MID] FAIL: %0d mismatches", errs); $finish; end diff --git a/test/tb_idma_otf_transpose.sv b/test/tb_idma_otf_transpose.sv index 86ebc80a..2bce85eb 100644 --- a/test/tb_idma_otf_transpose.sv +++ b/test/tb_idma_otf_transpose.sv @@ -41,6 +41,16 @@ module tb_idma_otf_transpose #( logic [StrbWidth-1:0] dout_strb; logic dout_valid, dout_ready; + // The stream driver and monitor share a clocking block so neither can race + // sequential DUT logic when changing valid/ready or sampling a transferred beat. + clocking stream_cb @(posedge clk); + default input #1step output #0; + input din_ready; + output din_data, din_valid; + input dout_data, dout_strb, dout_valid; + output dout_ready; + endclocking + idma_otf_transpose #( .StrbWidth (StrbWidth), .FullDuplex (FullDuplex) @@ -67,36 +77,48 @@ module tb_idma_otf_transpose #( task automatic drive_inputs(input int unsigned m, n, eb, ne, yt, nt); int unsigned beat = 0; - din_valid = 1'b0; din_data = '0; - @(posedge clk); + logic [StrbWidth-1:0][7:0] beat_data; + @(stream_cb); + stream_cb.din_valid <= 1'b0; + stream_cb.din_data <= '0; for (int unsigned ct = 0; ct < nt; ct++) for (int unsigned rt = 0; rt < yt; rt++) for (int unsigned row = 0; row < ne; row++) begin - if (backpressure) begin din_valid = 1'b0; repeat (beat % 3) @(posedge clk); end + if (backpressure) begin + stream_cb.din_valid <= 1'b0; + repeat (beat % 3) @(stream_cb); + end + beat_data = '0; for (int unsigned c = 0; c < ne; c++) begin automatic int unsigned gr = rt*ne + row; automatic int unsigned gc = ct*ne + c; for (int unsigned b = 0; b < eb; b++) - din_data[c*eb + b] = (gr < m && gc < n) ? inb[(gr*n + gc)*eb + b] : PAD; + beat_data[c*eb + b] = (gr < m && gc < n) ? inb[(gr*n + gc)*eb + b] : PAD; end - din_valid = 1'b1; - do @(posedge clk); while (!din_ready); + stream_cb.din_data <= beat_data; + stream_cb.din_valid <= 1'b1; + do @(stream_cb); while (!stream_cb.din_ready); beat++; end - din_valid = 1'b0; + stream_cb.din_valid <= 1'b0; + stream_cb.din_data <= '0; endtask task automatic capture_outputs(input int unsigned m, n, eb, ne, yt, nt); int unsigned beat = 0; - dout_ready = 1'b0; + @(stream_cb); + stream_cb.dout_ready <= 1'b0; for (int unsigned ct = 0; ct < nt; ct++) for (int unsigned rt = 0; rt < yt; rt++) for (int unsigned k = 0; k < ne; k++) begin - if (backpressure) begin dout_ready = 1'b0; repeat (beat % 4) @(posedge clk); end - dout_ready = 1'b1; - do @(posedge clk); while (!dout_valid); + if (backpressure) begin + stream_cb.dout_ready <= 1'b0; + repeat (beat % 4) @(stream_cb); + end + stream_cb.dout_ready <= 1'b1; + do @(stream_cb); while (!stream_cb.dout_valid); for (int unsigned e = 0; e < ne; e++) begin - if (dout_strb[e*eb]) begin // element e valid (element-granular mask) + if (stream_cb.dout_strb[e*eb]) begin // element e valid (element-granular mask) automatic int unsigned tr = ct*ne + k; // transposed row (= original col, 0..n-1) automatic int unsigned tc = rt*ne + e; // transposed col (= original row, 0..m-1) if (tr >= n || tc >= m) begin @@ -105,9 +127,11 @@ module tb_idma_otf_transpose #( end else begin for (int unsigned b = 0; b < eb; b++) begin automatic int gold = gm_get((tr*m + tc)*eb + b); - if (int'(dout_data[e*eb + b]) !== gold) begin + if (int'(stream_cb.dout_data[e*eb + b]) !== gold) begin errors++; - if (errors <= 16) $display("MISMATCH T(%0d,%0d).b%0d=%0d golden=%0d", tr, tc, b, dout_data[e*eb+b], gold); + if (errors <= 16) + $display("MISMATCH T(%0d,%0d).b%0d=%0d golden=%0d", + tr, tc, b, stream_cb.dout_data[e*eb+b], gold); end end wrote[tr*m + tc] = 1'b1; @@ -116,8 +140,7 @@ module tb_idma_otf_transpose #( end beat++; end - @(posedge clk); - dout_ready = 1'b0; + stream_cb.dout_ready <= 1'b0; endtask // Run one m x n transpose of eb-byte elements; returns the mismatch count. diff --git a/test/tb_idma_transpose_b2b.sv b/test/tb_idma_transpose_b2b.sv index 206fe408..8925267d 100644 --- a/test/tb_idma_transpose_b2b.sv +++ b/test/tb_idma_transpose_b2b.sv @@ -5,10 +5,10 @@ // Authors: // - Daniel Keller -// End-to-end back-to-back transpose regression: per geometry, two transposes of -// one source to DIFFERENT dst bases through the ND midend -> rw_axi backend -> -// axi_sim_mem. A stale base across transfers would leave the second dst -// untouched. Sweeps a geometry list internally (one run per bus width). +// End-to-end back-to-back transpose regression: per geometry, two transposes +// with different layout modes and destination bases pass through the transpose +// and ND midends, safe edge replay, rw_axi backend, and axi_sim_mem. This catches +// stale base addresses as well as stale compact/padded configuration. `include "axi/typedef.svh" `include "idma/typedef.svh" @@ -58,10 +58,12 @@ module tb_idma_transpose_b2b typedef struct packed { axi_write_meta_channel_t axi; } write_meta_channel_t; logic clk, rst_n; - idma_req_t idma_req; logic req_valid, req_ready; + idma_req_t nd_burst_req, idma_req; + logic nd_burst_valid, nd_burst_ready, req_valid, req_ready; idma_rsp_t idma_rsp; logic rsp_valid, rsp_ready; idma_eh_req_t idma_eh_req; logic eh_req_valid, eh_req_ready; - idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_nd_req_t transpose_req, nd_req; + logic transpose_req_valid, transpose_req_ready, nd_req_valid, nd_req_ready; idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; axi_req_t axi_read_req, axi_write_req, axi_req, axi_req_mem; axi_rsp_t axi_read_rsp, axi_write_rsp, axi_rsp, axi_rsp_mem; @@ -72,6 +74,15 @@ module tb_idma_transpose_b2b clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + // Keep testbench drives and samples out of the DUT's active clocking region. + clocking req_rsp_cb @(posedge clk); + default input #1step output #0; + input transpose_req_ready; + output transpose_req_valid; + output transpose_req; + input nd_rsp_valid, nd_rsp_ready; + endclocking + axi_rw_join #(.axi_req_t(axi_req_t), .axi_resp_t(axi_rsp_t)) i_axi_rw_join ( .clk_i(clk), .rst_ni(rst_n), .slv_read_req_i(axi_read_req), .slv_read_resp_o(axi_read_rsp), @@ -93,6 +104,14 @@ module tb_idma_transpose_b2b .mon_w_data_o(), .mon_w_addr_o(), .mon_w_valid_o() ); + // Convert the matrix dimensions and layout bit into the tiled ND walk. + idma_transpose_midend #( + .NumDim(NumDim), .StrbWidth(StrbWidth), .addr_t(addr_t), .idma_nd_req_t(idma_nd_req_t) + ) i_transpose_midend ( + .nd_req_i(transpose_req), .valid_i(transpose_req_valid), .ready_o(transpose_req_ready), + .nd_req_o(nd_req), .valid_o(nd_req_valid), .ready_i(nd_req_ready) + ); + idma_nd_midend #( .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) @@ -100,11 +119,21 @@ module tb_idma_transpose_b2b .clk_i(clk), .rst_ni(rst_n), .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), - .burst_req_o(idma_req), .burst_req_valid_o(req_valid), .burst_req_ready_i(req_ready), + .burst_req_o(nd_burst_req), .burst_req_valid_o(nd_burst_valid), + .burst_req_ready_i(nd_burst_ready), .burst_rsp_i(idma_rsp), .burst_rsp_valid_i(rsp_valid), .burst_rsp_ready_o(rsp_ready), .busy_o(nd_busy) ); + // Replay descriptors for nonexistent partial-tile rows before they reach AXI. + idma_transpose_req_replay #( + .StrbWidth(StrbWidth), .idma_req_t(idma_req_t) + ) i_transpose_req_replay ( + .clk_i(clk), .rst_ni(rst_n), + .req_i(nd_burst_req), .valid_i(nd_burst_valid), .ready_o(nd_burst_ready), + .req_o(idma_req), .valid_o(req_valid), .ready_i(req_ready) + ); + idma_backend_rw_axi #( .CombinedShifter(1'b0), .DataWidth(DataWidth), .AddrWidth(AddrWidth), .AxiIdWidth(AxiIdWidth), .UserWidth(UserWidth), .TFLenWidth(TFLenWidth), .MaskInvalidData(1'b1), .BufferDepth(3), @@ -134,59 +163,72 @@ module tb_idma_transpose_b2b return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; endfunction - // one m x n transpose of the source at sb -> dst base `db`; returns error count + task automatic send_transpose_req(input idma_nd_req_t req); + @(req_rsp_cb); + req_rsp_cb.transpose_req <= req; + req_rsp_cb.transpose_req_valid <= 1'b1; + do @(req_rsp_cb); while (!req_rsp_cb.transpose_req_ready); + req_rsp_cb.transpose_req <= '0; + req_rsp_cb.transpose_req_valid <= 1'b0; + endtask + + task automatic wait_nd_rsp; + while (!(req_rsp_cb.nd_rsp_valid && req_rsp_cb.nd_rsp_ready)) @(req_rsp_cb); + endtask + + // One transpose to `db` in the selected layout; returns its error count. task automatic do_transpose(input int unsigned m, input int unsigned n, input int unsigned eb, - input addr_t db, output int unsigned errs); + input bit compact, input addr_t db, output int unsigned errs); automatic int unsigned ne = StrbWidth / eb; automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; automatic int unsigned yt = (m + ne - 1) / ne; automatic int unsigned nt = (n + ne - 1) / ne; automatic int unsigned mp = yt * ne; + automatic int unsigned dp = compact ? m : mp; errs = 0; - // pre-fill full padded dst extent with sentinel + // Back the padded envelope in both modes. Bytes beyond the compact matrix + // become guards against stale strides or nonzero edge writes. for (int unsigned i = 0; i < nt*ne; i++) for (int unsigned j = 0; j < mp; j++) for (int unsigned b = 0; b < eb; b++) wr_mem(db + (i*mp + j)*eb + b, 8'hCC); - nd_req = '0; - nd_req.burst_req.length = tf_len_t'(ne*eb); - nd_req.burst_req.src_addr = sb; - nd_req.burst_req.dst_addr = db; - nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; - nd_req.burst_req.opt.dst_protocol = idma_pkg::AXI; - nd_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; - nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; - nd_req.burst_req.opt.beo.decouple_rw = 1'b1; - nd_req.burst_req.opt.beo.decouple_aw = 1'b1; - nd_req.burst_req.opt.compute.enable = 1'b1; - nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; - nd_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); - nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); - nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); - nd_req.burst_req.opt.last = 1'b1; - nd_req.d_req[0].reps = reps_t'(ne); nd_req.d_req[0].src_strides = addr_t'(int'(n*eb)); nd_req.d_req[0].dst_strides = addr_t'(int'(mp*eb)); - nd_req.d_req[1].reps = reps_t'(yt); nd_req.d_req[1].src_strides = addr_t'(int'(n*eb)); nd_req.d_req[1].dst_strides = addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb)); - nd_req.d_req[2].reps = reps_t'(nt); nd_req.d_req[2].src_strides = addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb)); nd_req.d_req[2].dst_strides = addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb)); - nd_req_valid = 1'b1; - do @(posedge clk); while (!nd_req_ready); // drop valid the cycle accept is seen (compliant) - nd_req_valid = 1'b0; - nd_req = '0; - while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); + transpose_req = '0; + transpose_req.burst_req.src_addr = sb; + transpose_req.burst_req.dst_addr = db; + transpose_req.burst_req.opt.src_protocol = idma_pkg::AXI; + transpose_req.burst_req.opt.dst_protocol = idma_pkg::AXI; + transpose_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + transpose_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + transpose_req.burst_req.opt.beo.decouple_rw = 1'b1; + transpose_req.burst_req.opt.beo.decouple_aw = 1'b1; + transpose_req.burst_req.opt.compute.enable = 1'b1; + transpose_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + transpose_req.burst_req.opt.compute.params.transpose.compact = compact; + transpose_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); + transpose_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); + transpose_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); + transpose_req.burst_req.opt.last = 1'b1; + + send_transpose_req(transpose_req); + wait_nd_rsp(); repeat (20) @(posedge clk); - // data + padding checks + // Check data at either compact or padded destination row pitch. for (int unsigned c = 0; c < n; c++) for (int unsigned r = 0; r < m; r++) for (int unsigned b = 0; b < eb; b++) - if (rd_mem(db + (c*mp + r)*eb + b) !== rd_mem(sb + (r*n + c)*eb + b)) begin + if (rd_mem(db + (c*dp + r)*eb + b) !== rd_mem(sb + (r*n + c)*eb + b)) begin errs++; if (errs <= 8) $display("[B2BT] @db=%0h MISMATCH out_T[%0d][%0d].b%0d", db, c, r, b); end - for (int unsigned i = 0; i < nt*ne; i++) - for (int unsigned j = 0; j < mp; j++) - if (i >= n || j >= m) - for (int unsigned b = 0; b < eb; b++) - if (rd_mem(db + (i*mp + j)*eb + b) !== 8'hCC) begin - errs++; if (errs <= 8) $display("[B2BT] @db=%0h PADDING CLOBBERED row=%0d col=%0d", db, i, j); - end + // Padded holes or the tail after a compact matrix must remain untouched. + for (int unsigned byte_idx = 0; byte_idx < nt*ne*mp*eb; byte_idx++) + if (byte_idx >= n*dp*eb || + (!compact && ((byte_idx / eb) / mp >= n || (byte_idx / eb) % mp >= m))) + if (rd_mem(db + byte_idx) !== 8'hCC) begin + errs++; + if (errs <= 8) + $display("[B2BT] @db=%0h UNUSED DESTINATION BYTE CLOBBERED at +0x%0h", + db, byte_idx); + end endtask initial begin @@ -194,7 +236,8 @@ module tb_idma_transpose_b2b automatic addr_t db1 = 'h0000_4000; automatic addr_t db2 = 'h0000_8000; // DIFFERENT base — a stale-addr bug misplaces xfer 2 automatic int unsigned m, n, eb; - nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + automatic bit first_compact; + transpose_req_valid = 1'b0; nd_rsp_ready = 1'b1; transpose_req = '0; @(posedge rst_n); repeat (5) @(posedge clk); @@ -206,15 +249,21 @@ module tb_idma_transpose_b2b for (int unsigned c = 0; c < n; c++) for (int unsigned b = 0; b < eb; b++) wr_mem(sb + (r*n + c)*eb + b, 8'((( (r*n+c)*eb + b )*7 + 3) & 8'hFF)); - $display("[B2BT] %0dx%0d EB=%0d: xfer1 -> db=%0h, xfer2 -> db=%0h", m, n, eb, db1, db2); - do_transpose(m, n, eb, db1, e1); - do_transpose(m, n, eb, db2, e2); // back-to-back, distinct base - if (e1 == 0 && e2 == 0) $display("[B2BT] PASS: %0dx%0d EB=%0d both back-to-back transposes correct", m, n, eb); + // Alternate the order so both padded->compact and compact->padded + // transitions are covered while retaining distinct destination bases. + first_compact = bit'(k & 1); + $display("[B2BT] %0dx%0d EB=%0d: compact=%0d -> db=%0h, compact=%0d -> db=%0h", + m, n, eb, first_compact, db1, !first_compact, db2); + do_transpose(m, n, eb, first_compact, db1, e1); + do_transpose(m, n, eb, !first_compact, db2, e2); + if (e1 == 0 && e2 == 0) + $display("[B2BT] PASS: %0dx%0d EB=%0d both layouts correct back-to-back", m, n, eb); else $display("[B2BT] FAIL: %0dx%0d EB=%0d xfer1=%0d xfer2=%0d", m, n, eb, e1, e2); total += e1 + e2; end - if (total == 0) $display("[B2BT] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + if (total == 0) $display("[B2BT] ALL PASS (%0d mixed-layout cases, StrbWidth=%0d)", + NCases, StrbWidth); else $fatal(1, "[B2BT] FAIL: %0d total mismatches", total); repeat (5) @(posedge clk); $finish(); diff --git a/test/tb_idma_transpose_nd.sv b/test/tb_idma_transpose_nd.sv index 77b9de01..bc6fa1d0 100644 --- a/test/tb_idma_transpose_nd.sv +++ b/test/tb_idma_transpose_nd.sv @@ -5,10 +5,10 @@ // Authors: // - Daniel Keller -// Self-checking multi-tile transpose testbench: idma_nd_midend (NumDim=4, -// transposed-stride program) -> idma_backend_rw_axi -> axi_sim_mem, exercising -// a full M x N transpose end-to-end. Reference: out_T[c][r] = in[r][c]. -// Sweeps a list of geometries internally (one elaboration per bus width). +// Self-checking multi-tile transpose testbench: transpose midend -> generic ND +// midend -> safe edge replay -> rw_axi backend -> axi_sim_mem. Exercises full +// M x N transposes with compact and tile-padded destinations. +// Reference: out_T[c][r] = in[r][c]. `include "axi/typedef.svh" `include "idma/typedef.svh" @@ -71,10 +71,12 @@ module tb_idma_transpose_nd // ── Signals ── logic clk, rst_n; - idma_req_t idma_req; logic req_valid, req_ready; + idma_req_t nd_burst_req, idma_req; + logic nd_burst_valid, nd_burst_ready, req_valid, req_ready; idma_rsp_t idma_rsp; logic rsp_valid, rsp_ready; idma_eh_req_t idma_eh_req; logic eh_req_valid, eh_req_ready; - idma_nd_req_t nd_req; logic nd_req_valid, nd_req_ready; + idma_nd_req_t transpose_req_to_midend, transpose_req_from_tb, nd_req; + logic transpose_req_valid, transpose_req_ready, nd_req_valid, nd_req_ready; idma_rsp_t nd_rsp; logic nd_rsp_valid, nd_rsp_ready; axi_req_t axi_read_req, axi_write_req, axi_req, axi_req_mem; axi_rsp_t axi_read_rsp, axi_write_rsp, axi_rsp, axi_rsp_mem; @@ -83,9 +85,24 @@ module tb_idma_transpose_nd assign idma_eh_req = '0; assign eh_req_valid = 1'b0; + initial begin + $dumpfile("dump.fst"); + $dumpvars(0, tb_idma_transpose_nd); + end + // ── Clock / reset ── clk_rst_gen #(.ClkPeriod(TCK), .RstClkCycles(1)) i_clk_rst_gen (.clk_o(clk), .rst_no(rst_n)); + // Sample DUT outputs immediately before the active edge and apply requests + // after it, so the driver cannot race the DUT's sequential handshake logic. + clocking req_rsp_cb @(posedge clk); + default input #1step output #0; + input transpose_req_ready; + output transpose_req_valid; + output transpose_req_to_midend; + input nd_rsp_valid, nd_rsp_ready; + endclocking + // ── AXI sim memory (read+write joined) ── axi_rw_join #(.axi_req_t(axi_req_t), .axi_resp_t(axi_rsp_t)) i_axi_rw_join ( .clk_i(clk), .rst_ni(rst_n), @@ -108,7 +125,16 @@ module tb_idma_transpose_nd .mon_w_data_o(), .mon_w_addr_o(), .mon_w_valid_o() ); - // ── ND midend: ND transpose descriptor -> 1D bursts ── + // Expand the transpose dimensions and selected destination layout into the + // four-dimensional walk consumed by the generic ND midend. + idma_transpose_midend #( + .NumDim(NumDim), .StrbWidth(StrbWidth), .addr_t(addr_t), .idma_nd_req_t(idma_nd_req_t) + ) i_transpose_midend ( + .nd_req_i(transpose_req_to_midend), .valid_i(transpose_req_valid), .ready_o(transpose_req_ready), + .nd_req_o(nd_req), .valid_o(nd_req_valid), .ready_i(nd_req_ready) + ); + + // ── ND midend: expanded transpose descriptor -> 1D bursts ── idma_nd_midend #( .NumDim(NumDim), .addr_t(addr_t), .idma_req_t(idma_req_t), .idma_rsp_t(idma_rsp_t), .idma_nd_req_t(idma_nd_req_t), .RepWidths(RepWidths) @@ -116,11 +142,22 @@ module tb_idma_transpose_nd .clk_i(clk), .rst_ni(rst_n), .nd_req_i(nd_req), .nd_req_valid_i(nd_req_valid), .nd_req_ready_o(nd_req_ready), .nd_rsp_o(nd_rsp), .nd_rsp_valid_o(nd_rsp_valid), .nd_rsp_ready_i(nd_rsp_ready), - .burst_req_o(idma_req), .burst_req_valid_o(req_valid), .burst_req_ready_i(req_ready), + .burst_req_o(nd_burst_req), .burst_req_valid_o(nd_burst_valid), + .burst_req_ready_i(nd_burst_ready), .burst_rsp_i(idma_rsp), .burst_rsp_valid_i(rsp_valid), .burst_rsp_ready_o(rsp_ready), .busy_o(nd_busy) ); + // Partial edge tiles still contain a full tile's descriptors. Redirect the + // descriptors for invalid rows to mapped addresses before issuing them. + idma_transpose_req_replay #( + .StrbWidth(StrbWidth), .idma_req_t(idma_req_t) + ) i_transpose_req_replay ( + .clk_i(clk), .rst_ni(rst_n), + .req_i(nd_burst_req), .valid_i(nd_burst_valid), .ready_o(nd_burst_ready), + .req_o(idma_req), .valid_o(req_valid), .ready_i(req_ready) + ); + // ── Backend (rw_axi) with transpose engine ── idma_backend_rw_axi #( .CombinedShifter(1'b0), .DataWidth(DataWidth), .AddrWidth(AddrWidth), .AxiIdWidth(AxiIdWidth), @@ -149,8 +186,9 @@ module tb_idma_transpose_nd addr_t sb = 'h0000_1000; addr_t db = 'h0000_4000; - // every AW (incl. wstrb=0 padding rows) must stay in the active case's padded - // dst allocation [chk_db, chk_aw_hi) — else a strict slave would DECERR + // Every AW (including zero-strobe edge rows) must stay in the backed padded + // envelope. Compact mode additionally checks that bytes outside its smaller + // logical footprint remain untouched. logic chk_active = 1'b0; addr_t chk_db, chk_aw_hi; always @(posedge clk) if (rst_n && chk_active && axi_write_req.aw_valid && axi_write_rsp.aw_ready) begin @@ -164,14 +202,28 @@ module tb_idma_transpose_nd return i_axi_sim_mem.mem.exists(a) ? i_axi_sim_mem.mem[a] : 8'hxx; endfunction - // Run one M x N transpose of EB-byte elements; returns the mismatch count. + task automatic send_transpose_req(input idma_nd_req_t req); + @(req_rsp_cb); + req_rsp_cb.transpose_req_to_midend <= req; + req_rsp_cb.transpose_req_valid <= 1'b1; + do @(req_rsp_cb); while (!req_rsp_cb.transpose_req_ready); + req_rsp_cb.transpose_req_to_midend <= '0; + req_rsp_cb.transpose_req_valid <= 1'b0; + endtask + + task automatic wait_nd_rsp; + while (!(req_rsp_cb.nd_rsp_valid && req_rsp_cb.nd_rsp_ready)) @(req_rsp_cb); + endtask + + // Run one M x N transpose in either compact or tile-padded destination layout. task automatic run_case(input int unsigned m, input int unsigned n, input int unsigned eb, - output int unsigned errs); + input bit compact, output int unsigned errs); automatic int unsigned ne = StrbWidth / eb; // tile side (elements) automatic int unsigned mode = (eb == 4) ? 2 : (eb == 2) ? 1 : 0; automatic int unsigned yt = (m + ne - 1) / ne; // row-tiles automatic int unsigned nt = (n + ne - 1) / ne; // col-tiles automatic int unsigned mp = yt * ne; // padded Aᵀ row pitch (StrbWidth-aligned) + automatic int unsigned dp = compact ? m : mp; // selected Aᵀ row pitch errs = 0; // init source matrix (row-major, m x n elements of eb bytes) @@ -180,7 +232,8 @@ module tb_idma_transpose_nd for (int unsigned b = 0; b < eb; b++) wr_mem(sb + (r*n + c)*eb + b, 8'((( (r*n+c)*eb + b )*7 + 3) & 8'hFF)); - // sentinel-fill the full padded Aᵀ extent; padding cols/rows must stay sentinel + // Always back the full padded envelope. Compact mode must only modify its + // n*m prefix; the remainder acts as a guard against stray edge writes. for (int unsigned i = 0; i < nt*ne; i++) for (int unsigned j = 0; j < mp; j++) for (int unsigned b = 0; b < eb; b++) @@ -191,90 +244,82 @@ module tb_idma_transpose_nd chk_aw_hi = db + addr_t'(nt*ne*mp*eb); chk_active = 1'b1; - // ── transposed-stride ND program (routing-plan §4.2) ── - nd_req = '0; - nd_req.burst_req.length = tf_len_t'(ne*eb); // one tile-row = StrbWidth bytes - nd_req.burst_req.src_addr = sb; - nd_req.burst_req.dst_addr = db; - nd_req.burst_req.opt.src_protocol = idma_pkg::AXI; - nd_req.burst_req.opt.dst_protocol = idma_pkg::AXI; - nd_req.burst_req.opt.src.burst = axi_pkg::BURST_INCR; - nd_req.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; - nd_req.burst_req.opt.beo.decouple_rw = 1'b1; - nd_req.burst_req.opt.beo.decouple_aw = 1'b1; - nd_req.burst_req.opt.beo.src_max_llen = '0; - nd_req.burst_req.opt.beo.dst_max_llen = '0; - nd_req.burst_req.opt.compute.enable = 1'b1; - nd_req.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; - nd_req.burst_req.opt.compute.params.transpose.mode = 2'(mode); - nd_req.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); - nd_req.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); - nd_req.burst_req.opt.last = 1'b1; - // ND midend strides are INCREMENTAL deltas (added on dim roll-over), NOT - // absolute pitches. Aᵀ uses padded pitch mp*eb (aligned writes); src keeps - // n*eb (misaligned reads coalesce in the pre-engine buffer). - nd_req.d_req[0].reps = reps_t'(ne); - nd_req.d_req[0].src_strides = addr_t'(int'(n*eb)); - nd_req.d_req[0].dst_strides = addr_t'(int'(mp*eb)); - nd_req.d_req[1].reps = reps_t'(yt); - nd_req.d_req[1].src_strides = addr_t'(int'(n*eb)); - nd_req.d_req[1].dst_strides = addr_t'(int'(ne*eb) - int'((ne-1)*mp*eb)); - nd_req.d_req[2].reps = reps_t'(nt); - nd_req.d_req[2].src_strides = addr_t'(int'(ne*eb) - int'((yt*ne-1)*n*eb)); - nd_req.d_req[2].dst_strides = addr_t'(int'(mp*eb) - int'((yt-1)*ne*eb)); - - $display("[TPN] case %0dx%0d EB=%0d (NE=%0d, %0dx%0d tiles)", m, n, eb, ne, yt, nt); - nd_req_valid = 1'b1; - // drop valid on accept; holding it one cycle past makes the midend re-walk the request - do @(posedge clk); while (!nd_req_ready); - nd_req_valid = 1'b0; - nd_req = '0; + // The transpose midend derives all reps and strides from this base request. + transpose_req_from_tb = '0; + transpose_req_from_tb.burst_req.src_addr = sb; + transpose_req_from_tb.burst_req.dst_addr = db; + transpose_req_from_tb.burst_req.opt.src_protocol = idma_pkg::AXI; + transpose_req_from_tb.burst_req.opt.dst_protocol = idma_pkg::AXI; + transpose_req_from_tb.burst_req.opt.src.burst = axi_pkg::BURST_INCR; + transpose_req_from_tb.burst_req.opt.dst.burst = axi_pkg::BURST_INCR; + transpose_req_from_tb.burst_req.opt.beo.decouple_rw = 1'b1; + transpose_req_from_tb.burst_req.opt.beo.decouple_aw = 1'b1; + transpose_req_from_tb.burst_req.opt.beo.src_max_llen = '0; + transpose_req_from_tb.burst_req.opt.beo.dst_max_llen = '0; + transpose_req_from_tb.burst_req.opt.compute.enable = 1'b1; + transpose_req_from_tb.burst_req.opt.compute.op = idma_pkg::COMPUTE_TRANSPOSE; + transpose_req_from_tb.burst_req.opt.compute.params.transpose.compact = compact; + transpose_req_from_tb.burst_req.opt.compute.params.transpose.mode = 2'(mode); + transpose_req_from_tb.burst_req.opt.compute.params.transpose.tensor_m = 12'(m); + transpose_req_from_tb.burst_req.opt.compute.params.transpose.tensor_n = 12'(n); + transpose_req_from_tb.burst_req.opt.last = 1'b1; + + $display("[TPN] case %0dx%0d EB=%0d compact=%0d (NE=%0d, %0dx%0d tiles)", + m, n, eb, compact, ne, yt, nt); + + send_transpose_req(transpose_req_from_tb); // wait for ND completion + drain - while (!(nd_rsp_valid && nd_rsp_ready)) @(posedge clk); + wait_nd_rsp(); repeat (20) @(posedge clk); chk_active = 1'b0; - // check 1 (data): out_T[c][r] == in[r][c], Aᵀ at padded pitch mp + // Check the transposed matrix using the selected destination row pitch. for (int unsigned c = 0; c < n; c++) for (int unsigned r = 0; r < m; r++) for (int unsigned b = 0; b < eb; b++) begin - automatic logic [7:0] got = rd_mem(db + (c*mp + r)*eb + b); + automatic logic [7:0] got = rd_mem(db + (c*dp + r)*eb + b); automatic logic [7:0] exp = rd_mem(sb + (r*n + c)*eb + b); if (got !== exp) begin errs++; if (errs <= 12) $display("[TPN] MISMATCH out_T[%0d][%0d].b%0d=%02h exp %02h", c, r, b, got, exp); end end - // check 2: padding cols [m,mp) and padding rows [n,nt*ne) must stay sentinel - for (int unsigned i = 0; i < nt*ne; i++) - for (int unsigned j = 0; j < mp; j++) - if (i >= n || j >= m) - for (int unsigned b = 0; b < eb; b++) begin - automatic logic [7:0] got = rd_mem(db + (i*mp + j)*eb + b); - if (got !== 8'hCC) begin - errs++; - if (errs <= 12) $display("[TPN] PADDING CLOBBERED at row=%0d col=%0d b%0d=%02h (exp CC)", i, j, b, got); - end - end + // Everything outside the selected logical layout remains sentinel. In + // padded mode this checks holes; in compact mode it checks the entire tail. + for (int unsigned byte_idx = 0; byte_idx < nt*ne*mp*eb; byte_idx++) + if (byte_idx >= n*dp*eb || + (!compact && ((byte_idx / eb) / mp >= n || (byte_idx / eb) % mp >= m))) + if (rd_mem(db + byte_idx) !== 8'hCC) begin + errs++; + if (errs <= 12) + $display("[TPN] UNUSED DESTINATION BYTE CLOBBERED at +0x%0h", byte_idx); + end endtask initial begin automatic int unsigned total = 0; automatic int unsigned ce; - nd_req_valid = 1'b0; nd_rsp_ready = 1'b1; nd_req = '0; + transpose_req_valid = 1'b0; nd_rsp_ready = 1'b1; transpose_req_from_tb = '0; @(posedge rst_n); repeat (5) @(posedge clk); for (int unsigned k = 0; k < NCases; k++) begin if (Cases[k][2] > StrbWidth) continue; // element must fit the bus - run_case(Cases[k][0], Cases[k][1], Cases[k][2], ce); - if (ce == 0) $display("[TPN] PASS: %0dx%0d EB=%0d", Cases[k][0], Cases[k][1], Cases[k][2]); - else $display("[TPN] FAIL: %0dx%0d EB=%0d (%0d mismatches)", Cases[k][0], Cases[k][1], Cases[k][2], ce); - total += ce; + for (int unsigned compact = 0; compact < 2; compact++) begin + run_case(Cases[k][0], Cases[k][1], Cases[k][2], bit'(compact), ce); + if (ce == 0) + $display("[TPN] PASS: %0dx%0d EB=%0d compact=%0d", + Cases[k][0], Cases[k][1], Cases[k][2], compact); + else + $display("[TPN] FAIL: %0dx%0d EB=%0d compact=%0d (%0d mismatches)", + Cases[k][0], Cases[k][1], Cases[k][2], compact, ce); + total += ce; + end end - if (total == 0) $display("[TPN] ALL PASS (%0d cases, StrbWidth=%0d)", NCases, StrbWidth); + if (total == 0) $display("[TPN] ALL PASS (%0d geometries x 2 layouts, StrbWidth=%0d)", + NCases, StrbWidth); else $fatal(1, "[TPN] FAIL: %0d total mismatches", total); repeat (5) @(posedge clk); $finish();