From 7b8f9552be3585425a82d6daa5156749f74b1dee Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Wed, 8 Jul 2026 09:12:23 +0200 Subject: [PATCH 01/14] Add ROB to support reordering of outstanding transactions --- rtl/redmule_ctrl.sv | 4 +- rtl/redmule_mm_wrap.sv | 2 + rtl/redmule_streamer.sv | 125 +++++++++++++++++++++++++++++++++++----- rtl/redmule_top.sv | 4 ++ 4 files changed, 118 insertions(+), 17 deletions(-) diff --git a/rtl/redmule_ctrl.sv b/rtl/redmule_ctrl.sv index df0e95b9..a490afde 100644 --- a/rtl/redmule_ctrl.sv +++ b/rtl/redmule_ctrl.sv @@ -15,7 +15,8 @@ module redmule_ctrl parameter int unsigned Height = MaxDim, parameter int unsigned Width = MaxDim, parameter int unsigned PipeRegs = MaxPipeRegs-1, - parameter int unsigned FpWidth = 16 + parameter int unsigned FpWidth = 16, + parameter bit EnableReordering = 1'b0 )( input logic clk_i , input logic rst_ni , @@ -123,6 +124,7 @@ module redmule_ctrl assign cntrl_scheduler_o.finished = current == REDMULE_FINISHED; assign latch_clear = current == REDMULE_LATCH_RST; + always_comb begin : controller_fsm cntrl_flags_o.idle = 1'b0; next = current; diff --git a/rtl/redmule_mm_wrap.sv b/rtl/redmule_mm_wrap.sv index 6b8769bd..abc35b38 100644 --- a/rtl/redmule_mm_wrap.sv +++ b/rtl/redmule_mm_wrap.sv @@ -20,6 +20,7 @@ module redmule_mm_wrap #( parameter int unsigned DataW = MaxDataW, parameter int unsigned MisalignedAccessSupport = MisalignedAccessSupportDefault, + parameter bit EnableReordering = 1'b0, parameter fp_format_e FpFormat = FP16, parameter int unsigned Height = MaxDim, parameter int unsigned Width = MaxDim, @@ -72,6 +73,7 @@ module redmule_mm_wrap redmule_top #( .DataW ( DataW ), .MisalignedAccessSupport ( MisalignedAccessSupport ), + .EnableReordering ( EnableReordering ), .FpFormat ( FpFormat ), .Height ( Height ), .Width ( Width ), diff --git a/rtl/redmule_streamer.sv b/rtl/redmule_streamer.sv index fccb07e3..d2cff1ae 100644 --- a/rtl/redmule_streamer.sv +++ b/rtl/redmule_streamer.sv @@ -5,6 +5,7 @@ // Yvan Tortorella // Andrea Belano // Arpan Suravi Prasad +// Marco Bertuletti // `include "hci_helpers.svh" @@ -17,6 +18,17 @@ module redmule_streamer #( parameter int unsigned DataW = MaxDataW, parameter int unsigned MisalignedAccessSupport = MisalignedAccessSupportDefault, + // Set to 1 to reorder responses of outstanding transactions on the X, W, and Y + // load channels (hci_core_rob) and arbitrate the shared TCDM port out-of-order + // (hci_core_mux_ooo). Requires a memory system that completes accepted requests + // in issue order, but possibly with variable (multi-cycle) latency. When 0, the + // streamer keeps the original fixed-priority, single-outstanding-request scheme. + parameter bit EnableReordering = 1'b0, + // Height/NumPipeRegs size the load-side response FIFO (RESP_FIFO_DEPTH) of + // hci_core_source when EnableReordering is set, matching the engine's + // in-flight response latency. + parameter int unsigned Height = MaxDim , + parameter int unsigned NumPipeRegs = MaxPipeRegs-1, parameter int unsigned FpFormat = FP16 , parameter int unsigned EccChunkSize = 32 , parameter fpnew_pkg::fmt_logic_t FpFmtConfig = 6'b001101, @@ -90,7 +102,7 @@ hci_core_intf #( .IW ( `HCI_SIZE_GET_IW(ldst_tcdm) ), .EW ( `HCI_SIZE_GET_EW(ldst_tcdm) ), .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) -) ldst_tcdm_pre_r_id ( .clk ( clk_i ) ); +) ldst_tcdm_pre_r_valid ( .clk ( clk_i ) ); hci_core_intf #( `ifndef SYNTHESIS @@ -104,7 +116,7 @@ hci_core_intf #( .IW ( `HCI_SIZE_GET_IW(ldst_tcdm) ), .EW ( `HCI_SIZE_GET_EW(ldst_tcdm) ), .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) -) ldst_tcdm_pre_r_valid ( .clk ( clk_i ) ); +) ldst_tcdm_pre_r_id ( .clk ( clk_i ) ); if (EW > 1) begin : gen_ecc_encoder logic [`HCI_SIZE_GET_DW(tcdm)/EccChunkSize-1:0] data_single_err, data_multi_err; @@ -143,6 +155,72 @@ hci_core_intf #( .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) ) virt_tcdm [0:NumStreamSources] ( .clk ( clk_i ) ); +// LD/ST arbitration onto the shared TCDM port. +// * EnableReordering = 0: fixed-priority, single-outstanding-request arbiter +// (redmule_mux) followed by a 1-cycle-latency r_valid filter, as before. +// * EnableReordering = 1: each of the X/W/Y load channels gets its own +// hci_core_rob so several requests can be in flight at once on that channel; +// the (already in-order) Z/store and spare channels pass through unchanged. +// All channels are then arbitrated out-of-order (hci_core_mux_ooo). +if (EnableReordering) begin : gen_outstanding_ldst_mux + + // Number of unique user-IDs the ROB and the out-of-order mux can track; + localparam int unsigned UW = `HCI_SIZE_GET_UW(ldst_tcdm); + localparam int unsigned ROB_NW = 1 << UW; + + hci_core_intf #( + `ifndef SYNTHESIS + .WAIVE_RSP3_ASSERT ( 1'b1 ), + .WAIVE_RSP5_ASSERT ( 1'b1 ), + .WAIVE_RQ3_ASSERT ( 1'b1 ), + .WAIVE_RQ4_ASSERT ( 1'b1 ), + `endif + .DW ( `HCI_SIZE_GET_DW(ldst_tcdm) ), + .AW ( `HCI_SIZE_GET_AW(ldst_tcdm) ), + .BW ( `HCI_SIZE_GET_BW(ldst_tcdm) ), + .UW ( `HCI_SIZE_GET_UW(ldst_tcdm) ), + .IW ( `HCI_SIZE_GET_IW(ldst_tcdm) ), + .EW ( `HCI_SIZE_GET_EW(ldst_tcdm) ), + .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) + ) virt_tcdm_rob [0:NumStreamSources] ( .clk ( clk_i ) ); + + for (genvar i = 0; i < NumStreamSources; i++) begin : gen_channel_rob + hci_core_rob #( + .ROB_NW ( ROB_NW ), + .`HCI_SIZE_PARAM(out) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + ) i_streamer_rob ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .in ( virt_tcdm[i] ), + .out ( virt_tcdm_rob[i] ) + ); + end + + // The Z/store channel does not need reordering: pass it through to the + // out-of-order mux unchanged. + hci_core_assign i_rob_bypass ( .tcdm_target (virt_tcdm[NumStreamSources]), .tcdm_initiator (virt_tcdm_rob[NumStreamSources]) ); + + logic [NumStreamSources:0][$clog2(NumStreamSources+1)-1:0] priority_encoding; + assign priority_encoding[0] = 0; + assign priority_encoding[1] = 1; + assign priority_encoding[2] = 2; + assign priority_encoding[3] = 3; + + hci_core_mux_ooo #( + .NB_CHAN ( NumStreamSources+1 ), + .`HCI_SIZE_PARAM(out) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + ) i_mux ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .clear_i ( clear_i ), + .priority_force_i ( 1'b1 ), + .priority_i ( priority_encoding ), + .in ( virt_tcdm_rob ), + .out ( ldst_tcdm ) + ); + +end else begin : gen_fixed_priority_ldst_mux + if (MuxPriority == MUX_PRIORITY_STATIC) begin : static_mux_gen redmule_mux #( .NB_CHAN (NumStreamSources+1), @@ -192,6 +270,9 @@ hci_core_r_id_filter #( .tcdm_initiator ( ldst_tcdm ) ); + +end + /************************************ Store Channel *************************************/ /* The store channel of the streamer connects the incoming stream interface (Z stream) * * to an HCI core sink module that translates the stream into a TCDM protocol. This * @@ -278,7 +359,7 @@ flags_fifo_t store_fifo_flags; // HCI store fifo. hci_core_fifo #( - .FIFO_DEPTH ( 2 ), + .FIFO_DEPTH ( EnableReordering ? Height*(NumPipeRegs+1) : 2 ), .`HCI_SIZE_PARAM(tcdm_initiator) ( `HCI_SIZE_PARAM(ldst_tcdm) ) ) i_store_fifo ( .clk_i ( clk_i ), @@ -372,18 +453,29 @@ for (genvar i = 0; i < NumStreamSources; i++) begin: gen_tcdm2stream hci_core_assign i_load_assign ( .tcdm_target (load_fifo_d[i]), .tcdm_initiator (virt_tcdm[i]) ); - hci_core_fifo #( - .FIFO_DEPTH ( 4 ), // to avoid protocol violations, as the consumer has a throughput - // of 1 packet over 4 cycles, we need a depth of 4 elements. - .`HCI_SIZE_PARAM(tcdm_initiator) ( `HCI_SIZE_PARAM(ldst_tcdm) ) - ) i_load_tcdm_fifo ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .clear_i ( clear_i ), - .flags_o ( ), - .tcdm_target ( load_fifo_q[i] ), - .tcdm_initiator ( load_fifo_d[i] ) - ); + // hci_core_fifo can retract a request it already raised without a grant + // (a documented RQ-4 protocol violation, harmless when grants are never + // delayed) whenever its response-side fill level changes while a request + // is pending - which happens routinely once the ROB lets requests wait + // for a grant across many cycles. The ROB already buffers/reorders on + // this channel, so the FIFO's buffering is redundant here: skip it and + // wire the cast unit straight to the ROB when outstanding is enabled. + if (EnableReordering) begin : gen_no_load_tcdm_fifo + hci_core_assign i_load_tcdm_bypass ( .tcdm_target (load_fifo_q[i]), .tcdm_initiator (load_fifo_d[i]) ); + end else begin : gen_load_tcdm_fifo + hci_core_fifo #( + .FIFO_DEPTH ( 4 ), // to avoid protocol violations, as the consumer has a throughput + // of 1 packet over 4 cycles, we need a depth of 4 elements. + .`HCI_SIZE_PARAM(tcdm_initiator) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + ) i_load_tcdm_fifo ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .clear_i ( clear_i ), + .flags_o ( ), + .tcdm_target ( load_fifo_q[i] ), + .tcdm_initiator ( load_fifo_d[i] ) + ); + end // Load cast unit // This unit uses only the data bus of the TCDM interface. The other buses @@ -435,7 +527,8 @@ for (genvar i = 0; i < NumStreamSources; i++) begin: gen_tcdm2stream hci_core_source #( .MISALIGNED_ACCESSES ( MisalignedAccessSupport ), - .`HCI_SIZE_PARAM(tcdm) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + .`HCI_SIZE_PARAM(tcdm) ( `HCI_SIZE_PARAM(ldst_tcdm) ), + .RESP_FIFO_DEPTH ( EnableReordering ? Height*(NumPipeRegs+1) : 0 ) ) i_stream_source ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), diff --git a/rtl/redmule_top.sv b/rtl/redmule_top.sv index 6f526d72..7704c750 100644 --- a/rtl/redmule_top.sv +++ b/rtl/redmule_top.sv @@ -16,6 +16,7 @@ module redmule_top #( parameter int unsigned DataW = MaxDataW, // TCDM port dimension (in bits) parameter int unsigned MisalignedAccessSupport = MisalignedAccessSupportDefault, // set to 1 to support misaligned accesses on TCDM + parameter bit EnableReordering = 1'b0, // set to 1 to enable ROB-based reordering of outstanding TCDM transactions parameter fp_format_e FpFormat = FP16, // Data format (default is FP16) parameter int unsigned Height = MaxDim, // Number of PEs within a row parameter int unsigned Width = MaxDim, // Number of parallel rows @@ -188,6 +189,9 @@ hwpe_stream_intf_stream #( .DATA_WIDTH ( DataW ) ) z_buffer_fifo ( .clk( cl redmule_streamer #( .DataW ( DataW ), .MisalignedAccessSupport ( MisalignedAccessSupport ), + .EnableReordering ( EnableReordering ), + .Height ( Height ), + .NumPipeRegs ( NumPipeRegs ), .EccChunkSize ( EccChunkSize ), .FpFormat ( FpFormat ), .FpFmtConfig ( FpFmtConfig ), From 1f09d8e94fa6acc2b462eb4a0b83a2e4fd6ef47c Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Wed, 8 Jul 2026 10:13:31 +0200 Subject: [PATCH 02/14] Bump common_cells --- Bender.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Bender.yml b/Bender.yml index 7268fd38..6d20fcd8 100644 --- a/Bender.yml +++ b/Bender.yml @@ -19,7 +19,7 @@ dependencies: hwpe-ctrl : { git: "https://github.com/pulp-platform/hwpe-ctrl.git" , version: 3.1.1 } hci : { git: "https://github.com/pulp-platform/hci.git" , version: 2.6.1 } fpnew : { git: "https://github.com/pulp-platform/cvfpu.git" , rev: "pulp-v0.1.3" } - common_cells : { git: "https://github.com/pulp-platform/common_cells.git" , version: 1.21.0 } + common_cells : { git: "https://github.com/pulp-platform/common_cells.git" , version: 1.38.0 } tech_cells_generic: { git: "https://github.com/pulp-platform/tech_cells_generic.git", version: 0.2.11 } sources: From 4bf57dfe04d326d81d37ffa05a428153da635a3f Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Thu, 9 Jul 2026 10:36:52 +0200 Subject: [PATCH 03/14] Update API with Register-RDL generated files --- rtl/ctrl/gen_regif.sh | 23 +++++- sw/archi_redmule.h | 77 +---------------- sw/hal_redmule.h | 69 +++++++--------- sw/hwpe_ctrl_target.h | 188 ++++++++++++++++++++++++++++++++++++++++++ sw/redmule.c | 4 +- 5 files changed, 246 insertions(+), 115 deletions(-) create mode 100644 sw/hwpe_ctrl_target.h diff --git a/rtl/ctrl/gen_regif.sh b/rtl/ctrl/gen_regif.sh index bd0de4f9..1196d6d5 100755 --- a/rtl/ctrl/gen_regif.sh +++ b/rtl/ctrl/gen_regif.sh @@ -5,7 +5,28 @@ peakrdl regblock redmule_regif.rdl -o regif/ --cpuif passthrough --default-reset arst_n --hwif-report --addr-width 32 peakrdl html redmule_regif.rdl -o regif/html/ -peakrdl c-header redmule_regif.rdl -o regif/hwpe_ctrl_target.h +peakrdl c-header redmule_regif.rdl -o ../../sw/hwpe_ctrl_target.h +awk ' + /#include / { + print + print "" + print "#if defined(__cplusplus)" + print "#define REDMULE_STATIC_ASSERT(cond, msg) static_assert(cond, msg)" + print "#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)" + print "#define REDMULE_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg)" + print "#else" + print "#define REDMULE_STATIC_ASSERT_CONCAT_(a, b) a##b" + print "#define REDMULE_STATIC_ASSERT_CONCAT(a, b) REDMULE_STATIC_ASSERT_CONCAT_(a, b)" + print "#define REDMULE_STATIC_ASSERT(cond, msg) \\" + print " typedef char REDMULE_STATIC_ASSERT_CONCAT(redmule_static_assert_, __LINE__)[(cond) ? 1 : -1]" + print "#endif" + next + } + { + gsub(/static_assert\(/, "REDMULE_STATIC_ASSERT(") + print + } +' ../../sw/hwpe_ctrl_target.h > ../../sw/hwpe_ctrl_target.h.tmp && mv ../../sw/hwpe_ctrl_target.h.tmp ../../sw/hwpe_ctrl_target.h # PeakRDL uses unpacked structs to avoid issues at compile time, which is commendable, but incompatible with FIFOing the output of the job! (use portable sed syntax that works on both Linux and macOS) sed -E 's/typedef[[:space:]]+struct([[:space:]])/typedef struct packed\1/g' regif/redmule_regif_pkg.sv > regif/redmule_regif_pkg.sv.tmp && mv regif/redmule_regif_pkg.sv.tmp regif/redmule_regif_pkg.sv diff --git a/sw/archi_redmule.h b/sw/archi_redmule.h index bf740add..d33e0f4a 100644 --- a/sw/archi_redmule.h +++ b/sw/archi_redmule.h @@ -8,86 +8,13 @@ #ifndef __ARCHI_REDMULE_H__ #define __ARCHI_REDMULE_H__ -/* - * Register map aligned to the SystemRDL interface (rtl/ctrl/redmule_regif.rdl). - * The job-dependent block now lives at offset 0x20 (was 0x40), configuration - * registers come first, and the operation/format selection is folded into - * mcnfig1 (there is no separate arithmetic register any more). - * - * |========================================================================| - * || || - * ||Control and generic configuration register layout || - * |========================================================================| - * || # reg | offset | bits | bitmask || content || - * ||-------+----------+---------+--------------++-------------------------|| - * || 0 | 0x0000 | 1: 0 | 0x00000003 || COMMIT_TRIGGER || - * || 1 | 0x0004 | 31: 0 | 0xFFFFFFFF || ACQUIRE || - * || 2 | 0x0008 | 31: 0 | 0xFFFFFFFF || reserved0 || - * || 3 | 0x000c | 31: 0 | 0xFFFFFFFF || STATUS || - * || 4 | 0x0010 | 7: 0 | 0x000000FF || RUNNING_JOB || - * || 5 | 0x0014 | 1: 0 | 0x00000003 || SOFT_CLEAR || - * |========================================================================| - * || || - * ||Job-dependent registers layout (base 0x20) || - * |========================================================================| - * || # reg | offset | bits | bitmask || content || - * ||-------+----------+---------+--------------++-------------------------|| - * || 0 | 0x0020 | | || Matrix Config 0 (mcnfig0)|| - * || | | 31:16 | 0xFFFF0000 || K Size (W Columns) || - * || | | 15: 0 | 0x0000FFFF || M Size (X Rows) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 1 | 0x0024 | | || Matrix Config 1 (mcnfig1)|| - * || | | 26:25 | 0x06000000 || Output format || - * || | | 24:23 | 0x01800000 || Input format || - * || | | 22:20 | 0x00700000 || Operation selection || - * || | | 19:16 | 0x000F0000 || send/receive stream || - * || | | 15: 0 | 0x0000FFFF || N Size (X Cols/W Rows) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 2 | 0x0028 | 31: 0 | 0xFFFFFFFF || Matrix Config 2 (mcnfig2)|| - * || | | | || Y offset (bias) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 3 | 0x002C | 31: 0 | 0xFFFFFFFF || X_ADDR (marith0) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 4 | 0x0030 | 31: 0 | 0xFFFFFFFF || W_ADDR (marith1) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 5 | 0x0034 | 31: 0 | 0xFFFFFFFF || Z_ADDR (marith2) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 6 | 0x0038 | 31: 0 | 0xFFFFFFFF || MOPCNT (RO) || - * |========================================================================| - * - */ - #define ARCHI_CL_EVT_ACC0 0 #define ARCHI_CL_EVT_ACC1 1 -// RedMulE architecture -#define ADDR_WIDTH 32 -#define DATA_WIDTH 256 -#define REDMULE_FMT 16 -#define ARRAY_HEIGHT 4 -#define PIPE_REGS 3 -#define ARRAY_WIDTH 12 /* Superior limit is ARRAY_HEIGHT*PIPE_REGS */ - // Base address +#ifndef REDMULE_BASE_ADD #define REDMULE_BASE_ADD 0x00100000 - -// Commands (mandatory control block @ 0x00) -#define REDMULE_TRIGGER 0x00 // commit_trigger[1:0]: writing 0 commits + starts the job -#define REDMULE_ACQUIRE 0x04 -#define REDMULE_FINISHED 0x08 // reserved0 in the RDL map -#define REDMULE_STATUS 0x0C -#define REDMULE_RUNNING_JOB 0x10 -#define REDMULE_SOFT_CLEAR 0x14 // soft_clear[1:0]: writing 0 clears everything (incl. regfile) - -// Job-dependent registers (base 0x20; config first, then X/W/Z addresses) -#define REDMULE_REG_OFFS 0x20 -#define REDMULE_MCFG0_PTR 0x00 // -> 0x20 mcnfig0: k_size[31:16], m_size[15:0] -#define REDMULE_MCFG1_PTR 0x04 // -> 0x24 mcnfig1: n[15:0], stream[19:16], ops[22:20], in_fmt[24:23], out_fmt[26:25] -#define REDMULE_MCFG2_PTR 0x08 // -> 0x28 mcnfig2: y_offs[31:0] -#define REDMULE_REG_X_PTR 0x0C // -> 0x2C marith0: x_addr -#define REDMULE_REG_W_PTR 0x10 // -> 0x30 marith1: w_addr -#define REDMULE_REG_Z_PTR 0x14 // -> 0x34 marith2: z_addr -#define REDMULE_MOPCNT_PTR 0x18 // -> 0x38 mopcnt (RO) +#endif // OPs definition #define MATMUL 0x0 diff --git a/sw/hal_redmule.h b/sw/hal_redmule.h index b00c767b..15e567cf 100644 --- a/sw/hal_redmule.h +++ b/sw/hal_redmule.h @@ -7,46 +7,47 @@ #ifndef __HAL_REDMULE_H__ #define __HAL_REDMULE_H__ + +#include "archi_redmule.h" +#include "hwpe_ctrl_target.h" #include -#include "tensor_dim.h" /* LOW-LEVEL HAL */ #define REDMULE_ADDR_BASE REDMULE_BASE_ADD #define REDMULE_ADDR_SPACE 0x00000100 -#define HWPE_WRITE(value, offset) *(int *)(REDMULE_ADDR_BASE + offset) = value -#define HWPE_READ(offset) *(int *)(REDMULE_ADDR_BASE + offset) +static inline volatile redmule_regif_t *redmule_regs(void) { + return (volatile redmule_regif_t *)REDMULE_ADDR_BASE; +} static inline void redmule_x_add_set(unsigned int value) { - HWPE_WRITE(value, REDMULE_REG_OFFS + REDMULE_REG_X_PTR); + redmule_regs()->hwpe_job_dep.marith0 = value; } static inline void redmule_w_add_set(unsigned int value) { - HWPE_WRITE(value, REDMULE_REG_OFFS + REDMULE_REG_W_PTR); + redmule_regs()->hwpe_job_dep.marith1 = value; } static inline void redmule_z_add_set(unsigned int value) { - HWPE_WRITE(value, REDMULE_REG_OFFS + REDMULE_REG_Z_PTR); + redmule_regs()->hwpe_job_dep.marith2 = value; } -static inline void redmule_mcfg_set(uint32_t mcfg0, uint32_t mcfg1, uint32_t mcfg2) { - HWPE_WRITE(mcfg0, REDMULE_REG_OFFS + REDMULE_MCFG0_PTR); - HWPE_WRITE(mcfg1, REDMULE_REG_OFFS + REDMULE_MCFG1_PTR); - HWPE_WRITE(mcfg2, REDMULE_REG_OFFS + REDMULE_MCFG2_PTR); +static inline void redmule_mcfg_set(uint32_t mcfg0, uint32_t mcfg1) { + redmule_regs()->hwpe_job_dep.mcnfig0 = mcfg0; + redmule_regs()->hwpe_job_dep.mcnfig1 = mcfg1; } -static inline void hwpe_trigger_job() { HWPE_WRITE(0, REDMULE_TRIGGER); } +static inline void hwpe_trigger_job() { redmule_regs()->hwpe_ctrl.commit_trigger = 0; } -static inline int hwpe_acquire_job() { return HWPE_READ(REDMULE_ACQUIRE); } +static inline int hwpe_acquire_job() { + return (int)redmule_regs()->hwpe_ctrl.acquire; +} static inline unsigned int hwpe_get_status() { - return HWPE_READ(REDMULE_STATUS); + return redmule_regs()->hwpe_ctrl.status; } -static inline void hwpe_soft_clear() { - volatile int i; - HWPE_WRITE(0, REDMULE_SOFT_CLEAR); -} +static inline void hwpe_soft_clear() { redmule_regs()->hwpe_ctrl.soft_clear = 0; } static inline void hwpe_cg_enable() { return; } @@ -57,27 +58,19 @@ void redmule_cfg(unsigned int x, unsigned int w, unsigned int z, uint16_t m_size uint32_t mcfg_reg0 = 0; uint32_t mcfg_reg1 = 0; - uint32_t mcfg_reg2 = 0; - - // mcnfig0: K size in [31:16], M size in [15:0] - mcfg_reg0 = ((uint32_t)k_size << 16) | ((uint32_t)m_size << 0); - - // mcnfig1: N size in [15:0]; op/format folded in (there is no separate ARITH reg). - // gemm_ops [22:20] - // gemm_input_fmt [24:23] - // gemm_output_fmt [26:25] - // stream-routing bits [19:16] left 0 (no external streaming in the MM scenario) - mcfg_reg1 = ((uint32_t)n_size << 0) | ((uint32_t)gemm_op << 20) | - ((uint32_t)gemm_fmt << 23) | ((uint32_t)gemm_fmt << 25); - - // mcnfig2: Y offset (bias). y_addr = z_addr + y_offs, so y_offs = 0 keeps the - // in-place GEMM behaviour (Y read from, and Z written to, the same buffer). - mcfg_reg2 = 0; - - redmule_mcfg_set((unsigned int)mcfg_reg0, (unsigned int)mcfg_reg1, (unsigned int)mcfg_reg2); - redmule_x_add_set((unsigned int)x); - redmule_w_add_set((unsigned int)w); - redmule_z_add_set((unsigned int)z); + + mcfg_reg0 = ((uint32_t)k_size << REDMULE_REGIF__MCNFIG0__K_SIZE_bp) | + ((uint32_t)m_size << REDMULE_REGIF__MCNFIG0__M_SIZE_bp); + // gemm_fmt is used for both the input and the output format. + mcfg_reg1 = ((uint32_t)n_size << REDMULE_REGIF__MCNFIG1__N_SIZE_bp) | + ((uint32_t)gemm_op << REDMULE_REGIF__MCNFIG1__GEMM_OPS_bp) | + ((uint32_t)gemm_fmt << REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bp) | + ((uint32_t)gemm_fmt << REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bp); + + redmule_x_add_set(x); + redmule_w_add_set(w); + redmule_z_add_set(z); + redmule_mcfg_set(mcfg_reg0, mcfg_reg1); } #endif diff --git a/sw/hwpe_ctrl_target.h b/sw/hwpe_ctrl_target.h new file mode 100644 index 00000000..711a8ad8 --- /dev/null +++ b/sw/hwpe_ctrl_target.h @@ -0,0 +1,188 @@ +// Generated by PeakRDL-cheader - A free and open-source header generator +// https://github.com/SystemRDL/PeakRDL-cheader + +#ifndef HWPE_CTRL_TARGET_H +#define HWPE_CTRL_TARGET_H + +#ifdef __cplusplus +extern "C" { +#endif +#include +#include + +#if defined(__cplusplus) +#define REDMULE_STATIC_ASSERT(cond, msg) static_assert(cond, msg) +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +#define REDMULE_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) +#else +#define REDMULE_STATIC_ASSERT_CONCAT_(a, b) a##b +#define REDMULE_STATIC_ASSERT_CONCAT(a, b) REDMULE_STATIC_ASSERT_CONCAT_(a, b) +#define REDMULE_STATIC_ASSERT(cond, msg) \ + typedef char REDMULE_STATIC_ASSERT_CONCAT(redmule_static_assert_, __LINE__)[(cond) ? 1 : -1] +#endif + +// reg - redmule_regif::hwpe_commit_trigger +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_bm 0x3 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_bp 0 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_bw 2 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_reset 0x0 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_bm 0xfffffffc +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_bp 2 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_bw 30 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_reset 0x0 + +// reg - redmule_regif::hwpe_acquire +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_bm 0xffffffff +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_bp 0 +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_bw 32 +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_reset 0x0 + +// reg - redmule_regif::hwpe_reserved +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_bm 0xffffffff +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_bp 0 +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_bw 32 +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_reset 0x0 + +// reg - redmule_regif::hwpe_status +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_bm 0xffffffff +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_bp 0 +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_bw 32 +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_reset 0x0 + +// reg - redmule_regif::hwpe_running_job +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_bm 0xff +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_bp 0 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_bw 8 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_reset 0x0 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_bm 0xffffff00 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_bp 8 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_bw 24 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_reset 0x0 + +// reg - redmule_regif::hwpe_soft_clear +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_bm 0x3 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_bp 0 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_bw 2 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_reset 0x0 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_bm 0xfffffffc +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_bp 2 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_bw 30 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_reset 0x0 + +// regfile - redmule_regif::hwpe_ctrl_mandatory +typedef struct __attribute__ ((__packed__)) { + uint32_t commit_trigger; + uint32_t acquire; + uint32_t reserved0; + uint32_t status; + uint32_t running_job; + uint32_t soft_clear; + uint32_t reserved1; + uint32_t reserved2; +} redmule_regif__hwpe_ctrl_mandatory_t; + +// reg - redmule_regif::mcnfig0 +#define REDMULE_REGIF__MCNFIG0__M_SIZE_bm 0xffff +#define REDMULE_REGIF__MCNFIG0__M_SIZE_bp 0 +#define REDMULE_REGIF__MCNFIG0__M_SIZE_bw 16 +#define REDMULE_REGIF__MCNFIG0__M_SIZE_reset 0x0 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_bm 0xffff0000 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_bp 16 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_bw 16 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_reset 0x0 + +// reg - redmule_regif::mcnfig1 +#define REDMULE_REGIF__MCNFIG1__N_SIZE_bm 0xffff +#define REDMULE_REGIF__MCNFIG1__N_SIZE_bp 0 +#define REDMULE_REGIF__MCNFIG1__N_SIZE_bw 16 +#define REDMULE_REGIF__MCNFIG1__N_SIZE_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_bm 0x10000 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_bp 16 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_bw 1 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__SEND_X_bm 0x20000 +#define REDMULE_REGIF__MCNFIG1__SEND_X_bp 17 +#define REDMULE_REGIF__MCNFIG1__SEND_X_bw 1 +#define REDMULE_REGIF__MCNFIG1__SEND_X_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_bm 0x40000 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_bp 18 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_bw 1 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__SEND_W_bm 0x80000 +#define REDMULE_REGIF__MCNFIG1__SEND_W_bp 19 +#define REDMULE_REGIF__MCNFIG1__SEND_W_bw 1 +#define REDMULE_REGIF__MCNFIG1__SEND_W_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_bm 0x700000 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_bp 20 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_bw 3 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bm 0x1800000 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bp 23 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bw 2 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bm 0x6000000 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bp 25 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bw 2 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_reset 0x0 + +// reg - redmule_regif::mcnfig2 +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_bm 0xffffffff +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_bp 0 +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_bw 32 +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_reset 0x0 + +// reg - redmule_regif::marith0 +#define REDMULE_REGIF__MARITH0__X_ADDR_bm 0xffffffff +#define REDMULE_REGIF__MARITH0__X_ADDR_bp 0 +#define REDMULE_REGIF__MARITH0__X_ADDR_bw 32 +#define REDMULE_REGIF__MARITH0__X_ADDR_reset 0x0 + +// reg - redmule_regif::marith1 +#define REDMULE_REGIF__MARITH1__W_ADDR_bm 0xffffffff +#define REDMULE_REGIF__MARITH1__W_ADDR_bp 0 +#define REDMULE_REGIF__MARITH1__W_ADDR_bw 32 +#define REDMULE_REGIF__MARITH1__W_ADDR_reset 0x0 + +// reg - redmule_regif::marith2 +#define REDMULE_REGIF__MARITH2__Z_ADDR_bm 0xffffffff +#define REDMULE_REGIF__MARITH2__Z_ADDR_bp 0 +#define REDMULE_REGIF__MARITH2__Z_ADDR_bw 32 +#define REDMULE_REGIF__MARITH2__Z_ADDR_reset 0x0 + +// reg - redmule_regif::mopcnt +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bm 0xffffffff +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bp 0 +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bw 32 +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_reset 0x0 + +// regfile - redmule_regif::hwpe_ctrl_job_dep +typedef struct __attribute__ ((__packed__)) { + uint32_t mcnfig0; + uint32_t mcnfig1; + uint32_t mcnfig2; + uint32_t marith0; + uint32_t marith1; + uint32_t marith2; + uint32_t mopcnt; +} redmule_regif__hwpe_ctrl_job_dep_t; + +// regfile - redmule_regif::hwpe_ctrl_job_indep +typedef struct __attribute__ ((__packed__)) { + uint32_t reserved; +} redmule_regif__hwpe_ctrl_job_indep_t; + +// addrmap - redmule_regif +typedef struct __attribute__ ((__packed__)) { + redmule_regif__hwpe_ctrl_mandatory_t hwpe_ctrl; + redmule_regif__hwpe_ctrl_job_dep_t hwpe_job_dep; + redmule_regif__hwpe_ctrl_job_indep_t hwpe_job_indep; +} redmule_regif_t; + + +REDMULE_STATIC_ASSERT(sizeof(redmule_regif_t) == 0x40, "Packing error"); + +#ifdef __cplusplus +} +#endif + +#endif /* HWPE_CTRL_TARGET_H */ diff --git a/sw/redmule.c b/sw/redmule.c index 6c0b2c87..19ff9113 100644 --- a/sw/redmule.c +++ b/sw/redmule.c @@ -5,11 +5,13 @@ // Yvan Tortorella // +#include + #include "archi_redmule.h" #include "hal_redmule.h" #include "redmule_utils.h" -#include +#include "tensor_dim.h" #include "golden.h" #include "w_input.h" #include "x_input.h" From c809a47de94151ede2be2ba6e94e6535fd420d9a Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Sat, 11 Jul 2026 15:17:25 +0200 Subject: [PATCH 04/14] Empty z fifo in outstanding mode --- rtl/redmule_ctrl.sv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rtl/redmule_ctrl.sv b/rtl/redmule_ctrl.sv index a490afde..ce44788c 100644 --- a/rtl/redmule_ctrl.sv +++ b/rtl/redmule_ctrl.sv @@ -45,6 +45,7 @@ module redmule_ctrl logic latch_clear; logic tiler_setback, tiler_valid; + logic fifo_z_empty; typedef enum logic [2:0] { REDMULE_LATCH_RST, @@ -123,7 +124,7 @@ module redmule_ctrl assign cntrl_scheduler_o.rst = current == REDMULE_FINISHED; assign cntrl_scheduler_o.finished = current == REDMULE_FINISHED; assign latch_clear = current == REDMULE_LATCH_RST; - + assign fifo_z_empty = EnableReordering ? flgs_streamer_i.store_fifo_empty : 1'b1; always_comb begin : controller_fsm cntrl_flags_o.idle = 1'b0; From 385562ac462e3426abea5dfc4adbd401258ad2f3 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Fri, 24 Oct 2025 12:11:28 +0200 Subject: [PATCH 05/14] Allow offset on W columns --- rtl/ctrl/redmule_regif.rdl | 10 +++++ rtl/ctrl/redmule_target_decoder.sv | 1 + rtl/ctrl/regif/redmule_regif.sv | 41 ++++++++++++++++++- rtl/ctrl/regif/redmule_regif_pkg.sv | 11 ++++- rtl/redmule_ctrl.sv | 48 +++++++++++++++++++--- rtl/redmule_inst_decoder.sv | 5 ++- rtl/redmule_pkg.sv | 1 + rtl/redmule_tiler.sv | 63 +++++++++++++++++++++++------ rtl/redmule_top.sv | 5 ++- sw/hal_redmule.h | 13 ++++-- sw/hwpe_ctrl_target.h | 9 ++++- sw/redmule.c | 2 +- 12 files changed, 180 insertions(+), 29 deletions(-) diff --git a/rtl/ctrl/redmule_regif.rdl b/rtl/ctrl/redmule_regif.rdl index 9e574f97..994f99e8 100644 --- a/rtl/ctrl/redmule_regif.rdl +++ b/rtl/ctrl/redmule_regif.rdl @@ -216,6 +216,15 @@ addrmap redmule_regif { sw = rw; } y_offs[31:0] = 0; }; + reg mcnfig3 { + name = "MCNFIG3"; + field { + name = "w_cols_offset"; + desc = "Column offset applied to W/Z before starting the first pass."; + hw = r; + sw = rw; + } w_cols_offset[31:0] = 0; + }; reg marith0 { name = "MARITH0"; field { @@ -263,6 +272,7 @@ addrmap redmule_regif { mcnfig0 mcnfig0 @ 0x00; mcnfig1 mcnfig1 @ 0x04; mcnfig2 mcnfig2 @ 0x08; + mcnfig3 mcnfig3 @ 0x1c; marith0 marith0 @ 0x0c; marith1 marith1 @ 0x10; marith2 marith2 @ 0x14; diff --git a/rtl/ctrl/redmule_target_decoder.sv b/rtl/ctrl/redmule_target_decoder.sv index 1f28d42b..66805e6c 100644 --- a/rtl/ctrl/redmule_target_decoder.sv +++ b/rtl/ctrl/redmule_target_decoder.sv @@ -137,6 +137,7 @@ module redmule_target_decoder assign config_o.receive_w = hwif_out.hwpe_job_dep.mcnfig1.receive_w.value; assign config_o.send_w = hwif_out.hwpe_job_dep.mcnfig1.send_w.value; assign config_o.y_offs = hwif_out.hwpe_job_dep.mcnfig2.y_offs.value; + assign config_o.w_cols_offset = hwif_out.hwpe_job_dep.mcnfig3.w_cols_offset.value[15:0]; assign config_o.x_addr = hwif_out.hwpe_job_dep.marith0.x_addr.value; assign config_o.w_addr = hwif_out.hwpe_job_dep.marith1.w_addr.value; assign config_o.z_addr = hwif_out.hwpe_job_dep.marith2.z_addr.value; diff --git a/rtl/ctrl/regif/redmule_regif.sv b/rtl/ctrl/regif/redmule_regif.sv index 3e92ab64..43e30a68 100644 --- a/rtl/ctrl/regif/redmule_regif.sv +++ b/rtl/ctrl/regif/redmule_regif.sv @@ -87,6 +87,7 @@ module redmule_regif ( logic marith1; logic marith2; logic mopcnt; + logic mcnfig3; } hwpe_job_dep; struct { logic reserved; @@ -120,7 +121,8 @@ module redmule_regif ( decoded_reg_strb.hwpe_job_dep.marith1 = cpuif_req_masked & (cpuif_addr == 32'h30); decoded_reg_strb.hwpe_job_dep.marith2 = cpuif_req_masked & (cpuif_addr == 32'h34); decoded_reg_strb.hwpe_job_dep.mopcnt = cpuif_req_masked & (cpuif_addr == 32'h38) & !cpuif_req_is_wr; - decoded_reg_strb.hwpe_job_indep.reserved = cpuif_req_masked & (cpuif_addr == 32'h3c) & !cpuif_req_is_wr; + decoded_reg_strb.hwpe_job_dep.mcnfig3 = cpuif_req_masked & (cpuif_addr == 32'h3c); + decoded_reg_strb.hwpe_job_indep.reserved = cpuif_req_masked & (cpuif_addr == 32'h40) & !cpuif_req_is_wr; decoded_err = '0; end @@ -224,6 +226,12 @@ module redmule_regif ( logic load_next; } z_addr; } marith2; + struct { + struct { + logic [31:0] next; + logic load_next; + } w_cols_offset; + } mcnfig3; } hwpe_job_dep; } field_combo_t; field_combo_t field_combo; @@ -301,6 +309,11 @@ module redmule_regif ( logic [31:0] value; } z_addr; } marith2; + struct { + struct { + logic [31:0] value; + } w_cols_offset; + } mcnfig3; } hwpe_job_dep; } field_storage_t; field_storage_t field_storage; @@ -705,6 +718,29 @@ module redmule_regif ( end end assign hwif_out.hwpe_job_dep.marith2.z_addr.value = field_storage.hwpe_job_dep.marith2.z_addr.value; + // Field: redmule_regif.hwpe_job_dep.mcnfig3.w_cols_offset + always_comb begin + automatic logic [31:0] next_c; + automatic logic load_next_c; + next_c = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value; + load_next_c = '0; + if(decoded_reg_strb.hwpe_job_dep.mcnfig3 && decoded_req_is_wr) begin // SW write + next_c = (field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value & ~decoded_wr_biten[31:0]) | (decoded_wr_data[31:0] & decoded_wr_biten[31:0]); + load_next_c = '1; + end + field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.next = next_c; + field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.load_next = load_next_c; + end + always_ff @(posedge clk or negedge arst_n) begin + if(~arst_n) begin + field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value <= 32'h0; + end else begin + if(field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.load_next) begin + field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value <= field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.next; + end + end + end + assign hwif_out.hwpe_job_dep.mcnfig3.w_cols_offset.value = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value; assign hwif_out.hwpe_job_indep.reserved.reserved.value = 32'h0; //-------------------------------------------------------------------------- @@ -783,6 +819,9 @@ module redmule_regif ( readback_data_var[31:0] = hwif_in.hwpe_job_dep.mopcnt.op_id_cnt.next; end if(rd_mux_addr == 32'h3c) begin + readback_data_var[31:0] = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value; + end + if(rd_mux_addr == 32'h40) begin readback_data_var[31:0] = 32'h0; end readback_data = readback_data_var; diff --git a/rtl/ctrl/regif/redmule_regif_pkg.sv b/rtl/ctrl/regif/redmule_regif_pkg.sv index a3b41159..658bc63f 100644 --- a/rtl/ctrl/regif/redmule_regif_pkg.sv +++ b/rtl/ctrl/regif/redmule_regif_pkg.sv @@ -8,7 +8,7 @@ package redmule_regif_pkg; localparam REDMULE_REGIF_DATA_WIDTH = 32; localparam REDMULE_REGIF_MIN_ADDR_WIDTH = 32; - localparam REDMULE_REGIF_SIZE = 'h40; + localparam REDMULE_REGIF_SIZE = 'h44; typedef struct packed { logic [31:0] next; @@ -220,6 +220,14 @@ package redmule_regif_pkg; redmule_regif__marith2__z_addr__out_t z_addr; } redmule_regif__marith2__out_t; + typedef struct packed { + logic [31:0] value; + } redmule_regif__mcnfig3__w_cols_offset__out_t; + + typedef struct packed { + redmule_regif__mcnfig3__w_cols_offset__out_t w_cols_offset; + } redmule_regif__mcnfig3__out_t; + typedef struct packed { redmule_regif__mcnfig0__out_t mcnfig0; redmule_regif__mcnfig1__out_t mcnfig1; @@ -227,6 +235,7 @@ package redmule_regif_pkg; redmule_regif__marith0__out_t marith0; redmule_regif__marith1__out_t marith1; redmule_regif__marith2__out_t marith2; + redmule_regif__mcnfig3__out_t mcnfig3; } redmule_regif__hwpe_ctrl_job_dep__out_t; typedef struct packed { diff --git a/rtl/redmule_ctrl.sv b/rtl/redmule_ctrl.sv index ce44788c..63925738 100644 --- a/rtl/redmule_ctrl.sv +++ b/rtl/redmule_ctrl.sv @@ -46,12 +46,14 @@ module redmule_ctrl logic latch_clear; logic tiler_setback, tiler_valid; logic fifo_z_empty; + logic set_offset_q, set_offset_d, loopback_reset; typedef enum logic [2:0] { REDMULE_LATCH_RST, REDMULE_IDLE, REDMULE_STARTING, REDMULE_COMPUTING, + REDMULE_LOOPBACK, REDMULE_FINISHED } redmule_ctrl_state_e; @@ -70,6 +72,7 @@ module redmule_ctrl .rst_ni ( rst_ni ), .clear_i ( target_clear_i ), .setback_i ( tiler_setback ), + .loopback_i ( loopback_reset ), .start_cfg_i ( start_cfg_i ), .valid_o ( tiler_valid ), .busy_o ( tiler_busy_o ), @@ -94,6 +97,18 @@ module redmule_ctrl end end + // Set offset flag + always_ff @(posedge clk_i or negedge rst_ni) begin + if(~rst_ni) begin + set_offset_q <= 1'b0; + end else begin + if (target_clear_i || latch_clear || current == REDMULE_FINISHED || loopback_reset) + set_offset_q <= 1'b0; + else + set_offset_q <= set_offset_d; + end + end + logic slave_start; always_ff @(posedge clk_i, negedge rst_ni) begin if (~rst_ni) begin @@ -119,9 +134,12 @@ module redmule_ctrl assign cntrl_scheduler_o.first_load = current == REDMULE_STARTING; assign tiler_setback = tiler_valid; - assign busy_o = slave_start | (current != REDMULE_LATCH_RST && current != REDMULE_IDLE && current != REDMULE_FINISHED); - assign flush_o = current == REDMULE_FINISHED; - assign cntrl_scheduler_o.rst = current == REDMULE_FINISHED; + // Keep the accelerator clocked while in FINISHED so the FSM can take the + // final transition back to IDLE before busy deasserts. + assign busy_o = slave_start | (current != REDMULE_LATCH_RST && current != REDMULE_IDLE); + assign loopback_reset = current == REDMULE_COMPUTING && next == REDMULE_LOOPBACK; + assign flush_o = current == REDMULE_FINISHED || loopback_reset; + assign cntrl_scheduler_o.rst = current == REDMULE_FINISHED || loopback_reset; assign cntrl_scheduler_o.finished = current == REDMULE_FINISHED; assign latch_clear = current == REDMULE_LATCH_RST; assign fifo_z_empty = EnableReordering ? flgs_streamer_i.store_fifo_empty : 1'b1; @@ -129,31 +147,47 @@ module redmule_ctrl always_comb begin : controller_fsm cntrl_flags_o.idle = 1'b0; next = current; + set_offset_d = set_offset_q; case (current) REDMULE_LATCH_RST: begin cntrl_flags_o.idle = 1'b1; + set_offset_d = 1'b0; next = REDMULE_IDLE; end REDMULE_IDLE: begin cntrl_flags_o.idle = 1'b1; + set_offset_d = config_i.w_cols_offset != '0; if ((slave_start & tiler_valid) || test_mode_i) begin next = REDMULE_STARTING; end end + REDMULE_LOOPBACK: begin + cntrl_flags_o.idle = 1'b1; + set_offset_d = 1'b0; + if (tiler_valid) begin + next = REDMULE_STARTING; + end + end + REDMULE_STARTING: begin if (w_loaded_i) begin next = REDMULE_COMPUTING; end end + REDMULE_COMPUTING: begin // busy_o gates clk_acc, so the job must not finish while stores are // still queued in the streamer's store FIFO, or they freeze unsent. if (flgs_streamer_i.z_stream_sink_flags.ready_start && fifo_empty_i && flgs_streamer_i.store_fifo_empty) begin - next = REDMULE_FINISHED; + if (set_offset_q) begin + next = REDMULE_LOOPBACK; + end else begin + next = REDMULE_FINISHED; + end end end @@ -166,7 +200,9 @@ module redmule_ctrl /*---------------------------------------------------------------------------------------------*/ /* Other combinational assigmnets */ /*---------------------------------------------------------------------------------------------*/ - assign evt_o = current == REDMULE_FINISHED; - assign clear_o = target_clear_i || latch_clear || current == REDMULE_FINISHED; + + assign evt_o = flgs_streamer_i.z_stream_sink_flags.done && ~set_offset_q; + + assign clear_o = target_clear_i || latch_clear || current == REDMULE_FINISHED || loopback_reset; endmodule : redmule_ctrl diff --git a/rtl/redmule_inst_decoder.sv b/rtl/redmule_inst_decoder.sv index e60e2847..d7852700 100644 --- a/rtl/redmule_inst_decoder.sv +++ b/rtl/redmule_inst_decoder.sv @@ -63,7 +63,7 @@ module redmule_inst_decoder // Per-hart FIFO status flags for instruction and register packets logic [XifNumHarts-1:0] issue_fifo_full, register_fifo_full, issue_fifo_empty, register_fifo_empty; - + // Hart ID of the currently executing operation (tracked through pipeline) logic [HartIdWidth-1:0] current_hartid_d, current_hartid_q; @@ -166,7 +166,7 @@ module redmule_inst_decoder // Output configuration from the winning hart to the RedMule tiler/controller assign config_o = config_d[winner]; - + // Configuration valid only for MARITH instructions when both FIFOs have data and CPU is ready // (MCNFIG updates config but doesn't trigger execution) assign config_valid_o = ~issue_fifo_empty[winner] && ~register_fifo_empty[winner] && x_result_ready_i && {cur_issue[winner].instr[26:25],cur_issue[winner].instr[14:12],cur_issue[winner].instr[6:0]} == MARITH; @@ -466,6 +466,7 @@ module redmule_inst_decoder config_d[i].send_w = cur_register[i].rs[1][19]; // Broadcast W to external stream config_d[i].gemm_ops = cur_register[i].rs[1][20] ? MATMUL : GEMM; config_d[i].y_offs = cur_register[i].rs[2][31:0]; // Y buffer offset for bias addition + config_d[i].w_cols_offset = '0; // XIF path does not currently expose W column offsets end MARITH: begin // Matrix arithmetic: extract memory addresses from rs1, rs2, rs3 diff --git a/rtl/redmule_pkg.sv b/rtl/redmule_pkg.sv index 96d33a23..6f640e95 100644 --- a/rtl/redmule_pkg.sv +++ b/rtl/redmule_pkg.sv @@ -184,6 +184,7 @@ package redmule_pkg; logic [15:0] x_rows_iter; logic [15:0] w_cols_iter; logic [15:0] w_rows_iter; + logic [15:0] w_cols_offset; logic [ 7:0] x_cols_lftovr; logic [ 7:0] x_rows_lftovr; logic [ 7:0] w_cols_lftovr; diff --git a/rtl/redmule_tiler.sv b/rtl/redmule_tiler.sv index 930c0b90..5fa07760 100644 --- a/rtl/redmule_tiler.sv +++ b/rtl/redmule_tiler.sv @@ -22,6 +22,7 @@ module redmule_tiler input logic rst_ni , input logic clear_i , input logic setback_i , + input logic loopback_i , input logic start_cfg_i, output logic valid_o , output logic busy_o , @@ -41,7 +42,9 @@ localparam int unsigned MinimumSizeN = MinimumSizeNFactor * Height; logic clk_en; logic clk_int; +redmule_config_t input_config_d, input_config_q; redmule_config_t config_d, config_q; +logic loopback_active_q, loopback_active; always_ff @(posedge clk_i, negedge rst_ni) begin: clock_gate_enabler if (~rst_ni) begin @@ -49,7 +52,7 @@ always_ff @(posedge clk_i, negedge rst_ni) begin: clock_gate_enabler end else begin if (clear_i || setback_i) begin clk_en <= 1'b0; - end else if (start_cfg_i && ready_i) begin + end else if ((start_cfg_i && ready_i) || loopback_i) begin clk_en <= 1'b1; end end @@ -64,12 +67,38 @@ tc_clk_gating i_tiler_clockg ( assign busy_o = clk_en || ~ready_i; -assign config_d.x_addr = config_i.x_addr; -assign config_d.w_addr = config_i.w_addr; -assign config_d.z_addr = config_i.z_addr; -assign config_d.m_size = config_i.m_size; -assign config_d.k_size = config_i.k_size; -assign config_d.n_size = config_i.n_size; // real N is carried downstream unchanged (used for operand gating) + +// Store loopback +assign loopback_active = (loopback_i || loopback_active_q); +always_ff @(posedge clk_i, negedge rst_ni) begin: loopback_ff + if (~rst_ni) begin + loopback_active_q <= 1'b0; + end else begin + loopback_active_q <= !(clear_i || setback_i) && loopback_active; + end +end + +always_ff @(posedge clk_i or negedge rst_ni) begin : input_config_ff + if (~rst_ni) begin + input_config_q <= '0; + end else if (clear_i) begin + input_config_q <= '0; + end else if (start_cfg_i && ready_i) begin + input_config_q <= config_i; + end +end + +assign input_config_d = loopback_active ? input_config_q : config_i; +assign config_d.x_addr = input_config_d.x_addr; +assign config_d.w_addr = loopback_active ? input_config_d.w_addr : + input_config_d.w_addr + input_config_d.w_cols_offset * (FpWidth/8); +assign config_d.z_addr = loopback_active ? input_config_d.z_addr : + input_config_d.z_addr + input_config_d.w_cols_offset * (FpWidth/8); +assign config_d.y_addr = loopback_active ? input_config_d.z_addr + input_config_d.y_offs : + input_config_d.z_addr + input_config_d.y_offs + input_config_d.w_cols_offset * (FpWidth/8); +assign config_d.m_size = input_config_d.m_size; +assign config_d.k_size = input_config_d.k_size; +assign config_d.n_size = input_config_d.n_size; // Effective N size used ONLY for the W-row loop length / streamer length: any job with // N <= Height is promoted to MinimumSizeN so the W-load takes as long as a full N-tile, restoring @@ -90,7 +119,6 @@ assign config_d.receive_x = config_i.receive_x; assign config_d.send_x = config_i.send_x; assign config_d.y_offs = config_i.y_offs; -assign config_d.y_addr = config_i.z_addr + config_i.y_offs; // Calculating the number of iterations alng the two dimensions of the X matrix logic [15:0] x_rows_iter_nolftovr; @@ -102,6 +130,8 @@ assign x_cols_iter_nolftovr = config_d.n_size/(Height*(PipeRegs + 1)); logic [15:0] w_cols_iter_nolftovr; logic [15:0] w_rows_iter_lftovr, w_rows_iter_nolftovr; +logic [15:0] w_cols_offset_tiles; +logic [15:0] w_cols_offset_effective; assign w_cols_iter_nolftovr = config_d.k_size/(Height*(PipeRegs + 1)); assign w_rows_iter_lftovr = w_rows_iter_nolftovr + Height - config_d.w_rows_lftovr; assign w_rows_iter_nolftovr = n_size_eff; // promoted N: W-row loop runs for a full N-tile when N <= Height @@ -114,8 +144,17 @@ assign config_d.x_cols_lftovr = config_d.n_size - (x_cols_iter_nolftovr*(Height* assign config_d.w_rows_lftovr = n_size_eff - (Height*(n_size_eff/Height)); // promoted N (0 when N <= Height -> full W-row loop) assign config_d.w_cols_lftovr = config_d.k_size - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))); -// Calculate w_cols, x_cols, x_rows iterations -assign config_d.w_cols_iter = config_d.w_cols_lftovr != '0 ? w_cols_iter_nolftovr + 1 : w_cols_iter_nolftovr; +// Convert the user-programmed column offset into whole RedMulE output tiles. +assign w_cols_offset_effective = loopback_active ? '0 : input_config_d.w_cols_offset; +assign w_cols_offset_tiles = input_config_d.w_cols_offset / (Height * (PipeRegs + 1)); +assign config_d.w_cols_offset = w_cols_offset_effective; + +// Calculate w_cols iterations +assign config_d.w_cols_iter = loopback_active ? w_cols_offset_tiles : + |config_d.w_cols_lftovr ? w_cols_iter_nolftovr - w_cols_offset_tiles + 1 : + w_cols_iter_nolftovr - w_cols_offset_tiles; + +// Calculate w_rows, x_cols, x_rows iterations assign config_d.w_rows_iter = config_d.w_rows_lftovr != '0 ? w_rows_iter_lftovr : w_rows_iter_nolftovr; assign config_d.x_cols_iter = config_d.x_cols_lftovr != '0 ? x_cols_iter_nolftovr + 1 : x_cols_iter_nolftovr; assign config_d.x_rows_iter = config_d.x_rows_lftovr != '0 ? x_rows_iter_nolftovr + 1 : x_rows_iter_nolftovr; @@ -124,7 +163,7 @@ assign config_d.x_rows_iter = config_d.x_rows_lftovr != '0 ? x_rows_iter_nolftov logic [31:0] x_rows_by_w_cols_iter_d, x_rows_by_w_cols_iter_q; logic x_rows_by_w_cols_iter_valid_d, x_rows_by_w_cols_iter_valid_q; -assign x_rows_by_w_cols_iter_d = start_cfg_i ? config_d.x_rows_iter * config_d.w_cols_iter : x_rows_by_w_cols_iter_q; +assign x_rows_by_w_cols_iter_d = (start_cfg_i || loopback_i) ? config_d.x_rows_iter * config_d.w_cols_iter : x_rows_by_w_cols_iter_q; always_ff @(posedge clk_i or negedge rst_ni) begin if (~rst_ni) begin @@ -138,7 +177,7 @@ always_ff @(posedge clk_i or negedge rst_ni) begin end end -assign x_rows_by_w_cols_iter_valid_d = start_cfg_i; +assign x_rows_by_w_cols_iter_valid_d = (start_cfg_i || loopback_i); always_ff @(posedge clk_i or negedge rst_ni) begin if (~rst_ni) begin diff --git a/rtl/redmule_top.sv b/rtl/redmule_top.sv index 7704c750..f89da9e9 100644 --- a/rtl/redmule_top.sv +++ b/rtl/redmule_top.sv @@ -565,7 +565,7 @@ if(CtrlIntfConfig == XIF) begin : xif_ctrl_intf_gen .rst_ni ( rst_ni ), .clear_i ( '0 ), // TODO: fixme, not having a software-based clear mechanism is a bad idea. .config_ready_i ( ~config_fifo_full ), - .op_done_i ( flgs_streamer.z_stream_sink_flags.done ), + .op_done_i ( evt_o ), .config_valid_o ( dec_config_valid ), .config_o ( dec_config ), .x_issue_req_i ( x_issue_req_i ), @@ -595,7 +595,7 @@ else begin : mm_ctrl_intf_gen .clear_i ( '0 ), // ORed internally with target_clear .target_clear_o ( target_clear ), .config_ready_i ( ~config_fifo_full ), - .op_done_i ( flgs_streamer.z_stream_sink_flags.done ), + .op_done_i ( evt_o ), .config_valid_o ( dec_config_valid ), .config_o ( dec_config ), .target ( target ) @@ -714,6 +714,7 @@ always_ff @(posedge clk_acc) begin $display("[redmule] gemm_output_fmt = %s", redmule_config.gemm_output_fmt.name()); $display("[redmule] x_cols_iter = 0x%h", redmule_config.x_cols_iter); $display("[redmule] x_rows_iter = 0x%h", redmule_config.x_rows_iter); + $display("[redmule] w_cols_offset = 0x%h", redmule_config.w_cols_offset); $display("[redmule] w_cols_iter = 0x%h", redmule_config.w_cols_iter); $display("[redmule] w_rows_iter = 0x%h", redmule_config.w_rows_iter); $display("[redmule] x_cols_lftovr = 0x%h", redmule_config.x_cols_lftovr); diff --git a/sw/hal_redmule.h b/sw/hal_redmule.h index 15e567cf..4ccfa961 100644 --- a/sw/hal_redmule.h +++ b/sw/hal_redmule.h @@ -32,9 +32,12 @@ static inline void redmule_z_add_set(unsigned int value) { redmule_regs()->hwpe_job_dep.marith2 = value; } -static inline void redmule_mcfg_set(uint32_t mcfg0, uint32_t mcfg1) { +static inline void redmule_mcfg_set(uint32_t mcfg0, uint32_t mcfg1, uint32_t mcfg2, + uint32_t mcfg3) { redmule_regs()->hwpe_job_dep.mcnfig0 = mcfg0; redmule_regs()->hwpe_job_dep.mcnfig1 = mcfg1; + redmule_regs()->hwpe_job_dep.mcnfig2 = mcfg2; + redmule_regs()->hwpe_job_dep.mcnfig3 = mcfg3; } static inline void hwpe_trigger_job() { redmule_regs()->hwpe_ctrl.commit_trigger = 0; } @@ -54,10 +57,12 @@ static inline void hwpe_cg_enable() { return; } static inline void hwpe_cg_disable() { return; } void redmule_cfg(unsigned int x, unsigned int w, unsigned int z, uint16_t m_size, uint16_t n_size, - uint16_t k_size, uint8_t gemm_op, uint8_t gemm_fmt) { + uint16_t k_size, uint16_t w_col_off, uint8_t gemm_op, uint8_t gemm_fmt) { uint32_t mcfg_reg0 = 0; uint32_t mcfg_reg1 = 0; + uint32_t mcfg_reg2 = 0; + uint32_t mcfg_reg3 = 0; mcfg_reg0 = ((uint32_t)k_size << REDMULE_REGIF__MCNFIG0__K_SIZE_bp) | ((uint32_t)m_size << REDMULE_REGIF__MCNFIG0__M_SIZE_bp); @@ -66,11 +71,13 @@ void redmule_cfg(unsigned int x, unsigned int w, unsigned int z, uint16_t m_size ((uint32_t)gemm_op << REDMULE_REGIF__MCNFIG1__GEMM_OPS_bp) | ((uint32_t)gemm_fmt << REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bp) | ((uint32_t)gemm_fmt << REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bp); + mcfg_reg2 = 0; + mcfg_reg3 = ((uint32_t)w_col_off << REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bp); redmule_x_add_set(x); redmule_w_add_set(w); redmule_z_add_set(z); - redmule_mcfg_set(mcfg_reg0, mcfg_reg1); + redmule_mcfg_set(mcfg_reg0, mcfg_reg1, mcfg_reg2, mcfg_reg3); } #endif diff --git a/sw/hwpe_ctrl_target.h b/sw/hwpe_ctrl_target.h index 711a8ad8..3d205f37 100644 --- a/sw/hwpe_ctrl_target.h +++ b/sw/hwpe_ctrl_target.h @@ -155,6 +155,12 @@ typedef struct __attribute__ ((__packed__)) { #define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bw 32 #define REDMULE_REGIF__MOPCNT__OP_ID_CNT_reset 0x0 +// reg - redmule_regif::mcnfig3 +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bm 0xffffffff +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bp 0 +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bw 32 +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_reset 0x0 + // regfile - redmule_regif::hwpe_ctrl_job_dep typedef struct __attribute__ ((__packed__)) { uint32_t mcnfig0; @@ -164,6 +170,7 @@ typedef struct __attribute__ ((__packed__)) { uint32_t marith1; uint32_t marith2; uint32_t mopcnt; + uint32_t mcnfig3; } redmule_regif__hwpe_ctrl_job_dep_t; // regfile - redmule_regif::hwpe_ctrl_job_indep @@ -179,7 +186,7 @@ typedef struct __attribute__ ((__packed__)) { } redmule_regif_t; -REDMULE_STATIC_ASSERT(sizeof(redmule_regif_t) == 0x40, "Packing error"); +REDMULE_STATIC_ASSERT(sizeof(redmule_regif_t) == 0x44, "Packing error"); #ifdef __cplusplus } diff --git a/sw/redmule.c b/sw/redmule.c index 19ff9113..89c93969 100644 --- a/sw/redmule.c +++ b/sw/redmule.c @@ -54,7 +54,7 @@ int main() { int pace_ops = 1; // int pace_ops = 0; - redmule_cfg((unsigned int)x, (unsigned int)w, (unsigned int)y, m_size, n_size, k_size, + redmule_cfg((unsigned int)x, (unsigned int)w, (unsigned int)y, m_size, n_size, k_size, 0, (uint8_t)gemm_ops, float_fmt); // Start RedMulE operation and sleeping until the end of computation From 0230b4962bd3d5fd689bb0eec644a4753d017f78 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Wed, 15 Jul 2026 14:07:24 +0200 Subject: [PATCH 06/14] Stop pop of next configuration when loopback is active --- rtl/ctrl/redmule_target_decoder.sv | 1 + rtl/redmule_inst_decoder.sv | 1 + rtl/redmule_pkg.sv | 1 + rtl/redmule_tiler.sv | 25 +++++++++++++++---------- rtl/redmule_top.sv | 4 ++-- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/rtl/ctrl/redmule_target_decoder.sv b/rtl/ctrl/redmule_target_decoder.sv index 66805e6c..89647993 100644 --- a/rtl/ctrl/redmule_target_decoder.sv +++ b/rtl/ctrl/redmule_target_decoder.sv @@ -144,6 +144,7 @@ module redmule_target_decoder assign config_o.gemm_ops = gemm_op_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_ops.value); assign config_o.gemm_input_fmt = gemm_fmt_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_input_fmt.value); assign config_o.gemm_output_fmt = gemm_fmt_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_output_fmt.value); + assign config_o.loopback_w = 1'b0; // Operation ID counter: // op_id_counter_in_q: Increments when operations are issued (tags for tracking) diff --git a/rtl/redmule_inst_decoder.sv b/rtl/redmule_inst_decoder.sv index d7852700..2f7b9d28 100644 --- a/rtl/redmule_inst_decoder.sv +++ b/rtl/redmule_inst_decoder.sv @@ -453,6 +453,7 @@ module redmule_inst_decoder // Decode instruction and extract configuration parameters from register file values always_comb begin : config_assignment config_d[i] = config_q[i]; // Default: retain previous configuration + config_d[i].loopback_w = 1'b0; unique case ({cur_issue[i].instr[26:25],cur_issue[i].instr[14:12],cur_issue[i].instr[6:0]}) MCNFIG: begin diff --git a/rtl/redmule_pkg.sv b/rtl/redmule_pkg.sv index 6f640e95..4702f8c9 100644 --- a/rtl/redmule_pkg.sv +++ b/rtl/redmule_pkg.sv @@ -207,6 +207,7 @@ package redmule_pkg; fpu_fmt_e input_format; fpu_fmt_e computing_format; logic gemm_selection; + logic loopback_w; logic send_w; logic receive_w; logic send_x; diff --git a/rtl/redmule_tiler.sv b/rtl/redmule_tiler.sv index 5fa07760..531e28e6 100644 --- a/rtl/redmule_tiler.sv +++ b/rtl/redmule_tiler.sv @@ -109,16 +109,21 @@ assign config_d.n_size = input_config_d.n_size; // step with the promoted N, and both padded operands are zeroed (X via x_cols_lftovr, // W via the cntrl_w_buffer_o.height gating in redmule_scheduler.sv). logic [15:0] n_size_eff; -assign n_size_eff = (config_i.n_size <= Height) ? MinimumSizeN[15:0] : config_i.n_size; -assign config_d.gemm_ops = config_i.gemm_ops; -assign config_d.gemm_input_fmt = config_i.gemm_input_fmt; -assign config_d.gemm_output_fmt = config_i.gemm_output_fmt; -assign config_d.receive_w = config_i.receive_w; -assign config_d.send_w = config_i.send_w; -assign config_d.receive_x = config_i.receive_x; -assign config_d.send_x = config_i.send_x; - -assign config_d.y_offs = config_i.y_offs; +assign n_size_eff = (input_config_d.n_size <= Height) ? MinimumSizeN[15:0] : input_config_d.n_size; +// Sourced from input_config_d (not config_i directly) so the loopback pass keeps +// the job's original operation/format even after dec_config_q's read pointer has +// advanced past its single valid entry (config_i can no longer be trusted once +// the first pass's completion has popped it). +assign config_d.gemm_ops = input_config_d.gemm_ops; +assign config_d.gemm_input_fmt = input_config_d.gemm_input_fmt; +assign config_d.gemm_output_fmt = input_config_d.gemm_output_fmt; +assign config_d.receive_w = input_config_d.receive_w; +assign config_d.send_w = input_config_d.send_w; +assign config_d.loopback_w = loopback_active; +assign config_d.receive_x = input_config_d.receive_x; +assign config_d.send_x = input_config_d.send_x; + +assign config_d.y_offs = input_config_d.y_offs; // Calculating the number of iterations alng the two dimensions of the X matrix logic [15:0] x_rows_iter_nolftovr; diff --git a/rtl/redmule_top.sv b/rtl/redmule_top.sv index f89da9e9..0eb74d0f 100644 --- a/rtl/redmule_top.sv +++ b/rtl/redmule_top.sv @@ -623,7 +623,7 @@ redmule_config_fifo #( .data_i ( dec_config ), .push_i ( dec_config_valid ), .data_o ( dec_config_q ), - .pop_i ( cfg_complete ) + .pop_i ( cfg_complete && !redmule_config.loopback_w ) ); /*---------------------------------------------------------------*/ @@ -699,7 +699,7 @@ redmule_scheduler #( `ifndef SYNTHESIS always_ff @(posedge clk_acc) begin - if (cfg_complete) begin + if (cfg_complete && !redmule_config.loopback_w) begin $display("[redmule] Configuration loaded at %t", $time); $display("[redmule] x_addr = 0x%h", redmule_config.x_addr); $display("[redmule] w_addr = 0x%h", redmule_config.w_addr); From 4a8641d2533ccd9f79018fa486d9a3928aef6615 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Wed, 15 Jul 2026 17:44:41 +0200 Subject: [PATCH 07/14] Add license to header file of register interface --- rtl/ctrl/gen_regif.sh | 2 +- sw/hwpe_ctrl_target.h | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/rtl/ctrl/gen_regif.sh b/rtl/ctrl/gen_regif.sh index 1196d6d5..de944a08 100755 --- a/rtl/ctrl/gen_regif.sh +++ b/rtl/ctrl/gen_regif.sh @@ -34,6 +34,6 @@ sed -E 's/typedef[[:space:]]+struct([[:space:]])/typedef struct packed\1/g' regi HEADER='// Copyright 2025 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51' -for f in regif/redmule_regif.sv regif/redmule_regif_pkg.sv; do +for f in regif/redmule_regif.sv regif/redmule_regif_pkg.sv ../../sw/hwpe_ctrl_target.h; do printf '%s\n' "$HEADER" | cat - "$f" > "$f.tmp" && mv "$f.tmp" "$f" done diff --git a/sw/hwpe_ctrl_target.h b/sw/hwpe_ctrl_target.h index 3d205f37..0b1377d7 100644 --- a/sw/hwpe_ctrl_target.h +++ b/sw/hwpe_ctrl_target.h @@ -1,3 +1,6 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 // Generated by PeakRDL-cheader - A free and open-source header generator // https://github.com/SystemRDL/PeakRDL-cheader From 3b010f5f1285bf9fe7cd55d7638cd3676576e491 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Wed, 15 Jul 2026 17:52:28 +0200 Subject: [PATCH 08/14] Suppress linting on third-party generated files --- .github/workflows/lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 46b44cab..d70f6730 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,6 +45,8 @@ jobs: exclude_paths: | rtl/redmule_noncomp.sv rtl/redmule_fma.sv + rtl/ctrl/regif/redmule_regif.sv + rtl/ctrl/regif/redmule_regif_pkg.sv extra_args: "--waiver_files .github/verible.waiver" github_token: ${{ secrets.GITHUB_TOKEN }} fail_on_error: true From 1fb84936153534c57d1fe0b27d380d3810e96706 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Mon, 20 Jul 2026 08:44:36 +0200 Subject: [PATCH 09/14] Add regression tests for reordering feature --- Makefile | 4 +++- scripts/regression.yml | 9 +++++++++ target/sim/src/redmule_tb.sv | 18 ++++++++++++++---- target/sim/src/redmule_tb_wrap.sv | 10 +++++++--- target/sim/verilator/verilator.mk | 3 ++- target/sim/vsim/vsim.mk | 14 ++++++++------ 6 files changed, 43 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 119d970c..9fd17a01 100644 --- a/Makefile +++ b/Makefile @@ -24,12 +24,14 @@ else Bender ?= bender Gcc ?= endif + OP ?= gemm fp_fmt ?= FP16 M ?= 24 N ?= 16 K ?= 16 -TEST_ID ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,) +EnableReordering ?= 0 +TEST_ID ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,)$(if $(filter 1,$(EnableReordering)),_reord,) INC_DIR ?= $(SW)/inc/$(TEST_ID) BUILD_DIR ?= $(SW)/build/$(TEST_ID) ISA ?= riscv diff --git a/scripts/regression.yml b/scripts/regression.yml index 792d53de..5940a263 100644 --- a/scripts/regression.yml +++ b/scripts/regression.yml @@ -292,3 +292,12 @@ redmule_regression: M33_N33_K33: path: . command: make golden M=33 N=33 K=33 && make sw-clean sw-build M=33 N=33 K=33 REDMULE_COMPLEX=0 Gcc= && make hw-run M=33 N=33 K=33 REDMULE_COMPLEX=0 target=$Target | tee target/sim/$Target/transcript_33_33_33 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_33_33_33 + M48_N32_K32_REORD: # reordered path, aligned and multi-tile baseline + path: . + command: make golden M=48 N=32 K=32 EnableReordering=1 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_48_32_32_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_reord + M16_N32_K18_REORD: # reordered path with K-leftover and N-tiling + path: . + command: make golden M=16 N=32 K=18 EnableReordering=1 && make sw-clean sw-build M=16 N=32 K=18 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=16 N=32 K=18 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_16_32_18_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_16_32_18_reord + M1_N32_K8_REORD: # reordered path on a tiny M edge case + path: . + command: make golden M=1 N=32 K=8 EnableReordering=1 && make sw-clean sw-build M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_1_32_8_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_1_32_8_reord diff --git a/target/sim/src/redmule_tb.sv b/target/sim/src/redmule_tb.sv index e99c499b..d4968520 100644 --- a/target/sim/src/redmule_tb.sv +++ b/target/sim/src/redmule_tb.sv @@ -22,6 +22,8 @@ module redmule_tb parameter TT = 0.8ns, // test time parameter int unsigned Height = 8, parameter int unsigned Width = 8, + parameter bit EnableReordering = 1'b0, + parameter int unsigned RobSlots = 16, parameter real PROB_STALL = 0.0 )( input logic clk_i, @@ -38,16 +40,18 @@ module redmule_tb localparam int unsigned RedmuleDataW = Height*(NumPipeRegs+1)*16; // = D*16 localparam int unsigned DW = RedmuleDataW + 32; // TCDM data width including MisalignedAccessSupport=1 (+32b word) localparam int unsigned MP = DW/32; + localparam int unsigned UW = EnableReordering ? $clog2(RobSlots) : hci_package::DEFAULT_UW; // HCI size parameter for RedMulE's TCDM port. It must be forwarded to // redmule_mm_wrap (redmule_top forwards it verbatim to the streamer, with no // fallback), otherwise the internal OoO multiplexer sees BW=0 and fails to - // elaborate. Fields other than DW take the hci_core_intf defaults, matching - // the `redmule_tcdm` interface declared below (DW-only override). + // elaborate. The `redmule_tcdm` interface below must match these dimensions. + // When reordering is enabled, widen UW so the HCI-side ROB can track more + // outstanding transactions (ROB_NW = 2**UW in rtl/redmule_streamer.sv). localparam hci_size_parameter_t HciSizeTcdm = '{ DW: DW, AW: hci_package::DEFAULT_AW, BW: hci_package::DEFAULT_BW, - UW: hci_package::DEFAULT_UW, + UW: UW, IW: hci_package::DEFAULT_IW, EW: hci_package::DEFAULT_EW, EHW: hci_package::DEFAULT_EHW @@ -72,7 +76,10 @@ module redmule_tb hwpe_stream_intf_tcdm tcdm [MP:0] (.clk(clk_i)); // RedMulE modern interfaces - hci_core_intf #(.DW(DW)) redmule_tcdm (.clk(clk_i)); + hci_core_intf #( + .DW ( DW ), + .UW ( UW ) + ) redmule_tcdm (.clk(clk_i)); hwpe_ctrl_intf_periph #(.ID_WIDTH(ID)) periph (.clk(clk_i)); logic [MP-1:0] tcdm_gnt; @@ -173,6 +180,7 @@ module redmule_tb .HCI_SIZE_tcdm ( HciSizeTcdm ), .DataW ( RedmuleDataW ), .MisalignedAccessSupport ( 1 ), + .EnableReordering ( EnableReordering ), .Height ( Height ), .Width ( Width ), .NumPipeRegs ( NumPipeRegs ) @@ -329,6 +337,8 @@ module redmule_tb if (!$value$plusargs("STIM_DATA=%s", stim_data)) stim_data = "../../../sw/build/stim_data.txt"; $display("Height = %d", Height); $display("Width = %d", Width); + $display("EnableReordering = %0d", EnableReordering); + $display("RobSlots = %0d", RobSlots); $display("PROB_STALL = %f", PROB_STALL); test_mode = 1'b0; diff --git a/target/sim/src/redmule_tb_wrap.sv b/target/sim/src/redmule_tb_wrap.sv index 334d5c19..0eb7810f 100644 --- a/target/sim/src/redmule_tb_wrap.sv +++ b/target/sim/src/redmule_tb_wrap.sv @@ -12,6 +12,8 @@ module redmule_tb_wrap #( parameter int unsigned Height = 8, parameter int unsigned Width = 8, + parameter bit EnableReordering = 1'b0, + parameter int unsigned RobSlots = 16, parameter real PROB_STALL = 0.0 ); @@ -25,9 +27,11 @@ module redmule_tb_wrap .TCP ( TCP ), .TA ( TA ), .TT ( TT ), - .Height ( Height ), - .Width ( Width ), - .PROB_STALL ( PROB_STALL ) + .Height ( Height ), + .Width ( Width ), + .EnableReordering ( EnableReordering ), + .RobSlots ( RobSlots ), + .PROB_STALL ( PROB_STALL ) ) i_redmule_tb ( .clk_i ( clk ), .rst_ni ( rst_n ), diff --git a/target/sim/verilator/verilator.mk b/target/sim/verilator/verilator.mk index 76ab959e..0b09a65f 100644 --- a/target/sim/verilator/verilator.mk +++ b/target/sim/verilator/verilator.mk @@ -25,6 +25,7 @@ VerilatorCompileScript := $(VerilatorDir)/compile.$(target).tcl VerilatorWaves := $(VerilatorDir)/redmule.vcd RedmuleHeight ?= 8 RedmuleWidth ?= 8 +EnableReordering ?= 0 ProbStall ?= 0.05 # Parallelism for hw-build. With --binary this covers both verilation and the # C++ compile of the model. @@ -45,7 +46,7 @@ hw-build: hw-script -Wall -Wno-fatal --Wno-lint --Wno-UNOPTFLAT --Wno-MODDUP -Wno-BLKANDNBLK -Wno-ENUMVALUE \ -j $(VerilatorJobs) \ --x-assign unique --x-initial unique --top-module $(Module)_tb_wrap --Mdir $(VerilatorAbsObjDir) \ - -GHeight=$(RedmuleHeight) -GWidth=$(RedmuleWidth) -GPROB_STALL=$(ProbStall) \ + -GHeight=$(RedmuleHeight) -GWidth=$(RedmuleWidth) -GEnableReordering=$(EnableReordering) -GPROB_STALL=$(ProbStall) \ -CFLAGS "-DTbName=$(Vmodule)_tb_wrap -DWafeformPath=$(VerilatorWaves)" --binary \ -sv -cc -f $(VerilatorCompileScript) diff --git a/target/sim/vsim/vsim.mk b/target/sim/vsim/vsim.mk index 7a414284..1fb3f02e 100644 --- a/target/sim/vsim/vsim.mk +++ b/target/sim/vsim/vsim.mk @@ -9,6 +9,7 @@ Questa ?= Module := redmule ProbStall ?= 0.05 +EnableReordering ?= 0 VsimDir := $(SimDir)/$(target) VsimCompileScript := $(VsimDir)/compile.$(target).tcl VsimWaves := $(VsimDir)/wave.tcl @@ -60,12 +61,13 @@ hw-build: hw-script hw-run: mkdir -p $(BUILD_DIR) ln -sfn $(VsimDir)/work $(BUILD_DIR)/work - cd $(BUILD_DIR); \ - $(QUESTA) $(target) $(Tb)_opt \ - $(VsimFlags) \ - -gPROB_STALL=$(ProbStall) \ - +STIM_INSTR=$(STIM_INSTR) \ - +STIM_DATA=$(STIM_DATA) \ + cd $(BUILD_DIR); \ + $(QUESTA) $(target) $(Tb)_opt \ + $(VsimFlags) \ + -gEnableReordering=$(EnableReordering) \ + -gPROB_STALL=$(ProbStall) \ + +STIM_INSTR=$(STIM_INSTR) \ + +STIM_DATA=$(STIM_DATA) \ -do "run -a" hw-all: hw-clean hw-script hw-build hw-run From 383cf9b8fc195c4efe9e631d08912f8df166dca5 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Mon, 20 Jul 2026 11:02:03 +0200 Subject: [PATCH 10/14] Store response ID and user for ROB operation --- target/sim/src/redmule_tb.sv | 65 +++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/target/sim/src/redmule_tb.sv b/target/sim/src/redmule_tb.sv index d4968520..d81725d0 100644 --- a/target/sim/src/redmule_tb.sv +++ b/target/sim/src/redmule_tb.sv @@ -41,6 +41,7 @@ module redmule_tb localparam int unsigned DW = RedmuleDataW + 32; // TCDM data width including MisalignedAccessSupport=1 (+32b word) localparam int unsigned MP = DW/32; localparam int unsigned UW = EnableReordering ? $clog2(RobSlots) : hci_package::DEFAULT_UW; + localparam int unsigned UserFifoDepth = 1 << UW; // HCI size parameter for RedMulE's TCDM port. It must be forwarded to // redmule_mm_wrap (redmule_top forwards it verbatim to the streamer, with no // fallback), otherwise the internal OoO multiplexer sees BW=0 and fails to @@ -85,6 +86,15 @@ module redmule_tb logic [MP-1:0] tcdm_gnt; logic [MP-1:0][31:0] tcdm_r_data; logic [MP-1:0] tcdm_r_valid; + typedef logic [hci_package::iomsb(UW):0] hci_user_t; + typedef logic [hci_package::iomsb(hci_package::DEFAULT_IW):0] hci_id_t; + hci_user_t user_fifo_rdata; + hci_id_t id_fifo_rdata; + logic user_fifo_full, user_fifo_empty; + logic [$clog2(UserFifoDepth)-1:0] user_fifo_usage; + logic id_fifo_full, id_fifo_empty; + logic [$clog2(UserFifoDepth)-1:0] id_fifo_usage; + logic redmule_req_fire, redmule_rsp_fire; logic instr_req; logic instr_gnt; @@ -155,7 +165,46 @@ module redmule_tb assign redmule_tcdm.r_data = { >> {tcdm_r_data} }; assign redmule_tcdm.r_valid = &tcdm_r_valid; assign redmule_tcdm.r_opc = '0; - assign redmule_tcdm.r_user = '0; + assign redmule_tcdm.r_user = user_fifo_rdata; + assign redmule_tcdm.r_id = id_fifo_rdata; + assign redmule_req_fire = redmule_tcdm.req & redmule_tcdm.gnt; + assign redmule_rsp_fire = redmule_tcdm.r_valid & redmule_tcdm.r_ready; + + fifo_v3 #( + .FALL_THROUGH ( 1'b0 ), + .DEPTH ( UserFifoDepth ), + .dtype ( hci_user_t ) + ) i_user_fifo ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .flush_i ( 1'b0 ), + .testmode_i ( 1'b0 ), + .full_o ( user_fifo_full ), + .empty_o ( user_fifo_empty ), + .usage_o ( user_fifo_usage ), + .data_i ( redmule_tcdm.user ), + .push_i ( redmule_req_fire ), + .data_o ( user_fifo_rdata ), + .pop_i ( redmule_rsp_fire ) + ); + + fifo_v3 #( + .FALL_THROUGH ( 1'b0 ), + .DEPTH ( UserFifoDepth ), + .dtype ( hci_id_t ) + ) i_id_fifo ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .flush_i ( 1'b0 ), + .testmode_i ( 1'b0 ), + .full_o ( id_fifo_full ), + .empty_o ( id_fifo_empty ), + .usage_o ( id_fifo_usage ), + .data_i ( redmule_tcdm.id ), + .push_i ( redmule_req_fire ), + .data_o ( id_fifo_rdata ), + .pop_i ( redmule_rsp_fire ) + ); // Core data-side port (last bank of the data memory). assign tcdm[MP].req = data_req & (data_addr[31:24] != '0) & (data_addr[31:24] != 8'h80) & ~data_addr[HWPE_ADDR_BASE_BIT]; @@ -177,13 +226,13 @@ module redmule_tb other_r_valid ; redmule_mm_wrap #( - .HCI_SIZE_tcdm ( HciSizeTcdm ), - .DataW ( RedmuleDataW ), - .MisalignedAccessSupport ( 1 ), - .EnableReordering ( EnableReordering ), - .Height ( Height ), - .Width ( Width ), - .NumPipeRegs ( NumPipeRegs ) + .HCI_SIZE_tcdm ( HciSizeTcdm ), + .DataW ( RedmuleDataW ), + .MisalignedAccessSupport ( 1 ), + .EnableReordering ( EnableReordering ), + .Height ( Height ), + .Width ( Width ), + .NumPipeRegs ( NumPipeRegs ) ) i_redmule_wrap ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), From 1d9429bd45ed066d4399d8013a413b9f53efcef0 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Mon, 20 Jul 2026 14:56:00 +0200 Subject: [PATCH 11/14] Disable misaligned access when enablereordering is on --- target/sim/src/redmule_tb.sv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/sim/src/redmule_tb.sv b/target/sim/src/redmule_tb.sv index d81725d0..28a4b94a 100644 --- a/target/sim/src/redmule_tb.sv +++ b/target/sim/src/redmule_tb.sv @@ -228,7 +228,7 @@ module redmule_tb redmule_mm_wrap #( .HCI_SIZE_tcdm ( HciSizeTcdm ), .DataW ( RedmuleDataW ), - .MisalignedAccessSupport ( 1 ), + .MisalignedAccessSupport ( EnableReordering ? 0 : 1 ), .EnableReordering ( EnableReordering ), .Height ( Height ), .Width ( Width ), From fb22e147cd635d0f0218e47b30ca471bb95e1f3c Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Fri, 24 Jul 2026 11:38:56 +0200 Subject: [PATCH 12/14] Fix compilation warnings --- sw/redmule.c | 14 +++++++------- sw/redmule_complex.c | 10 +++++----- sw/utils/tinyprintf.h | 3 ++- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/sw/redmule.c b/sw/redmule.c index 89c93969..3ba2818f 100644 --- a/sw/redmule.c +++ b/sw/redmule.c @@ -26,11 +26,11 @@ int main() { uint16_t n_size = N_SIZE; uint16_t k_size = K_SIZE; - uint8_t *x = x_inp; - uint8_t *w = w_inp; - uint8_t *y = y_inp; + uint16_t *x = x_inp; + uint16_t *w = w_inp; + uint16_t *y = y_inp; - uint8_t *z = z_oup; // golden_out //1c010000 + uint16_t *z = z_oup; // golden_out //1c010000 uint8_t float_fmt = (SRC_FMT == FP8) ? (uint8_t)Float8 : (SRC_FMT == FP8ALT) ? (uint8_t)Float8Alt @@ -71,11 +71,11 @@ int main() { if (float_fmt == Float16 || float_fmt == Float16Alt) if (gemm_ops == PACE) - errors = redmule16_compare_int(y, golden, K_SIZE/2, 0); + errors = redmule16_compare_int((uint32_t *)y, golden, K_SIZE/2, 0); else - errors = redmule16_compare_int(y, golden, m_size * k_size / 2, ERR); + errors = redmule16_compare_int((uint32_t *)y, golden, m_size * k_size / 2, ERR); else if (float_fmt == Float8 || float_fmt == Float8Alt) - errors = redmule8_compare_int(y, golden, m_size * k_size / 4, ERR); + errors = redmule8_compare_int((uint32_t *)y, golden, m_size * k_size / 4, ERR); *(int *)0x80000000 = errors; diff --git a/sw/redmule_complex.c b/sw/redmule_complex.c index 21771051..8ed0c765 100644 --- a/sw/redmule_complex.c +++ b/sw/redmule_complex.c @@ -17,10 +17,10 @@ int main() { - uint8_t *x = x_inp; - uint8_t *w = w_inp; - uint8_t *y = y_inp; - uint8_t *z = z_oup; // golden_out //1c010000 + uint16_t *x = x_inp; + uint16_t *w = w_inp; + uint16_t *y = y_inp; + uint16_t *z = z_oup; // golden_out //1c010000 uint16_t m_size = M_SIZE; uint16_t n_size = N_SIZE; @@ -78,7 +78,7 @@ int main() { // Wait for end of computation asm volatile("wfi" ::: "memory"); - errors = redmule16_compare_int(y, golden, m_size * k_size / 2); + errors = redmule16_compare_int((uint32_t *)y, golden, m_size * k_size / 2, 0); *(int *)0x80000000 = errors; diff --git a/sw/utils/tinyprintf.h b/sw/utils/tinyprintf.h index 85ddf69a..4f7cb92a 100644 --- a/sw/utils/tinyprintf.h +++ b/sw/utils/tinyprintf.h @@ -107,7 +107,7 @@ For further details see source code. regs Kusti, 23.10.2004 */ -void putf(char *null, char c) { +void putf(void *null, char c) { *(volatile int *) (0x80000004) = (int)c; } @@ -559,6 +559,7 @@ void tfp_format(void *putp, putcf putf__, const char *fmt, va_list va) lng = 2; # endif #endif + __attribute__((fallthrough)); #endif case 'x': case 'X': From 9c5221800d3bd2ba7207d58f3fdaba8d94571003 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Fri, 24 Jul 2026 12:01:34 +0200 Subject: [PATCH 13/14] Add regression tests for W columns offset --- Makefile | 27 ++++++++++++++++----------- scripts/regression.yml | 6 ++++++ sw/redmule.c | 8 ++++++-- 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 9fd17a01..7b129e2f 100644 --- a/Makefile +++ b/Makefile @@ -25,18 +25,21 @@ else Gcc ?= endif -OP ?= gemm -fp_fmt ?= FP16 -M ?= 24 -N ?= 16 -K ?= 16 +OP ?= gemm +fp_fmt ?= FP16 +M ?= 24 +N ?= 16 +K ?= 16 +WColsOffset ?= 0 EnableReordering ?= 0 -TEST_ID ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,)$(if $(filter 1,$(EnableReordering)),_reord,) -INC_DIR ?= $(SW)/inc/$(TEST_ID) -BUILD_DIR ?= $(SW)/build/$(TEST_ID) -ISA ?= riscv -ARCH ?= rv -XLEN ?= 32 + +TEST_ID ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter-out 0,$(WColsOffset)),_w_cols_offset_$(WColsOffset),)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,)$(if $(filter 1,$(EnableReordering)),_reord,) +INC_DIR ?= $(SW)/inc/$(TEST_ID) +BUILD_DIR ?= $(SW)/build/$(TEST_ID) +ISA ?= riscv +ARCH ?= rv +XLEN ?= 32 + # Local PULP GCC toolchains are based on older GCC and bundle # Zicsr together with the I extension. For GitHub CI we use # a newer version of GCC @@ -79,6 +82,8 @@ ifeq ($(debug),1) FLAGS += -DDEBUG endif +FLAGS += -DW_COLS_OFFSET=$(WColsOffset) + # Include directories INC += -I$(SW) INC += -I$(INC_DIR) diff --git a/scripts/regression.yml b/scripts/regression.yml index 5940a263..cb06ef80 100644 --- a/scripts/regression.yml +++ b/scripts/regression.yml @@ -301,3 +301,9 @@ redmule_regression: M1_N32_K8_REORD: # reordered path on a tiny M edge case path: . command: make golden M=1 N=32 K=8 EnableReordering=1 && make sw-clean sw-build M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_1_32_8_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_1_32_8_reord + M48_N32_K64_W_OFFSET16: # offset on W columns + path: . + command: make golden M=48 N=32 K=64 WColsOffset=16 && make sw-clean sw-build M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=16 Gcc= && make hw-run M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=16 target=$Target | tee target/sim/$Target/transcript_48_32_64_w_cols_offset_16 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_64_w_cols_offset_16 + M48_N32_K64_W_OFFSET32: # offset on W columns + path: . + command: make golden M=48 N=32 K=64 WColsOffset=32 && make sw-clean sw-build M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=32 Gcc= && make hw-run M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=32 target=$Target | tee target/sim/$Target/transcript_48_32_64_w_cols_offset_32 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_64_w_cols_offset_32 \ No newline at end of file diff --git a/sw/redmule.c b/sw/redmule.c index 3ba2818f..b72c612c 100644 --- a/sw/redmule.c +++ b/sw/redmule.c @@ -20,6 +20,10 @@ #include "z_output.h" #define ERR 0x0000 +#ifndef W_COLS_OFFSET +#define W_COLS_OFFSET 0 +#endif + int main() { uint16_t m_size = M_SIZE; @@ -54,8 +58,8 @@ int main() { int pace_ops = 1; // int pace_ops = 0; - redmule_cfg((unsigned int)x, (unsigned int)w, (unsigned int)y, m_size, n_size, k_size, 0, - (uint8_t)gemm_ops, float_fmt); + redmule_cfg((unsigned int)x, (unsigned int)w, (unsigned int)y, m_size, n_size, k_size, + (uint16_t)W_COLS_OFFSET, (uint8_t)gemm_ops, float_fmt); // Start RedMulE operation and sleeping until the end of computation printf("Triggering accelerator and going to sleep...\n"); From 15c30936c6b2be26e90a8062b0c739c6a37b15f1 Mon Sep 17 00:00:00 2001 From: mbertuletti Date: Tue, 28 Jul 2026 16:26:07 +0200 Subject: [PATCH 14/14] W columns offset on uneven dimensions --- rtl/redmule_tiler.sv | 22 ++++++++-------------- scripts/regression.yml | 11 +++++++---- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/rtl/redmule_tiler.sv b/rtl/redmule_tiler.sv index 531e28e6..44f98435 100644 --- a/rtl/redmule_tiler.sv +++ b/rtl/redmule_tiler.sv @@ -122,7 +122,8 @@ assign config_d.send_w = input_config_d.send_w; assign config_d.loopback_w = loopback_active; assign config_d.receive_x = input_config_d.receive_x; assign config_d.send_x = input_config_d.send_x; - +// Convert the user-programmed column offset into whole RedMulE output tiles. +assign config_d.w_cols_offset = loopback_active ? '0 : input_config_d.w_cols_offset; assign config_d.y_offs = input_config_d.y_offs; // Calculating the number of iterations alng the two dimensions of the X matrix @@ -135,9 +136,8 @@ assign x_cols_iter_nolftovr = config_d.n_size/(Height*(PipeRegs + 1)); logic [15:0] w_cols_iter_nolftovr; logic [15:0] w_rows_iter_lftovr, w_rows_iter_nolftovr; -logic [15:0] w_cols_offset_tiles; -logic [15:0] w_cols_offset_effective; -assign w_cols_iter_nolftovr = config_d.k_size/(Height*(PipeRegs + 1)); +assign w_cols_iter_nolftovr = loopback_active ? (input_config_d.w_cols_offset/(Height*(PipeRegs + 1))) : + (config_d.k_size - input_config_d.w_cols_offset)/(Height*(PipeRegs + 1)); assign w_rows_iter_lftovr = w_rows_iter_nolftovr + Height - config_d.w_rows_lftovr; assign w_rows_iter_nolftovr = n_size_eff; // promoted N: W-row loop runs for a full N-tile when N <= Height @@ -147,17 +147,11 @@ assign config_d.x_cols_lftovr = config_d.n_size - (x_cols_iter_nolftovr*(Height* // Calculating the residuals along the weight dimensions assign config_d.w_rows_lftovr = n_size_eff - (Height*(n_size_eff/Height)); // promoted N (0 when N <= Height -> full W-row loop) -assign config_d.w_cols_lftovr = config_d.k_size - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))); - -// Convert the user-programmed column offset into whole RedMulE output tiles. -assign w_cols_offset_effective = loopback_active ? '0 : input_config_d.w_cols_offset; -assign w_cols_offset_tiles = input_config_d.w_cols_offset / (Height * (PipeRegs + 1)); -assign config_d.w_cols_offset = w_cols_offset_effective; +assign config_d.w_cols_lftovr = loopback_active ? input_config_d.w_cols_offset - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))) : + (config_d.k_size - input_config_d.w_cols_offset) - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))); // Calculate w_cols iterations -assign config_d.w_cols_iter = loopback_active ? w_cols_offset_tiles : - |config_d.w_cols_lftovr ? w_cols_iter_nolftovr - w_cols_offset_tiles + 1 : - w_cols_iter_nolftovr - w_cols_offset_tiles; +assign config_d.w_cols_iter = config_d.w_cols_lftovr != '0 ? w_cols_iter_nolftovr + 1 : w_cols_iter_nolftovr; // Calculate w_rows, x_cols, x_rows iterations assign config_d.w_rows_iter = config_d.w_rows_lftovr != '0 ? w_rows_iter_lftovr : w_rows_iter_nolftovr; @@ -307,7 +301,7 @@ assign config_d.gemm_selection = config_d.gemm_ops == MATMUL ? 1'b0 : 1'b1; assign config_d.x_d1_stride = ((4*FpWidth)/AddrWidth)*(((DataW/FpWidth)*x_cols_iter_nolftovr) + config_d.x_cols_lftovr); assign config_d.x_rows_offs = Width*config_d.x_d1_stride; assign config_d.w_tot_len = x_rows_by_w_cols_by_w_rows_iter_q[31:0]; -assign config_d.w_d0_stride = ((4*FpWidth)/AddrWidth)*(((DataW/FpWidth)*w_cols_iter_nolftovr) + config_d.w_cols_lftovr); +assign config_d.w_d0_stride = ((4*FpWidth)/AddrWidth)*((DataW/FpWidth) * (config_d.k_size)/(Height*(PipeRegs + 1))); assign config_d.yz_tot_len = Width*x_rows_by_w_cols_iter_q[15:0]; assign config_d.yz_d0_stride = config_d.w_d0_stride; assign config_d.yz_d2_stride = Width*config_d.w_d0_stride; diff --git a/scripts/regression.yml b/scripts/regression.yml index cb06ef80..abefd8b1 100644 --- a/scripts/regression.yml +++ b/scripts/regression.yml @@ -301,9 +301,12 @@ redmule_regression: M1_N32_K8_REORD: # reordered path on a tiny M edge case path: . command: make golden M=1 N=32 K=8 EnableReordering=1 && make sw-clean sw-build M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_1_32_8_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_1_32_8_reord - M48_N32_K64_W_OFFSET16: # offset on W columns + M48_N32_K32_W_OFFSET16: # offset on W columns path: . - command: make golden M=48 N=32 K=64 WColsOffset=16 && make sw-clean sw-build M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=16 Gcc= && make hw-run M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=16 target=$Target | tee target/sim/$Target/transcript_48_32_64_w_cols_offset_16 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_64_w_cols_offset_16 - M48_N32_K64_W_OFFSET32: # offset on W columns + command: make golden M=48 N=32 K=32 WColsOffset=16 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=16 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=16 target=$Target | tee target/sim/$Target/transcript_48_32_32_w_cols_offset_16 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_w_cols_offset_16 + M48_N32_K32_W_OFFSET17: # offset on W columns path: . - command: make golden M=48 N=32 K=64 WColsOffset=32 && make sw-clean sw-build M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=32 Gcc= && make hw-run M=48 N=32 K=64 REDMULE_COMPLEX=0 WColsOffset=32 target=$Target | tee target/sim/$Target/transcript_48_32_64_w_cols_offset_32 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_64_w_cols_offset_32 \ No newline at end of file + command: make golden M=48 N=32 K=32 WColsOffset=17 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=17 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=17 target=$Target | tee target/sim/$Target/transcript_48_32_32_w_cols_offset_17 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_w_cols_offset_17 + M48_N32_K32_W_OFFSET7: # offset on W columns + path: . + command: make golden M=48 N=32 K=32 WColsOffset=7 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=7 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=7 target=$Target | tee target/sim/$Target/transcript_48_32_32_w_cols_offset_7 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_w_cols_offset_7