ROBs in Streamer for MemPool integration

.github/workflows/lint.yml

@@ -45,6 +45,8 @@ jobs:
         exclude_paths: |
           rtl/redmule_noncomp.sv
           rtl/redmule_fma.sv
+          rtl/ctrl/regif/redmule_regif.sv
+          rtl/ctrl/regif/redmule_regif_pkg.sv
         extra_args: "--waiver_files .github/verible.waiver"
         github_token: ${{ secrets.GITHUB_TOKEN }}
         fail_on_error: true

Bender.yml

@@ -19,7 +19,7 @@ dependencies:
   hwpe-ctrl         : { git: "https://github.com/pulp-platform/hwpe-ctrl.git"         , version: 3.1.1                                } 
   hci               : { git: "https://github.com/pulp-platform/hci.git"               , version: 2.6.1                                }
   fpnew             : { git: "https://github.com/pulp-platform/cvfpu.git"             , rev: "pulp-v0.1.3"                            }
-  common_cells      : { git: "https://github.com/pulp-platform/common_cells.git"      , version: 1.21.0                               }
+  common_cells      : { git: "https://github.com/pulp-platform/common_cells.git"      , version: 1.38.0                               }
   tech_cells_generic: { git: "https://github.com/pulp-platform/tech_cells_generic.git", version: 0.2.11                               }
 
 sources:

Makefile

@@ -24,17 +24,22 @@ else
     Bender ?= bender
     Gcc    ?= 
 endif
-OP     ?= gemm
-fp_fmt ?= FP16
-M      ?= 24
-N      ?= 16
-K      ?= 16
-TEST_ID   ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,)
-INC_DIR   ?= $(SW)/inc/$(TEST_ID)
-BUILD_DIR  ?= $(SW)/build/$(TEST_ID)
-ISA        ?= riscv
-ARCH       ?= rv
-XLEN       ?= 32
+
+OP          ?= gemm
+fp_fmt      ?= FP16
+M           ?= 24
+N           ?= 16
+K           ?= 16
+WColsOffset ?= 0
+EnableReordering ?= 0
+
+TEST_ID     ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter-out 0,$(WColsOffset)),_w_cols_offset_$(WColsOffset),)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,)$(if $(filter 1,$(EnableReordering)),_reord,)
+INC_DIR     ?= $(SW)/inc/$(TEST_ID)
+BUILD_DIR   ?= $(SW)/build/$(TEST_ID)
+ISA         ?= riscv
+ARCH        ?= rv
+XLEN        ?= 32
+
 # Local PULP GCC toolchains are based on older GCC and bundle
 # Zicsr together with the I extension. For GitHub CI we use
 # a newer version of GCC
@@ -77,6 +82,8 @@ ifeq ($(debug),1)
 	FLAGS += -DDEBUG
 endif
 
+FLAGS += -DW_COLS_OFFSET=$(WColsOffset)
+
 # Include directories
 INC += -I$(SW)
 INC += -I$(INC_DIR)

rtl/ctrl/gen_regif.sh

@@ -5,14 +5,35 @@
 
 peakrdl regblock  redmule_regif.rdl -o regif/ --cpuif passthrough --default-reset arst_n --hwif-report --addr-width 32
 peakrdl html      redmule_regif.rdl -o regif/html/
-peakrdl c-header  redmule_regif.rdl -o regif/hwpe_ctrl_target.h
+peakrdl c-header  redmule_regif.rdl -o ../../sw/hwpe_ctrl_target.h
+awk '
+  /#include <assert.h>/ {
+    print
+    print ""
+    print "#if defined(__cplusplus)"
+    print "#define REDMULE_STATIC_ASSERT(cond, msg) static_assert(cond, msg)"
+    print "#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)"
+    print "#define REDMULE_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg)"
+    print "#else"
+    print "#define REDMULE_STATIC_ASSERT_CONCAT_(a, b) a##b"
+    print "#define REDMULE_STATIC_ASSERT_CONCAT(a, b) REDMULE_STATIC_ASSERT_CONCAT_(a, b)"
+    print "#define REDMULE_STATIC_ASSERT(cond, msg) \\"
+    print "  typedef char REDMULE_STATIC_ASSERT_CONCAT(redmule_static_assert_, __LINE__)[(cond) ? 1 : -1]"
+    print "#endif"
+    next
+  }
+  {
+    gsub(/static_assert\(/, "REDMULE_STATIC_ASSERT(")
+    print
+  }
+' ../../sw/hwpe_ctrl_target.h > ../../sw/hwpe_ctrl_target.h.tmp && mv ../../sw/hwpe_ctrl_target.h.tmp ../../sw/hwpe_ctrl_target.h
 # PeakRDL uses unpacked structs to avoid issues at compile time, which is commendable, but incompatible with FIFOing the output of the job! (use portable sed syntax that works on both Linux and macOS)
 sed -E 's/typedef[[:space:]]+struct([[:space:]])/typedef struct packed\1/g' regif/redmule_regif_pkg.sv > regif/redmule_regif_pkg.sv.tmp && mv regif/redmule_regif_pkg.sv.tmp regif/redmule_regif_pkg.sv
 
 # PeakRDL does not emit a license header; prepend the repository's SPDX header to the generated SystemVerilog.
 HEADER='// Copyright 2025 ETH Zurich and University of Bologna.
 // Solderpad Hardware License, Version 0.51, see LICENSE for details.
 // SPDX-License-Identifier: SHL-0.51'
-for f in regif/redmule_regif.sv regif/redmule_regif_pkg.sv; do
+for f in regif/redmule_regif.sv regif/redmule_regif_pkg.sv ../../sw/hwpe_ctrl_target.h; do
   printf '%s\n' "$HEADER" | cat - "$f" > "$f.tmp" && mv "$f.tmp" "$f"
 done

rtl/ctrl/redmule_regif.rdl

@@ -216,6 +216,15 @@ addrmap redmule_regif {
             sw = rw;
         } y_offs[31:0] = 0;
     };
+    reg mcnfig3 {
+        name = "MCNFIG3";
+        field {
+            name = "w_cols_offset";
+            desc = "Column offset applied to W/Z before starting the first pass.";
+            hw = r;
+            sw = rw;
+        } w_cols_offset[31:0] = 0;
+    };
     reg marith0 {
         name = "MARITH0";
         field {
@@ -263,6 +272,7 @@ addrmap redmule_regif {
         mcnfig0 mcnfig0 @ 0x00;
         mcnfig1 mcnfig1 @ 0x04;
         mcnfig2 mcnfig2 @ 0x08;
+        mcnfig3 mcnfig3 @ 0x1c;
         marith0 marith0 @ 0x0c;
         marith1 marith1 @ 0x10;
         marith2 marith2 @ 0x14;

rtl/ctrl/redmule_target_decoder.sv

@@ -137,12 +137,14 @@ module redmule_target_decoder
   assign config_o.receive_w       = hwif_out.hwpe_job_dep.mcnfig1.receive_w.value;
   assign config_o.send_w          = hwif_out.hwpe_job_dep.mcnfig1.send_w.value;
   assign config_o.y_offs          = hwif_out.hwpe_job_dep.mcnfig2.y_offs.value;
+  assign config_o.w_cols_offset   = hwif_out.hwpe_job_dep.mcnfig3.w_cols_offset.value[15:0];
   assign config_o.x_addr          = hwif_out.hwpe_job_dep.marith0.x_addr.value;
   assign config_o.w_addr          = hwif_out.hwpe_job_dep.marith1.w_addr.value;
   assign config_o.z_addr          = hwif_out.hwpe_job_dep.marith2.z_addr.value;
   assign config_o.gemm_ops        = gemm_op_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_ops.value);
   assign config_o.gemm_input_fmt  = gemm_fmt_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_input_fmt.value);
   assign config_o.gemm_output_fmt = gemm_fmt_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_output_fmt.value);
+  assign config_o.loopback_w      = 1'b0;
 
   // Operation ID counter:
   // op_id_counter_in_q:  Increments when operations are issued (tags for tracking)

rtl/ctrl/regif/redmule_regif.sv

@@ -87,6 +87,7 @@ module redmule_regif (
             logic marith1;
             logic marith2;
             logic mopcnt;
+            logic mcnfig3;
         } hwpe_job_dep;
         struct {
             logic reserved;
@@ -120,7 +121,8 @@ module redmule_regif (
         decoded_reg_strb.hwpe_job_dep.marith1 = cpuif_req_masked & (cpuif_addr == 32'h30);
         decoded_reg_strb.hwpe_job_dep.marith2 = cpuif_req_masked & (cpuif_addr == 32'h34);
         decoded_reg_strb.hwpe_job_dep.mopcnt = cpuif_req_masked & (cpuif_addr == 32'h38) & !cpuif_req_is_wr;
-        decoded_reg_strb.hwpe_job_indep.reserved = cpuif_req_masked & (cpuif_addr == 32'h3c) & !cpuif_req_is_wr;
+        decoded_reg_strb.hwpe_job_dep.mcnfig3 = cpuif_req_masked & (cpuif_addr == 32'h3c);
+        decoded_reg_strb.hwpe_job_indep.reserved = cpuif_req_masked & (cpuif_addr == 32'h40) & !cpuif_req_is_wr;
         decoded_err = '0;
     end
 
@@ -224,6 +226,12 @@ module redmule_regif (
                     logic load_next;
                 } z_addr;
             } marith2;
+            struct {
+                struct {
+                    logic [31:0] next;
+                    logic load_next;
+                } w_cols_offset;
+            } mcnfig3;
         } hwpe_job_dep;
     } field_combo_t;
     field_combo_t field_combo;
@@ -301,6 +309,11 @@ module redmule_regif (
                     logic [31:0] value;
                 } z_addr;
             } marith2;
+            struct {
+                struct {
+                    logic [31:0] value;
+                } w_cols_offset;
+            } mcnfig3;
         } hwpe_job_dep;
     } field_storage_t;
     field_storage_t field_storage;
@@ -705,6 +718,29 @@ module redmule_regif (
         end
     end
     assign hwif_out.hwpe_job_dep.marith2.z_addr.value = field_storage.hwpe_job_dep.marith2.z_addr.value;
+    // Field: redmule_regif.hwpe_job_dep.mcnfig3.w_cols_offset
+    always_comb begin
+        automatic logic [31:0] next_c;
+        automatic logic load_next_c;
+        next_c = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value;
+        load_next_c = '0;
+        if(decoded_reg_strb.hwpe_job_dep.mcnfig3 && decoded_req_is_wr) begin // SW write
+            next_c = (field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value & ~decoded_wr_biten[31:0]) | (decoded_wr_data[31:0] & decoded_wr_biten[31:0]);
+            load_next_c = '1;
+        end
+        field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.next = next_c;
+        field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.load_next = load_next_c;
+    end
+    always_ff @(posedge clk or negedge arst_n) begin
+        if(~arst_n) begin
+            field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value <= 32'h0;
+        end else begin
+            if(field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.load_next) begin
+                field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value <= field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.next;
+            end
+        end
+    end
+    assign hwif_out.hwpe_job_dep.mcnfig3.w_cols_offset.value = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value;
     assign hwif_out.hwpe_job_indep.reserved.reserved.value = 32'h0;
 
     //--------------------------------------------------------------------------
@@ -783,6 +819,9 @@ module redmule_regif (
             readback_data_var[31:0] = hwif_in.hwpe_job_dep.mopcnt.op_id_cnt.next;
         end
         if(rd_mux_addr == 32'h3c) begin
+            readback_data_var[31:0] = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value;
+        end
+        if(rd_mux_addr == 32'h40) begin
             readback_data_var[31:0] = 32'h0;
         end
         readback_data = readback_data_var;

rtl/ctrl/regif/redmule_regif_pkg.sv

@@ -8,7 +8,7 @@ package redmule_regif_pkg;
 
     localparam REDMULE_REGIF_DATA_WIDTH = 32;
     localparam REDMULE_REGIF_MIN_ADDR_WIDTH = 32;
-    localparam REDMULE_REGIF_SIZE = 'h40;
+    localparam REDMULE_REGIF_SIZE = 'h44;
 
     typedef struct packed {
         logic [31:0] next;
@@ -220,13 +220,22 @@ package redmule_regif_pkg;
         redmule_regif__marith2__z_addr__out_t z_addr;
     } redmule_regif__marith2__out_t;
 
+    typedef struct packed {
+        logic [31:0] value;
+    } redmule_regif__mcnfig3__w_cols_offset__out_t;
+
+    typedef struct packed {
+        redmule_regif__mcnfig3__w_cols_offset__out_t w_cols_offset;
+    } redmule_regif__mcnfig3__out_t;
+
     typedef struct packed {
         redmule_regif__mcnfig0__out_t mcnfig0;
         redmule_regif__mcnfig1__out_t mcnfig1;
         redmule_regif__mcnfig2__out_t mcnfig2;
         redmule_regif__marith0__out_t marith0;
         redmule_regif__marith1__out_t marith1;
         redmule_regif__marith2__out_t marith2;
+        redmule_regif__mcnfig3__out_t mcnfig3;
     } redmule_regif__hwpe_ctrl_job_dep__out_t;
 
     typedef struct packed {

rtl/redmule_ctrl.sv

@@ -15,7 +15,8 @@ module redmule_ctrl
   parameter int unsigned Height = MaxDim,
   parameter int unsigned Width = MaxDim,
   parameter int unsigned PipeRegs = MaxPipeRegs-1,
-  parameter int unsigned FpWidth = 16
+  parameter int unsigned FpWidth = 16,
+  parameter bit          EnableReordering = 1'b0
 )(
   input  logic                    clk_i             ,
   input  logic                    rst_ni            ,
@@ -44,12 +45,15 @@ module redmule_ctrl
 
   logic        latch_clear;
   logic        tiler_setback, tiler_valid;
+  logic        fifo_z_empty;
+  logic        set_offset_q, set_offset_d, loopback_reset;
 
   typedef enum logic [2:0] {
     REDMULE_LATCH_RST,
     REDMULE_IDLE,
     REDMULE_STARTING,
     REDMULE_COMPUTING,
+    REDMULE_LOOPBACK,
     REDMULE_FINISHED
   } redmule_ctrl_state_e;
 
@@ -68,6 +72,7 @@ module redmule_ctrl
     .rst_ni      ( rst_ni         ),
     .clear_i     ( target_clear_i ),
     .setback_i   ( tiler_setback  ),
+    .loopback_i  ( loopback_reset ),
     .start_cfg_i ( start_cfg_i    ),
     .valid_o     ( tiler_valid    ),
     .busy_o      ( tiler_busy_o   ),
@@ -92,6 +97,18 @@ module redmule_ctrl
     end
   end
 
+  // Set offset flag
+  always_ff @(posedge clk_i or negedge rst_ni) begin
+    if(~rst_ni) begin
+       set_offset_q <= 1'b0;
+    end else begin
+      if (target_clear_i || latch_clear || current == REDMULE_FINISHED || loopback_reset)
+        set_offset_q <= 1'b0;
+      else
+        set_offset_q <= set_offset_d;
+    end
+  end
+
   logic slave_start;
   always_ff @(posedge clk_i, negedge rst_ni) begin
     if (~rst_ni) begin
@@ -117,40 +134,60 @@ module redmule_ctrl
 
   assign cntrl_scheduler_o.first_load = current == REDMULE_STARTING;
   assign tiler_setback                = tiler_valid;
-  assign busy_o                       = slave_start | (current != REDMULE_LATCH_RST && current != REDMULE_IDLE && current != REDMULE_FINISHED);
-  assign flush_o                      = current == REDMULE_FINISHED;
-  assign cntrl_scheduler_o.rst        = current == REDMULE_FINISHED;
+  // Keep the accelerator clocked while in FINISHED so the FSM can take the
+  // final transition back to IDLE before busy deasserts.
+  assign busy_o                       = slave_start | (current != REDMULE_LATCH_RST && current != REDMULE_IDLE);
+  assign loopback_reset               = current == REDMULE_COMPUTING && next == REDMULE_LOOPBACK;
+  assign flush_o                      = current == REDMULE_FINISHED || loopback_reset;
+  assign cntrl_scheduler_o.rst        = current == REDMULE_FINISHED || loopback_reset;
   assign cntrl_scheduler_o.finished   = current == REDMULE_FINISHED;
   assign latch_clear                  = current == REDMULE_LATCH_RST;
+  assign fifo_z_empty                 = EnableReordering ? flgs_streamer_i.store_fifo_empty : 1'b1;
 
   always_comb begin : controller_fsm
     cntrl_flags_o.idle = 1'b0;
     next = current;
+    set_offset_d = set_offset_q;
 
     case (current)
       REDMULE_LATCH_RST: begin
         cntrl_flags_o.idle = 1'b1;
+        set_offset_d = 1'b0;
         next = REDMULE_IDLE;
       end
 
       REDMULE_IDLE: begin
         cntrl_flags_o.idle = 1'b1;
+        set_offset_d = config_i.w_cols_offset != '0;
         if ((slave_start & tiler_valid) || test_mode_i) begin
           next = REDMULE_STARTING;
         end
       end
 
+      REDMULE_LOOPBACK: begin
+        cntrl_flags_o.idle = 1'b1;
+        set_offset_d = 1'b0;
+        if (tiler_valid) begin
+          next = REDMULE_STARTING;
+        end
+      end
+
       REDMULE_STARTING: begin
         if (w_loaded_i) begin
           next = REDMULE_COMPUTING;
         end
       end
+
       REDMULE_COMPUTING: begin
         // busy_o gates clk_acc, so the job must not finish while stores are
         // still queued in the streamer's store FIFO, or they freeze unsent.
         if (flgs_streamer_i.z_stream_sink_flags.ready_start && fifo_empty_i
             && flgs_streamer_i.store_fifo_empty) begin
-          next = REDMULE_FINISHED;
+          if (set_offset_q) begin
+            next = REDMULE_LOOPBACK;
+          end else begin
+            next = REDMULE_FINISHED;
+          end
         end
       end
 
@@ -163,7 +200,9 @@ module redmule_ctrl
   /*---------------------------------------------------------------------------------------------*/
   /*                            Other combinational assigmnets                                   */
   /*---------------------------------------------------------------------------------------------*/
-  assign evt_o   = current == REDMULE_FINISHED;
-  assign clear_o = target_clear_i || latch_clear || current == REDMULE_FINISHED;
+
+  assign evt_o   = flgs_streamer_i.z_stream_sink_flags.done && ~set_offset_q;
+
+  assign clear_o = target_clear_i || latch_clear || current == REDMULE_FINISHED || loopback_reset;
 
 endmodule : redmule_ctrl