diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 46b44cab..d70f6730 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -45,6 +45,8 @@ jobs: exclude_paths: | rtl/redmule_noncomp.sv rtl/redmule_fma.sv + rtl/ctrl/regif/redmule_regif.sv + rtl/ctrl/regif/redmule_regif_pkg.sv extra_args: "--waiver_files .github/verible.waiver" github_token: ${{ secrets.GITHUB_TOKEN }} fail_on_error: true diff --git a/Bender.yml b/Bender.yml index 7268fd38..6d20fcd8 100644 --- a/Bender.yml +++ b/Bender.yml @@ -19,7 +19,7 @@ dependencies: hwpe-ctrl : { git: "https://github.com/pulp-platform/hwpe-ctrl.git" , version: 3.1.1 } hci : { git: "https://github.com/pulp-platform/hci.git" , version: 2.6.1 } fpnew : { git: "https://github.com/pulp-platform/cvfpu.git" , rev: "pulp-v0.1.3" } - common_cells : { git: "https://github.com/pulp-platform/common_cells.git" , version: 1.21.0 } + common_cells : { git: "https://github.com/pulp-platform/common_cells.git" , version: 1.38.0 } tech_cells_generic: { git: "https://github.com/pulp-platform/tech_cells_generic.git", version: 0.2.11 } sources: diff --git a/Makefile b/Makefile index 119d970c..7b129e2f 100644 --- a/Makefile +++ b/Makefile @@ -24,17 +24,22 @@ else Bender ?= bender Gcc ?= endif -OP ?= gemm -fp_fmt ?= FP16 -M ?= 24 -N ?= 16 -K ?= 16 -TEST_ID ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,) -INC_DIR ?= $(SW)/inc/$(TEST_ID) -BUILD_DIR ?= $(SW)/build/$(TEST_ID) -ISA ?= riscv -ARCH ?= rv -XLEN ?= 32 + +OP ?= gemm +fp_fmt ?= FP16 +M ?= 24 +N ?= 16 +K ?= 16 +WColsOffset ?= 0 +EnableReordering ?= 0 + +TEST_ID ?= $(OP)_$(fp_fmt)_$(M)x$(N)x$(K)$(if $(filter-out 0,$(WColsOffset)),_w_cols_offset_$(WColsOffset),)$(if $(filter 1,$(REDMULE_COMPLEX)),_cplx,)$(if $(filter 1,$(EnableReordering)),_reord,) +INC_DIR ?= $(SW)/inc/$(TEST_ID) +BUILD_DIR ?= $(SW)/build/$(TEST_ID) +ISA ?= riscv +ARCH ?= rv +XLEN ?= 32 + # Local PULP GCC toolchains are based on older GCC and bundle # Zicsr together with the I extension. For GitHub CI we use # a newer version of GCC @@ -77,6 +82,8 @@ ifeq ($(debug),1) FLAGS += -DDEBUG endif +FLAGS += -DW_COLS_OFFSET=$(WColsOffset) + # Include directories INC += -I$(SW) INC += -I$(INC_DIR) diff --git a/rtl/ctrl/gen_regif.sh b/rtl/ctrl/gen_regif.sh index bd0de4f9..de944a08 100755 --- a/rtl/ctrl/gen_regif.sh +++ b/rtl/ctrl/gen_regif.sh @@ -5,7 +5,28 @@ peakrdl regblock redmule_regif.rdl -o regif/ --cpuif passthrough --default-reset arst_n --hwif-report --addr-width 32 peakrdl html redmule_regif.rdl -o regif/html/ -peakrdl c-header redmule_regif.rdl -o regif/hwpe_ctrl_target.h +peakrdl c-header redmule_regif.rdl -o ../../sw/hwpe_ctrl_target.h +awk ' + /#include / { + print + print "" + print "#if defined(__cplusplus)" + print "#define REDMULE_STATIC_ASSERT(cond, msg) static_assert(cond, msg)" + print "#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)" + print "#define REDMULE_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg)" + print "#else" + print "#define REDMULE_STATIC_ASSERT_CONCAT_(a, b) a##b" + print "#define REDMULE_STATIC_ASSERT_CONCAT(a, b) REDMULE_STATIC_ASSERT_CONCAT_(a, b)" + print "#define REDMULE_STATIC_ASSERT(cond, msg) \\" + print " typedef char REDMULE_STATIC_ASSERT_CONCAT(redmule_static_assert_, __LINE__)[(cond) ? 1 : -1]" + print "#endif" + next + } + { + gsub(/static_assert\(/, "REDMULE_STATIC_ASSERT(") + print + } +' ../../sw/hwpe_ctrl_target.h > ../../sw/hwpe_ctrl_target.h.tmp && mv ../../sw/hwpe_ctrl_target.h.tmp ../../sw/hwpe_ctrl_target.h # PeakRDL uses unpacked structs to avoid issues at compile time, which is commendable, but incompatible with FIFOing the output of the job! (use portable sed syntax that works on both Linux and macOS) sed -E 's/typedef[[:space:]]+struct([[:space:]])/typedef struct packed\1/g' regif/redmule_regif_pkg.sv > regif/redmule_regif_pkg.sv.tmp && mv regif/redmule_regif_pkg.sv.tmp regif/redmule_regif_pkg.sv @@ -13,6 +34,6 @@ sed -E 's/typedef[[:space:]]+struct([[:space:]])/typedef struct packed\1/g' regi HEADER='// Copyright 2025 ETH Zurich and University of Bologna. // Solderpad Hardware License, Version 0.51, see LICENSE for details. // SPDX-License-Identifier: SHL-0.51' -for f in regif/redmule_regif.sv regif/redmule_regif_pkg.sv; do +for f in regif/redmule_regif.sv regif/redmule_regif_pkg.sv ../../sw/hwpe_ctrl_target.h; do printf '%s\n' "$HEADER" | cat - "$f" > "$f.tmp" && mv "$f.tmp" "$f" done diff --git a/rtl/ctrl/redmule_regif.rdl b/rtl/ctrl/redmule_regif.rdl index 9e574f97..994f99e8 100644 --- a/rtl/ctrl/redmule_regif.rdl +++ b/rtl/ctrl/redmule_regif.rdl @@ -216,6 +216,15 @@ addrmap redmule_regif { sw = rw; } y_offs[31:0] = 0; }; + reg mcnfig3 { + name = "MCNFIG3"; + field { + name = "w_cols_offset"; + desc = "Column offset applied to W/Z before starting the first pass."; + hw = r; + sw = rw; + } w_cols_offset[31:0] = 0; + }; reg marith0 { name = "MARITH0"; field { @@ -263,6 +272,7 @@ addrmap redmule_regif { mcnfig0 mcnfig0 @ 0x00; mcnfig1 mcnfig1 @ 0x04; mcnfig2 mcnfig2 @ 0x08; + mcnfig3 mcnfig3 @ 0x1c; marith0 marith0 @ 0x0c; marith1 marith1 @ 0x10; marith2 marith2 @ 0x14; diff --git a/rtl/ctrl/redmule_target_decoder.sv b/rtl/ctrl/redmule_target_decoder.sv index 1f28d42b..89647993 100644 --- a/rtl/ctrl/redmule_target_decoder.sv +++ b/rtl/ctrl/redmule_target_decoder.sv @@ -137,12 +137,14 @@ module redmule_target_decoder assign config_o.receive_w = hwif_out.hwpe_job_dep.mcnfig1.receive_w.value; assign config_o.send_w = hwif_out.hwpe_job_dep.mcnfig1.send_w.value; assign config_o.y_offs = hwif_out.hwpe_job_dep.mcnfig2.y_offs.value; + assign config_o.w_cols_offset = hwif_out.hwpe_job_dep.mcnfig3.w_cols_offset.value[15:0]; assign config_o.x_addr = hwif_out.hwpe_job_dep.marith0.x_addr.value; assign config_o.w_addr = hwif_out.hwpe_job_dep.marith1.w_addr.value; assign config_o.z_addr = hwif_out.hwpe_job_dep.marith2.z_addr.value; assign config_o.gemm_ops = gemm_op_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_ops.value); assign config_o.gemm_input_fmt = gemm_fmt_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_input_fmt.value); assign config_o.gemm_output_fmt = gemm_fmt_e'(hwif_out.hwpe_job_dep.mcnfig1.gemm_output_fmt.value); + assign config_o.loopback_w = 1'b0; // Operation ID counter: // op_id_counter_in_q: Increments when operations are issued (tags for tracking) diff --git a/rtl/ctrl/regif/redmule_regif.sv b/rtl/ctrl/regif/redmule_regif.sv index 3e92ab64..43e30a68 100644 --- a/rtl/ctrl/regif/redmule_regif.sv +++ b/rtl/ctrl/regif/redmule_regif.sv @@ -87,6 +87,7 @@ module redmule_regif ( logic marith1; logic marith2; logic mopcnt; + logic mcnfig3; } hwpe_job_dep; struct { logic reserved; @@ -120,7 +121,8 @@ module redmule_regif ( decoded_reg_strb.hwpe_job_dep.marith1 = cpuif_req_masked & (cpuif_addr == 32'h30); decoded_reg_strb.hwpe_job_dep.marith2 = cpuif_req_masked & (cpuif_addr == 32'h34); decoded_reg_strb.hwpe_job_dep.mopcnt = cpuif_req_masked & (cpuif_addr == 32'h38) & !cpuif_req_is_wr; - decoded_reg_strb.hwpe_job_indep.reserved = cpuif_req_masked & (cpuif_addr == 32'h3c) & !cpuif_req_is_wr; + decoded_reg_strb.hwpe_job_dep.mcnfig3 = cpuif_req_masked & (cpuif_addr == 32'h3c); + decoded_reg_strb.hwpe_job_indep.reserved = cpuif_req_masked & (cpuif_addr == 32'h40) & !cpuif_req_is_wr; decoded_err = '0; end @@ -224,6 +226,12 @@ module redmule_regif ( logic load_next; } z_addr; } marith2; + struct { + struct { + logic [31:0] next; + logic load_next; + } w_cols_offset; + } mcnfig3; } hwpe_job_dep; } field_combo_t; field_combo_t field_combo; @@ -301,6 +309,11 @@ module redmule_regif ( logic [31:0] value; } z_addr; } marith2; + struct { + struct { + logic [31:0] value; + } w_cols_offset; + } mcnfig3; } hwpe_job_dep; } field_storage_t; field_storage_t field_storage; @@ -705,6 +718,29 @@ module redmule_regif ( end end assign hwif_out.hwpe_job_dep.marith2.z_addr.value = field_storage.hwpe_job_dep.marith2.z_addr.value; + // Field: redmule_regif.hwpe_job_dep.mcnfig3.w_cols_offset + always_comb begin + automatic logic [31:0] next_c; + automatic logic load_next_c; + next_c = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value; + load_next_c = '0; + if(decoded_reg_strb.hwpe_job_dep.mcnfig3 && decoded_req_is_wr) begin // SW write + next_c = (field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value & ~decoded_wr_biten[31:0]) | (decoded_wr_data[31:0] & decoded_wr_biten[31:0]); + load_next_c = '1; + end + field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.next = next_c; + field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.load_next = load_next_c; + end + always_ff @(posedge clk or negedge arst_n) begin + if(~arst_n) begin + field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value <= 32'h0; + end else begin + if(field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.load_next) begin + field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value <= field_combo.hwpe_job_dep.mcnfig3.w_cols_offset.next; + end + end + end + assign hwif_out.hwpe_job_dep.mcnfig3.w_cols_offset.value = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value; assign hwif_out.hwpe_job_indep.reserved.reserved.value = 32'h0; //-------------------------------------------------------------------------- @@ -783,6 +819,9 @@ module redmule_regif ( readback_data_var[31:0] = hwif_in.hwpe_job_dep.mopcnt.op_id_cnt.next; end if(rd_mux_addr == 32'h3c) begin + readback_data_var[31:0] = field_storage.hwpe_job_dep.mcnfig3.w_cols_offset.value; + end + if(rd_mux_addr == 32'h40) begin readback_data_var[31:0] = 32'h0; end readback_data = readback_data_var; diff --git a/rtl/ctrl/regif/redmule_regif_pkg.sv b/rtl/ctrl/regif/redmule_regif_pkg.sv index a3b41159..658bc63f 100644 --- a/rtl/ctrl/regif/redmule_regif_pkg.sv +++ b/rtl/ctrl/regif/redmule_regif_pkg.sv @@ -8,7 +8,7 @@ package redmule_regif_pkg; localparam REDMULE_REGIF_DATA_WIDTH = 32; localparam REDMULE_REGIF_MIN_ADDR_WIDTH = 32; - localparam REDMULE_REGIF_SIZE = 'h40; + localparam REDMULE_REGIF_SIZE = 'h44; typedef struct packed { logic [31:0] next; @@ -220,6 +220,14 @@ package redmule_regif_pkg; redmule_regif__marith2__z_addr__out_t z_addr; } redmule_regif__marith2__out_t; + typedef struct packed { + logic [31:0] value; + } redmule_regif__mcnfig3__w_cols_offset__out_t; + + typedef struct packed { + redmule_regif__mcnfig3__w_cols_offset__out_t w_cols_offset; + } redmule_regif__mcnfig3__out_t; + typedef struct packed { redmule_regif__mcnfig0__out_t mcnfig0; redmule_regif__mcnfig1__out_t mcnfig1; @@ -227,6 +235,7 @@ package redmule_regif_pkg; redmule_regif__marith0__out_t marith0; redmule_regif__marith1__out_t marith1; redmule_regif__marith2__out_t marith2; + redmule_regif__mcnfig3__out_t mcnfig3; } redmule_regif__hwpe_ctrl_job_dep__out_t; typedef struct packed { diff --git a/rtl/redmule_ctrl.sv b/rtl/redmule_ctrl.sv index df0e95b9..63925738 100644 --- a/rtl/redmule_ctrl.sv +++ b/rtl/redmule_ctrl.sv @@ -15,7 +15,8 @@ module redmule_ctrl parameter int unsigned Height = MaxDim, parameter int unsigned Width = MaxDim, parameter int unsigned PipeRegs = MaxPipeRegs-1, - parameter int unsigned FpWidth = 16 + parameter int unsigned FpWidth = 16, + parameter bit EnableReordering = 1'b0 )( input logic clk_i , input logic rst_ni , @@ -44,12 +45,15 @@ module redmule_ctrl logic latch_clear; logic tiler_setback, tiler_valid; + logic fifo_z_empty; + logic set_offset_q, set_offset_d, loopback_reset; typedef enum logic [2:0] { REDMULE_LATCH_RST, REDMULE_IDLE, REDMULE_STARTING, REDMULE_COMPUTING, + REDMULE_LOOPBACK, REDMULE_FINISHED } redmule_ctrl_state_e; @@ -68,6 +72,7 @@ module redmule_ctrl .rst_ni ( rst_ni ), .clear_i ( target_clear_i ), .setback_i ( tiler_setback ), + .loopback_i ( loopback_reset ), .start_cfg_i ( start_cfg_i ), .valid_o ( tiler_valid ), .busy_o ( tiler_busy_o ), @@ -92,6 +97,18 @@ module redmule_ctrl end end + // Set offset flag + always_ff @(posedge clk_i or negedge rst_ni) begin + if(~rst_ni) begin + set_offset_q <= 1'b0; + end else begin + if (target_clear_i || latch_clear || current == REDMULE_FINISHED || loopback_reset) + set_offset_q <= 1'b0; + else + set_offset_q <= set_offset_d; + end + end + logic slave_start; always_ff @(posedge clk_i, negedge rst_ni) begin if (~rst_ni) begin @@ -117,40 +134,60 @@ module redmule_ctrl assign cntrl_scheduler_o.first_load = current == REDMULE_STARTING; assign tiler_setback = tiler_valid; - assign busy_o = slave_start | (current != REDMULE_LATCH_RST && current != REDMULE_IDLE && current != REDMULE_FINISHED); - assign flush_o = current == REDMULE_FINISHED; - assign cntrl_scheduler_o.rst = current == REDMULE_FINISHED; + // Keep the accelerator clocked while in FINISHED so the FSM can take the + // final transition back to IDLE before busy deasserts. + assign busy_o = slave_start | (current != REDMULE_LATCH_RST && current != REDMULE_IDLE); + assign loopback_reset = current == REDMULE_COMPUTING && next == REDMULE_LOOPBACK; + assign flush_o = current == REDMULE_FINISHED || loopback_reset; + assign cntrl_scheduler_o.rst = current == REDMULE_FINISHED || loopback_reset; assign cntrl_scheduler_o.finished = current == REDMULE_FINISHED; assign latch_clear = current == REDMULE_LATCH_RST; + assign fifo_z_empty = EnableReordering ? flgs_streamer_i.store_fifo_empty : 1'b1; always_comb begin : controller_fsm cntrl_flags_o.idle = 1'b0; next = current; + set_offset_d = set_offset_q; case (current) REDMULE_LATCH_RST: begin cntrl_flags_o.idle = 1'b1; + set_offset_d = 1'b0; next = REDMULE_IDLE; end REDMULE_IDLE: begin cntrl_flags_o.idle = 1'b1; + set_offset_d = config_i.w_cols_offset != '0; if ((slave_start & tiler_valid) || test_mode_i) begin next = REDMULE_STARTING; end end + REDMULE_LOOPBACK: begin + cntrl_flags_o.idle = 1'b1; + set_offset_d = 1'b0; + if (tiler_valid) begin + next = REDMULE_STARTING; + end + end + REDMULE_STARTING: begin if (w_loaded_i) begin next = REDMULE_COMPUTING; end end + REDMULE_COMPUTING: begin // busy_o gates clk_acc, so the job must not finish while stores are // still queued in the streamer's store FIFO, or they freeze unsent. if (flgs_streamer_i.z_stream_sink_flags.ready_start && fifo_empty_i && flgs_streamer_i.store_fifo_empty) begin - next = REDMULE_FINISHED; + if (set_offset_q) begin + next = REDMULE_LOOPBACK; + end else begin + next = REDMULE_FINISHED; + end end end @@ -163,7 +200,9 @@ module redmule_ctrl /*---------------------------------------------------------------------------------------------*/ /* Other combinational assigmnets */ /*---------------------------------------------------------------------------------------------*/ - assign evt_o = current == REDMULE_FINISHED; - assign clear_o = target_clear_i || latch_clear || current == REDMULE_FINISHED; + + assign evt_o = flgs_streamer_i.z_stream_sink_flags.done && ~set_offset_q; + + assign clear_o = target_clear_i || latch_clear || current == REDMULE_FINISHED || loopback_reset; endmodule : redmule_ctrl diff --git a/rtl/redmule_inst_decoder.sv b/rtl/redmule_inst_decoder.sv index e60e2847..2f7b9d28 100644 --- a/rtl/redmule_inst_decoder.sv +++ b/rtl/redmule_inst_decoder.sv @@ -63,7 +63,7 @@ module redmule_inst_decoder // Per-hart FIFO status flags for instruction and register packets logic [XifNumHarts-1:0] issue_fifo_full, register_fifo_full, issue_fifo_empty, register_fifo_empty; - + // Hart ID of the currently executing operation (tracked through pipeline) logic [HartIdWidth-1:0] current_hartid_d, current_hartid_q; @@ -166,7 +166,7 @@ module redmule_inst_decoder // Output configuration from the winning hart to the RedMule tiler/controller assign config_o = config_d[winner]; - + // Configuration valid only for MARITH instructions when both FIFOs have data and CPU is ready // (MCNFIG updates config but doesn't trigger execution) assign config_valid_o = ~issue_fifo_empty[winner] && ~register_fifo_empty[winner] && x_result_ready_i && {cur_issue[winner].instr[26:25],cur_issue[winner].instr[14:12],cur_issue[winner].instr[6:0]} == MARITH; @@ -453,6 +453,7 @@ module redmule_inst_decoder // Decode instruction and extract configuration parameters from register file values always_comb begin : config_assignment config_d[i] = config_q[i]; // Default: retain previous configuration + config_d[i].loopback_w = 1'b0; unique case ({cur_issue[i].instr[26:25],cur_issue[i].instr[14:12],cur_issue[i].instr[6:0]}) MCNFIG: begin @@ -466,6 +467,7 @@ module redmule_inst_decoder config_d[i].send_w = cur_register[i].rs[1][19]; // Broadcast W to external stream config_d[i].gemm_ops = cur_register[i].rs[1][20] ? MATMUL : GEMM; config_d[i].y_offs = cur_register[i].rs[2][31:0]; // Y buffer offset for bias addition + config_d[i].w_cols_offset = '0; // XIF path does not currently expose W column offsets end MARITH: begin // Matrix arithmetic: extract memory addresses from rs1, rs2, rs3 diff --git a/rtl/redmule_mm_wrap.sv b/rtl/redmule_mm_wrap.sv index 6b8769bd..abc35b38 100644 --- a/rtl/redmule_mm_wrap.sv +++ b/rtl/redmule_mm_wrap.sv @@ -20,6 +20,7 @@ module redmule_mm_wrap #( parameter int unsigned DataW = MaxDataW, parameter int unsigned MisalignedAccessSupport = MisalignedAccessSupportDefault, + parameter bit EnableReordering = 1'b0, parameter fp_format_e FpFormat = FP16, parameter int unsigned Height = MaxDim, parameter int unsigned Width = MaxDim, @@ -72,6 +73,7 @@ module redmule_mm_wrap redmule_top #( .DataW ( DataW ), .MisalignedAccessSupport ( MisalignedAccessSupport ), + .EnableReordering ( EnableReordering ), .FpFormat ( FpFormat ), .Height ( Height ), .Width ( Width ), diff --git a/rtl/redmule_pkg.sv b/rtl/redmule_pkg.sv index 96d33a23..4702f8c9 100644 --- a/rtl/redmule_pkg.sv +++ b/rtl/redmule_pkg.sv @@ -184,6 +184,7 @@ package redmule_pkg; logic [15:0] x_rows_iter; logic [15:0] w_cols_iter; logic [15:0] w_rows_iter; + logic [15:0] w_cols_offset; logic [ 7:0] x_cols_lftovr; logic [ 7:0] x_rows_lftovr; logic [ 7:0] w_cols_lftovr; @@ -206,6 +207,7 @@ package redmule_pkg; fpu_fmt_e input_format; fpu_fmt_e computing_format; logic gemm_selection; + logic loopback_w; logic send_w; logic receive_w; logic send_x; diff --git a/rtl/redmule_streamer.sv b/rtl/redmule_streamer.sv index fccb07e3..d2cff1ae 100644 --- a/rtl/redmule_streamer.sv +++ b/rtl/redmule_streamer.sv @@ -5,6 +5,7 @@ // Yvan Tortorella // Andrea Belano // Arpan Suravi Prasad +// Marco Bertuletti // `include "hci_helpers.svh" @@ -17,6 +18,17 @@ module redmule_streamer #( parameter int unsigned DataW = MaxDataW, parameter int unsigned MisalignedAccessSupport = MisalignedAccessSupportDefault, + // Set to 1 to reorder responses of outstanding transactions on the X, W, and Y + // load channels (hci_core_rob) and arbitrate the shared TCDM port out-of-order + // (hci_core_mux_ooo). Requires a memory system that completes accepted requests + // in issue order, but possibly with variable (multi-cycle) latency. When 0, the + // streamer keeps the original fixed-priority, single-outstanding-request scheme. + parameter bit EnableReordering = 1'b0, + // Height/NumPipeRegs size the load-side response FIFO (RESP_FIFO_DEPTH) of + // hci_core_source when EnableReordering is set, matching the engine's + // in-flight response latency. + parameter int unsigned Height = MaxDim , + parameter int unsigned NumPipeRegs = MaxPipeRegs-1, parameter int unsigned FpFormat = FP16 , parameter int unsigned EccChunkSize = 32 , parameter fpnew_pkg::fmt_logic_t FpFmtConfig = 6'b001101, @@ -90,7 +102,7 @@ hci_core_intf #( .IW ( `HCI_SIZE_GET_IW(ldst_tcdm) ), .EW ( `HCI_SIZE_GET_EW(ldst_tcdm) ), .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) -) ldst_tcdm_pre_r_id ( .clk ( clk_i ) ); +) ldst_tcdm_pre_r_valid ( .clk ( clk_i ) ); hci_core_intf #( `ifndef SYNTHESIS @@ -104,7 +116,7 @@ hci_core_intf #( .IW ( `HCI_SIZE_GET_IW(ldst_tcdm) ), .EW ( `HCI_SIZE_GET_EW(ldst_tcdm) ), .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) -) ldst_tcdm_pre_r_valid ( .clk ( clk_i ) ); +) ldst_tcdm_pre_r_id ( .clk ( clk_i ) ); if (EW > 1) begin : gen_ecc_encoder logic [`HCI_SIZE_GET_DW(tcdm)/EccChunkSize-1:0] data_single_err, data_multi_err; @@ -143,6 +155,72 @@ hci_core_intf #( .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) ) virt_tcdm [0:NumStreamSources] ( .clk ( clk_i ) ); +// LD/ST arbitration onto the shared TCDM port. +// * EnableReordering = 0: fixed-priority, single-outstanding-request arbiter +// (redmule_mux) followed by a 1-cycle-latency r_valid filter, as before. +// * EnableReordering = 1: each of the X/W/Y load channels gets its own +// hci_core_rob so several requests can be in flight at once on that channel; +// the (already in-order) Z/store and spare channels pass through unchanged. +// All channels are then arbitrated out-of-order (hci_core_mux_ooo). +if (EnableReordering) begin : gen_outstanding_ldst_mux + + // Number of unique user-IDs the ROB and the out-of-order mux can track; + localparam int unsigned UW = `HCI_SIZE_GET_UW(ldst_tcdm); + localparam int unsigned ROB_NW = 1 << UW; + + hci_core_intf #( + `ifndef SYNTHESIS + .WAIVE_RSP3_ASSERT ( 1'b1 ), + .WAIVE_RSP5_ASSERT ( 1'b1 ), + .WAIVE_RQ3_ASSERT ( 1'b1 ), + .WAIVE_RQ4_ASSERT ( 1'b1 ), + `endif + .DW ( `HCI_SIZE_GET_DW(ldst_tcdm) ), + .AW ( `HCI_SIZE_GET_AW(ldst_tcdm) ), + .BW ( `HCI_SIZE_GET_BW(ldst_tcdm) ), + .UW ( `HCI_SIZE_GET_UW(ldst_tcdm) ), + .IW ( `HCI_SIZE_GET_IW(ldst_tcdm) ), + .EW ( `HCI_SIZE_GET_EW(ldst_tcdm) ), + .EHW ( `HCI_SIZE_GET_EHW(ldst_tcdm) ) + ) virt_tcdm_rob [0:NumStreamSources] ( .clk ( clk_i ) ); + + for (genvar i = 0; i < NumStreamSources; i++) begin : gen_channel_rob + hci_core_rob #( + .ROB_NW ( ROB_NW ), + .`HCI_SIZE_PARAM(out) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + ) i_streamer_rob ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .in ( virt_tcdm[i] ), + .out ( virt_tcdm_rob[i] ) + ); + end + + // The Z/store channel does not need reordering: pass it through to the + // out-of-order mux unchanged. + hci_core_assign i_rob_bypass ( .tcdm_target (virt_tcdm[NumStreamSources]), .tcdm_initiator (virt_tcdm_rob[NumStreamSources]) ); + + logic [NumStreamSources:0][$clog2(NumStreamSources+1)-1:0] priority_encoding; + assign priority_encoding[0] = 0; + assign priority_encoding[1] = 1; + assign priority_encoding[2] = 2; + assign priority_encoding[3] = 3; + + hci_core_mux_ooo #( + .NB_CHAN ( NumStreamSources+1 ), + .`HCI_SIZE_PARAM(out) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + ) i_mux ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .clear_i ( clear_i ), + .priority_force_i ( 1'b1 ), + .priority_i ( priority_encoding ), + .in ( virt_tcdm_rob ), + .out ( ldst_tcdm ) + ); + +end else begin : gen_fixed_priority_ldst_mux + if (MuxPriority == MUX_PRIORITY_STATIC) begin : static_mux_gen redmule_mux #( .NB_CHAN (NumStreamSources+1), @@ -192,6 +270,9 @@ hci_core_r_id_filter #( .tcdm_initiator ( ldst_tcdm ) ); + +end + /************************************ Store Channel *************************************/ /* The store channel of the streamer connects the incoming stream interface (Z stream) * * to an HCI core sink module that translates the stream into a TCDM protocol. This * @@ -278,7 +359,7 @@ flags_fifo_t store_fifo_flags; // HCI store fifo. hci_core_fifo #( - .FIFO_DEPTH ( 2 ), + .FIFO_DEPTH ( EnableReordering ? Height*(NumPipeRegs+1) : 2 ), .`HCI_SIZE_PARAM(tcdm_initiator) ( `HCI_SIZE_PARAM(ldst_tcdm) ) ) i_store_fifo ( .clk_i ( clk_i ), @@ -372,18 +453,29 @@ for (genvar i = 0; i < NumStreamSources; i++) begin: gen_tcdm2stream hci_core_assign i_load_assign ( .tcdm_target (load_fifo_d[i]), .tcdm_initiator (virt_tcdm[i]) ); - hci_core_fifo #( - .FIFO_DEPTH ( 4 ), // to avoid protocol violations, as the consumer has a throughput - // of 1 packet over 4 cycles, we need a depth of 4 elements. - .`HCI_SIZE_PARAM(tcdm_initiator) ( `HCI_SIZE_PARAM(ldst_tcdm) ) - ) i_load_tcdm_fifo ( - .clk_i ( clk_i ), - .rst_ni ( rst_ni ), - .clear_i ( clear_i ), - .flags_o ( ), - .tcdm_target ( load_fifo_q[i] ), - .tcdm_initiator ( load_fifo_d[i] ) - ); + // hci_core_fifo can retract a request it already raised without a grant + // (a documented RQ-4 protocol violation, harmless when grants are never + // delayed) whenever its response-side fill level changes while a request + // is pending - which happens routinely once the ROB lets requests wait + // for a grant across many cycles. The ROB already buffers/reorders on + // this channel, so the FIFO's buffering is redundant here: skip it and + // wire the cast unit straight to the ROB when outstanding is enabled. + if (EnableReordering) begin : gen_no_load_tcdm_fifo + hci_core_assign i_load_tcdm_bypass ( .tcdm_target (load_fifo_q[i]), .tcdm_initiator (load_fifo_d[i]) ); + end else begin : gen_load_tcdm_fifo + hci_core_fifo #( + .FIFO_DEPTH ( 4 ), // to avoid protocol violations, as the consumer has a throughput + // of 1 packet over 4 cycles, we need a depth of 4 elements. + .`HCI_SIZE_PARAM(tcdm_initiator) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + ) i_load_tcdm_fifo ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .clear_i ( clear_i ), + .flags_o ( ), + .tcdm_target ( load_fifo_q[i] ), + .tcdm_initiator ( load_fifo_d[i] ) + ); + end // Load cast unit // This unit uses only the data bus of the TCDM interface. The other buses @@ -435,7 +527,8 @@ for (genvar i = 0; i < NumStreamSources; i++) begin: gen_tcdm2stream hci_core_source #( .MISALIGNED_ACCESSES ( MisalignedAccessSupport ), - .`HCI_SIZE_PARAM(tcdm) ( `HCI_SIZE_PARAM(ldst_tcdm) ) + .`HCI_SIZE_PARAM(tcdm) ( `HCI_SIZE_PARAM(ldst_tcdm) ), + .RESP_FIFO_DEPTH ( EnableReordering ? Height*(NumPipeRegs+1) : 0 ) ) i_stream_source ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), diff --git a/rtl/redmule_tiler.sv b/rtl/redmule_tiler.sv index 930c0b90..44f98435 100644 --- a/rtl/redmule_tiler.sv +++ b/rtl/redmule_tiler.sv @@ -22,6 +22,7 @@ module redmule_tiler input logic rst_ni , input logic clear_i , input logic setback_i , + input logic loopback_i , input logic start_cfg_i, output logic valid_o , output logic busy_o , @@ -41,7 +42,9 @@ localparam int unsigned MinimumSizeN = MinimumSizeNFactor * Height; logic clk_en; logic clk_int; +redmule_config_t input_config_d, input_config_q; redmule_config_t config_d, config_q; +logic loopback_active_q, loopback_active; always_ff @(posedge clk_i, negedge rst_ni) begin: clock_gate_enabler if (~rst_ni) begin @@ -49,7 +52,7 @@ always_ff @(posedge clk_i, negedge rst_ni) begin: clock_gate_enabler end else begin if (clear_i || setback_i) begin clk_en <= 1'b0; - end else if (start_cfg_i && ready_i) begin + end else if ((start_cfg_i && ready_i) || loopback_i) begin clk_en <= 1'b1; end end @@ -64,12 +67,38 @@ tc_clk_gating i_tiler_clockg ( assign busy_o = clk_en || ~ready_i; -assign config_d.x_addr = config_i.x_addr; -assign config_d.w_addr = config_i.w_addr; -assign config_d.z_addr = config_i.z_addr; -assign config_d.m_size = config_i.m_size; -assign config_d.k_size = config_i.k_size; -assign config_d.n_size = config_i.n_size; // real N is carried downstream unchanged (used for operand gating) + +// Store loopback +assign loopback_active = (loopback_i || loopback_active_q); +always_ff @(posedge clk_i, negedge rst_ni) begin: loopback_ff + if (~rst_ni) begin + loopback_active_q <= 1'b0; + end else begin + loopback_active_q <= !(clear_i || setback_i) && loopback_active; + end +end + +always_ff @(posedge clk_i or negedge rst_ni) begin : input_config_ff + if (~rst_ni) begin + input_config_q <= '0; + end else if (clear_i) begin + input_config_q <= '0; + end else if (start_cfg_i && ready_i) begin + input_config_q <= config_i; + end +end + +assign input_config_d = loopback_active ? input_config_q : config_i; +assign config_d.x_addr = input_config_d.x_addr; +assign config_d.w_addr = loopback_active ? input_config_d.w_addr : + input_config_d.w_addr + input_config_d.w_cols_offset * (FpWidth/8); +assign config_d.z_addr = loopback_active ? input_config_d.z_addr : + input_config_d.z_addr + input_config_d.w_cols_offset * (FpWidth/8); +assign config_d.y_addr = loopback_active ? input_config_d.z_addr + input_config_d.y_offs : + input_config_d.z_addr + input_config_d.y_offs + input_config_d.w_cols_offset * (FpWidth/8); +assign config_d.m_size = input_config_d.m_size; +assign config_d.k_size = input_config_d.k_size; +assign config_d.n_size = input_config_d.n_size; // Effective N size used ONLY for the W-row loop length / streamer length: any job with // N <= Height is promoted to MinimumSizeN so the W-load takes as long as a full N-tile, restoring @@ -80,17 +109,22 @@ assign config_d.n_size = config_i.n_size; // real N is carried downstre // step with the promoted N, and both padded operands are zeroed (X via x_cols_lftovr, // W via the cntrl_w_buffer_o.height gating in redmule_scheduler.sv). logic [15:0] n_size_eff; -assign n_size_eff = (config_i.n_size <= Height) ? MinimumSizeN[15:0] : config_i.n_size; -assign config_d.gemm_ops = config_i.gemm_ops; -assign config_d.gemm_input_fmt = config_i.gemm_input_fmt; -assign config_d.gemm_output_fmt = config_i.gemm_output_fmt; -assign config_d.receive_w = config_i.receive_w; -assign config_d.send_w = config_i.send_w; -assign config_d.receive_x = config_i.receive_x; -assign config_d.send_x = config_i.send_x; - -assign config_d.y_offs = config_i.y_offs; -assign config_d.y_addr = config_i.z_addr + config_i.y_offs; +assign n_size_eff = (input_config_d.n_size <= Height) ? MinimumSizeN[15:0] : input_config_d.n_size; +// Sourced from input_config_d (not config_i directly) so the loopback pass keeps +// the job's original operation/format even after dec_config_q's read pointer has +// advanced past its single valid entry (config_i can no longer be trusted once +// the first pass's completion has popped it). +assign config_d.gemm_ops = input_config_d.gemm_ops; +assign config_d.gemm_input_fmt = input_config_d.gemm_input_fmt; +assign config_d.gemm_output_fmt = input_config_d.gemm_output_fmt; +assign config_d.receive_w = input_config_d.receive_w; +assign config_d.send_w = input_config_d.send_w; +assign config_d.loopback_w = loopback_active; +assign config_d.receive_x = input_config_d.receive_x; +assign config_d.send_x = input_config_d.send_x; +// Convert the user-programmed column offset into whole RedMulE output tiles. +assign config_d.w_cols_offset = loopback_active ? '0 : input_config_d.w_cols_offset; +assign config_d.y_offs = input_config_d.y_offs; // Calculating the number of iterations alng the two dimensions of the X matrix logic [15:0] x_rows_iter_nolftovr; @@ -102,7 +136,8 @@ assign x_cols_iter_nolftovr = config_d.n_size/(Height*(PipeRegs + 1)); logic [15:0] w_cols_iter_nolftovr; logic [15:0] w_rows_iter_lftovr, w_rows_iter_nolftovr; -assign w_cols_iter_nolftovr = config_d.k_size/(Height*(PipeRegs + 1)); +assign w_cols_iter_nolftovr = loopback_active ? (input_config_d.w_cols_offset/(Height*(PipeRegs + 1))) : + (config_d.k_size - input_config_d.w_cols_offset)/(Height*(PipeRegs + 1)); assign w_rows_iter_lftovr = w_rows_iter_nolftovr + Height - config_d.w_rows_lftovr; assign w_rows_iter_nolftovr = n_size_eff; // promoted N: W-row loop runs for a full N-tile when N <= Height @@ -112,10 +147,13 @@ assign config_d.x_cols_lftovr = config_d.n_size - (x_cols_iter_nolftovr*(Height* // Calculating the residuals along the weight dimensions assign config_d.w_rows_lftovr = n_size_eff - (Height*(n_size_eff/Height)); // promoted N (0 when N <= Height -> full W-row loop) -assign config_d.w_cols_lftovr = config_d.k_size - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))); +assign config_d.w_cols_lftovr = loopback_active ? input_config_d.w_cols_offset - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))) : + (config_d.k_size - input_config_d.w_cols_offset) - (w_cols_iter_nolftovr*(Height*(PipeRegs + 1))); -// Calculate w_cols, x_cols, x_rows iterations +// Calculate w_cols iterations assign config_d.w_cols_iter = config_d.w_cols_lftovr != '0 ? w_cols_iter_nolftovr + 1 : w_cols_iter_nolftovr; + +// Calculate w_rows, x_cols, x_rows iterations assign config_d.w_rows_iter = config_d.w_rows_lftovr != '0 ? w_rows_iter_lftovr : w_rows_iter_nolftovr; assign config_d.x_cols_iter = config_d.x_cols_lftovr != '0 ? x_cols_iter_nolftovr + 1 : x_cols_iter_nolftovr; assign config_d.x_rows_iter = config_d.x_rows_lftovr != '0 ? x_rows_iter_nolftovr + 1 : x_rows_iter_nolftovr; @@ -124,7 +162,7 @@ assign config_d.x_rows_iter = config_d.x_rows_lftovr != '0 ? x_rows_iter_nolftov logic [31:0] x_rows_by_w_cols_iter_d, x_rows_by_w_cols_iter_q; logic x_rows_by_w_cols_iter_valid_d, x_rows_by_w_cols_iter_valid_q; -assign x_rows_by_w_cols_iter_d = start_cfg_i ? config_d.x_rows_iter * config_d.w_cols_iter : x_rows_by_w_cols_iter_q; +assign x_rows_by_w_cols_iter_d = (start_cfg_i || loopback_i) ? config_d.x_rows_iter * config_d.w_cols_iter : x_rows_by_w_cols_iter_q; always_ff @(posedge clk_i or negedge rst_ni) begin if (~rst_ni) begin @@ -138,7 +176,7 @@ always_ff @(posedge clk_i or negedge rst_ni) begin end end -assign x_rows_by_w_cols_iter_valid_d = start_cfg_i; +assign x_rows_by_w_cols_iter_valid_d = (start_cfg_i || loopback_i); always_ff @(posedge clk_i or negedge rst_ni) begin if (~rst_ni) begin @@ -263,7 +301,7 @@ assign config_d.gemm_selection = config_d.gemm_ops == MATMUL ? 1'b0 : 1'b1; assign config_d.x_d1_stride = ((4*FpWidth)/AddrWidth)*(((DataW/FpWidth)*x_cols_iter_nolftovr) + config_d.x_cols_lftovr); assign config_d.x_rows_offs = Width*config_d.x_d1_stride; assign config_d.w_tot_len = x_rows_by_w_cols_by_w_rows_iter_q[31:0]; -assign config_d.w_d0_stride = ((4*FpWidth)/AddrWidth)*(((DataW/FpWidth)*w_cols_iter_nolftovr) + config_d.w_cols_lftovr); +assign config_d.w_d0_stride = ((4*FpWidth)/AddrWidth)*((DataW/FpWidth) * (config_d.k_size)/(Height*(PipeRegs + 1))); assign config_d.yz_tot_len = Width*x_rows_by_w_cols_iter_q[15:0]; assign config_d.yz_d0_stride = config_d.w_d0_stride; assign config_d.yz_d2_stride = Width*config_d.w_d0_stride; diff --git a/rtl/redmule_top.sv b/rtl/redmule_top.sv index 6f526d72..0eb74d0f 100644 --- a/rtl/redmule_top.sv +++ b/rtl/redmule_top.sv @@ -16,6 +16,7 @@ module redmule_top #( parameter int unsigned DataW = MaxDataW, // TCDM port dimension (in bits) parameter int unsigned MisalignedAccessSupport = MisalignedAccessSupportDefault, // set to 1 to support misaligned accesses on TCDM + parameter bit EnableReordering = 1'b0, // set to 1 to enable ROB-based reordering of outstanding TCDM transactions parameter fp_format_e FpFormat = FP16, // Data format (default is FP16) parameter int unsigned Height = MaxDim, // Number of PEs within a row parameter int unsigned Width = MaxDim, // Number of parallel rows @@ -188,6 +189,9 @@ hwpe_stream_intf_stream #( .DATA_WIDTH ( DataW ) ) z_buffer_fifo ( .clk( cl redmule_streamer #( .DataW ( DataW ), .MisalignedAccessSupport ( MisalignedAccessSupport ), + .EnableReordering ( EnableReordering ), + .Height ( Height ), + .NumPipeRegs ( NumPipeRegs ), .EccChunkSize ( EccChunkSize ), .FpFormat ( FpFormat ), .FpFmtConfig ( FpFmtConfig ), @@ -561,7 +565,7 @@ if(CtrlIntfConfig == XIF) begin : xif_ctrl_intf_gen .rst_ni ( rst_ni ), .clear_i ( '0 ), // TODO: fixme, not having a software-based clear mechanism is a bad idea. .config_ready_i ( ~config_fifo_full ), - .op_done_i ( flgs_streamer.z_stream_sink_flags.done ), + .op_done_i ( evt_o ), .config_valid_o ( dec_config_valid ), .config_o ( dec_config ), .x_issue_req_i ( x_issue_req_i ), @@ -591,7 +595,7 @@ else begin : mm_ctrl_intf_gen .clear_i ( '0 ), // ORed internally with target_clear .target_clear_o ( target_clear ), .config_ready_i ( ~config_fifo_full ), - .op_done_i ( flgs_streamer.z_stream_sink_flags.done ), + .op_done_i ( evt_o ), .config_valid_o ( dec_config_valid ), .config_o ( dec_config ), .target ( target ) @@ -619,7 +623,7 @@ redmule_config_fifo #( .data_i ( dec_config ), .push_i ( dec_config_valid ), .data_o ( dec_config_q ), - .pop_i ( cfg_complete ) + .pop_i ( cfg_complete && !redmule_config.loopback_w ) ); /*---------------------------------------------------------------*/ @@ -695,7 +699,7 @@ redmule_scheduler #( `ifndef SYNTHESIS always_ff @(posedge clk_acc) begin - if (cfg_complete) begin + if (cfg_complete && !redmule_config.loopback_w) begin $display("[redmule] Configuration loaded at %t", $time); $display("[redmule] x_addr = 0x%h", redmule_config.x_addr); $display("[redmule] w_addr = 0x%h", redmule_config.w_addr); @@ -710,6 +714,7 @@ always_ff @(posedge clk_acc) begin $display("[redmule] gemm_output_fmt = %s", redmule_config.gemm_output_fmt.name()); $display("[redmule] x_cols_iter = 0x%h", redmule_config.x_cols_iter); $display("[redmule] x_rows_iter = 0x%h", redmule_config.x_rows_iter); + $display("[redmule] w_cols_offset = 0x%h", redmule_config.w_cols_offset); $display("[redmule] w_cols_iter = 0x%h", redmule_config.w_cols_iter); $display("[redmule] w_rows_iter = 0x%h", redmule_config.w_rows_iter); $display("[redmule] x_cols_lftovr = 0x%h", redmule_config.x_cols_lftovr); diff --git a/scripts/regression.yml b/scripts/regression.yml index 792d53de..abefd8b1 100644 --- a/scripts/regression.yml +++ b/scripts/regression.yml @@ -292,3 +292,21 @@ redmule_regression: M33_N33_K33: path: . command: make golden M=33 N=33 K=33 && make sw-clean sw-build M=33 N=33 K=33 REDMULE_COMPLEX=0 Gcc= && make hw-run M=33 N=33 K=33 REDMULE_COMPLEX=0 target=$Target | tee target/sim/$Target/transcript_33_33_33 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_33_33_33 + M48_N32_K32_REORD: # reordered path, aligned and multi-tile baseline + path: . + command: make golden M=48 N=32 K=32 EnableReordering=1 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_48_32_32_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_reord + M16_N32_K18_REORD: # reordered path with K-leftover and N-tiling + path: . + command: make golden M=16 N=32 K=18 EnableReordering=1 && make sw-clean sw-build M=16 N=32 K=18 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=16 N=32 K=18 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_16_32_18_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_16_32_18_reord + M1_N32_K8_REORD: # reordered path on a tiny M edge case + path: . + command: make golden M=1 N=32 K=8 EnableReordering=1 && make sw-clean sw-build M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 Gcc= && make hw-run M=1 N=32 K=8 REDMULE_COMPLEX=0 EnableReordering=1 target=$Target | tee target/sim/$Target/transcript_1_32_8_reord && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_1_32_8_reord + M48_N32_K32_W_OFFSET16: # offset on W columns + path: . + command: make golden M=48 N=32 K=32 WColsOffset=16 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=16 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=16 target=$Target | tee target/sim/$Target/transcript_48_32_32_w_cols_offset_16 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_w_cols_offset_16 + M48_N32_K32_W_OFFSET17: # offset on W columns + path: . + command: make golden M=48 N=32 K=32 WColsOffset=17 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=17 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=17 target=$Target | tee target/sim/$Target/transcript_48_32_32_w_cols_offset_17 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_w_cols_offset_17 + M48_N32_K32_W_OFFSET7: # offset on W columns + path: . + command: make golden M=48 N=32 K=32 WColsOffset=7 && make sw-clean sw-build M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=7 Gcc= && make hw-run M=48 N=32 K=32 REDMULE_COMPLEX=0 WColsOffset=7 target=$Target | tee target/sim/$Target/transcript_48_32_32_w_cols_offset_7 && grep -q '\[TB\] - Success!' target/sim/$Target/transcript_48_32_32_w_cols_offset_7 diff --git a/sw/archi_redmule.h b/sw/archi_redmule.h index bf740add..d33e0f4a 100644 --- a/sw/archi_redmule.h +++ b/sw/archi_redmule.h @@ -8,86 +8,13 @@ #ifndef __ARCHI_REDMULE_H__ #define __ARCHI_REDMULE_H__ -/* - * Register map aligned to the SystemRDL interface (rtl/ctrl/redmule_regif.rdl). - * The job-dependent block now lives at offset 0x20 (was 0x40), configuration - * registers come first, and the operation/format selection is folded into - * mcnfig1 (there is no separate arithmetic register any more). - * - * |========================================================================| - * || || - * ||Control and generic configuration register layout || - * |========================================================================| - * || # reg | offset | bits | bitmask || content || - * ||-------+----------+---------+--------------++-------------------------|| - * || 0 | 0x0000 | 1: 0 | 0x00000003 || COMMIT_TRIGGER || - * || 1 | 0x0004 | 31: 0 | 0xFFFFFFFF || ACQUIRE || - * || 2 | 0x0008 | 31: 0 | 0xFFFFFFFF || reserved0 || - * || 3 | 0x000c | 31: 0 | 0xFFFFFFFF || STATUS || - * || 4 | 0x0010 | 7: 0 | 0x000000FF || RUNNING_JOB || - * || 5 | 0x0014 | 1: 0 | 0x00000003 || SOFT_CLEAR || - * |========================================================================| - * || || - * ||Job-dependent registers layout (base 0x20) || - * |========================================================================| - * || # reg | offset | bits | bitmask || content || - * ||-------+----------+---------+--------------++-------------------------|| - * || 0 | 0x0020 | | || Matrix Config 0 (mcnfig0)|| - * || | | 31:16 | 0xFFFF0000 || K Size (W Columns) || - * || | | 15: 0 | 0x0000FFFF || M Size (X Rows) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 1 | 0x0024 | | || Matrix Config 1 (mcnfig1)|| - * || | | 26:25 | 0x06000000 || Output format || - * || | | 24:23 | 0x01800000 || Input format || - * || | | 22:20 | 0x00700000 || Operation selection || - * || | | 19:16 | 0x000F0000 || send/receive stream || - * || | | 15: 0 | 0x0000FFFF || N Size (X Cols/W Rows) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 2 | 0x0028 | 31: 0 | 0xFFFFFFFF || Matrix Config 2 (mcnfig2)|| - * || | | | || Y offset (bias) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 3 | 0x002C | 31: 0 | 0xFFFFFFFF || X_ADDR (marith0) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 4 | 0x0030 | 31: 0 | 0xFFFFFFFF || W_ADDR (marith1) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 5 | 0x0034 | 31: 0 | 0xFFFFFFFF || Z_ADDR (marith2) || - * ||-------+----------+---------+--------------++-------------------------|| - * || 6 | 0x0038 | 31: 0 | 0xFFFFFFFF || MOPCNT (RO) || - * |========================================================================| - * - */ - #define ARCHI_CL_EVT_ACC0 0 #define ARCHI_CL_EVT_ACC1 1 -// RedMulE architecture -#define ADDR_WIDTH 32 -#define DATA_WIDTH 256 -#define REDMULE_FMT 16 -#define ARRAY_HEIGHT 4 -#define PIPE_REGS 3 -#define ARRAY_WIDTH 12 /* Superior limit is ARRAY_HEIGHT*PIPE_REGS */ - // Base address +#ifndef REDMULE_BASE_ADD #define REDMULE_BASE_ADD 0x00100000 - -// Commands (mandatory control block @ 0x00) -#define REDMULE_TRIGGER 0x00 // commit_trigger[1:0]: writing 0 commits + starts the job -#define REDMULE_ACQUIRE 0x04 -#define REDMULE_FINISHED 0x08 // reserved0 in the RDL map -#define REDMULE_STATUS 0x0C -#define REDMULE_RUNNING_JOB 0x10 -#define REDMULE_SOFT_CLEAR 0x14 // soft_clear[1:0]: writing 0 clears everything (incl. regfile) - -// Job-dependent registers (base 0x20; config first, then X/W/Z addresses) -#define REDMULE_REG_OFFS 0x20 -#define REDMULE_MCFG0_PTR 0x00 // -> 0x20 mcnfig0: k_size[31:16], m_size[15:0] -#define REDMULE_MCFG1_PTR 0x04 // -> 0x24 mcnfig1: n[15:0], stream[19:16], ops[22:20], in_fmt[24:23], out_fmt[26:25] -#define REDMULE_MCFG2_PTR 0x08 // -> 0x28 mcnfig2: y_offs[31:0] -#define REDMULE_REG_X_PTR 0x0C // -> 0x2C marith0: x_addr -#define REDMULE_REG_W_PTR 0x10 // -> 0x30 marith1: w_addr -#define REDMULE_REG_Z_PTR 0x14 // -> 0x34 marith2: z_addr -#define REDMULE_MOPCNT_PTR 0x18 // -> 0x38 mopcnt (RO) +#endif // OPs definition #define MATMUL 0x0 diff --git a/sw/hal_redmule.h b/sw/hal_redmule.h index b00c767b..4ccfa961 100644 --- a/sw/hal_redmule.h +++ b/sw/hal_redmule.h @@ -7,77 +7,77 @@ #ifndef __HAL_REDMULE_H__ #define __HAL_REDMULE_H__ + +#include "archi_redmule.h" +#include "hwpe_ctrl_target.h" #include -#include "tensor_dim.h" /* LOW-LEVEL HAL */ #define REDMULE_ADDR_BASE REDMULE_BASE_ADD #define REDMULE_ADDR_SPACE 0x00000100 -#define HWPE_WRITE(value, offset) *(int *)(REDMULE_ADDR_BASE + offset) = value -#define HWPE_READ(offset) *(int *)(REDMULE_ADDR_BASE + offset) +static inline volatile redmule_regif_t *redmule_regs(void) { + return (volatile redmule_regif_t *)REDMULE_ADDR_BASE; +} static inline void redmule_x_add_set(unsigned int value) { - HWPE_WRITE(value, REDMULE_REG_OFFS + REDMULE_REG_X_PTR); + redmule_regs()->hwpe_job_dep.marith0 = value; } static inline void redmule_w_add_set(unsigned int value) { - HWPE_WRITE(value, REDMULE_REG_OFFS + REDMULE_REG_W_PTR); + redmule_regs()->hwpe_job_dep.marith1 = value; } static inline void redmule_z_add_set(unsigned int value) { - HWPE_WRITE(value, REDMULE_REG_OFFS + REDMULE_REG_Z_PTR); + redmule_regs()->hwpe_job_dep.marith2 = value; } -static inline void redmule_mcfg_set(uint32_t mcfg0, uint32_t mcfg1, uint32_t mcfg2) { - HWPE_WRITE(mcfg0, REDMULE_REG_OFFS + REDMULE_MCFG0_PTR); - HWPE_WRITE(mcfg1, REDMULE_REG_OFFS + REDMULE_MCFG1_PTR); - HWPE_WRITE(mcfg2, REDMULE_REG_OFFS + REDMULE_MCFG2_PTR); +static inline void redmule_mcfg_set(uint32_t mcfg0, uint32_t mcfg1, uint32_t mcfg2, + uint32_t mcfg3) { + redmule_regs()->hwpe_job_dep.mcnfig0 = mcfg0; + redmule_regs()->hwpe_job_dep.mcnfig1 = mcfg1; + redmule_regs()->hwpe_job_dep.mcnfig2 = mcfg2; + redmule_regs()->hwpe_job_dep.mcnfig3 = mcfg3; } -static inline void hwpe_trigger_job() { HWPE_WRITE(0, REDMULE_TRIGGER); } +static inline void hwpe_trigger_job() { redmule_regs()->hwpe_ctrl.commit_trigger = 0; } -static inline int hwpe_acquire_job() { return HWPE_READ(REDMULE_ACQUIRE); } +static inline int hwpe_acquire_job() { + return (int)redmule_regs()->hwpe_ctrl.acquire; +} static inline unsigned int hwpe_get_status() { - return HWPE_READ(REDMULE_STATUS); + return redmule_regs()->hwpe_ctrl.status; } -static inline void hwpe_soft_clear() { - volatile int i; - HWPE_WRITE(0, REDMULE_SOFT_CLEAR); -} +static inline void hwpe_soft_clear() { redmule_regs()->hwpe_ctrl.soft_clear = 0; } static inline void hwpe_cg_enable() { return; } static inline void hwpe_cg_disable() { return; } void redmule_cfg(unsigned int x, unsigned int w, unsigned int z, uint16_t m_size, uint16_t n_size, - uint16_t k_size, uint8_t gemm_op, uint8_t gemm_fmt) { + uint16_t k_size, uint16_t w_col_off, uint8_t gemm_op, uint8_t gemm_fmt) { uint32_t mcfg_reg0 = 0; uint32_t mcfg_reg1 = 0; uint32_t mcfg_reg2 = 0; - - // mcnfig0: K size in [31:16], M size in [15:0] - mcfg_reg0 = ((uint32_t)k_size << 16) | ((uint32_t)m_size << 0); - - // mcnfig1: N size in [15:0]; op/format folded in (there is no separate ARITH reg). - // gemm_ops [22:20] - // gemm_input_fmt [24:23] - // gemm_output_fmt [26:25] - // stream-routing bits [19:16] left 0 (no external streaming in the MM scenario) - mcfg_reg1 = ((uint32_t)n_size << 0) | ((uint32_t)gemm_op << 20) | - ((uint32_t)gemm_fmt << 23) | ((uint32_t)gemm_fmt << 25); - - // mcnfig2: Y offset (bias). y_addr = z_addr + y_offs, so y_offs = 0 keeps the - // in-place GEMM behaviour (Y read from, and Z written to, the same buffer). + uint32_t mcfg_reg3 = 0; + + mcfg_reg0 = ((uint32_t)k_size << REDMULE_REGIF__MCNFIG0__K_SIZE_bp) | + ((uint32_t)m_size << REDMULE_REGIF__MCNFIG0__M_SIZE_bp); + // gemm_fmt is used for both the input and the output format. + mcfg_reg1 = ((uint32_t)n_size << REDMULE_REGIF__MCNFIG1__N_SIZE_bp) | + ((uint32_t)gemm_op << REDMULE_REGIF__MCNFIG1__GEMM_OPS_bp) | + ((uint32_t)gemm_fmt << REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bp) | + ((uint32_t)gemm_fmt << REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bp); mcfg_reg2 = 0; + mcfg_reg3 = ((uint32_t)w_col_off << REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bp); - redmule_mcfg_set((unsigned int)mcfg_reg0, (unsigned int)mcfg_reg1, (unsigned int)mcfg_reg2); - redmule_x_add_set((unsigned int)x); - redmule_w_add_set((unsigned int)w); - redmule_z_add_set((unsigned int)z); + redmule_x_add_set(x); + redmule_w_add_set(w); + redmule_z_add_set(z); + redmule_mcfg_set(mcfg_reg0, mcfg_reg1, mcfg_reg2, mcfg_reg3); } #endif diff --git a/sw/hwpe_ctrl_target.h b/sw/hwpe_ctrl_target.h new file mode 100644 index 00000000..0b1377d7 --- /dev/null +++ b/sw/hwpe_ctrl_target.h @@ -0,0 +1,198 @@ +// Copyright 2025 ETH Zurich and University of Bologna. +// Solderpad Hardware License, Version 0.51, see LICENSE for details. +// SPDX-License-Identifier: SHL-0.51 +// Generated by PeakRDL-cheader - A free and open-source header generator +// https://github.com/SystemRDL/PeakRDL-cheader + +#ifndef HWPE_CTRL_TARGET_H +#define HWPE_CTRL_TARGET_H + +#ifdef __cplusplus +extern "C" { +#endif +#include +#include + +#if defined(__cplusplus) +#define REDMULE_STATIC_ASSERT(cond, msg) static_assert(cond, msg) +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +#define REDMULE_STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) +#else +#define REDMULE_STATIC_ASSERT_CONCAT_(a, b) a##b +#define REDMULE_STATIC_ASSERT_CONCAT(a, b) REDMULE_STATIC_ASSERT_CONCAT_(a, b) +#define REDMULE_STATIC_ASSERT(cond, msg) \ + typedef char REDMULE_STATIC_ASSERT_CONCAT(redmule_static_assert_, __LINE__)[(cond) ? 1 : -1] +#endif + +// reg - redmule_regif::hwpe_commit_trigger +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_bm 0x3 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_bp 0 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_bw 2 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__COMMIT_TRIGGER_reset 0x0 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_bm 0xfffffffc +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_bp 2 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_bw 30 +#define REDMULE_REGIF__HWPE_COMMIT_TRIGGER__R0_reset 0x0 + +// reg - redmule_regif::hwpe_acquire +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_bm 0xffffffff +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_bp 0 +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_bw 32 +#define REDMULE_REGIF__HWPE_ACQUIRE__ACQUIRE_reset 0x0 + +// reg - redmule_regif::hwpe_reserved +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_bm 0xffffffff +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_bp 0 +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_bw 32 +#define REDMULE_REGIF__HWPE_RESERVED__RESERVED_reset 0x0 + +// reg - redmule_regif::hwpe_status +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_bm 0xffffffff +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_bp 0 +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_bw 32 +#define REDMULE_REGIF__HWPE_STATUS__STATUS0_reset 0x0 + +// reg - redmule_regif::hwpe_running_job +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_bm 0xff +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_bp 0 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_bw 8 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__RUNNING_JOB_reset 0x0 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_bm 0xffffff00 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_bp 8 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_bw 24 +#define REDMULE_REGIF__HWPE_RUNNING_JOB__R0_reset 0x0 + +// reg - redmule_regif::hwpe_soft_clear +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_bm 0x3 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_bp 0 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_bw 2 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__SOFT_CLEAR_reset 0x0 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_bm 0xfffffffc +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_bp 2 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_bw 30 +#define REDMULE_REGIF__HWPE_SOFT_CLEAR__R0_reset 0x0 + +// regfile - redmule_regif::hwpe_ctrl_mandatory +typedef struct __attribute__ ((__packed__)) { + uint32_t commit_trigger; + uint32_t acquire; + uint32_t reserved0; + uint32_t status; + uint32_t running_job; + uint32_t soft_clear; + uint32_t reserved1; + uint32_t reserved2; +} redmule_regif__hwpe_ctrl_mandatory_t; + +// reg - redmule_regif::mcnfig0 +#define REDMULE_REGIF__MCNFIG0__M_SIZE_bm 0xffff +#define REDMULE_REGIF__MCNFIG0__M_SIZE_bp 0 +#define REDMULE_REGIF__MCNFIG0__M_SIZE_bw 16 +#define REDMULE_REGIF__MCNFIG0__M_SIZE_reset 0x0 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_bm 0xffff0000 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_bp 16 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_bw 16 +#define REDMULE_REGIF__MCNFIG0__K_SIZE_reset 0x0 + +// reg - redmule_regif::mcnfig1 +#define REDMULE_REGIF__MCNFIG1__N_SIZE_bm 0xffff +#define REDMULE_REGIF__MCNFIG1__N_SIZE_bp 0 +#define REDMULE_REGIF__MCNFIG1__N_SIZE_bw 16 +#define REDMULE_REGIF__MCNFIG1__N_SIZE_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_bm 0x10000 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_bp 16 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_bw 1 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_X_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__SEND_X_bm 0x20000 +#define REDMULE_REGIF__MCNFIG1__SEND_X_bp 17 +#define REDMULE_REGIF__MCNFIG1__SEND_X_bw 1 +#define REDMULE_REGIF__MCNFIG1__SEND_X_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_bm 0x40000 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_bp 18 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_bw 1 +#define REDMULE_REGIF__MCNFIG1__RECEIVE_W_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__SEND_W_bm 0x80000 +#define REDMULE_REGIF__MCNFIG1__SEND_W_bp 19 +#define REDMULE_REGIF__MCNFIG1__SEND_W_bw 1 +#define REDMULE_REGIF__MCNFIG1__SEND_W_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_bm 0x700000 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_bp 20 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_bw 3 +#define REDMULE_REGIF__MCNFIG1__GEMM_OPS_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bm 0x1800000 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bp 23 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_bw 2 +#define REDMULE_REGIF__MCNFIG1__GEMM_INPUT_FMT_reset 0x0 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bm 0x6000000 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bp 25 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_bw 2 +#define REDMULE_REGIF__MCNFIG1__GEMM_OUTPUT_FMT_reset 0x0 + +// reg - redmule_regif::mcnfig2 +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_bm 0xffffffff +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_bp 0 +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_bw 32 +#define REDMULE_REGIF__MCNFIG2__Y_OFFS_reset 0x0 + +// reg - redmule_regif::marith0 +#define REDMULE_REGIF__MARITH0__X_ADDR_bm 0xffffffff +#define REDMULE_REGIF__MARITH0__X_ADDR_bp 0 +#define REDMULE_REGIF__MARITH0__X_ADDR_bw 32 +#define REDMULE_REGIF__MARITH0__X_ADDR_reset 0x0 + +// reg - redmule_regif::marith1 +#define REDMULE_REGIF__MARITH1__W_ADDR_bm 0xffffffff +#define REDMULE_REGIF__MARITH1__W_ADDR_bp 0 +#define REDMULE_REGIF__MARITH1__W_ADDR_bw 32 +#define REDMULE_REGIF__MARITH1__W_ADDR_reset 0x0 + +// reg - redmule_regif::marith2 +#define REDMULE_REGIF__MARITH2__Z_ADDR_bm 0xffffffff +#define REDMULE_REGIF__MARITH2__Z_ADDR_bp 0 +#define REDMULE_REGIF__MARITH2__Z_ADDR_bw 32 +#define REDMULE_REGIF__MARITH2__Z_ADDR_reset 0x0 + +// reg - redmule_regif::mopcnt +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bm 0xffffffff +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bp 0 +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_bw 32 +#define REDMULE_REGIF__MOPCNT__OP_ID_CNT_reset 0x0 + +// reg - redmule_regif::mcnfig3 +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bm 0xffffffff +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bp 0 +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_bw 32 +#define REDMULE_REGIF__MCNFIG3__W_COLS_OFFSET_reset 0x0 + +// regfile - redmule_regif::hwpe_ctrl_job_dep +typedef struct __attribute__ ((__packed__)) { + uint32_t mcnfig0; + uint32_t mcnfig1; + uint32_t mcnfig2; + uint32_t marith0; + uint32_t marith1; + uint32_t marith2; + uint32_t mopcnt; + uint32_t mcnfig3; +} redmule_regif__hwpe_ctrl_job_dep_t; + +// regfile - redmule_regif::hwpe_ctrl_job_indep +typedef struct __attribute__ ((__packed__)) { + uint32_t reserved; +} redmule_regif__hwpe_ctrl_job_indep_t; + +// addrmap - redmule_regif +typedef struct __attribute__ ((__packed__)) { + redmule_regif__hwpe_ctrl_mandatory_t hwpe_ctrl; + redmule_regif__hwpe_ctrl_job_dep_t hwpe_job_dep; + redmule_regif__hwpe_ctrl_job_indep_t hwpe_job_indep; +} redmule_regif_t; + + +REDMULE_STATIC_ASSERT(sizeof(redmule_regif_t) == 0x44, "Packing error"); + +#ifdef __cplusplus +} +#endif + +#endif /* HWPE_CTRL_TARGET_H */ diff --git a/sw/redmule.c b/sw/redmule.c index 6c0b2c87..b72c612c 100644 --- a/sw/redmule.c +++ b/sw/redmule.c @@ -5,11 +5,13 @@ // Yvan Tortorella // +#include + #include "archi_redmule.h" #include "hal_redmule.h" #include "redmule_utils.h" -#include +#include "tensor_dim.h" #include "golden.h" #include "w_input.h" #include "x_input.h" @@ -18,17 +20,21 @@ #include "z_output.h" #define ERR 0x0000 +#ifndef W_COLS_OFFSET +#define W_COLS_OFFSET 0 +#endif + int main() { uint16_t m_size = M_SIZE; uint16_t n_size = N_SIZE; uint16_t k_size = K_SIZE; - uint8_t *x = x_inp; - uint8_t *w = w_inp; - uint8_t *y = y_inp; + uint16_t *x = x_inp; + uint16_t *w = w_inp; + uint16_t *y = y_inp; - uint8_t *z = z_oup; // golden_out //1c010000 + uint16_t *z = z_oup; // golden_out //1c010000 uint8_t float_fmt = (SRC_FMT == FP8) ? (uint8_t)Float8 : (SRC_FMT == FP8ALT) ? (uint8_t)Float8Alt @@ -53,7 +59,7 @@ int main() { // int pace_ops = 0; redmule_cfg((unsigned int)x, (unsigned int)w, (unsigned int)y, m_size, n_size, k_size, - (uint8_t)gemm_ops, float_fmt); + (uint16_t)W_COLS_OFFSET, (uint8_t)gemm_ops, float_fmt); // Start RedMulE operation and sleeping until the end of computation printf("Triggering accelerator and going to sleep...\n"); @@ -69,11 +75,11 @@ int main() { if (float_fmt == Float16 || float_fmt == Float16Alt) if (gemm_ops == PACE) - errors = redmule16_compare_int(y, golden, K_SIZE/2, 0); + errors = redmule16_compare_int((uint32_t *)y, golden, K_SIZE/2, 0); else - errors = redmule16_compare_int(y, golden, m_size * k_size / 2, ERR); + errors = redmule16_compare_int((uint32_t *)y, golden, m_size * k_size / 2, ERR); else if (float_fmt == Float8 || float_fmt == Float8Alt) - errors = redmule8_compare_int(y, golden, m_size * k_size / 4, ERR); + errors = redmule8_compare_int((uint32_t *)y, golden, m_size * k_size / 4, ERR); *(int *)0x80000000 = errors; diff --git a/sw/redmule_complex.c b/sw/redmule_complex.c index 21771051..8ed0c765 100644 --- a/sw/redmule_complex.c +++ b/sw/redmule_complex.c @@ -17,10 +17,10 @@ int main() { - uint8_t *x = x_inp; - uint8_t *w = w_inp; - uint8_t *y = y_inp; - uint8_t *z = z_oup; // golden_out //1c010000 + uint16_t *x = x_inp; + uint16_t *w = w_inp; + uint16_t *y = y_inp; + uint16_t *z = z_oup; // golden_out //1c010000 uint16_t m_size = M_SIZE; uint16_t n_size = N_SIZE; @@ -78,7 +78,7 @@ int main() { // Wait for end of computation asm volatile("wfi" ::: "memory"); - errors = redmule16_compare_int(y, golden, m_size * k_size / 2); + errors = redmule16_compare_int((uint32_t *)y, golden, m_size * k_size / 2, 0); *(int *)0x80000000 = errors; diff --git a/sw/utils/tinyprintf.h b/sw/utils/tinyprintf.h index 85ddf69a..4f7cb92a 100644 --- a/sw/utils/tinyprintf.h +++ b/sw/utils/tinyprintf.h @@ -107,7 +107,7 @@ For further details see source code. regs Kusti, 23.10.2004 */ -void putf(char *null, char c) { +void putf(void *null, char c) { *(volatile int *) (0x80000004) = (int)c; } @@ -559,6 +559,7 @@ void tfp_format(void *putp, putcf putf__, const char *fmt, va_list va) lng = 2; # endif #endif + __attribute__((fallthrough)); #endif case 'x': case 'X': diff --git a/target/sim/src/redmule_tb.sv b/target/sim/src/redmule_tb.sv index e99c499b..28a4b94a 100644 --- a/target/sim/src/redmule_tb.sv +++ b/target/sim/src/redmule_tb.sv @@ -22,6 +22,8 @@ module redmule_tb parameter TT = 0.8ns, // test time parameter int unsigned Height = 8, parameter int unsigned Width = 8, + parameter bit EnableReordering = 1'b0, + parameter int unsigned RobSlots = 16, parameter real PROB_STALL = 0.0 )( input logic clk_i, @@ -38,16 +40,19 @@ module redmule_tb localparam int unsigned RedmuleDataW = Height*(NumPipeRegs+1)*16; // = D*16 localparam int unsigned DW = RedmuleDataW + 32; // TCDM data width including MisalignedAccessSupport=1 (+32b word) localparam int unsigned MP = DW/32; + localparam int unsigned UW = EnableReordering ? $clog2(RobSlots) : hci_package::DEFAULT_UW; + localparam int unsigned UserFifoDepth = 1 << UW; // HCI size parameter for RedMulE's TCDM port. It must be forwarded to // redmule_mm_wrap (redmule_top forwards it verbatim to the streamer, with no // fallback), otherwise the internal OoO multiplexer sees BW=0 and fails to - // elaborate. Fields other than DW take the hci_core_intf defaults, matching - // the `redmule_tcdm` interface declared below (DW-only override). + // elaborate. The `redmule_tcdm` interface below must match these dimensions. + // When reordering is enabled, widen UW so the HCI-side ROB can track more + // outstanding transactions (ROB_NW = 2**UW in rtl/redmule_streamer.sv). localparam hci_size_parameter_t HciSizeTcdm = '{ DW: DW, AW: hci_package::DEFAULT_AW, BW: hci_package::DEFAULT_BW, - UW: hci_package::DEFAULT_UW, + UW: UW, IW: hci_package::DEFAULT_IW, EW: hci_package::DEFAULT_EW, EHW: hci_package::DEFAULT_EHW @@ -72,12 +77,24 @@ module redmule_tb hwpe_stream_intf_tcdm tcdm [MP:0] (.clk(clk_i)); // RedMulE modern interfaces - hci_core_intf #(.DW(DW)) redmule_tcdm (.clk(clk_i)); + hci_core_intf #( + .DW ( DW ), + .UW ( UW ) + ) redmule_tcdm (.clk(clk_i)); hwpe_ctrl_intf_periph #(.ID_WIDTH(ID)) periph (.clk(clk_i)); logic [MP-1:0] tcdm_gnt; logic [MP-1:0][31:0] tcdm_r_data; logic [MP-1:0] tcdm_r_valid; + typedef logic [hci_package::iomsb(UW):0] hci_user_t; + typedef logic [hci_package::iomsb(hci_package::DEFAULT_IW):0] hci_id_t; + hci_user_t user_fifo_rdata; + hci_id_t id_fifo_rdata; + logic user_fifo_full, user_fifo_empty; + logic [$clog2(UserFifoDepth)-1:0] user_fifo_usage; + logic id_fifo_full, id_fifo_empty; + logic [$clog2(UserFifoDepth)-1:0] id_fifo_usage; + logic redmule_req_fire, redmule_rsp_fire; logic instr_req; logic instr_gnt; @@ -148,7 +165,46 @@ module redmule_tb assign redmule_tcdm.r_data = { >> {tcdm_r_data} }; assign redmule_tcdm.r_valid = &tcdm_r_valid; assign redmule_tcdm.r_opc = '0; - assign redmule_tcdm.r_user = '0; + assign redmule_tcdm.r_user = user_fifo_rdata; + assign redmule_tcdm.r_id = id_fifo_rdata; + assign redmule_req_fire = redmule_tcdm.req & redmule_tcdm.gnt; + assign redmule_rsp_fire = redmule_tcdm.r_valid & redmule_tcdm.r_ready; + + fifo_v3 #( + .FALL_THROUGH ( 1'b0 ), + .DEPTH ( UserFifoDepth ), + .dtype ( hci_user_t ) + ) i_user_fifo ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .flush_i ( 1'b0 ), + .testmode_i ( 1'b0 ), + .full_o ( user_fifo_full ), + .empty_o ( user_fifo_empty ), + .usage_o ( user_fifo_usage ), + .data_i ( redmule_tcdm.user ), + .push_i ( redmule_req_fire ), + .data_o ( user_fifo_rdata ), + .pop_i ( redmule_rsp_fire ) + ); + + fifo_v3 #( + .FALL_THROUGH ( 1'b0 ), + .DEPTH ( UserFifoDepth ), + .dtype ( hci_id_t ) + ) i_id_fifo ( + .clk_i ( clk_i ), + .rst_ni ( rst_ni ), + .flush_i ( 1'b0 ), + .testmode_i ( 1'b0 ), + .full_o ( id_fifo_full ), + .empty_o ( id_fifo_empty ), + .usage_o ( id_fifo_usage ), + .data_i ( redmule_tcdm.id ), + .push_i ( redmule_req_fire ), + .data_o ( id_fifo_rdata ), + .pop_i ( redmule_rsp_fire ) + ); // Core data-side port (last bank of the data memory). assign tcdm[MP].req = data_req & (data_addr[31:24] != '0) & (data_addr[31:24] != 8'h80) & ~data_addr[HWPE_ADDR_BASE_BIT]; @@ -170,12 +226,13 @@ module redmule_tb other_r_valid ; redmule_mm_wrap #( - .HCI_SIZE_tcdm ( HciSizeTcdm ), - .DataW ( RedmuleDataW ), - .MisalignedAccessSupport ( 1 ), - .Height ( Height ), - .Width ( Width ), - .NumPipeRegs ( NumPipeRegs ) + .HCI_SIZE_tcdm ( HciSizeTcdm ), + .DataW ( RedmuleDataW ), + .MisalignedAccessSupport ( EnableReordering ? 0 : 1 ), + .EnableReordering ( EnableReordering ), + .Height ( Height ), + .Width ( Width ), + .NumPipeRegs ( NumPipeRegs ) ) i_redmule_wrap ( .clk_i ( clk_i ), .rst_ni ( rst_ni ), @@ -329,6 +386,8 @@ module redmule_tb if (!$value$plusargs("STIM_DATA=%s", stim_data)) stim_data = "../../../sw/build/stim_data.txt"; $display("Height = %d", Height); $display("Width = %d", Width); + $display("EnableReordering = %0d", EnableReordering); + $display("RobSlots = %0d", RobSlots); $display("PROB_STALL = %f", PROB_STALL); test_mode = 1'b0; diff --git a/target/sim/src/redmule_tb_wrap.sv b/target/sim/src/redmule_tb_wrap.sv index 334d5c19..0eb7810f 100644 --- a/target/sim/src/redmule_tb_wrap.sv +++ b/target/sim/src/redmule_tb_wrap.sv @@ -12,6 +12,8 @@ module redmule_tb_wrap #( parameter int unsigned Height = 8, parameter int unsigned Width = 8, + parameter bit EnableReordering = 1'b0, + parameter int unsigned RobSlots = 16, parameter real PROB_STALL = 0.0 ); @@ -25,9 +27,11 @@ module redmule_tb_wrap .TCP ( TCP ), .TA ( TA ), .TT ( TT ), - .Height ( Height ), - .Width ( Width ), - .PROB_STALL ( PROB_STALL ) + .Height ( Height ), + .Width ( Width ), + .EnableReordering ( EnableReordering ), + .RobSlots ( RobSlots ), + .PROB_STALL ( PROB_STALL ) ) i_redmule_tb ( .clk_i ( clk ), .rst_ni ( rst_n ), diff --git a/target/sim/verilator/verilator.mk b/target/sim/verilator/verilator.mk index 76ab959e..0b09a65f 100644 --- a/target/sim/verilator/verilator.mk +++ b/target/sim/verilator/verilator.mk @@ -25,6 +25,7 @@ VerilatorCompileScript := $(VerilatorDir)/compile.$(target).tcl VerilatorWaves := $(VerilatorDir)/redmule.vcd RedmuleHeight ?= 8 RedmuleWidth ?= 8 +EnableReordering ?= 0 ProbStall ?= 0.05 # Parallelism for hw-build. With --binary this covers both verilation and the # C++ compile of the model. @@ -45,7 +46,7 @@ hw-build: hw-script -Wall -Wno-fatal --Wno-lint --Wno-UNOPTFLAT --Wno-MODDUP -Wno-BLKANDNBLK -Wno-ENUMVALUE \ -j $(VerilatorJobs) \ --x-assign unique --x-initial unique --top-module $(Module)_tb_wrap --Mdir $(VerilatorAbsObjDir) \ - -GHeight=$(RedmuleHeight) -GWidth=$(RedmuleWidth) -GPROB_STALL=$(ProbStall) \ + -GHeight=$(RedmuleHeight) -GWidth=$(RedmuleWidth) -GEnableReordering=$(EnableReordering) -GPROB_STALL=$(ProbStall) \ -CFLAGS "-DTbName=$(Vmodule)_tb_wrap -DWafeformPath=$(VerilatorWaves)" --binary \ -sv -cc -f $(VerilatorCompileScript) diff --git a/target/sim/vsim/vsim.mk b/target/sim/vsim/vsim.mk index 7a414284..1fb3f02e 100644 --- a/target/sim/vsim/vsim.mk +++ b/target/sim/vsim/vsim.mk @@ -9,6 +9,7 @@ Questa ?= Module := redmule ProbStall ?= 0.05 +EnableReordering ?= 0 VsimDir := $(SimDir)/$(target) VsimCompileScript := $(VsimDir)/compile.$(target).tcl VsimWaves := $(VsimDir)/wave.tcl @@ -60,12 +61,13 @@ hw-build: hw-script hw-run: mkdir -p $(BUILD_DIR) ln -sfn $(VsimDir)/work $(BUILD_DIR)/work - cd $(BUILD_DIR); \ - $(QUESTA) $(target) $(Tb)_opt \ - $(VsimFlags) \ - -gPROB_STALL=$(ProbStall) \ - +STIM_INSTR=$(STIM_INSTR) \ - +STIM_DATA=$(STIM_DATA) \ + cd $(BUILD_DIR); \ + $(QUESTA) $(target) $(Tb)_opt \ + $(VsimFlags) \ + -gEnableReordering=$(EnableReordering) \ + -gPROB_STALL=$(ProbStall) \ + +STIM_INSTR=$(STIM_INSTR) \ + +STIM_DATA=$(STIM_DATA) \ -do "run -a" hw-all: hw-clean hw-script hw-build hw-run