Smooth out cce mode switch transition

bp_me/src/v/cce/bp_cce_msg.v

@@ -121,20 +121,6 @@ module bp_cce_msg
   bp_cce_mem_msg_s                               mem_resp_from_uc;
   logic                                          mem_resp_v_from_uc, mem_resp_yumi_from_uc;
 
-  logic [$bits(bp_cce_mode_e)-1:0] cce_mode_r;
-  bp_cce_mode_e cce_mode_lo;
-  bsg_dff_reset_en
-   #(.width_p($bits(bp_cce_mode_e)))
-   cce_mode_reg
-    (.clk_i(clk_i)
-     ,.reset_i(reset_i)
-     ,.en_i(mem_resp_yumi_o)
-
-     ,.data_i(cfg_bus_cast_i.cce_mode)
-     ,.data_o(cce_mode_r)
-     );
-  assign cce_mode_lo = {cce_mode_r};
-
   // Message unit
   bp_cce_msg_cached
     #(.bp_params_p(bp_params_p))
@@ -143,7 +129,6 @@ module bp_cce_msg
       ,.reset_i(reset_i)
 
       ,.cce_id_i(cfg_bus_cast_i.cce_id)
-      ,.cce_mode_i(cce_mode_lo)
 
       // To CCE
       ,.lce_req_i(lce_req_from_msg)
@@ -194,7 +179,6 @@ module bp_cce_msg
       ,.reset_i(reset_i)
 
       ,.cce_id_i(cfg_bus_cast_i.cce_id)
-      ,.cce_mode_i(cce_mode_lo)
 
       // To CCE
       ,.lce_req_i(lce_req_from_uc)
@@ -217,6 +201,16 @@ module bp_cce_msg
       ,.mem_cmd_ready_i(mem_cmd_ready_from_uc)
       );
 
+  // Need to resolve the last outstanding msg during a mode switch
+  logic uncached_outstanding;
+  always_ff @(posedge clk_i)
+    begin
+      if (reset_i)
+        uncached_outstanding <= '0;
+      else if (mem_cmd_v_from_uc | mem_resp_yumi_from_uc)
+        uncached_outstanding <= ~mem_resp_yumi_from_uc;
+    end
+
   // Output Message Formation
   //
   // Input messages to the CCE are buffered by two element FIFOs in bp_cce_buffered.v, thus
@@ -231,7 +225,7 @@ module bp_cce_msg
 
     {lce_req_from_msg, lce_req_v_from_msg, lce_resp_from_msg, lce_resp_v_from_msg, lce_cmd_ready_from_msg} = '0;
     {mem_cmd_ready_from_msg, mem_resp_from_msg, mem_resp_v_from_msg} = '0;
-    if (cce_mode_lo == e_cce_mode_uncached) begin
+    if (uncached_outstanding || (cfg_bus_cast_i.cce_mode == e_cce_mode_uncached)) begin
       lce_req_from_uc = lce_req_i;
       lce_req_v_from_uc = lce_req_v_i;
       lce_req_yumi_o = lce_req_yumi_from_uc;

bp_me/src/v/cce/bp_cce_msg_cached.v

@@ -36,8 +36,7 @@ module bp_cce_msg_cached
   (input                                               clk_i
    , input                                             reset_i
 
-   , input [cce_id_width_p-1:0]                         cce_id_i
-   , input bp_cce_mode_e                               cce_mode_i
+   , input [cce_id_width_p-1:0]                        cce_id_i
 
    // LCE-CCE Interface
    // inbound: valid->ready (a.k.a., valid->yumi), demanding consumer (connects to FIFO)

bp_me/src/v/cce/bp_cce_msg_uncached.v

@@ -38,7 +38,6 @@ module bp_cce_msg_uncached
    , input                                             reset_i
 
    , input [cce_id_width_p-1:0]                        cce_id_i
-   , input bp_cce_mode_e                               cce_mode_i
 
    // LCE-CCE Interface
    // inbound: valid->ready (a.k.a., valid->yumi), demanding consumer (connects to FIFO)
@@ -90,7 +89,7 @@ module bp_cce_msg_uncached
 
   always_ff @(posedge clk_i) begin
     // This module only operates when reset is low and CCE is in uncached mode
-    if (reset_i | (cce_mode_i != e_cce_mode_uncached)) begin
+    if (reset_i) begin
       uc_state <= READY;
       lce_req_r <= '0;
     end else begin
@@ -120,8 +119,7 @@ module bp_cce_msg_uncached
 
     uc_state_n = READY;
 
-    // only operate if not in reset and cce mode is uncached
-    if (~reset_i & (cce_mode_i == e_cce_mode_uncached)) begin
+    if (~reset_i) begin
       case (uc_state)
       READY: begin
         uc_state_n = READY;

bp_me/test/common/bp_cce_mmio_cfg_loader.v

@@ -87,6 +87,22 @@ module bp_cce_mmio_cfg_loader
     ,DONE
   } state_n, state_r;
 
+  logic [`BSG_WIDTH(io_noc_max_credits_p)-1:0] credit_count_lo;
+  bsg_flow_counter
+   #(.els_p(io_noc_max_credits_p))
+   cfg_counter
+    (.clk_i(clk_i)
+     ,.reset_i(reset_i)
+
+     ,.v_i(io_cmd_yumi_i)
+     ,.ready_i(1'b1)
+
+     ,.yumi_i(io_resp_v_i)
+     ,.count_o(credit_count_lo)
+     );
+  wire credits_full_lo = (credit_count_lo == io_noc_max_credits_p);
+  wire credits_empty_lo = (credit_count_lo == '0);
+
   logic [cfg_addr_width_p-1:0] sync_cnt_r;
   logic sync_cnt_clr, sync_cnt_inc;
   bsg_counter_clear_up
@@ -167,7 +183,7 @@ module bp_cce_mmio_cfg_loader
 
   always_comb
     begin
-      io_cmd_v_o = cfg_w_v_lo | cfg_r_v_lo;
+      io_cmd_v_o = (cfg_w_v_lo | cfg_r_v_lo) & ~credits_full_lo;
 
       // uncached store
       io_cmd_cast_o.msg_type      = cfg_w_v_lo ? e_cce_io_wr : e_cce_io_rd;
@@ -280,10 +296,10 @@ module bp_cce_mmio_cfg_loader
         SEND_CCE_NORMAL: begin
           state_n = core_prog_done ? WAIT_FOR_SYNC : SEND_CCE_NORMAL;
 
-          core_cnt_inc = ~core_prog_done;
-          core_cnt_clr = core_prog_done;
+          core_cnt_inc = ~core_prog_done & credits_empty_lo;
+          core_cnt_clr = core_prog_done & credits_empty_lo;
 
-          cfg_w_v_lo = 1'b1;
+          cfg_w_v_lo = credits_empty_lo;
           cfg_addr_lo = bp_cfg_reg_cce_mode_gp;
           cfg_data_lo = dword_width_p'(e_cce_mode_normal);
         end

bp_me/test/tb/bp_cce/testbench.v

@@ -154,7 +154,7 @@ bp_me_nonsynth_mock_lce #(
   ,.lce_cmd_ready_i(lce_cmd_ready_li)
 );
 
-// Transduce between mem and io
+// TODO: Transduce between mem and io
 assign cfg_mem_cmd_lo = cfg_io_cmd_lo;
 assign cfg_mem_cmd_v_lo = cfg_io_cmd_v_lo;
 assign cfg_io_cmd_yumi_li = cfg_mem_cmd_yumi_li;

bp_top/test/common/bp_nonsynth_nbf_loader.v

@@ -44,6 +44,22 @@ module bp_nonsynth_nbf_loader
   wire unused_resp = &{io_resp_i, io_resp_v_i};
   assign io_resp_ready_o = 1'b1;
 
+  logic [`BSG_WIDTH(io_noc_max_credits_p)-1:0] credit_count_lo;
+  bsg_flow_counter
+   #(.els_p(io_noc_max_credits_p))
+   nbf_counter
+    (.clk_i(clk_i)
+     ,.reset_i(reset_i)
+
+     ,.v_i(io_cmd_yumi_i)
+     ,.ready_i(1'b1)
+
+     ,.yumi_i(io_resp_v_i)
+     ,.count_o(credit_count_lo)
+     );
+  wire credits_full_lo = (credit_count_lo == io_noc_max_credits_p);
+  wire credits_empty_lo = (credit_count_lo == '0);
+
   // bp_nbf packet
   typedef struct packed {
     logic [nbf_opcode_width_p-1:0] opcode;
@@ -84,7 +100,7 @@ module bp_nonsynth_nbf_loader
   initial $readmemh(nbf_filename_p, nbf);
 
   logic done_r, done_n;
-  assign done_o = done_r;
+  assign done_o = done_r & credits_empty_lo;
 
  // combinational
   always_comb 
@@ -100,7 +116,7 @@ module bp_nonsynth_nbf_loader
           end
         else 
           begin
-            io_cmd_v_lo = 1'b1;
+            io_cmd_v_lo = ~credits_full_lo;
             nbf_index_n = nbf_index_r + io_cmd_yumi_i;
           end
       end

bp_top/test/tb/bp_top_trace_demo/testbench.v

@@ -472,7 +472,7 @@ always_comb
   if (nbf_done_lo)
     begin
       load_cmd_lo = cfg_cmd_lo;
-      load_cmd_v_lo = cfg_cmd_v_lo;
+      load_cmd_v_lo = load_cmd_ready_li & cfg_cmd_v_lo;
 
       nbf_cmd_ready_li = 1'b0;
       cfg_cmd_ready_li = load_cmd_ready_li;
@@ -481,14 +481,14 @@ always_comb
       nbf_resp_v_li = 1'b0;
 
       cfg_resp_li = load_resp_li;
-      cfg_resp_v_li = load_resp_v_li;
+      cfg_resp_v_li = load_resp_v_li & load_resp_ready_lo & cfg_resp_ready_lo;
 
       load_resp_ready_lo = cfg_resp_ready_lo;
     end
   else
     begin
       load_cmd_lo = nbf_cmd_lo;
-      load_cmd_v_lo = nbf_cmd_v_lo;
+      load_cmd_v_lo = load_cmd_ready_li & nbf_cmd_v_lo;
 
       nbf_cmd_ready_li = load_cmd_ready_li;
       cfg_cmd_ready_li = 1'b0;