Reland ring buffer (#14568)

* #111795: Revert "Revert Kernel Ring Buffer Changes (#14244)"

For tensix only (disabled for eth for now)
Enables execute in place (XIP) so kernels can be loaded anywhere
Moves kernels from FW address "holes" to (larger) ring buffer
Still runs synchronously.

This reverts commit e966f77.

tests/tt_metal/tt_metal/test_compile_sets_kernel_binaries.cpp

@@ -204,7 +204,7 @@ int main(int argc, char **argv) {
                             dm_class_idx,
                             0,
                             get_latest_kernel_binary_path(mask, riscv0_kernel));
-                        ll_api::memory brisc_binary = llrt::get_risc_binary(brisc_hex_path, 0, llrt::PackSpans::PACK);
+                        ll_api::memory brisc_binary = llrt::get_risc_binary(brisc_hex_path, 0, ll_api::memory::PackSpans::PACK, ll_api::memory::Relocate::XIP);
                         TT_FATAL(
                             brisc_binary == brisc_binaries.at(mask).at(0),
                             "Expected saved BRISC binary to be the same as binary in persistent cache");
@@ -213,7 +213,11 @@ int main(int argc, char **argv) {
                             dm_class_idx,
                             1,
                             get_latest_kernel_binary_path(mask, riscv1_kernel));
-                        ll_api::memory ncrisc_binary = llrt::get_risc_binary(ncrisc_hex_path, 1, llrt::PackSpans::PACK);
+                        ll_api::memory::Relocate relo_type =
+                            (device->arch() == tt::ARCH::GRAYSKULL || device->arch() == tt::ARCH::WORMHOLE_B0) ?
+                            ll_api::memory::Relocate::NONE : ll_api::memory::Relocate::XIP;
+
+                        ll_api::memory ncrisc_binary = llrt::get_risc_binary(ncrisc_hex_path, 1, ll_api::memory::PackSpans::PACK, relo_type);
                         TT_FATAL(
                             ncrisc_binary == ncrisc_binaries.at(mask).at(0),
                             "Expected saved NCRISC binary to be the same as binary in persistent cache");
@@ -224,7 +228,7 @@ int main(int argc, char **argv) {
                                 compute_class_idx,
                                 trisc_id,
                                 get_latest_kernel_binary_path(mask, compute_kernel));
-                            ll_api::memory trisc_binary = llrt::get_risc_binary(trisc_hex_path, 2, llrt::PackSpans::PACK);
+                            ll_api::memory trisc_binary = llrt::get_risc_binary(trisc_hex_path, 2, ll_api::memory::PackSpans::PACK, ll_api::memory::Relocate::XIP);
                             TT_FATAL(
                                 trisc_binary == compute_binaries.at(mask).at(trisc_id),
                                 "Expected saved TRISC binary for {} to be the same as binary in persistent cache", trisc_id_str);

tests/tt_metal/tt_metal/unit_tests/circular_buffer/test_CircularBuffer_creation.cpp

@@ -76,6 +76,7 @@ TEST_F(DeviceFixture, TestCreateCircularBufferAtValidIndices) {
     auto cb = CreateCircularBuffer(program, cr_set, config);
 
     for (unsigned int id = 0; id < num_devices_; id++) {
+        detail::CompileProgram(devices_.at(id), program);
         program.finalize(devices_.at(id));
         EXPECT_TRUE(test_cb_config_written_to_core(program, this->devices_.at(id), cr_set, golden_cb_config));
     }

tt_metal/hostdevcommon/common_runtime_address_map.h

@@ -13,11 +13,9 @@
 * This file contains addresses that are visible to both host and device compiled code.
 */
 
-// Kernel config buffer is WIP
-// Size is presently based on the old sizes of the RTAs + CB config + Sems
-// plus some extra space freed up in the mem map
+// TODO: move this to the memory manager, make configurable through the API
 constexpr static std::uint32_t L1_KERNEL_CONFIG_BASE = MEM_MAP_END;
-constexpr static std::uint32_t L1_KERNEL_CONFIG_SIZE = 4 * 1024 + 256 + 128 + 512;
+constexpr static std::uint32_t L1_KERNEL_CONFIG_SIZE = 69 * 1024;
 
 constexpr static std::uint32_t NUM_CIRCULAR_BUFFERS = 32;
 constexpr static std::uint32_t UINT32_WORDS_PER_CIRCULAR_BUFFER_CONFIG = 4;

tt_metal/hw/firmware/src/brisc.cc

@@ -169,13 +169,13 @@ void set_deassert_addresses() {
 #endif
 }
 
-void l1_to_ncrisc_iram_copy(uint16_t size, uint32_t address_offset = 0) {
+void l1_to_ncrisc_iram_copy(uint32_t src_addr, uint16_t size, uint32_t address_offset = 0) {
 #ifdef NCRISC_HAS_IRAM
     // Always copy ncrisc even if its size is 0 (save branch)...
     // Copy NCRISC firmware from L1 to local IRAM using tensix DMA
     tdma_xmov(
         TDMA_MOVER0,
-        (MEM_NCRISC_INIT_IRAM_L1_BASE >> 4) + address_offset,
+        src_addr,
         MEM_MOVER_VIEW_IRAM_BASE_ADDR + address_offset,
         size,
         XMOV_L1_TO_L0);
@@ -267,16 +267,22 @@ void init_sync_registers() {
     }
 }
 
-inline void deassert_ncrisc_trisc() {
-    // Below sets ncrisc to go so we can wait until it is cleared on first iteration
-    mailboxes->slave_sync.all = RUN_SYNC_MSG_ALL_SLAVES_DONE;
-
+inline void init_ncrisc_iram() {
+#ifdef NCRISC_HAS_IRAM
     uint16_t fw_size16 = mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.ncrisc_kernel_size16;
     ncrisc_kernel_start_offset16 = fw_size16;
 
     // Copies from L1 to IRAM on chips where NCRISC has IRAM
-    l1_to_ncrisc_iram_copy(fw_size16);
+    l1_to_ncrisc_iram_copy(MEM_NCRISC_INIT_IRAM_L1_BASE >> 4, fw_size16);
     l1_to_ncrisc_iram_copy_wait();
+#endif
+}
+
+inline void deassert_ncrisc_trisc() {
+    // Below sets ncrisc to go so we can wait until it is cleared on first iteration
+    mailboxes->slave_sync.all = RUN_SYNC_MSG_ALL_SLAVES_DONE;
+
+    init_ncrisc_iram();
 
     // Bring ncrisc/triscs out of reset
     deassert_all_reset();
@@ -400,8 +406,13 @@ int main() {
             DeviceValidateProfiler(launch_msg_address->kernel_config.enables);
             DeviceZoneSetCounter(launch_msg_address->kernel_config.host_assigned_id);
             // Copies from L1 to IRAM on chips where NCRISC has IRAM
-            l1_to_ncrisc_iram_copy(launch_msg_address->kernel_config.ncrisc_kernel_size16, ncrisc_kernel_start_offset16);
-
+            uint32_t kernel_config_base = firmware_config_init(mailboxes, ProgrammableCoreType::TENSIX, DISPATCH_CLASS_TENSIX_DM0);
+            int ncrisc_index = static_cast<std::underlying_type<TensixProcessorTypes>::type>(TensixProcessorTypes::DM1);
+            uint32_t ncrisc_kernel_src_address =
+                kernel_config_base + launch_msg_address->kernel_config.kernel_text_offset[ncrisc_index];
+            l1_to_ncrisc_iram_copy(ncrisc_kernel_src_address >> 4,
+                launch_msg_address->kernel_config.ncrisc_kernel_size16,
+                ncrisc_kernel_start_offset16);
             // Invalidate the i$ now the kernels have loaded and before running
             volatile tt_reg_ptr uint32_t* cfg_regs = core.cfg_regs_base(0);
             cfg_regs[RISCV_IC_INVALIDATE_InvalidateAll_ADDR32] = RISCV_IC_BRISC_MASK | RISCV_IC_TRISC_ALL_MASK | RISCV_IC_NCRISC_MASK;
@@ -423,7 +434,6 @@ int main() {
             }
             prev_noc_mode = noc_mode;
 
-            uint32_t kernel_config_base = firmware_config_init(mailboxes, ProgrammableCoreType::TENSIX, DISPATCH_CLASS_TENSIX_DM0);
             uint32_t tt_l1_ptr *cb_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
                 launch_msg_address->kernel_config.cb_offset);
             setup_cb_read_write_interfaces(cb_l1_base, 0, num_cbs_to_early_init, true, true, false);
@@ -433,10 +443,13 @@ int main() {
             WAYPOINT("R");
             if (enables & DISPATCH_CLASS_MASK_TENSIX_ENABLE_DM0) {
                 setup_cb_read_write_interfaces(cb_l1_base, num_cbs_to_early_init, launch_msg_address->kernel_config.max_cb_index, true, true, false);
-                kernel_init();
+                int index = static_cast<std::underlying_type<TensixProcessorTypes>::type>(TensixProcessorTypes::DM0);
+                void (*kernel_address)(uint32_t) = (void (*)(uint32_t))
+                    (kernel_config_base + launch_msg_address->kernel_config.kernel_text_offset[index]);
+                (*kernel_address)((uint32_t)kernel_address);
                 RECORD_STACK_USAGE();
             } else {
-                // This was not initialized in kernel_init
+                // This was not initialized in the kernel
                 if (noc_mode == DM_DEDICATED_NOC) {
                     noc_local_state_init(noc_index);
                 }

tt_metal/hw/firmware/src/brisck.cc

@@ -19,16 +19,17 @@
 #include <kernel_includes.hpp>
 
 extern uint32_t __kernel_init_local_l1_base[];
+extern uint32_t __fw_export_end_text[];
 
-void kernel_launch() {
+void kernel_launch(uint32_t kernel_base_addr) {
 
 #if defined(DEBUG_NULL_KERNELS) && !defined(DISPATCH_KERNEL)
 #ifdef KERNEL_RUN_TIME
     uint64_t end_time = c_tensix_core::read_wall_clock() + KERNEL_RUN_TIME;
     while (c_tensix_core::read_wall_clock() < end_time);
 #endif
 #else
-    firmware_kernel_common_init((void tt_l1_ptr *)(__kernel_init_local_l1_base));
+    firmware_kernel_common_init((void tt_l1_ptr *)(kernel_base_addr + (uint32_t) __kernel_init_local_l1_base - (uint32_t)__fw_export_end_text));
 
     if constexpr (NOC_MODE == DM_DEDICATED_NOC) {
         noc_local_state_init(NOC_INDEX);

tt_metal/hw/firmware/src/erisc.cc

@@ -72,10 +72,13 @@ void __attribute__((noinline)) Application(void) {
             launch_msg_t* launch_msg_address = &(mailboxes->launch[launch_msg_rd_ptr]);
             DeviceValidateProfiler(launch_msg_address->kernel_config.enables);
             DeviceZoneSetCounter(launch_msg_address->kernel_config.host_assigned_id);
+            // Note that a core may get "GO" w/ enable false to keep its launch_msg's in sync
             enum dispatch_core_processor_masks enables = (enum dispatch_core_processor_masks)launch_msg_address->kernel_config.enables;
             if (enables & DISPATCH_CLASS_MASK_ETH_DM0) {
+                WAYPOINT("R");
                 firmware_config_init(mailboxes, ProgrammableCoreType::ACTIVE_ETH, DISPATCH_CLASS_ETH_DM0);
-                kernel_init();
+                kernel_init(0);
+                WAYPOINT("D");
             }
             mailboxes->go_message.signal = RUN_MSG_DONE;
 
@@ -89,7 +92,6 @@ void __attribute__((noinline)) Application(void) {
                 // Only executed if watcher is enabled. Ensures that we don't report stale data due to invalid launch messages in the ring buffer
                 CLEAR_PREVIOUS_LAUNCH_MESSAGE_ENTRY_FOR_WATCHER();
             }
-            WAYPOINT("R");
 
         } else if (go_message_signal == RUN_MSG_RESET_READ_PTR) {
             // Reset the launch message buffer read ptr

tt_metal/hw/firmware/src/erisck.cc

@@ -23,7 +23,7 @@
 
 CBInterface cb_interface[NUM_CIRCULAR_BUFFERS];
 
-void kernel_launch() {
+void kernel_launch(uint32_t) {
     DeviceZoneScopedMainChildN("ERISC-KERNEL");
     rtos_context_switch_ptr = (void (*)())RtosTable[0];
 

tt_metal/hw/firmware/src/idle_erisc.cc

@@ -25,7 +25,6 @@
 
 #include "debug/watcher_common.h"
 #include "debug/waypoint.h"
-#include "debug/dprint.h"
 #include "debug/stack_usage.h"
 
 uint8_t noc_index;
@@ -135,7 +134,10 @@ int main() {
 
             // Run the ERISC kernel
             WAYPOINT("R");
-            kernel_init();
+            int index = static_cast<std::underlying_type<EthProcessorTypes>::type>(EthProcessorTypes::DM0);
+            void (*kernel_address)(uint32_t) = (void (*)(uint32_t))
+                (kernel_config_base + mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.kernel_text_offset[index]);
+            (*kernel_address)((uint32_t)kernel_address);
             RECORD_STACK_USAGE();
             WAYPOINT("D");
             mailboxes->go_message.signal = RUN_MSG_DONE;

tt_metal/hw/firmware/src/idle_erisck.cc

@@ -22,10 +22,12 @@
 #include <kernel_includes.hpp>
 
 extern uint32_t __kernel_init_local_l1_base[];
+extern uint32_t __fw_export_end_text[];
 
-void kernel_launch() {
+void kernel_launch(uint32_t kernel_base_addr) {
     DeviceZoneScopedMainChildN("ERISC-KERNEL");
-    firmware_kernel_common_init((void tt_l1_ptr *)__kernel_init_local_l1_base);
+
+    firmware_kernel_common_init((void tt_l1_ptr *)(kernel_base_addr + (uint32_t) __kernel_init_local_l1_base - (uint32_t)__fw_export_end_text));
 
     noc_local_state_init(NOC_INDEX);
 

tt_metal/hw/firmware/src/ncrisc.cc

@@ -91,13 +91,23 @@ int main(int argc, char *argv[]) {
         notify_brisc_and_wait();
         DeviceZoneScopedMainN("NCRISC-FW");
 
+        uint32_t launch_msg_rd_ptr = mailboxes->launch_msg_rd_ptr;
+        launch_msg_t* launch_msg = &(mailboxes->launch[launch_msg_rd_ptr]);
+
         uint32_t kernel_config_base = firmware_config_init(mailboxes, ProgrammableCoreType::TENSIX, DISPATCH_CLASS_TENSIX_DM1);
         uint32_t tt_l1_ptr *cb_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
-            mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.cb_offset);
-        setup_cb_read_write_interfaces(cb_l1_base, 0, mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.max_cb_index, true, true, false);
-
+            launch_msg->kernel_config.cb_offset);
+        setup_cb_read_write_interfaces(cb_l1_base, 0, launch_msg->kernel_config.max_cb_index, true, true, false);
         WAYPOINT("R");
-        kernel_init();
+
+        int index = static_cast<std::underlying_type<TensixProcessorTypes>::type>(TensixProcessorTypes::DM1);
+        void (*kernel_address)(uint32_t) = (void (*)(uint32_t))
+            (kernel_config_base + launch_msg->kernel_config.kernel_text_offset[index]);
+#ifdef ARCH_BLACKHOLE
+        (*kernel_address)((uint32_t)kernel_address);
+#else
+        kernel_init((uint32_t)kernel_address);
+#endif
         RECORD_STACK_USAGE();
         WAYPOINT("D");
 

tt_metal/hw/firmware/src/ncrisck.cc

@@ -27,8 +27,9 @@ uint32_t noc_nonposted_atomics_acked[NUM_NOCS];
 uint32_t noc_posted_writes_num_issued[NUM_NOCS];
 
 extern uint32_t __kernel_init_local_l1_base[];
+extern uint32_t __fw_export_end_text[];
 
-void kernel_launch() {
+void kernel_launch(uint32_t kernel_base_addr) {
 
   DeviceZoneScopedMainChildN("NCRISC-KERNEL");
 #if defined(DEBUG_NULL_KERNELS) && !defined(DISPATCH_KERNEL)
@@ -37,11 +38,8 @@ void kernel_launch() {
     while (c_tensix_core::read_wall_clock() < KERNEL_RUN_TIME);
 #endif
 #else
-#ifdef ARCH_BLACKHOLE
-    firmware_kernel_common_init((void tt_l1_ptr *)__kernel_init_local_l1_base);
-#else
-    firmware_kernel_common_init((void tt_l1_ptr *)(MEM_NCRISC_INIT_IRAM_L1_BASE + (uint32_t)__kernel_init_local_l1_base - MEM_NCRISC_IRAM_BASE));
-#endif
+
+    firmware_kernel_common_init((void tt_l1_ptr *)(kernel_base_addr + (uint32_t) __kernel_init_local_l1_base - (uint32_t)__fw_export_end_text));
 
     if constexpr (NOC_MODE == DM_DEDICATED_NOC) {
         noc_local_state_init(NOC_INDEX);

tt_metal/hw/firmware/src/trisc.cc

@@ -94,21 +94,27 @@ int main(int argc, char *argv[]) {
         while (*trisc_run != RUN_SYNC_MSG_GO);
         DeviceZoneScopedMainN("TRISC-FW");
 
-        uint32_t kernel_config_base = mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.kernel_config_base[ProgrammableCoreType::TENSIX];
+        uint32_t launch_msg_rd_ptr = mailboxes->launch_msg_rd_ptr;
+        launch_msg_t* launch_msg = &(mailboxes->launch[launch_msg_rd_ptr]);
+
+        uint32_t kernel_config_base = launch_msg->kernel_config.kernel_config_base[ProgrammableCoreType::TENSIX];
 
 #if !defined(UCK_CHLKC_MATH)
         uint32_t tt_l1_ptr *cb_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
-            mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.cb_offset);
-        setup_cb_read_write_interfaces(cb_l1_base, 0, mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.max_cb_index, cb_init_read, cb_init_write, cb_init_write);
+            launch_msg->kernel_config.cb_offset);
+        setup_cb_read_write_interfaces(cb_l1_base, 0, launch_msg->kernel_config.max_cb_index, cb_init_read, cb_init_write, cb_init_write);
 #endif
 
         rta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
-            mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.rta_offset[DISPATCH_CLASS_TENSIX_COMPUTE].rta_offset);
+            launch_msg->kernel_config.rta_offset[DISPATCH_CLASS_TENSIX_COMPUTE].rta_offset);
         crta_l1_base = (uint32_t tt_l1_ptr *)(kernel_config_base +
-            mailboxes->launch[mailboxes->launch_msg_rd_ptr].kernel_config.rta_offset[DISPATCH_CLASS_TENSIX_COMPUTE].crta_offset);
+            launch_msg->kernel_config.rta_offset[DISPATCH_CLASS_TENSIX_COMPUTE].crta_offset);
 
         WAYPOINT("R");
-        kernel_init();
+        int index = static_cast<std::underlying_type<TensixProcessorTypes>::type>(TensixProcessorTypes::MATH0) + thread_id;
+        void (*kernel_address)(uint32_t) = (void (*)(uint32_t))
+            (kernel_config_base + launch_msg->kernel_config.kernel_text_offset[index]);
+        (*kernel_address)((uint32_t)kernel_address);
         RECORD_STACK_USAGE();
         WAYPOINT("D");
 

tt_metal/hw/firmware/src/trisck.cc

@@ -34,16 +34,17 @@ volatile tt_reg_ptr uint * mailbox_base[4] = {
 }
 
 extern uint32_t __kernel_init_local_l1_base[];
+extern uint32_t __fw_export_end_text[];
 
-void kernel_launch()
+void kernel_launch(uint32_t kernel_base_addr)
 {
   DeviceZoneScopedMainChildN("TRISC-KERNEL");
 #if defined(DEBUG_NULL_KERNELS) && !defined(DISPATCH_KERNEL)
 #ifdef KERNEL_RUN_TIME
     ckernel::wait(KERNEL_RUN_TIME);
 #endif
 #else
-    firmware_kernel_common_init((void tt_l1_ptr *)(__kernel_init_local_l1_base));
+    firmware_kernel_common_init((void tt_l1_ptr *)(kernel_base_addr + (uint32_t) __kernel_init_local_l1_base - (uint32_t)__fw_export_end_text));
 
 #if defined(UCK_CHLKC_UNPACK)
     // Make sure DBG_FEATURE_DISABLE register is cleared before every kernel is executed

tt_metal/hw/inc/blackhole/dev_mem_map.h

@@ -43,11 +43,19 @@
 
 /////////////
 // Firmware/kernel code holes
-#define MEM_BRISC_FIRMWARE_SIZE (10 * 1024 + MEM_BRISC_LOCAL_SIZE)
-#define MEM_NCRISC_FIRMWARE_SIZE (16 * 1024 + MEM_NCRISC_LOCAL_SIZE)
-#define MEM_TRISC0_FIRMWARE_SIZE (16 * 1024 + MEM_TRISC_LOCAL_SIZE)
-#define MEM_TRISC1_FIRMWARE_SIZE (16 * 1024 + MEM_TRISC_LOCAL_SIZE)
-#define MEM_TRISC2_FIRMWARE_SIZE (16 * 1024 + MEM_TRISC_LOCAL_SIZE)
+#define MEM_BRISC_FIRMWARE_SIZE (5 * 1024)
+// TODO: perhaps put NCRISC FW in the scratch area and free 1.5K after init (GS/WH)
+#define MEM_NCRISC_FIRMWARE_SIZE 1536
+#define MEM_TRISC0_FIRMWARE_SIZE 1536
+#define MEM_TRISC1_FIRMWARE_SIZE 1536
+#define MEM_TRISC2_FIRMWARE_SIZE 1536
+
+#define MEM_BRISC_KERNEL_SIZE  (24 * 1024)
+#define MEM_NCRISC_KERNEL_SIZE (24 * 1024)
+#define MEM_TRISC0_KERNEL_SIZE (24 * 1024)
+#define MEM_TRISC1_KERNEL_SIZE (24 * 1024)
+#define MEM_TRISC2_KERNEL_SIZE (24 * 1024)
+
 #define MEM_ZEROS_SIZE 512
 
 #define MEM_BOOT_CODE_BASE 0

tt_metal/hw/inc/dev_msgs.h

@@ -91,15 +91,15 @@ struct kernel_config_msg_t {
     volatile uint16_t watcher_kernel_ids[DISPATCH_CLASS_MAX];
     volatile uint16_t ncrisc_kernel_size16;  // size in 16 byte units
 
-    volatile uint16_t host_assigned_id;
-
     // Ring buffer of kernel configuration data
     volatile uint32_t kernel_config_base[static_cast<int>(ProgrammableCoreType::COUNT)];
     volatile uint16_t sem_offset[static_cast<int>(ProgrammableCoreType::COUNT)];
     volatile uint16_t cb_offset;
     rta_offset_t rta_offset[DISPATCH_CLASS_MAX];
     volatile uint32_t kernel_text_offset[MaxProcessorsPerCoreType];
 
+    volatile uint16_t host_assigned_id;
+
     volatile uint8_t mode;                   // dispatch mode host/dev
     volatile uint8_t brisc_noc_id;
     volatile uint8_t brisc_noc_mode;

tt_metal/hw/inc/firmware_common.h

@@ -21,8 +21,8 @@ extern uint32_t __ldm_data_end[];
 extern void (* __init_array_start[])();
 extern void (* __init_array_end[])();
 
-extern void kernel_init();
-extern void kernel_launch();
+extern void kernel_init(uint32_t kernel_init);
+extern void kernel_launch(uint32_t kernel_base_addr);
 
 inline void l1_to_local_mem_copy(uint32_t *local_mem_addr, uint32_t tt_l1_ptr *l1_addr, int32_t len) {
     // Cover L1 load latency of 6 cycles for the bulk of the copy