Implement DMA to/from psram on esp32s3 (#1827)

* initial non-working attemt for dma from psram on esp32s3

* flush cache - now works for extmem as source but not for extmem as destination

* use cache_invalidate_addr on destination address

* update changelog

* require dma transfers to/from psram to be aligned to dcache line size

* cache_writeback_addr() should suspend/resume dcache autoload

* no need for cfg(esp32s3) in esp32s3 specific module

* dma: document alignment requirements for DMA to/from PSRAM

* fix doc typos

esp-hal/CHANGELOG.md

@@ -12,6 +12,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Peripheral driver constructors don't take `InterruptHandler`s anymore. Use `set_interrupt_handler` to explicitly set the interrupt handler now. (#1819)
 
+- Allow DMA to/from psram for esp32s3 (#1827)
+
 ### Fixed
 - Improve error detection in the I2C driver (#1847)
 

esp-hal/ld/esp32s3/rom-functions.x

@@ -14,8 +14,13 @@ PROVIDE(esp_rom_spiflash_select_qio_pins = 0x40000a68 );
 PROVIDE(esp_rom_spi_set_op_mode = 0x400008a0 );
 PROVIDE(esp_rom_spi_cmd_start = 0x40000888);
 PROVIDE(esp_rom_spi_cmd_config = 0x4000087c);
+PROVIDE(Cache_Get_DCache_Line_Size = 0x40001608 );
+PROVIDE(Cache_Invalidate_Addr = 0x400016b0 );
 PROVIDE(Cache_Suspend_DCache = 0x400018b4 );
 PROVIDE(Cache_Resume_DCache = 0x400018c0 );
+PROVIDE(Cache_Suspend_DCache_Autoload = 0x40001734 );
+PROVIDE(Cache_Resume_DCache_Autoload = 0x40001740 );
+PROVIDE(Cache_WriteBack_Addr = 0x400016c8 );
 PROVIDE(rom_config_data_cache_mode = 0x40001a28 );
 PROVIDE(rom_config_instruction_cache_mode = 0x40001a1c );
 PROVIDE(ets_efuse_get_wp_pad = 0x40001fa4);

esp-hal/src/dma/gdma.rs

@@ -93,6 +93,13 @@ impl<const N: u8> RegisterAccess for Channel<N> {
             .modify(|_, w| w.mem_trans_en().bit(value));
     }
 
+    #[cfg(esp32s3)]
+    fn set_out_ext_mem_block_size(size: DmaExtMemBKSize) {
+        Self::ch()
+            .out_conf1()
+            .modify(|_, w| unsafe { w.out_ext_mem_bk_size().bits(size as u8) });
+    }
+
     fn set_out_burstmode(burst_mode: bool) {
         Self::ch().out_conf0().modify(|_, w| {
             w.out_data_burst_en()
@@ -206,6 +213,13 @@ impl<const N: u8> RegisterAccess for Channel<N> {
             .write(|w| w.out_eof().clear_bit_by_one());
     }
 
+    #[cfg(esp32s3)]
+    fn set_in_ext_mem_block_size(size: DmaExtMemBKSize) {
+        Self::ch()
+            .in_conf1()
+            .modify(|_, w| unsafe { w.in_ext_mem_bk_size().bits(size as u8) });
+    }
+
     fn set_in_burstmode(burst_mode: bool) {
         Self::ch().in_conf0().modify(|_, w| {
             w.in_data_burst_en()
@@ -626,6 +640,8 @@ pub use m2m::*;
 mod m2m {
     use embedded_dma::{ReadBuffer, WriteBuffer};
 
+    #[cfg(esp32s3)]
+    use crate::dma::DmaExtMemBKSize;
     use crate::dma::{
         dma_private::{DmaSupport, DmaSupportRx},
         Channel,
@@ -751,6 +767,21 @@ mod m2m {
                     .prepare_transfer_without_start(self.peripheral, &self.rx_chain)?;
                 self.channel.rx.set_mem2mem_mode(true);
             }
+            #[cfg(esp32s3)]
+            {
+                let align = match unsafe { crate::soc::cache_get_dcache_line_size() } {
+                    16 => DmaExtMemBKSize::Size16,
+                    32 => DmaExtMemBKSize::Size32,
+                    64 => DmaExtMemBKSize::Size64,
+                    _ => panic!("unsupported cache line size"),
+                };
+                if crate::soc::is_valid_psram_address(tx_ptr as u32) {
+                    self.channel.tx.set_ext_mem_block_size(align);
+                }
+                if crate::soc::is_valid_psram_address(rx_ptr as u32) {
+                    self.channel.rx.set_ext_mem_block_size(align);
+                }
+            }
             self.channel.tx.start_transfer()?;
             self.channel.rx.start_transfer()?;
             Ok(DmaTransferRx::new(self))

esp-hal/src/dma/mod.rs

@@ -43,6 +43,11 @@
 //! ⚠️ Note: Descriptors should be sized as `(max_transfer_size + CHUNK_SIZE - 1) / CHUNK_SIZE`.
 //! I.e., to transfer buffers of size `1..=CHUNK_SIZE`, you need 1 descriptor.
 //!
+//! ⚠️ Note: For chips that support DMA to/from PSRAM (esp32s3) DMA transfers to/from PSRAM
+//! have extra alignment requirements. The address and size of the buffer pointed to by
+//! each descriptor must be a multiple of the cache line (block) size. This is 32 bytes
+//! on esp32s3.
+//!
 //! For convenience you can use the [crate::dma_buffers] macro.
 #![warn(missing_docs)]
 
@@ -66,7 +71,7 @@ impl Debug for DmaDescriptorFlags {
             .field("size", &self.size())
             .field("length", &self.length())
             .field("suc_eof", &self.suc_eof())
-            .field("owner", &self.owner())
+            .field("owner", &(if self.owner() { "DMA" } else { "CPU" }))
             .finish()
     }
 }
@@ -95,6 +100,11 @@ impl DmaDescriptor {
         self.flags.set_length(len as u16)
     }
 
+    #[allow(unused)]
+    fn size(&self) -> usize {
+        self.flags.size() as usize
+    }
+
     fn len(&self) -> usize {
         self.flags.length() as usize
     }
@@ -567,8 +577,8 @@ impl DescriptorChain {
     ) -> Result<(), DmaError> {
         if !crate::soc::is_valid_ram_address(self.first() as u32)
             || !crate::soc::is_valid_ram_address(self.last() as u32)
-            || !crate::soc::is_valid_ram_address(data as u32)
-            || !crate::soc::is_valid_ram_address(unsafe { data.add(len) } as u32)
+            || !crate::soc::is_valid_memory_address(data as u32)
+            || !crate::soc::is_valid_memory_address(unsafe { data.add(len) } as u32)
         {
             return Err(DmaError::UnsupportedMemoryRegion);
         }
@@ -639,8 +649,8 @@ impl DescriptorChain {
     ) -> Result<(), DmaError> {
         if !crate::soc::is_valid_ram_address(self.first() as u32)
             || !crate::soc::is_valid_ram_address(self.last() as u32)
-            || !crate::soc::is_valid_ram_address(data as u32)
-            || !crate::soc::is_valid_ram_address(unsafe { data.add(len) } as u32)
+            || !crate::soc::is_valid_memory_address(data as u32)
+            || !crate::soc::is_valid_memory_address(unsafe { data.add(len) } as u32)
         {
             return Err(DmaError::UnsupportedMemoryRegion);
         }
@@ -707,6 +717,15 @@ impl DescriptorChain {
     }
 }
 
+/// Block size for transfers to/from psram
+#[derive(Copy, Clone, Debug, PartialEq)]
+#[allow(missing_docs)]
+pub enum DmaExtMemBKSize {
+    Size16 = 0,
+    Size32 = 1,
+    Size64 = 2,
+}
+
 pub(crate) struct TxCircularState {
     write_offset: usize,
     write_descr_ptr: *mut DmaDescriptor,
@@ -967,6 +986,9 @@ pub trait RxPrivate: crate::private::Sealed {
 
     fn start_transfer(&mut self) -> Result<(), DmaError>;
 
+    #[cfg(esp32s3)]
+    fn set_ext_mem_block_size(&self, size: DmaExtMemBKSize);
+
     #[cfg(gdma)]
     fn set_mem2mem_mode(&mut self, value: bool);
 
@@ -1119,13 +1141,37 @@ where
             return Err(DmaError::InvalidAlignment);
         }
 
+        // for esp32s3 we check each descriptor buffer that points to psram for
+        // alignment and invalidate the cache for that buffer
+        // NOTE: for RX the `buffer` and `size` need to be aligned but the `len` does
+        // not. TRM section 3.4.9
+        #[cfg(esp32s3)]
+        for des in chain.descriptors.iter() {
+            // we are forcing the DMA alignment to the cache line size
+            // required when we are using dcache
+            let alignment = crate::soc::cache_get_dcache_line_size() as usize;
+            if crate::soc::is_valid_psram_address(des.buffer as u32) {
+                // both the size and address of the buffer must be aligned
+                if des.buffer as usize % alignment != 0 && des.size() % alignment != 0 {
+                    return Err(DmaError::InvalidAlignment);
+                }
+                // TODO: make this optional?
+                crate::soc::cache_invalidate_addr(des.buffer as u32, des.size() as u32);
+            }
+        }
+
         self.rx_impl.prepare_transfer_without_start(chain, peri)
     }
 
     fn start_transfer(&mut self) -> Result<(), DmaError> {
         self.rx_impl.start_transfer()
     }
 
+    #[cfg(esp32s3)]
+    fn set_ext_mem_block_size(&self, size: DmaExtMemBKSize) {
+        CH::Channel::set_in_ext_mem_block_size(size);
+    }
+
     #[cfg(gdma)]
     fn set_mem2mem_mode(&mut self, value: bool) {
         CH::Channel::set_mem2mem_mode(value);
@@ -1244,6 +1290,9 @@ pub trait TxPrivate: crate::private::Sealed {
 
     fn start_transfer(&mut self) -> Result<(), DmaError>;
 
+    #[cfg(esp32s3)]
+    fn set_ext_mem_block_size(&self, size: DmaExtMemBKSize);
+
     fn clear_ch_out_done(&self);
 
     fn is_ch_out_done_set(&self) -> bool;
@@ -1403,13 +1452,33 @@ where
         peri: DmaPeripheral,
         chain: &DescriptorChain,
     ) -> Result<(), DmaError> {
+        // for esp32s3 we check each descriptor buffer that points to psram for
+        // alignment and writeback the cache for that buffer
+        #[cfg(esp32s3)]
+        for des in chain.descriptors.iter() {
+            // we are forcing the DMA alignment to the cache line size
+            // required when we are using dcache
+            let alignment = crate::soc::cache_get_dcache_line_size() as usize;
+            if crate::soc::is_valid_psram_address(des.buffer as u32) {
+                // both the size and address of the buffer must be aligned
+                if des.buffer as usize % alignment != 0 && des.size() % alignment != 0 {
+                    return Err(DmaError::InvalidAlignment);
+                }
+                crate::soc::cache_writeback_addr(des.buffer as u32, des.size() as u32);
+            }
+        }
         self.tx_impl.prepare_transfer_without_start(chain, peri)
     }
 
     fn start_transfer(&mut self) -> Result<(), DmaError> {
         self.tx_impl.start_transfer()
     }
 
+    #[cfg(esp32s3)]
+    fn set_ext_mem_block_size(&self, size: DmaExtMemBKSize) {
+        CH::Channel::set_out_ext_mem_block_size(size);
+    }
+
     fn clear_ch_out_done(&self) {
         self.tx_impl.clear_ch_out_done();
     }
@@ -1489,6 +1558,8 @@ pub trait RegisterAccess: crate::private::Sealed {
     fn init_channel();
     #[cfg(gdma)]
     fn set_mem2mem_mode(value: bool);
+    #[cfg(esp32s3)]
+    fn set_out_ext_mem_block_size(size: DmaExtMemBKSize);
     fn set_out_burstmode(burst_mode: bool);
     fn set_out_priority(priority: DmaPriority);
     fn clear_out_interrupts();
@@ -1507,6 +1578,8 @@ pub trait RegisterAccess: crate::private::Sealed {
     fn reset_out_eof_interrupt();
     fn last_out_dscr_address() -> usize;
 
+    #[cfg(esp32s3)]
+    fn set_in_ext_mem_block_size(size: DmaExtMemBKSize);
     fn set_in_burstmode(burst_mode: bool);
     fn set_in_priority(priority: DmaPriority);
     fn clear_in_interrupts();

esp-hal/src/soc/esp32s3/mod.rs

@@ -170,3 +170,35 @@ unsafe fn post_init() {
     Wdt::<TIMG0, crate::Blocking>::set_wdt_enabled(false);
     Wdt::<TIMG1, crate::Blocking>::set_wdt_enabled(false);
 }
+
+#[doc(hidden)]
+#[link_section = ".rwtext"]
+pub unsafe fn cache_writeback_addr(addr: u32, size: u32) {
+    extern "C" {
+        fn Cache_WriteBack_Addr(addr: u32, size: u32);
+        fn Cache_Suspend_DCache_Autoload() -> u32;
+        fn Cache_Resume_DCache_Autoload(value: u32);
+    }
+    // suspend autoload, avoid load cachelines being written back
+    let autoload = Cache_Suspend_DCache_Autoload();
+    Cache_WriteBack_Addr(addr, size);
+    Cache_Resume_DCache_Autoload(autoload);
+}
+
+#[doc(hidden)]
+#[link_section = ".rwtext"]
+pub unsafe fn cache_invalidate_addr(addr: u32, size: u32) {
+    extern "C" {
+        fn Cache_Invalidate_Addr(addr: u32, size: u32);
+    }
+    Cache_Invalidate_Addr(addr, size);
+}
+
+#[doc(hidden)]
+#[link_section = ".rwtext"]
+pub unsafe fn cache_get_dcache_line_size() -> u32 {
+    extern "C" {
+        fn Cache_Get_DCache_Line_Size() -> u32;
+    }
+    Cache_Get_DCache_Line_Size()
+}

esp-hal/src/soc/mod.rs

@@ -71,3 +71,20 @@ impl self::efuse::Efuse {
 pub(crate) fn is_valid_ram_address(address: u32) -> bool {
     (self::constants::SOC_DRAM_LOW..=self::constants::SOC_DRAM_HIGH).contains(&address)
 }
+
+#[allow(unused)]
+pub(crate) fn is_valid_psram_address(address: u32) -> bool {
+    #[cfg(psram)]
+    {
+        let start = crate::psram::psram_vaddr_start() as u32;
+        let end = start + crate::psram::PSRAM_BYTES as u32;
+        (start..=end).contains(&address)
+    }
+    #[cfg(not(psram))]
+    false
+}
+
+#[allow(unused)]
+pub(crate) fn is_valid_memory_address(address: u32) -> bool {
+    is_valid_ram_address(address) || is_valid_psram_address(address)
+}

examples/Cargo.toml

@@ -7,6 +7,7 @@ publish = false
 
 [dependencies]
 aes                 = "0.8.4"
+aligned             = { version = "0.4.2", optional = true }
 bleps               = { git = "https://github.com/bjoernQ/bleps", package = "bleps", rev = "a5148d8ae679e021b78f53fd33afb8bb35d0b62e", features = [ "macros", "async"] }
 cfg-if              = "1.0.0"
 critical-section    = "1.1.2"