Skip to content

Commit

Permalink
Send preprocessed tensorflow input over serial
Browse files Browse the repository at this point in the history
Signed-off-by: Stefan Gloor <code@stefan-gloor.ch>
  • Loading branch information
stgloorious committed Jun 7, 2024
1 parent 1909f3a commit f7865f5
Show file tree
Hide file tree
Showing 11 changed files with 324 additions and 35 deletions.
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,6 @@
[submodule "third_party/tflite-micro"]
path = third_party/tflite-micro
url = https://github.com/tensorflow/tflite-micro.git
[submodule "third_party/libcrc"]
path = third_party/libcrc
url = https://github.com/lammertb/libcrc.git
16 changes: 15 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ include_directories(${CMAKE_SOURCE_DIR}/third_party/tflite-micro/tensorflow/lite
include_directories(${CMAKE_SOURCE_DIR}/third_party/tflite-micro/tensorflow/lite/micro/tools/make/downloads/ruy)
include_directories(${CMAKE_SOURCE_DIR}/third_party/tflite-micro/tensorflow/lite/micro/tools/make/downloads/gemmlowp)

# Libcrc32 includes
include_directories(${CMAKE_SOURCE_DIR}/third_party/libcrc/include)

# Link
FILE(GLOB linker_script ld/*.ld)
add_link_options(-T ${linker_script})
Expand Down Expand Up @@ -111,6 +114,17 @@ add_library(tflm
${CMAKE_SOURCE_DIR}/third_party/tflite-micro/tensorflow/lite/micro/tools/make/downloads
)

# This target generates the CRC32 table required for libcrc32
# This will NOT be cross-compiled to work around compilation errors,
# as the generated header is architecture agnostic.
add_custom_target(gentab32 COMMAND make -C
${CMAKE_SOURCE_DIR}/third_party/libcrc tab/gentab32.inc
)

# This creates the cmake library representation of libcrc
add_library(crc32 STATIC ${CMAKE_SOURCE_DIR}/third_party/libcrc/src/crc32.c)
add_dependencies(crc32 gentab32)

# Project sources
FILE(GLOB demo_srcs src/*.c
src/*.cc
Expand All @@ -120,7 +134,7 @@ FILE(GLOB demo_srcs src/*.c
)

add_executable(demo.elf ${demo_srcs})
target_link_libraries(demo.elf hal tflm)
target_link_libraries(demo.elf hal tflm crc32)

# st-util wants a binary-only format, not an ELF
add_custom_target(bin ALL DEPENDS demo.elf
Expand Down
2 changes: 0 additions & 2 deletions include/models/sample_input.h

This file was deleted.

49 changes: 49 additions & 0 deletions include/serial.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/**
* @file serial.h
* @brief Simple serial protocol for transferring preprocessed input data
*/

/*
* Copyright (C) 2024 Stefan Gloor
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/

#define SERIAL_BLOCK_SIZE 256
#define SERIAL_CMD_START_TRANSACTION "START"
#define SERIAL_CMD_ACK "A"
#define SERIAL_CMD_NACK "N"


/**
* @brief Receive up to len bytes
* @returns number of bytes received
*
*/
#ifdef __cplusplus
extern "C"
#endif
int serial_recv(char* buf, size_t len);



4 changes: 2 additions & 2 deletions ld/stm32l475vgtx.ld
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ ENTRY(Reset_Handler)
/* Highest address of the user mode stack */
_estack = ORIGIN(RAM) + LENGTH(RAM); /* end of "RAM" Ram type memory */

_Min_Heap_Size = 0x100; /* required amount of heap */
_Min_Stack_Size = 0x1000; /* required amount of stack */
_Min_Heap_Size = 0x200; /* required amount of heap */
_Min_Stack_Size = 0x2000; /* required amount of stack */

/* Memories definition */
MEMORY
Expand Down
13 changes: 2 additions & 11 deletions ml/train.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,9 +231,9 @@ def representative_dataset():
# Finally, verify the model's prediction output using an input audio file of someone saying "no". How well does your model perform?

#x = data_dir/'no/01bb6a2a_nohash_0.wav'
#x = data_dir/'yes/5184ed3e_nohash_0.wav'
x = data_dir/'yes/5184ed3e_nohash_0.wav'
#x = data_dir/'no/0cd323ec_nohash_1.wav'
x = data_dir/'no/2da58b32_nohash_4.wav'
#x = data_dir/'no/2da58b32_nohash_4.wav'
#x = data_dir/'yes/1a673010_nohash_0.wav'
#x = data_dir/'yes/01bb6a2a_nohash_4.wav'
#x = data_dir/'left/0e5193e6_nohash_0.wav'
Expand Down Expand Up @@ -268,15 +268,6 @@ def representative_dataset():
with open('sample_input.bin', 'wb') as f:
f.write(preprocessed_input_data)

os.system('xxd -i sample_input.bin > sample_input.cc')
os.system('sed -i \'s/unsigned char/const unsigned char/\' sample_input.cc')
os.system('sed -i \'s/unsigned int/const unsigned int/\' sample_input.cc')
os.system('sed -i \'1i #include <sample_input.h>\n\' sample_input.cc')

if not os.path.exists('../src/models'):
os.makedirs('../src/models')
shutil.copyfile('sample_input.cc', '../src/models/sample_input.cc')

print(f'input shape {preprocessed_input_data.shape}')
print(f'input bytes {len(preprocessed_input_data)}')

Expand Down
40 changes: 25 additions & 15 deletions src/debug_io.c
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,17 @@

UART_HandleTypeDef uart_hd_debug_uart;

#define UART_RX_FIFO_SIZE 64
static uint8_t uart_rx_fifo[UART_RX_FIFO_SIZE];
static uint8_t uart_rx_fifo_enqueue = 0;
static uint8_t uart_rx_fifo_dequeue = 1;
#define UART_RX_FIFO_SIZE 16
volatile static uint8_t uart_rx_fifo[UART_RX_FIFO_SIZE];
volatile static uint8_t uart_rx_fifo_enqueue = 0;
volatile static uint8_t uart_rx_fifo_dequeue = 0;
volatile static uint32_t uart_rx_fifo_count = 0;

void HAL_UART_RxCpltCallback(UART_HandleTypeDef *uart_hd)
{
HAL_UART_Receive_IT(uart_hd, uart_rx_fifo + uart_rx_fifo_enqueue, 1);
uart_rx_fifo_enqueue = (uart_rx_fifo_enqueue + 1) % UART_RX_FIFO_SIZE;
uart_rx_fifo_count++;
HAL_UART_Receive_IT(uart_hd, (uint8_t*)uart_rx_fifo + uart_rx_fifo_enqueue, 1);
}

/* attach the _read,_write syscall to debug UART */
Expand All @@ -67,18 +69,26 @@ int _write(int fd, const void *buf, int cnt)

int _read(int fd, uint8_t *buf, int cnt)
{
/* wait as long as fifo is empty */
while (uart_rx_fifo_enqueue == uart_rx_fifo_dequeue)
;
size_t start = HAL_GetTick();
const size_t timeout = 500;

if (fd != STDIN_FILENO){
return 0;
}
/* wait as long as fifo is empty or timeout expires */
while (uart_rx_fifo_count == 0){
if ((HAL_GetTick() - start) > timeout){
return 0;
}
}

int i = 0;
while (uart_rx_fifo_enqueue != uart_rx_fifo_dequeue) {
buf[i++] = uart_rx_fifo[uart_rx_fifo_dequeue - 1];
while (uart_rx_fifo_count > 0) {
// Read at the dequeue index
buf[i++] = uart_rx_fifo[uart_rx_fifo_dequeue];
uart_rx_fifo_dequeue =
(uart_rx_fifo_dequeue + 1) % UART_RX_FIFO_SIZE;
if (buf[i] == '\n') {
break;
}
uart_rx_fifo_count--;
if (i == cnt) {
break;
}
Expand Down Expand Up @@ -129,8 +139,8 @@ int uart_debug_init()
HAL_NVIC_SetPriority(USART1_IRQn, 10, 10);
HAL_NVIC_EnableIRQ(USART1_IRQn);

HAL_UART_Receive_IT(&uart_hd_debug_uart,
uart_rx_fifo + uart_rx_fifo_enqueue, 1);
HAL_UART_Receive_IT(&uart_hd_debug_uart, (uint8_t*)uart_rx_fifo +
uart_rx_fifo_enqueue, 1);
return 0;
}

Expand Down
15 changes: 11 additions & 4 deletions src/main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ limitations under the License.

#include <stdio.h>
#include <string.h>
#include <unistd.h>

#include <models/model_tflite.h>
#include <models/sample_input.h>
#include <tensorflow/lite/core/c/common.h>
#include <tensorflow/lite/micro/micro_interpreter.h>
#include <tensorflow/lite/micro/micro_log.h>
Expand All @@ -28,12 +28,16 @@ limitations under the License.
#include <tensorflow/lite/micro/system_setup.h>
#include <tensorflow/lite/schema/schema_generated.h>

#include <serial.h>

#include "mic.h"
int dfsdm_conversion_done;

const int kTensorArenaSize = 86 * 1024;
const int kTensorArenaSize = 48 * 1024;
alignas(16) static uint8_t tensor_arena[kTensorArenaSize] = { 0x55 };

static char input_tensor[16500];

#define DEBUG_PRINTF(...) \
{ \
printf("[i] "); \
Expand Down Expand Up @@ -120,6 +124,9 @@ int main(int argc, char *argv[])
microphone.dump_recording();
*/

size_t input_tensor_len = serial_recv(input_tensor, sizeof(input_tensor));
DEBUG_PRINTF("Received %u bytes.\n", input_tensor_len);

const tflite::Model *model = tflite::GetModel(model_tflite);
DEBUG_PRINTF("Model architecture:\n");
DEBUG_PRINTF("==============================================\n");
Expand Down Expand Up @@ -172,10 +179,10 @@ int main(int argc, char *argv[])
input->dims->data[1] = 124;
input->dims->data[2] = 129;
input->dims->data[3] = 1;
input->bytes = sample_input_bin_len;
input->bytes = input_tensor_len;
const char input_name[] = "Input";
input->name = input_name;
memcpy(input->data.uint8, sample_input_bin, input->bytes);
memcpy(input->data.uint8, input_tensor, input->bytes);
print_shape(input);

// Perform inference
Expand Down
114 changes: 114 additions & 0 deletions src/serial.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
/**
* @file serial.c
* @brief Simple serial protocol for transferring preprocessed input data
*/

/*
* Copyright (C) 2024 Stefan Gloor
*
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*
*
*/

#include <stdlib.h>
#include <stddef.h>
#include <string.h>
#include <stdio.h>
#include <unistd.h>

#include <serial.h>
#include <checksum.h>

static uint32_t bytes_received = 0;

int serial_recv(char* out, size_t len){
int i = 0;
// Wait for transaction to start
do {
i += read(STDIN_FILENO, out+i, strlen(SERIAL_CMD_START_TRANSACTION));
} while ((strstr(out, SERIAL_CMD_START_TRANSACTION) == NULL) && (i < len));
if (i >= len - 1){
write(STDOUT_FILENO, SERIAL_CMD_NACK, strlen(SERIAL_CMD_NACK));
return 0;
}
// Send "START" back
write(STDOUT_FILENO, SERIAL_CMD_START_TRANSACTION, strlen(SERIAL_CMD_START_TRANSACTION));

// Send expected block size
char blocksize[5];
snprintf(blocksize, sizeof(blocksize), "%04u", SERIAL_BLOCK_SIZE);
write(STDOUT_FILENO, blocksize, 4);

// Receive file size
i = 0;
while (i < 8){
i += read(STDIN_FILENO, out+i, 8-i);
}
out[8] = '\0';
size_t filesize = atoi(out);
size_t rounded_filesize = (filesize + SERIAL_BLOCK_SIZE - 1)
/ SERIAL_BLOCK_SIZE * SERIAL_BLOCK_SIZE;

if ((rounded_filesize > len) || (filesize == 0)){
write(STDOUT_FILENO, SERIAL_CMD_NACK, strlen(SERIAL_CMD_NACK));
return 0;
}
write(STDOUT_FILENO, SERIAL_CMD_ACK, strlen(SERIAL_CMD_ACK));

// Receive data
bytes_received = 0;
unsigned char blockbuf[SERIAL_BLOCK_SIZE];
do {
int j = 0;
char checksum[9];
// 32 bit checksum first
while (j < 8){
j += read(STDIN_FILENO, checksum+j, 1);
}
checksum[8] = '\0';
uint32_t expected_crc32 = strtoll(checksum, NULL, 16);

j = 0;
// actual data block
while (j < SERIAL_BLOCK_SIZE){
size_t len = read(STDIN_FILENO, blockbuf+j, 1);
// An error occurred (e.g., timeout)
if (len == 0){
break;
}
else {
j += len;
}
}

if (crc_32(blockbuf, SERIAL_BLOCK_SIZE) == expected_crc32) {
write(STDOUT_FILENO, SERIAL_CMD_ACK, strlen(SERIAL_CMD_ACK));
memcpy(out + bytes_received, blockbuf, SERIAL_BLOCK_SIZE);
bytes_received += SERIAL_BLOCK_SIZE;
}
else {
write(STDOUT_FILENO, SERIAL_CMD_NACK, strlen(SERIAL_CMD_NACK));
}
} while (bytes_received < rounded_filesize);

return bytes_received;
}
1 change: 1 addition & 0 deletions third_party/libcrc
Submodule libcrc added at 7719e2
Loading

0 comments on commit f7865f5

Please sign in to comment.