加入中文UI

CMakeLists.txt

@@ -4,7 +4,7 @@
 # CMakeLists in this exact order for cmake to work correctly
 cmake_minimum_required(VERSION 3.16)
 
-set(PROJECT_VER "0.8.3")
+set(PROJECT_VER "0.9.0")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)
 project(xiaozhi)

README.md

@@ -21,7 +21,7 @@
 - Wi-Fi 配网
 - 支持 BOOT 键唤醒和打断
 - 离线语音唤醒（乐鑫方案）
-- 流式语音对话（WebSocket 协议）
+- 流式语音对话（WebSocket 或 UDP 协议）
 - 支持国语、粤语、英语、日语、韩语 5 种语言识别（SenseVoice 方案）
 - 声纹识别（识别是谁在喊 AI 的名字，[3D Speaker 项目](https://github.com/modelscope/3D-Speaker)）
 - 使用大模型 TTS（火山引擎与 CosyVoice 方案）

main/CMakeLists.txt

@@ -1,23 +1,26 @@
-set(SOURCES "audio_codec.cc"
+set(SOURCES "audio_codecs/audio_codec.cc"
             "audio_codecs/no_audio_codec.cc"
             "audio_codecs/box_audio_codec.cc"
-            "display.cc"
+            "display/display.cc"
             "display/no_display.cc"
             "display/st7789_display.cc"
             "display/ssd1306_display.cc"
-            "board.cc"
-            "protocol.cc"
+            "protocols/protocol.cc"
             "protocols/mqtt_protocol.cc"
             "protocols/websocket_protocol.cc"
             "system_info.cc"
             "application.cc"
-            "button.cc"
-            "led.cc"
             "ota.cc"
             "settings.cc"
             "main.cc"
             )
-set(INCLUDE_DIRS ".")
+
+set(INCLUDE_DIRS "." "display" "audio_codecs" "protocols")
+
+# 字体
+file(GLOB FONT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/fonts/*.c)
+list(APPEND SOURCES ${FONT_SOURCES})
+list(APPEND INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/fonts)
 
 # 添加板级公共文件
 file(GLOB BOARD_COMMON_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/boards/common/*.cc)

main/application.cc

@@ -2,8 +2,9 @@
 #include "system_info.h"
 #include "ml307_ssl_transport.h"
 #include "audio_codec.h"
-#include "protocols/mqtt_protocol.h"
-#include "protocols/websocket_protocol.h"
+#include "mqtt_protocol.h"
+#include "websocket_protocol.h"
+#include "font_awesome_symbols.h"
 
 #include <cstring>
 #include <esp_log.h>
@@ -43,30 +44,37 @@ Application::~Application() {
 }
 
 void Application::CheckNewVersion() {
+    auto& board = Board::GetInstance();
+    auto display = board.GetDisplay();
     // Check if there is a new firmware version available
-    ota_.SetPostData(Board::GetInstance().GetJson());
+    ota_.SetPostData(board.GetJson());
 
     while (true) {
         if (ota_.CheckVersion()) {
             if (ota_.HasNewVersion()) {
                 // Wait for the chat state to be idle
-                while (chat_state_ != kChatStateIdle) {
-                    vTaskDelay(100);
-                }
+                do {
+                    vTaskDelay(pdMS_TO_TICKS(3000));
+                } while (GetChatState() != kChatStateIdle);
 
                 SetChatState(kChatStateUpgrading);
-                ota_.StartUpgrade([](int progress, size_t speed) {
+
+                display->SetIcon(FONT_AWESOME_DOWNLOAD);
+                display->SetStatus("新版本 " + ota_.GetFirmwareVersion());
+                board.GetAudioCodec()->EnableOutput(false);
+
+                ota_.StartUpgrade([display](int progress, size_t speed) {
                     char buffer[64];
-                    snprintf(buffer, sizeof(buffer), "Upgrading...\n %d%% %zuKB/s", progress, speed / 1024);
-                    auto display = Board::GetInstance().GetDisplay();
-                    display->SetText(buffer);
+                    snprintf(buffer, sizeof(buffer), "%d%% %zuKB/s", progress, speed / 1024);
+                    display->SetStatus(buffer);
                 });
 
                 // If upgrade success, the device will reboot and never reach here
                 ESP_LOGI(TAG, "Firmware upgrade failed...");
                 SetChatState(kChatStateIdle);
             } else {
                 ota_.MarkCurrentVersionValid();
+                display->ShowNotification("版本 " + ota_.GetCurrentVersion());
             }
             return;
         }
@@ -79,7 +87,7 @@ void Application::CheckNewVersion() {
 void Application::Alert(const std::string&& title, const std::string&& message) {
     ESP_LOGW(TAG, "Alert: %s, %s", title.c_str(), message.c_str());
     auto display = Board::GetInstance().GetDisplay();
-    display->ShowNotification(std::string(title + "\n" + message));
+    display->ShowNotification(message);
 
     if (message == "PIN is not ready") {
         PlayLocalFile(p3_err_pin_start, p3_err_pin_end - p3_err_pin_start);
@@ -137,7 +145,6 @@ void Application::Start() {
 
     /* Setup the display */
     auto display = board.GetDisplay();
-    display->SetupUI();
 
     /* Setup the audio codec */
     auto codec = board.GetAudioCodec();
@@ -230,9 +237,9 @@ void Application::Start() {
             auto builtin_led = Board::GetInstance().GetBuiltinLed();
             if (chat_state_ == kChatStateListening) {
                 if (speaking) {
-                    builtin_led->SetRed(32);
+                    builtin_led->SetRed(HIGH_BRIGHTNESS);
                 } else {
-                    builtin_led->SetRed(8);
+                    builtin_led->SetRed(LOW_BRIGHTNESS);
                 }
                 builtin_led->TurnOn();
             }
@@ -269,7 +276,7 @@ void Application::Start() {
 #endif
 
     // Initialize the protocol
-    display->SetText("Starting protocol...");
+    display->SetStatus("初始化协议");
 #ifdef CONFIG_CONNECTION_TYPE_WEBSOCKET
     protocol_ = new WebsocketProtocol();
 #else
@@ -294,7 +301,7 @@ void Application::Start() {
         });
         board.SetPowerSaveMode(true);
     });
-    protocol_->OnIncomingJson([this](const cJSON* root) {
+    protocol_->OnIncomingJson([this, display](const cJSON* root) {
         // Parse JSON data
         auto type = cJSON_GetObjectItem(root, "type");
         if (strcmp(type->valuestring, "tts") == 0) {
@@ -315,28 +322,30 @@ void Application::Start() {
             } else if (strcmp(state->valuestring, "sentence_start") == 0) {
                 auto text = cJSON_GetObjectItem(root, "text");
                 if (text != NULL) {
-                    ESP_LOGI(TAG, ">> %s", text->valuestring);
+                    ESP_LOGI(TAG, "<< %s", text->valuestring);
+                    display->SetChatMessage("assistant", text->valuestring);
                 }
             }
         } else if (strcmp(type->valuestring, "stt") == 0) {
             auto text = cJSON_GetObjectItem(root, "text");
             if (text != NULL) {
                 ESP_LOGI(TAG, ">> %s", text->valuestring);
+                display->SetChatMessage("user", text->valuestring);
             }
         } else if (strcmp(type->valuestring, "llm") == 0) {
             auto emotion = cJSON_GetObjectItem(root, "emotion");
             if (emotion != NULL) {
-                ESP_LOGD(TAG, "EMOTION: %s", emotion->valuestring);
+                display->SetEmotion(emotion->valuestring);
             }
         }
     });
 
     // Blink the LED to indicate the device is running
+    display->SetStatus("待命");
     builtin_led->SetGreen();
     builtin_led->BlinkOnce();
 
     SetChatState(kChatStateIdle);
-    display->UpdateDisplay();
 }
 
 void Application::Schedule(std::function<void()> callback) {
@@ -394,28 +403,30 @@ void Application::SetChatState(ChatState state) {
         case kChatStateUnknown:
         case kChatStateIdle:
             builtin_led->TurnOff();
-            display->SetText("I'm\nIdle.");
+            display->SetStatus("待命");
+            display->SetEmotion("neutral");
 #ifdef CONFIG_USE_AFE_SR
             audio_processor_.Stop();
 #endif
             break;
         case kChatStateConnecting:
             builtin_led->SetBlue();
             builtin_led->TurnOn();
-            display->SetText("I'm\nConnecting...");
+            display->SetStatus("连接中...");
             break;
         case kChatStateListening:
             builtin_led->SetRed();
             builtin_led->TurnOn();
-            display->SetText("I'm\nListening...");
+            display->SetStatus("聆听中...");
+            display->SetEmotion("neutral");
 #ifdef CONFIG_USE_AFE_SR
             audio_processor_.Start();
 #endif
             break;
         case kChatStateSpeaking:
             builtin_led->SetGreen();
             builtin_led->TurnOn();
-            display->SetText("I'm\nSpeaking...");
+            display->SetStatus("说话中...");
 #ifdef CONFIG_USE_AFE_SR
             audio_processor_.Stop();
 #endif

main/audio_codec.h → main/audio_codecs/audio_codec.h

@@ -23,6 +23,8 @@ class AudioCodec {
     virtual ~AudioCodec();
 
     virtual void SetOutputVolume(int volume);
+    virtual void EnableInput(bool enable);
+    virtual void EnableOutput(bool enable);
 
     void Start();
     void OnInputData(std::function<void(std::vector<int16_t>&& data)> callback);
@@ -67,8 +69,6 @@ class AudioCodec {
 
     virtual int Read(int16_t* dest, int samples) = 0;
     virtual int Write(const int16_t* data, int samples) = 0;
-    virtual void EnableInput(bool enable);
-    virtual void EnableOutput(bool enable);
 };
 
 #endif // _AUDIO_CODEC_H

main/audio_codecs/box_audio_codec.h

@@ -22,8 +22,6 @@ class BoxAudioCodec : public AudioCodec {
 
     virtual int Read(int16_t* dest, int samples) override;
     virtual int Write(const int16_t* data, int samples) override;
-    virtual void EnableInput(bool enable) override;
-    virtual void EnableOutput(bool enable) override;
 
 public:
     BoxAudioCodec(void* i2c_master_handle, int input_sample_rate, int output_sample_rate,
@@ -32,6 +30,8 @@ class BoxAudioCodec : public AudioCodec {
     virtual ~BoxAudioCodec();
 
     virtual void SetOutputVolume(int volume) override;
+    virtual void EnableInput(bool enable) override;
+    virtual void EnableOutput(bool enable) override;
 };
 
 #endif // _BOX_AUDIO_CODEC_H

main/boards/bread-compact-ml307/compact_ml307_board.cc

@@ -48,13 +48,13 @@ class CompactMl307Board : public Ml307Board {
                 volume = 100;
             }
             codec->SetOutputVolume(volume);
-            GetDisplay()->ShowNotification("Volume\n" + std::to_string(volume));
+            GetDisplay()->ShowNotification("音量 " + std::to_string(volume));
         });
 
         volume_up_button_.OnLongPress([this]() {
             auto codec = GetAudioCodec();
             codec->SetOutputVolume(100);
-            GetDisplay()->ShowNotification("Volume\n100");
+            GetDisplay()->ShowNotification("最大音量");
         });
 
         volume_down_button_.OnClick([this]() {
@@ -64,13 +64,13 @@ class CompactMl307Board : public Ml307Board {
                 volume = 0;
             }
             codec->SetOutputVolume(volume);
-            GetDisplay()->ShowNotification("Volume\n" + std::to_string(volume));
+            GetDisplay()->ShowNotification("音量 " + std::to_string(volume));
         });
 
         volume_down_button_.OnLongPress([this]() {
             auto codec = GetAudioCodec();
             codec->SetOutputVolume(0);
-            GetDisplay()->ShowNotification("Volume\n0");
+            GetDisplay()->ShowNotification("已静音");
         });
     }
 

main/boards/bread-compact-wifi/compact_wifi_board.cc

@@ -48,13 +48,13 @@ class CompactWifiBoard : public WifiBoard {
                 volume = 100;
             }
             codec->SetOutputVolume(volume);
-            GetDisplay()->ShowNotification("Volume\n" + std::to_string(volume));
+            GetDisplay()->ShowNotification("音量 " + std::to_string(volume));
         });
 
         volume_up_button_.OnLongPress([this]() {
             auto codec = GetAudioCodec();
             codec->SetOutputVolume(100);
-            GetDisplay()->ShowNotification("Volume\n100");
+            GetDisplay()->ShowNotification("最大音量");
         });
 
         volume_down_button_.OnClick([this]() {
@@ -64,13 +64,13 @@ class CompactWifiBoard : public WifiBoard {
                 volume = 0;
             }
             codec->SetOutputVolume(volume);
-            GetDisplay()->ShowNotification("Volume\n" + std::to_string(volume));
+            GetDisplay()->ShowNotification("音量 " + std::to_string(volume));
         });
 
         volume_down_button_.OnLongPress([this]() {
             auto codec = GetAudioCodec();
             codec->SetOutputVolume(0);
-            GetDisplay()->ShowNotification("Volume\n0");
+            GetDisplay()->ShowNotification("已静音");
         });
     }
 

main/board.h → main/boards/common/board.h

@@ -41,6 +41,7 @@ class Board {
     virtual Mqtt* CreateMqtt() = 0;
     virtual Udp* CreateUdp() = 0;
     virtual bool GetNetworkState(std::string& network_name, int& signal_quality, std::string& signal_quality_text) = 0;
+    virtual const char* GetNetworkStateIcon() = 0;
     virtual bool GetBatteryLevel(int &level, bool& charging);
     virtual std::string GetJson();
     virtual void SetPowerSaveMode(bool enabled) = 0;

main/led.h → main/boards/common/led.h

@@ -11,7 +11,9 @@
 #define BLINK_TASK_STOPPED_BIT BIT0
 #define BLINK_TASK_RUNNING_BIT BIT1
 
-#define DEFAULT_BRIGHTNESS 16
+#define DEFAULT_BRIGHTNESS 4
+#define HIGH_BRIGHTNESS 16
+#define LOW_BRIGHTNESS 2
 
 class Led {
 public:

main/boards/common/ml307_board.cc

@@ -1,5 +1,6 @@
 #include "ml307_board.h"
 #include "application.h"
+#include "font_awesome_symbols.h"
 
 #include <esp_log.h>
 #include <esp_timer.h>
@@ -34,7 +35,7 @@ Ml307Board::Ml307Board(gpio_num_t tx_pin, gpio_num_t rx_pin, size_t rx_buffer_si
 
 void Ml307Board::StartNetwork() {
     auto display = Board::GetInstance().GetDisplay();
-    display->SetText(std::string("Starting modem"));
+    display->SetStatus("初始化模块");
     modem_.SetDebug(false);
     modem_.SetBaudRate(921600);
 
@@ -54,7 +55,7 @@ void Ml307Board::StartNetwork() {
 void Ml307Board::WaitForNetworkReady() {
     auto& application = Application::GetInstance();
     auto display = Board::GetInstance().GetDisplay();
-    display->SetText(std::string("Wait for network\n"));
+    display->SetStatus("等待网络...");
     int result = modem_.WaitForNetworkReady();
     if (result == -1) {
         application.Alert("Error", "PIN is not ready");
@@ -103,6 +104,29 @@ bool Ml307Board::GetNetworkState(std::string& network_name, int& signal_quality,
     return signal_quality != -1;
 }
 
+const char* Ml307Board::GetNetworkStateIcon() {
+    if (!modem_.network_ready()) {
+        return FONT_AWESOME_SIGNAL_OFF;
+    }
+    int csq = modem_.GetCsq();
+    if (csq == -1) {
+        return FONT_AWESOME_SIGNAL_OFF;
+    } else if (csq >= 0 && csq <= 9) {
+        return FONT_AWESOME_SIGNAL_1;
+    } else if (csq >= 10 && csq <= 14) {
+        return FONT_AWESOME_SIGNAL_2;
+    } else if (csq >= 15 && csq <= 19) {
+        return FONT_AWESOME_SIGNAL_3;
+    } else if (csq >= 20 && csq <= 24) {
+        return FONT_AWESOME_SIGNAL_4;
+    } else if (csq >= 25 && csq <= 31) {
+        return FONT_AWESOME_SIGNAL_FULL;
+    }
+
+    ESP_LOGW(TAG, "Invalid CSQ: %d", csq);
+    return FONT_AWESOME_SIGNAL_OFF;
+}
+
 std::string Ml307Board::GetBoardJson() {
     // Set the board type for OTA
     std::string board_type = BOARD_TYPE;