[mirotalksfu] - add Video AI

miroslavpejic85 · May 29, 2024 · 3a50fe1 · 3a50fe1
1 parent 11185ba
commit 3a50fe1
Show file tree

Hide file tree

Showing 12 changed files with 844 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -57,6 +57,7 @@
 -   Snapshot video frames and save them as PNG images.
 -   Chat with an Emoji Picker for expressing feelings, private messages, Markdown support, and conversation saving.
 -   ChatGPT (powered by OpenAI) for answering questions, providing information, and connecting users to relevant resources.
+-   VideoAI enables users to customize AI avatars to deliver messages, perform tasks, or act out scripts.
 -   Speech recognition, execute the app features simply with your voice.
 -   Push-to-talk functionality, similar to a walkie-talkie.
 -   Advanced collaborative whiteboard for teachers.

diff --git a/app/src/Room.js b/app/src/Room.js
@@ -40,6 +40,7 @@ module.exports = class Room {
         };
         this.survey = config.survey;
         this.redirect = config.redirect;
+        this.videoAIEnabled = config?.videoAI?.enabled || false;
         this.peers = new Map();
         this.bannedPeers = [];
         this.webRtcTransport = config.mediasoup.webRtcTransport;
@@ -65,6 +66,7 @@ module.exports = class Room {
             moderator: this._moderator,
             survey: this.survey,
             redirect: this.redirect,
+            videoAIEnabled: this.videoAIEnabled,
             peers: JSON.stringify([...this.peers]),
         };
     }

diff --git a/app/src/Server.js b/app/src/Server.js
@@ -42,7 +42,7 @@ dependencies: {
  * @license For commercial or closed source, contact us at license.mirotalk@gmail.com or purchase directly via CodeCanyon
  * @license CodeCanyon: https://codecanyon.net/item/mirotalk-sfu-webrtc-realtime-video-conferences/40769970
  * @author  Miroslav Pejic - miroslav.pejic.85@gmail.com
- * @version 1.4.36
+ * @version 1.4.40
  *
  */
 
@@ -1870,6 +1870,219 @@ function startServer() {
             }
         });
 
+        // https://docs.heygen.com/reference/overview-copy
+
+        socket.on('getAvatarList', async ({}, cb) => {
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+            try {
+                const response = await axios.get(`${config.videoAI.basePath}/v1/avatar.list`, {
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-Api-Key': config.videoAI.apiKey,
+                    },
+                });
+
+                const data = { response: response.data.data };
+
+                //log.debug('getAvatarList', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.response?.status === 500 ? 'Internal server error' : error.message });
+            }
+        });
+
+        socket.on('getVoiceList', async ({}, cb) => {
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+            try {
+                const response = await axios.get(`${config.videoAI.basePath}/v1/voice.list`, {
+                    headers: {
+                        'Content-Type': 'application/json',
+                        'X-Api-Key': config.videoAI.apiKey,
+                    },
+                });
+
+                const data = { response: response.data.data };
+
+                //log.debug('getVoiceList', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.response?.status === 500 ? 'Internal server error' : error.message });
+            }
+        });
+
+        socket.on('streamingNew', async ({ quality, avatar_name, voice_id }, cb) => {
+            if (!roomList.has(socket.room_id)) return;
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+            try {
+                const response = await axios.post(
+                    `${config.videoAI.basePath}/v1/streaming.new`,
+                    {
+                        quality,
+                        avatar_name,
+                        voice: {
+                            voice_id: voice_id,
+                        },
+                    },
+                    {
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'X-Api-Key': config.videoAI.apiKey,
+                        },
+                    },
+                );
+
+                const data = { response: response.data };
+
+                log.debug('streamingNew', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.response?.status === 500 ? 'Internal server error' : error });
+            }
+        });
+
+        socket.on('streamingStart', async ({ session_id, sdp }, cb) => {
+            if (!roomList.has(socket.room_id)) return;
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+
+            try {
+                const response = await axios.post(
+                    `${config.videoAI.basePath}/v1/streaming.start`,
+                    { session_id, sdp },
+                    {
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'X-Api-Key': config.videoAI.apiKey,
+                        },
+                    },
+                );
+
+                const data = { response: response.data.data };
+
+                log.debug('startSessionAi', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.response?.status === 500 ? 'server error' : error });
+            }
+        });
+
+        socket.on('streamingICE', async ({ session_id, candidate }, cb) => {
+            if (!roomList.has(socket.room_id)) return;
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+
+            try {
+                const response = await axios.post(
+                    `${config.videoAI.basePath}/v1/streaming.ice`,
+                    { session_id, candidate },
+                    {
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'X-Api-Key': config.videoAI.apiKey,
+                        },
+                    },
+                );
+
+                const data = { response: response.data };
+
+                log.debug('streamingICE', data);
+
+                cb(data);
+            } catch (error) {
+                log.error('Error in streamingICE:', error.response?.data || error.message); // Log detailed error
+                cb({ error: error.response?.status === 500 ? 'Internal server error' : error });
+            }
+        });
+
+        socket.on('streamingTask', async ({ session_id, text }, cb) => {
+            if (!roomList.has(socket.room_id)) return;
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+            try {
+                const response = await axios.post(
+                    `${config.videoAI.basePath}/v1/streaming.task`,
+                    {
+                        session_id,
+                        text,
+                    },
+                    {
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'X-Api-Key': config.videoAI.apiKey,
+                        },
+                    },
+                );
+
+                const data = { response: response.data };
+
+                log.debug('streamingTask', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.response?.status === 500 ? 'server error' : error });
+            }
+        });
+
+        socket.on('talkToOpenAI', async ({ text, context }, cb) => {
+            if (!roomList.has(socket.room_id)) return;
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+            try {
+                const systemLimit = config.videoAI.systemLimit;
+                const arr = {
+                    messages: [...context, { role: 'system', content: systemLimit }, { role: 'user', content: text }],
+                    model: 'gpt-3.5-turbo',
+                };
+                const chatCompletion = await chatGPT.chat.completions.create(arr);
+                const chatText = chatCompletion.choices[0].message.content;
+                context.push({ role: 'system', content: chatText });
+                context.push({ role: 'assistant', content: chatText });
+
+                const data = { response: chatText, context: context };
+
+                log.debug('talkToOpenAI', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.message });
+            }
+        });
+
+        socket.on('streamingStop', async ({ session_id }, cb) => {
+            if (!roomList.has(socket.room_id)) return;
+            if (!config.videoAI.enabled || !config.videoAI.apiKey)
+                return cb({ error: 'Video AI seems disabled, try later!' });
+            try {
+                const response = await axios.post(
+                    `${config.videoAI.basePath}/v1/streaming.stop`,
+                    {
+                        session_id,
+                    },
+                    {
+                        headers: {
+                            'Content-Type': 'application/json',
+                            'X-Api-Key': config.videoAI.apiKey,
+                        },
+                    },
+                );
+
+                const data = { response: response.data };
+
+                log.debug('streamingStop', data);
+
+                cb(data);
+            } catch (error) {
+                cb({ error: error.response?.status === 500 ? 'Internal server error' : error });
+            }
+        });
+
         socket.on('disconnect', async () => {
             if (!roomList.has(socket.room_id)) return;
 

diff --git a/app/src/config.template.js b/app/src/config.template.js
@@ -171,6 +171,19 @@ module.exports = {
         max_tokens: 1000,
         temperature: 0,
     },
+    videoAI: {
+        /*
+        HeyGen Video AI
+            1. Goto  https://app.heygen.com
+            2. Create your account
+            3. Generate your APIKey https://app.heygen.com/settings?nav=API
+         */
+        enabled: false,
+        basePath: 'https://api.heygen.com',
+        apiKey: '',
+        systemLimit:
+            'You are a streaming avatar from MiroTalk SFU, an industry-leading product that specialize in videos communications. Audience will try to have a conversation with you, please try answer the questions or respond their comments naturally, and concisely. - please try your best to response with short answers, and only answer the last question.',
+    },
     email: {
         /*
             Configure email settings for notifications or alerts

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
     "name": "mirotalksfu",
-    "version": "1.4.36",
+    "version": "1.4.40",
     "description": "WebRTC SFU browser-based video calls",
     "main": "Server.js",
     "scripts": {
@@ -43,7 +43,7 @@
     "dependencies": {
         "@sentry/integrations": "7.114.0",
         "@sentry/node": "7.114.0",
-        "axios": "^1.7.1",
+        "axios": "^1.7.2",
         "body-parser": "1.20.2",
         "colors": "1.4.0",
         "compression": "1.7.4",
@@ -67,7 +67,7 @@
     },
     "devDependencies": {
         "node-fetch": "^3.3.2",
-        "nodemon": "^3.1.0",
+        "nodemon": "^3.1.1",
         "prettier": "3.2.5"
     }
 }
diff --git a/public/css/Room.css b/public/css/Room.css
@@ -1320,6 +1320,70 @@ hr {
     }
 }
 
+/*--------------------------------------------------------------
+# Video AI Avatars
+--------------------------------------------------------------*/
+
+.avatarsVideoAI {
+    min-height: 480px;
+    max-height: 600px;
+    color: white;
+    overflow: auto;
+    justify-content: center; /* Center horizontally */
+    align-items: center; /* Center vertically */
+}
+
+/* Custom avatar audio UI */
+
+#audio-container {
+    background: var(--select-bg);
+    color: #fff;
+    padding: 10px;
+    border-radius: 5px;
+}
+
+#audio-container audio {
+    width: 100%;
+    outline: none;
+}
+
+#audio-container audio::-webkit-media-controls-panel {
+    background: var(--body-bg);
+    border-radius: 5px;
+    padding: 5px;
+}
+
+#audio-container audio::-webkit-media-controls-timeline {
+    background: var(--body-bg);
+    border-radius: 5px;
+}
+
+/* Styling for the audio volume controls */
+#audio-container audio::-webkit-media-controls-volume-slider-container {
+    color: #fff;
+}
+
+#audio-container audio::-webkit-media-controls-volume-slider {
+    background: var(--body-bg);
+    border-radius: 5px;
+}
+
+/* Styling for the play/pause button */
+#audio-container audio::-webkit-media-controls-play-button,
+#audio-container audio::-webkit-media-controls-pause-button {
+    background-color: #fff;
+    border-radius: 50%;
+    width: 30px;
+    height: 30px;
+    margin: 0 10px;
+}
+
+/* Styling for the time display */
+#audio-container audio::-webkit-media-controls-current-time-display,
+#audio-container audio::-webkit-media-controls-time-remaining-display {
+    color: #fff;
+}
+
 /*
 z-index:
     - 1 videoMediaContainer

diff --git a/public/css/VideoGrid.css b/public/css/VideoGrid.css
@@ -191,6 +191,14 @@ video {
     cursor: pointer;
 }
 
+#canvasAIElement {
+    width: 100%;
+    height: 100%;
+    object-fit: var(--videoObjFit);
+    border-radius: 10px;
+    cursor: pointer;
+}
+
 video:hover {
     opacity: 0.9;
 }

diff --git a/public/images/virtual/1.jpg b/public/images/virtual/1.jpg
diff --git a/public/images/virtual/2.jpg b/public/images/virtual/2.jpg