add azure vision

agents/ten_packages/extension/azure_vision_python/BUILD.gn

@@ -0,0 +1,21 @@
+#
+#
+# Agora Real Time Engagement
+# Created by Wei Hu in 2022-11.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+import("//build/feature/ten_package.gni")
+
+ten_package("azure_vision_python") {
+  package_kind = "extension"
+
+  resources = [
+    "__init__.py",
+    "addon.py",
+    "extension.py",
+    "log.py",
+    "manifest.json",
+    "property.json",
+  ]
+}

agents/ten_packages/extension/azure_vision_python/README.md

@@ -0,0 +1,29 @@
+# azure_vision_python
+
+<!-- brief introduction for the extension -->
+
+## Features
+
+<!-- main features introduction -->
+
+- xxx feature
+
+## API
+
+Refer to `api` definition in [manifest.json] and default values in [property.json](property.json).
+
+<!-- Additional API.md can be referred to if extra introduction needed -->
+
+## Development
+
+### Build
+
+<!-- build dependencies and steps -->
+
+### Unit test
+
+<!-- how to do unit test for the extension -->
+
+## Misc
+
+<!-- others if applicable -->

agents/ten_packages/extension/azure_vision_python/__init__.py

@@ -0,0 +1,11 @@
+#
+#
+# Agora Real Time Engagement
+# Created by Wei Hu in 2024-08.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+from . import addon
+from .log import logger
+
+logger.info("azure_vision_python extension loaded")

agents/ten_packages/extension/azure_vision_python/addon.py

@@ -0,0 +1,22 @@
+#
+#
+# Agora Real Time Engagement
+# Created by Wei Hu in 2024-08.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+from ten import (
+    Addon,
+    register_addon_as_extension,
+    TenEnv,
+)
+from .extension import AzureVisionExtension
+from .log import logger
+
+
+@register_addon_as_extension("azure_vision_python")
+class AzureVisionExtensionAddon(Addon):
+
+    def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None:
+        logger.info("AzureVisionExtensionAddon on_create_instance")
+        ten_env.on_create_instance_done(AzureVisionExtension(name), context)

agents/ten_packages/extension/azure_vision_python/extension.py

@@ -0,0 +1,153 @@
+#
+#
+# Agora Real Time Engagement
+# Created by Wei Hu in 2024-08.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+import json
+
+from typing import Any
+from azure.ai.vision.imageanalysis import ImageAnalysisClient
+from azure.ai.vision.imageanalysis.models import VisualFeatures
+from azure.core.credentials import AzureKeyCredential
+
+from ten import (
+    AudioFrame,
+    VideoFrame,
+    Extension,
+    TenEnv,
+    Cmd,
+    StatusCode,
+    CmdResult,
+    Data,
+)
+from .log import logger
+
+PROPERTY_KEY = "key"
+PROPERTY_ENDPOINT = "endpoint"
+
+CMD_IMAGE_ANALYZE = "image_analyze"
+
+class AzureVisionExtension(Extension):
+    key: str = ""
+    endpoint: str = "https://tenagentvision.cognitiveservices.azure.com/"
+
+    def on_init(self, ten_env: TenEnv) -> None:
+        logger.info("AzureVisionExtension on_init")
+        ten_env.on_init_done()
+
+    def on_start(self, ten_env: TenEnv) -> None:
+        logger.info("AzureVisionExtension on_start")
+
+        try:
+            self.key = ten_env.get_property_string(PROPERTY_KEY)
+        except Exception as err:
+            logger.error(f"GetProperty optional {PROPERTY_KEY} error: {err}")
+            return
+
+        try:
+            self.endpoint = ten_env.get_property_string(PROPERTY_ENDPOINT)
+        except Exception as err:
+            logger.info(f"GetProperty optional {PROPERTY_ENDPOINT} error: {err}")
+
+        ten_env.on_start_done()
+
+    def on_stop(self, ten_env: TenEnv) -> None:
+        logger.info("AzureVisionExtension on_stop")
+        ten_env.on_stop_done()
+
+    def on_deinit(self, ten_env: TenEnv) -> None:
+        logger.info("AzureVisionExtension on_deinit")
+        ten_env.on_deinit_done()
+
+    def on_cmd(self, ten_env: TenEnv, cmd: Cmd) -> None:
+        cmd_name = cmd.get_name()
+        logger.info("on_cmd name {}".format(cmd_name))
+
+        if cmd_name == CMD_IMAGE_ANALYZE:
+            try:
+                image_data = cmd.get_property_buf("image_data")
+                resp = self._analyze_image(image_data)
+                cmd_result = CmdResult.create(StatusCode.OK)
+                cmd_result.set_property_string("response", json.dumps(resp))
+                ten_env.return_result(cmd_result, cmd)
+                return
+            except:
+                logger.exception("Failed to handle analyze")
+
+        cmd_result = CmdResult.create(StatusCode.OK)
+        ten_env.return_result(cmd_result, cmd)
+
+    def on_data(self, ten_env: TenEnv, data: Data) -> None:
+        pass
+
+    def on_audio_frame(self, ten_env: TenEnv, audio_frame: AudioFrame) -> None:
+        pass
+
+    def on_video_frame(self, ten_env: TenEnv, video_frame: VideoFrame) -> None:
+        pass
+
+    def _analyze_image(self, image_data: bytes) -> Any:
+        client = ImageAnalysisClient(
+            endpoint=self.endpoint,
+            credential=AzureKeyCredential(self.key)
+        )
+
+        # Get a caption for the image. This will be a synchronously (blocking) call.
+        result = client.analyze(
+            image_data=image_data,
+            visual_features=[VisualFeatures.TAGS, VisualFeatures.CAPTION, VisualFeatures.READ, VisualFeatures.PEOPLE, VisualFeatures.OBJECTS],
+            gender_neutral_caption=True,
+        )
+
+        logger.info(f"before return {result}")
+
+        rst = {}
+        if result.tags is not None:
+            tags = []
+            for tag in result.tags.list:
+                tags.append({
+                    "name": tag.name,
+                    "confidence": tag.confidence
+                })
+            rst["tags"] = tags
+
+        if result.caption is not None:
+            rst["caption"] = {
+                "text": result.caption.text,
+                "confidence": result.caption.confidence
+            }
+
+        if result.read is not None:
+            lines = []
+            for block in result.read.blocks:
+                for line in block.lines:
+                    lines.append({
+                        "text": line.text,
+                        "bounding_box": str(line.bounding_polygon),
+                    })
+            rst["read"] = lines
+
+        if result.objects is not None:
+            objects = []
+            for object in result.objects.list:
+                objects.append({
+                    "name": object.tags[0].name,
+                    "bounding_box": str(object.bounding_box),
+                    "confidence": object.tags[0].confidence
+                })
+            rst["objects"] = objects
+
+        if result.people is not None:
+            people = []
+            for person in result.people.list:
+                people.append({
+                    "bounding_box": str(person.bounding_box),
+                    "confidence": person.confidence
+                })
+            rst["people"] = people
+
+        logger.info(f"after parse {rst}")
+
+        return rst

agents/ten_packages/extension/azure_vision_python/log.py

@@ -0,0 +1,22 @@
+#
+#
+# Agora Real Time Engagement
+# Created by Wei Hu in 2024-08.
+# Copyright (c) 2024 Agora IO. All rights reserved.
+#
+#
+import logging
+
+logger = logging.getLogger("azure_vision_python")
+logger.setLevel(logging.INFO)
+
+formatter_str = (
+    "%(asctime)s - %(name)s - %(levelname)s - %(process)d - "
+    "[%(filename)s:%(lineno)d] - %(message)s"
+)
+formatter = logging.Formatter(formatter_str)
+
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(formatter)
+
+logger.addHandler(console_handler)

agents/ten_packages/extension/azure_vision_python/manifest.json

@@ -0,0 +1,23 @@
+{
+  "type": "extension",
+  "name": "azure_vision_python",
+  "version": "0.1.0",
+  "dependencies": [
+    {
+      "type": "system",
+      "name": "ten_runtime_python",
+      "version": "0.2"
+    }
+  ],
+  "package": {
+    "include": [
+      "manifest.json",
+      "property.json",
+      "BUILD.gn",
+      "**.tent",
+      "**.py",
+      "README.md"
+    ]
+  },
+  "api": {}
+}

agents/ten_packages/extension/azure_vision_python/property.json

@@ -0,0 +1 @@
+{}

agents/ten_packages/extension/azure_vision_python/requirements.txt

@@ -0,0 +1 @@
+azure-ai-vision-imageanalysis

agents/ten_packages/extension/openai_v2v_python/extension.py

@@ -172,6 +172,9 @@ def on_data(self, ten_env: TenEnv, data: Data) -> None:
 
     def on_config_changed(self) -> None:
         # update session again
+        if self._update_session:
+            logger.info("update session after config changed")
+            self._update_session()
         return
 
     async def _init_connection(self):