Feat: 使用Langchain实现的'Jarvis'管家插件，可替换NLU模块，支持拓展功能函数

plugins/Jarvis.py

@@ -0,0 +1,193 @@
+# -*- coding:utf-8 -*-
+import requests
+import json
+import re
+import os
+os.environ["OPENAI_API_VERSION"] = "2023-05-15"
+# os.environ["http_proxy"] = "http://127.0.0.1:20172"
+# os.environ["https_proxy"] = "http://127.0.0.1:20172"
+from robot import logging
+from robot import config
+from robot.sdk.AbstractPlugin import AbstractPlugin
+from langchain import hub
+from langchain.agents import AgentExecutor, create_openai_tools_agent, create_structured_chat_agent
+from langchain_community.tools.tavily_search import TavilySearchResults
+from langchain_openai import ChatOpenAI
+from langchain_openai import AzureChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.agents import tool, Tool, initialize_agent, load_tools
+from langchain.tools import BaseTool, StructuredTool, tool
+from langchain.pydantic_v1 import BaseModel, Field
+
+
+logger = logging.getLogger(__name__)
+
+
+hass_url = config.get('jarvis')['hass']['host']
+hass_port = config.get('jarvis')['hass']['port']
+hass_headers = {'Authorization': config.get('jarvis')['hass']['key'], 'content-type': 'application/json'}
+
+class BrightnessControlInput(BaseModel):
+    entity_id: str 
+    brightness_pct: int
+
+class FeederOutInput(BaseModel):
+    entity_id: str
+    nums: int
+
+class HvacControlInput(BaseModel):
+    entity_id: str
+    input_dict: dict
+
+
+class Plugin(AbstractPlugin):
+
+    SLUG = "jarvis"
+    DEVICES = None
+    PRIORITY = config.get('jarvis')['priority']
+
+    def __init__(self, con):
+        super().__init__(con)
+        self.profile = config.get()
+        self.langchain_init()
+
+    def langchain_init(self):
+        self.llm = AzureChatOpenAI(azure_deployment="gpt-35-turbo")
+        from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+        structured_chat_prompt = hub.pull("hwchase17/structured-chat-agent")
+
+        addtional_system_message = """You can control the devices and answer any other questions. In my House, the devices are as blow (in the dict, the value is use purpose, the key is the entity_id):
+        {device_list}. You can control the devices by using the given tools. You must use the correct parameters when using the tools. Sometimes before you change the value of some device, 
+        you should first query the current state of the device to confirm how to change the value. I'm in '{location}' now. ALWAYS outputs the final result to {language}."""
+        structured_chat_system = structured_chat_prompt.messages[0].prompt.template
+        structured_chat_human = structured_chat_prompt.messages[2].prompt.template
+        prompt = ChatPromptTemplate.from_messages([
+            ('system', structured_chat_system+ addtional_system_message),
+            structured_chat_human
+            ]
+        )
+
+        brightness_control_tool = StructuredTool(
+            name="brightness_control",
+            description="Control the brightness of a light. the brightness_pct must be between 10 and 100 when you just ajust the brightness, but if you want to turn off the light, brightness should be set to 0. input: brightness_pct: int, entity_id: str, output: bool.",
+            func=self.brightness_control,
+            args_schema=BrightnessControlInput
+        )
+
+        feeder_out_tool = StructuredTool(
+            name="feeder_out",
+            description="Control the pet feeder. You can Only use this tool when you need to feed. The nums must be between 1 and 10, input: nums: int, entity_id: str, output: bool.",
+            func=self.feeder_out,
+            args_schema=FeederOutInput
+        )
+
+        get_attr_tool = Tool(
+            name="get_attributes",
+            description="Get the attributes of a device. input: entity_id: str, output: dict.",
+            func=self.get_attributes
+        )
+
+        hvac_control_tool = StructuredTool(
+            name="hvac_control",
+            description="""Control the hvac. input: entity_id: str, input_dict: dict, output: bool. input_dict include: operation (set_hvac_mode, set_fan_mode, set_temperature), 
+            hvac_mode (off, auto, cool, heat, dry, fan_only), temperature, fan_mode ('Fan Speed Down', 'Fan Speed Up'), You must choose at least one operation and Pass the corresponding parameter (ONLY ONE) as needed.
+            """,
+            func=self.hvac_control,
+            args_schema=HvacControlInput
+        )
+
+        internal_tools = load_tools(["openweathermap-api", "google-search"], self.llm)
+
+
+        tools = [brightness_control_tool, feeder_out_tool, get_attr_tool, hvac_control_tool
+                 ] + internal_tools
+        agent = create_structured_chat_agent(self.llm, tools, prompt)
+        self.agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True, max_iterations=3, handle_parsing_errors=True)
+        self.device_dict = self.profile['jarvis']['entity_ids']
+
+    def handle(self, text, parsed):
+        handle_result = self.agent_executor.invoke({"input": f"{text}", "device_list": self.profile["jarvis"]['entity_ids'],
+                                                    "location": self.profile['location'], 
+                                                    "language": f"{self.profile['jarvis']['language']}"})
+        output_text = handle_result["output"]
+        self.say(output_text, cache=True)
+
+    @staticmethod
+    def brightness_control(entity_id, brightness_pct):
+        data = {"entity_id": entity_id,
+                "brightness_pct": brightness_pct
+        }
+        p = json.dumps(data)
+        domain = entity_id.split(".")[0]
+        s = "/api/services/" + domain + "/"
+        url_s = hass_url + ":" + hass_port + s + "turn_on"
+        request = requests.post(url_s, headers=hass_headers, data=p)
+        if format(request.status_code) == "200" or \
+            format(request.status_code) == "201": 
+            return True
+        else:
+            logger.error(format(request))
+            return False
+
+    @staticmethod
+    def hvac_control(entity_id, input_dict:dict):
+        data = {"entity_id": entity_id
+                }
+        operation = input_dict['operation']
+        if input_dict.get("hvac_mode"):
+            data["hvac_mode"] = input_dict.get("hvac_mode")
+        if input_dict.get("temperature"):
+            data["temperature"] = input_dict.get("temperature")
+        if input_dict.get("fan_mode"):
+            data["fan_mode"] = input_dict.get("fan_mode")
+        p = json.dumps(data)
+        domain = entity_id.split(".")[0]
+        s = "/api/services/" + domain + "/"
+        url_s = hass_url + ":" + hass_port + s + operation
+        logger.info(f"url_s: {url_s}, data: {p}")
+        request = requests.post(url_s, headers=hass_headers, data=p)
+        if format(request.status_code) == "200" or \
+            format(request.status_code) == "201": 
+            return True
+        else:
+            logger.error(format(request))
+            return False
+
+    @staticmethod
+    def feeder_out(entity_id, nums):
+        domain = entity_id.split(".")[0]
+        s = "/api/services/" + domain + "/"
+        url_s = hass_url + ":" + hass_port + s + "turn_on"
+        data = {
+            "entity_id": entity_id,
+            "variables": {"nums": nums}
+        }
+        p = json.dumps(data)
+        request = requests.post(url_s, headers=hass_headers, data=p)
+        if format(request.status_code) == "200" or \
+            format(request.status_code) == "201": 
+            return True
+        else:
+            logger.error(format(request))
+            return False
+
+    @staticmethod
+    def get_attributes(entity_id):
+        url_entity = hass_url + ":" + hass_port + "/api/states/" + entity_id
+        device_state = requests.get(url_entity, headers=hass_headers).json()
+        attributes = device_state['attributes']
+        return attributes
+
+    def isValid(self, text, parsed):
+
+        return True
+
+if __name__ == "__main__":
+    # ajust_brightness()
+    # get_state()
+    # ajust_color_temp()
+    # pet_feeder()
+    # refresh_devices()
+    pass
+    # profile = config.get()
+    # print(profile)

robot/Conversation.py

@@ -157,7 +157,12 @@ def doResponse(self, query, UUID="", onSay=None, onStream=None):
 
         lastImmersiveMode = self.immersiveMode
 
-        parsed = self.doParse(query)
+        # 如果开启了jarvis，并且优先级设置了>0，则把nlu的任务直接跳过。
+        if_jarvis = config.get("jarvis")['enable'] and config.get("jarvis")['priority'] > 0
+        if if_jarvis:
+            parsed = {}
+        else:
+            parsed = self.doParse(query)
         if self._InGossip(query) or not self.brain.query(query, parsed):
             # 进入闲聊
             if self.nlu.hasIntent(parsed, "PAUSE") or "闭嘴" in query:

static/default.yml

@@ -362,4 +362,21 @@ weather:
     enable: false
     key: '心知天气 API Key'
 
-
+# Jarvis管家模式，使用Langchain完成意图揣测+执行，支持拓展不同的函数工具
+# 目前仅在插件中，实现了亮度调节、空调控制、宠物喂食器控制、谷歌查询、OpenWeather天气；
+# 如果需要使用谷歌查询工具，配置GOOGLE_CSE_ID, GOOGLE_API_KEY，可以去Google Cloud Platform申请 Custom Search JSON API。开通有门槛，但是使用是有每日免费调用次数的；
+# 如果需要使用OpenWeather天气，配置OPENWEATHERMAP_API_KEY，可以去OpenWeather官网申请，有每月免费额度；
+jarvis:
+    enable: false
+    priority: 1 # 优先级，越大越优先, 如果希望 Jarvis 优先处理，可以设置为 1. 注意：如果设置了优先级，wukong-robot会跳过NLU任务，直接交给jarvis插件处理，意味着其他所有插件都会失效，请确保jarvis中设置了合适的处理函数，并且满足你的需求。
+    language: Chinese # 语言，最后输出的语言
+    hass:
+        host: "http://192.168.0.100" # home assistant 地址
+        port: "8123" # home assistant 端口
+        key: "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiI4OWMzMmU1ZGYxZGU0M2Q3ODE1MDA2ODE2NTE2NjdjOSIsImlhdCI6MTcxMDA3NjkzMiwiZXhwIjoyMDI1NDM2OTMyfQ.n_R7IIWLq7ZDUC3CiiU4DYsniOEj0AVwhkFOmCFxBUo"
+    # hass 中，对Jarvis可见的设备
+    # 进入到home assistant的开发者工具，找到对应设备的entity_id，写上设备的描述即可
+    entity_ids:
+        light.yeelink_bslamp2_1401_light: "卧室床头灯"
+        script.pet_feeder_out: "宠物喂食器"
+        climate.miir_ir02_5728_ir_aircondition_control: "卧室空调"