From 3f03be35443ace7f7d38283d13a0c5187bf8ef79 Mon Sep 17 00:00:00 2001 From: lvliang-intel Date: Fri, 26 Apr 2024 19:48:18 +0800 Subject: [PATCH] Add service infrastructure (#8) * Add MicroService Infrastructure Signed-off-by: lvliang-intel * rename directory Signed-off-by: lvliang-intel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix ci issue Signed-off-by: lvliang-intel * fix ci issue Signed-off-by: lvliang-intel * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * refine microservice class Signed-off-by: lvliang-intel --------- Signed-off-by: lvliang-intel Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../{embedding => embeddings}/README.md | 0 GenAIComps/embeddings/__init__.py | 13 ++ GenAIComps/embeddings/langchain/__init__.py | 13 ++ .../langchain/embedding.py | 0 GenAIComps/llms/vllm/README.md | 5 +- GenAIComps/mega/constants.py | 22 +++ .../mega/{service.py => http_service.py} | 0 GenAIComps/mega/micro_service.py | 90 +++++++++ GenAIComps/mega/utils.py | 176 +++++++++++++++++- .../{vectordb => vectorstores}/README.md | 0 GenAIComps/vectorstores/__init__.py | 13 ++ .../langchain/chroma/README.md | 0 .../vectorstores/langchain/chroma/__init__.py | 13 ++ .../langchain/qdrant/README.md | 0 .../vectorstores/langchain/qdrant/__init__.py | 13 ++ .../langchain/redis/README.md | 0 .../vectorstores/langchain/redis/__init__.py | 13 ++ .../langchain/redis/config.py | 0 .../langchain/redis/docker-compose-redis.yml | 0 .../langchain/redis/schema.yml | 0 .../langchain/redis/schema_dim_1024.yml | 0 .../langchain/redis/schema_dim_768.yml | 0 .../langchain/redis/schema_lcdocs_dim_768.yml | 0 23 files changed, 366 insertions(+), 5 deletions(-) rename GenAIComps/{embedding => embeddings}/README.md (100%) create mode 100644 GenAIComps/embeddings/__init__.py create mode 100644 GenAIComps/embeddings/langchain/__init__.py rename GenAIComps/{embedding => embeddings}/langchain/embedding.py (100%) create mode 100644 GenAIComps/mega/constants.py rename GenAIComps/mega/{service.py => http_service.py} (100%) create mode 100644 GenAIComps/mega/micro_service.py rename GenAIComps/{vectordb => vectorstores}/README.md (100%) create mode 100644 GenAIComps/vectorstores/__init__.py rename GenAIComps/{vectordb => vectorstores}/langchain/chroma/README.md (100%) create mode 100644 GenAIComps/vectorstores/langchain/chroma/__init__.py rename GenAIComps/{vectordb => vectorstores}/langchain/qdrant/README.md (100%) create mode 100644 GenAIComps/vectorstores/langchain/qdrant/__init__.py rename GenAIComps/{vectordb => vectorstores}/langchain/redis/README.md (100%) create mode 100644 GenAIComps/vectorstores/langchain/redis/__init__.py rename GenAIComps/{vectordb => vectorstores}/langchain/redis/config.py (100%) rename GenAIComps/{vectordb => vectorstores}/langchain/redis/docker-compose-redis.yml (100%) rename GenAIComps/{vectordb => vectorstores}/langchain/redis/schema.yml (100%) rename GenAIComps/{vectordb => vectorstores}/langchain/redis/schema_dim_1024.yml (100%) rename GenAIComps/{vectordb => vectorstores}/langchain/redis/schema_dim_768.yml (100%) rename GenAIComps/{vectordb => vectorstores}/langchain/redis/schema_lcdocs_dim_768.yml (100%) diff --git a/GenAIComps/embedding/README.md b/GenAIComps/embeddings/README.md similarity index 100% rename from GenAIComps/embedding/README.md rename to GenAIComps/embeddings/README.md diff --git a/GenAIComps/embeddings/__init__.py b/GenAIComps/embeddings/__init__.py new file mode 100644 index 0000000000..28f108cb63 --- /dev/null +++ b/GenAIComps/embeddings/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/embeddings/langchain/__init__.py b/GenAIComps/embeddings/langchain/__init__.py new file mode 100644 index 0000000000..28f108cb63 --- /dev/null +++ b/GenAIComps/embeddings/langchain/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/embedding/langchain/embedding.py b/GenAIComps/embeddings/langchain/embedding.py similarity index 100% rename from GenAIComps/embedding/langchain/embedding.py rename to GenAIComps/embeddings/langchain/embedding.py diff --git a/GenAIComps/llms/vllm/README.md b/GenAIComps/llms/vllm/README.md index d82d8e5ea2..b25fd3e457 100644 --- a/GenAIComps/llms/vllm/README.md +++ b/GenAIComps/llms/vllm/README.md @@ -1,5 +1,6 @@ # vLLM Endpoint Serive -[vLLM](https://github.com/vllm-project/vllm) is a RESTful service that provides a simple way to interact with the vLLM API, which is eary to use for LLM inference and serving on [Intel products](https://www.intel.com/content/www/us/en/products/overview.html). Currently, the vLLM is supporting Intel CPU, and will be extended to Intel Gaudi accelerators soon. + +[vLLM](https://github.com/vllm-project/vllm) is a RESTful service that provides a simple way to interact with the vLLM API, which is eerie to use for LLM inference and serving on [Intel products](https://www.intel.com/content/www/us/en/products/overview.html). Currently, the vLLM is supporting Intel CPU, and will be extended to Intel Gaudi accelerators soon. ## Getting Started @@ -44,4 +45,4 @@ You have the flexibility to customize twp parameters according to your specific ```bash export vLLM_LLM_ENDPOINT="http://xxx.xxx.xxx.xxx:8080" export LLM_MODEL= # example: export LLM_MODEL="mistralai/Mistral-7B-v0.1" -``` \ No newline at end of file +``` diff --git a/GenAIComps/mega/constants.py b/GenAIComps/mega/constants.py new file mode 100644 index 0000000000..d641c9afb2 --- /dev/null +++ b/GenAIComps/mega/constants.py @@ -0,0 +1,22 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from enum import Enum + + +class ServiceRoleType(Enum): + """The enum of a service role.""" + + MICROSERVICE = 0 + MEGASERVICE = 1 diff --git a/GenAIComps/mega/service.py b/GenAIComps/mega/http_service.py similarity index 100% rename from GenAIComps/mega/service.py rename to GenAIComps/mega/http_service.py diff --git a/GenAIComps/mega/micro_service.py b/GenAIComps/mega/micro_service.py new file mode 100644 index 0000000000..634e7e82ad --- /dev/null +++ b/GenAIComps/mega/micro_service.py @@ -0,0 +1,90 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from typing import Dict, Optional + +from constants import ServiceRoleType +from utils import check_ports_availability + + +class MicroService: + """MicroService class to create a microservice.""" + + def __init__(self, args: Optional[Dict] = None): + """Init the microservice.""" + self.args = args + if args.get("name", None): + self.name = f'{args.get("name")}/{self.__class__.__name__}' + else: + self.name = self.__class__.__name__ + self.service_role = args.get("service_role", ServiceRoleType.MICROSERVICE) + self.protocol = args.get("protocol", "http") + + self.host = args.get("host", "localhost") + self.port = args.get("port", 8080) + self.replicas = args.get("replicas", 1) + self.provider = args.get("provider", None) + self.provider_endpoint = args.get("provider_endpoint", None) + + self.server = self._get_server() + self.app = self.server.app + self.event_loop = asyncio.new_event_loop() + self.event_loop.run_until_complete(self.async_setup()) + + def _get_server(self): + """Get the server instance based on the protocol. + + This method currently only supports HTTP services. It creates an instance of the HTTPService class with the + necessary arguments. + In the future, it will also support gRPC services. + """ + from http_service import HTTPService + + runtime_args = { + "protocol": self.protocol, + "host": self.host, + "port": self.port, + "title": self.name, + "description": "OPEA Microservice Infrastructure", + } + + return HTTPService( + uvicorn_kwargs=self.args.get("uvicorn_kwargs", None), + runtime_args=runtime_args, + cors=self.args.get("cors", None), + ) + + async def async_setup(self): + """The async method setup the runtime. + + This method is responsible for setting up the server. It first checks if the port is available, then it gets + the server instance and initializes it. + """ + if self.protocol.lower() == "http": + if not (check_ports_availability(self.host, self.port)): + raise RuntimeError(f"port:{self.port}") + + await self.server.initialize_server() + + async def async_run_forever(self): + """Running method of the server.""" + await self.server.execute_server() + + def run(self): + """Running method to block the main thread. + + This method runs the event loop until a Future is done. It is designed to be called in the main thread to keep it busy. + """ + self.event_loop.run_until_complete(self.async_run_forever()) diff --git a/GenAIComps/mega/utils.py b/GenAIComps/mega/utils.py index ab39278b66..76f76655a1 100644 --- a/GenAIComps/mega/utils.py +++ b/GenAIComps/mega/utils.py @@ -12,11 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import ipaddress +import multiprocessing +import os +import random from socket import AF_INET, SOCK_STREAM, socket -from typing import List, Union +from typing import List, Optional, Union -def _is_port_free(host: str, port: int) -> bool: +def is_port_free(host: str, port: int) -> bool: """Check if a given port on a host is free. :param host: The host to check. @@ -37,4 +41,170 @@ def check_ports_availability(host: Union[str, List[str]], port: Union[int, List[ hosts = [host] if isinstance(host, str) else host ports = [port] if isinstance(port, int) else port - return all(_is_port_free(h, p) for h in hosts for p in ports) + return all(is_port_free(h, p) for h in hosts for p in ports) + + +def get_internal_ip(): + """Return the private IP address of the gateway in the same network. + + :return: Private IP address. + """ + import socket + + ip = "127.0.0.1" + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.connect(("10.255.255.255", 1)) + ip = s.getsockname()[0] + except Exception: + pass + return ip + + +def get_public_ip(timeout: float = 0.3): + """Return the public IP address of the gateway in the public network.""" + import urllib.request + + def _get_public_ip(url): + try: + req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, timeout=timeout) as fp: + _ip = fp.read().decode().strip() + return _ip + + except: + pass + + ip_lookup_services = [ + "https://api.ipify.org", + "https://ident.me", + "https://checkip.amazonaws.com/", + ] + + for _, url in enumerate(ip_lookup_services): + ip = _get_public_ip(url) + if ip: + return ip + + +def typename(obj): + """Get the typename of object.""" + if not isinstance(obj, type): + obj = obj.__class__ + try: + return f"{obj.__module__}.{obj.__name__}" + except AttributeError: + return str(obj) + + +def get_event(obj) -> multiprocessing.Event: + if isinstance(obj, multiprocessing.Process) or isinstance(obj, multiprocessing.context.ForkProcess): + return multiprocessing.Event() + elif isinstance(obj, multiprocessing.context.SpawnProcess): + return multiprocessing.get_context("spawn").Event() + else: + raise TypeError(f'{obj} is not an instance of "multiprocessing.Process"') + + +def in_docker(): + """Checks if the current process is running inside Docker.""" + path = "/proc/self/cgroup" + if os.path.exists("/.dockerenv"): + return True + if os.path.isfile(path): + with open(path, encoding="utf-8") as file: + return any("docker" in line for line in file) + return False + + +def host_is_local(hostname): + """Check if hostname is point to localhost.""" + import socket + + fqn = socket.getfqdn(hostname) + if fqn in ("localhost", "0.0.0.0") or hostname == "0.0.0.0": + return True + + try: + return ipaddress.ip_address(hostname).is_loopback + except ValueError: + return False + + +assigned_ports = set() +unassigned_ports = [] +DEFAULT_MIN_PORT = 49153 +MAX_PORT = 65535 + + +def reset_ports(): + def _get_unassigned_ports(): + # if we are running out of ports, lower default minimum port + if MAX_PORT - DEFAULT_MIN_PORT - len(assigned_ports) < 100: + min_port = int(os.environ.get("JINA_RANDOM_PORT_MIN", "16384")) + else: + min_port = int(os.environ.get("JINA_RANDOM_PORT_MIN", str(DEFAULT_MIN_PORT))) + max_port = int(os.environ.get("JINA_RANDOM_PORT_MAX", str(MAX_PORT))) + return set(range(min_port, max_port + 1)) - set(assigned_ports) + + unassigned_ports.clear() + assigned_ports.clear() + unassigned_ports.extend(_get_unassigned_ports()) + random.shuffle(unassigned_ports) + + +def random_port() -> Optional[int]: + """Get a random available port number. + + :return: A random port. + """ + + def _random_port(): + import socket + + def _check_bind(port): + with socket.socket() as s: + try: + s.bind(("", port)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return port + except OSError: + return None + + _port = None + if len(unassigned_ports) == 0: + reset_ports() + for idx, _port in enumerate(unassigned_ports): + if _check_bind(_port) is not None: + break + else: + raise OSError( + f"can not find an available port in {len(unassigned_ports)} unassigned ports, assigned already {len(assigned_ports)} ports" + ) + int_port = int(_port) + unassigned_ports.pop(idx) + assigned_ports.add(int_port) + return int_port + + try: + return _random_port() + except OSError: + assigned_ports.clear() + unassigned_ports.clear() + return _random_port() + + +class SafeContextManager: + """This context manager ensures that the `__exit__` method of the + sub context is called, even when there is an Exception in the + `__init__` method.""" + + def __init__(self, context_to_manage): + self.context_to_manage = context_to_manage + + def __enter__(self): + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type: + self.context_to_manage.__exit__(exc_type, exc_val, exc_tb) diff --git a/GenAIComps/vectordb/README.md b/GenAIComps/vectorstores/README.md similarity index 100% rename from GenAIComps/vectordb/README.md rename to GenAIComps/vectorstores/README.md diff --git a/GenAIComps/vectorstores/__init__.py b/GenAIComps/vectorstores/__init__.py new file mode 100644 index 0000000000..28f108cb63 --- /dev/null +++ b/GenAIComps/vectorstores/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/chroma/README.md b/GenAIComps/vectorstores/langchain/chroma/README.md similarity index 100% rename from GenAIComps/vectordb/langchain/chroma/README.md rename to GenAIComps/vectorstores/langchain/chroma/README.md diff --git a/GenAIComps/vectorstores/langchain/chroma/__init__.py b/GenAIComps/vectorstores/langchain/chroma/__init__.py new file mode 100644 index 0000000000..28f108cb63 --- /dev/null +++ b/GenAIComps/vectorstores/langchain/chroma/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/qdrant/README.md b/GenAIComps/vectorstores/langchain/qdrant/README.md similarity index 100% rename from GenAIComps/vectordb/langchain/qdrant/README.md rename to GenAIComps/vectorstores/langchain/qdrant/README.md diff --git a/GenAIComps/vectorstores/langchain/qdrant/__init__.py b/GenAIComps/vectorstores/langchain/qdrant/__init__.py new file mode 100644 index 0000000000..28f108cb63 --- /dev/null +++ b/GenAIComps/vectorstores/langchain/qdrant/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/redis/README.md b/GenAIComps/vectorstores/langchain/redis/README.md similarity index 100% rename from GenAIComps/vectordb/langchain/redis/README.md rename to GenAIComps/vectorstores/langchain/redis/README.md diff --git a/GenAIComps/vectorstores/langchain/redis/__init__.py b/GenAIComps/vectorstores/langchain/redis/__init__.py new file mode 100644 index 0000000000..28f108cb63 --- /dev/null +++ b/GenAIComps/vectorstores/langchain/redis/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/GenAIComps/vectordb/langchain/redis/config.py b/GenAIComps/vectorstores/langchain/redis/config.py similarity index 100% rename from GenAIComps/vectordb/langchain/redis/config.py rename to GenAIComps/vectorstores/langchain/redis/config.py diff --git a/GenAIComps/vectordb/langchain/redis/docker-compose-redis.yml b/GenAIComps/vectorstores/langchain/redis/docker-compose-redis.yml similarity index 100% rename from GenAIComps/vectordb/langchain/redis/docker-compose-redis.yml rename to GenAIComps/vectorstores/langchain/redis/docker-compose-redis.yml diff --git a/GenAIComps/vectordb/langchain/redis/schema.yml b/GenAIComps/vectorstores/langchain/redis/schema.yml similarity index 100% rename from GenAIComps/vectordb/langchain/redis/schema.yml rename to GenAIComps/vectorstores/langchain/redis/schema.yml diff --git a/GenAIComps/vectordb/langchain/redis/schema_dim_1024.yml b/GenAIComps/vectorstores/langchain/redis/schema_dim_1024.yml similarity index 100% rename from GenAIComps/vectordb/langchain/redis/schema_dim_1024.yml rename to GenAIComps/vectorstores/langchain/redis/schema_dim_1024.yml diff --git a/GenAIComps/vectordb/langchain/redis/schema_dim_768.yml b/GenAIComps/vectorstores/langchain/redis/schema_dim_768.yml similarity index 100% rename from GenAIComps/vectordb/langchain/redis/schema_dim_768.yml rename to GenAIComps/vectorstores/langchain/redis/schema_dim_768.yml diff --git a/GenAIComps/vectordb/langchain/redis/schema_lcdocs_dim_768.yml b/GenAIComps/vectorstores/langchain/redis/schema_lcdocs_dim_768.yml similarity index 100% rename from GenAIComps/vectordb/langchain/redis/schema_lcdocs_dim_768.yml rename to GenAIComps/vectorstores/langchain/redis/schema_lcdocs_dim_768.yml