From d7a1d8d2dc375425aa0a8892b9d24a42b05dd2c4 Mon Sep 17 00:00:00 2001 From: Letong Han <106566639+letonghan@users.noreply.github.com> Date: Mon, 30 Oct 2023 14:01:25 +0800 Subject: [PATCH] [NeuralChat] Integrate photoai backend into restful API (#478) --- .../unitTest/run_unit_test_neuralchat.sh | 4 + .../neural_chat/examples/photo_ai/README.md | 8 + .../examples/photo_ai/backend/README.md | 99 +++ .../examples/photo_ai/backend/photoai.py | 29 + .../examples/photo_ai/backend/photoai.yaml | 53 ++ .../examples/photo_ai/backend/run.sh | 38 + .../neural_chat/examples/photo_ai/frontend | 0 .../neural_chat/pipeline/plugins/ner/ner.py | 19 +- .../pipeline/plugins/ner/ner_int.py | 5 +- .../neural_chat/requirements.txt | 3 + .../neural_chat/requirements_cpu.txt | 3 + .../neural_chat/requirements_hpu.txt | 5 + .../neural_chat/server/README.md | 21 +- .../neural_chat/server/config/neuralchat.yaml | 4 +- .../neural_chat/server/restful/api.py | 4 +- .../neural_chat/server/restful/photoai_api.py | 491 +++++++++++ .../server/restful/photoai_services.py | 778 ++++++++++++++++++ .../server/restful/photoai_utils.py | 151 ++++ requirements.txt | 2 +- 19 files changed, 1702 insertions(+), 15 deletions(-) create mode 100644 intel_extension_for_transformers/neural_chat/examples/photo_ai/README.md create mode 100644 intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/README.md create mode 100644 intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.py create mode 100644 intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.yaml create mode 100644 intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/run.sh create mode 100644 intel_extension_for_transformers/neural_chat/examples/photo_ai/frontend create mode 100644 intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py create mode 100644 intel_extension_for_transformers/neural_chat/server/restful/photoai_services.py create mode 100644 intel_extension_for_transformers/neural_chat/server/restful/photoai_utils.py diff --git a/.github/workflows/script/unitTest/run_unit_test_neuralchat.sh b/.github/workflows/script/unitTest/run_unit_test_neuralchat.sh index 36ad4125b29..b93bcc11fe2 100644 --- a/.github/workflows/script/unitTest/run_unit_test_neuralchat.sh +++ b/.github/workflows/script/unitTest/run_unit_test_neuralchat.sh @@ -73,6 +73,10 @@ function main() { apt-get update apt-get install ffmpeg -y apt-get install lsof + apt-get install libgl1 + apt-get install -y libgl1-mesa-glx + apt-get install -y libgl1-mesa-dev + apt-get install libsm6 libxext6 -y wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb dpkg -i libssl1.1_1.1.1f-1ubuntu2.19_amd64.deb python -m pip install --upgrade --force-reinstall torch diff --git a/intel_extension_for_transformers/neural_chat/examples/photo_ai/README.md b/intel_extension_for_transformers/neural_chat/examples/photo_ai/README.md new file mode 100644 index 00000000000..3422fe18a35 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/examples/photo_ai/README.md @@ -0,0 +1,8 @@ +Welcome to Photo AI! This example introduces how to deploy the Text Chatbot system and guides you through setting up both the backend and frontend components. You can deploy this chatbot on various platforms, including Intel XEON Scalable Processors, Habana's Gaudi processors (HPU), Intel Data Center GPU and Client GPU, Nvidia Data Center GPU and Client GPU. + +| Section | Link | +| ---------------------| --------------------------| +| Backend Setup | [Backend README](./backend/README.md) | +| Frontend Setup | [Frontend README](./frontend/README.md) | + + diff --git a/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/README.md b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/README.md new file mode 100644 index 00000000000..e96af4f4b95 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/README.md @@ -0,0 +1,99 @@ +This README is intended to guide you through setting up the backend for a Photo AI demo using the NeuralChat framework. You can deploy it on various platforms, including Intel XEON Scalable Processors, Habana's Gaudi processors (HPU), Intel Data Center GPU and Client GPU, Nvidia Data Center GPU and Client GPU. + + +# Setup Environment + + +## Setup Conda + +First, you need to install and configure the Conda environment: + +```shell +# Download and install Miniconda +wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh +bash `Miniconda*.sh` +source ~/.bashrc +``` + +## Install numactl + +Next, install the numactl library: + +```shell +sudo apt install numactl +``` + +## Install Python dependencies + +Install the following Python dependencies using Conda: + +```shell +conda install astunparse ninja pyyaml mkl mkl-include setuptools cmake cffi typing_extensions future six requests dataclasses -y +conda install jemalloc gperftools -c conda-forge -y +conda install git-lfs -y +# install libGL.so.1 for opencv +sudo apt-get update +sudo apt-get install -y libgl1-mesa-glx +``` + +Install other dependencies using pip: + +```bash +pip install -r ../../../requirements.txt +``` + +## Install Models +```shell +git-lfs install +# download llama-2 model for NER plugin +git clone https://huggingface.co/meta-llama/Llama-2-7b-chat-hf +# download spacy model for NER post process +python -m spacy download en_core_web_lg +``` + + +# Setup Database +## Install MySQL +```shell +# install mysql +sudo apt-get install mysql-server +# start mysql server +systemctl status mysql +``` + +## Create Tables +```shell +cd ../../../utils/database/ +# login mysql +mysql +source ./init_db_ai_photos.sql +``` + +## Create Image Database +```shell +mkdir /home/nfs_images +export IMAGE_SERVER_IP="your.server.ip" +``` + +# Configurate photoai.yaml + +You can customize the configuration file 'photoai.yaml' to match your environment setup. Here's a table to help you understand the configurable options: + +| Item | Value | +| ------------------- | ---------------------------------------| +| host | 127.0.0.1 | +| port | 9000 | +| model_name_or_path | "./Llama-2-7b-chat-hf" | +| device | "auto" | +| asr.enable | true | +| tts.enable | true | +| ner.enable | true | +| tasks_list | ['voicechat', 'photoai'] | + + +# Run the PhotoAI server +To start the PhotoAI server, use the following command: + +```shell +nohup bash run.sh & +``` diff --git a/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.py b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.py new file mode 100644 index 00000000000..b4e2c17db43 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from intel_extension_for_transformers.neural_chat import NeuralChatServerExecutor + +def main(): + server_executor = NeuralChatServerExecutor() + server_executor( + config_file="./photoai.yaml", + log_file="./photoai.log") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.yaml b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.yaml new file mode 100644 index 00000000000..1f4ca940152 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/photoai.yaml @@ -0,0 +1,53 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the parameter configuration file for NeuralChat Serving. + +################################################################################# +# SERVER SETTING # +################################################################################# +host: 0.0.0.0 +port: 9000 + +model_name_or_path: "meta-llama/Llama-2-7b-chat-hf" +device: "auto" + +asr: + enable: true + args: + device: "cpu" + model_name_or_path: "openai/whisper-small" + bf16: false + +tts: + enable: true + args: + device: "cpu" + voice: "default" + stream_mode: true + output_audio_path: "./output_audio" + +ner: + enable: true + args: + device: "cpu" + model_path: "./Llama-2-7b-chat-hf" + spacy_model: "en_core_web_lg" + bf16: true + + +tasks_list: ['voicechat', 'photoai'] diff --git a/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/run.sh b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/run.sh new file mode 100644 index 00000000000..1b9858f0878 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/examples/photo_ai/backend/run.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Kill the exist and re-run +ps -ef |grep 'photoai' |awk '{print $2}' |xargs kill -9 + +# KMP +export KMP_BLOCKTIME=1 +export KMP_SETTINGS=1 +export KMP_AFFINITY=granularity=fine,compact,1,0 + +# OMP +export OMP_NUM_THREADS=56 +export LD_PRELOAD=${CONDA_PREFIX}/lib/libiomp5.so + +# tc malloc +export LD_PRELOAD=${LD_PRELOAD}:${CONDA_PREFIX}/lib/libtcmalloc.so + +# environment variables +export MYSQL_PASSWORD="root" +export MYSQL_HOST="127.0.0.1" +export MYSQL_DB="ai_photos" + +numactl -l -C 0-55 python -m photoai 2>&1 | tee run.log diff --git a/intel_extension_for_transformers/neural_chat/examples/photo_ai/frontend b/intel_extension_for_transformers/neural_chat/examples/photo_ai/frontend new file mode 100644 index 00000000000..e69de29bb2d diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py index d519f177562..5d60864f73e 100644 --- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py +++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner.py @@ -26,7 +26,6 @@ TextIteratorStreamer, AutoConfig, ) -import intel_extension_for_pytorch as intel_ipex from .utils.utils import ( enforce_stop_tokens, get_current_time @@ -41,11 +40,17 @@ class NamedEntityRecognition(): Set bf16=True if you want to inference with bf16 model. """ - def __init__(self, model_path="./Llama-2-7b-chat-hf/", spacy_model="en_core_web_lg", bf16: bool=False) -> None: + def __init__(self, + model_path="meta-llama/Llama-2-7b-chat-hf", + spacy_model="en_core_web_lg", + bf16: bool=False, + device="cpu") -> None: # initialize tokenizer and models self.nlp = spacy.load(spacy_model) config = AutoConfig.from_pretrained(model_path, trust_remote_code=True) config.init_device = 'cuda:0' if torch.cuda.is_available() else "cpu" + self.device = device + self.bf16 = False self.tokenizer = AutoTokenizer.from_pretrained( model_path, use_fast=False if (re.search("llama", model_path, re.IGNORECASE) @@ -59,9 +64,15 @@ def __init__(self, model_path="./Llama-2-7b-chat-hf/", spacy_model="en_core_web_ device_map="auto", trust_remote_code=True ) - self.bf16 = bf16 + # make sure ipex is available on current server + try: + import intel_extension_for_pytorch as intel_ipex + self.is_ipex_available = True + except ImportError: + self.is_ipex_available = False # optimize model with ipex if bf16 - if bf16: + if bf16 and self.is_ipex_available: + self.bf16 = bf16 self.model = intel_ipex.optimize( self.model.eval(), dtype=torch.bfloat16, diff --git a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner_int.py b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner_int.py index ab7921d7862..002cf0f4af7 100644 --- a/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner_int.py +++ b/intel_extension_for_transformers/neural_chat/pipeline/plugins/ner/ner_int.py @@ -38,10 +38,11 @@ class NamedEntityRecognitionINT(): """ def __init__(self, - model_path="/home/tme/Llama-2-7b-chat-hf/", + model_path="meta-llama/Llama-2-7b-chat-hf", spacy_model="en_core_web_lg", compute_dtype="fp32", - weight_dtype="int8") -> None: + weight_dtype="int8", + device="cpu") -> None: self.nlp = spacy.load(spacy_model) config = WeightOnlyQuantConfig(compute_dtype=compute_dtype, weight_dtype=weight_dtype) self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) diff --git a/intel_extension_for_transformers/neural_chat/requirements.txt b/intel_extension_for_transformers/neural_chat/requirements.txt index 1144b5bb05c..3b313622206 100644 --- a/intel_extension_for_transformers/neural_chat/requirements.txt +++ b/intel_extension_for_transformers/neural_chat/requirements.txt @@ -39,6 +39,9 @@ tiktoken==0.4.0 lm_eval accelerate cchardet +pymysql +deepface +exifread spacy neural-compressor==2.3.1 pymysql diff --git a/intel_extension_for_transformers/neural_chat/requirements_cpu.txt b/intel_extension_for_transformers/neural_chat/requirements_cpu.txt index cb6b526a0d2..996140a9f12 100644 --- a/intel_extension_for_transformers/neural_chat/requirements_cpu.txt +++ b/intel_extension_for_transformers/neural_chat/requirements_cpu.txt @@ -42,3 +42,6 @@ torchaudio==2.1.0 spacy neural-compressor==2.3.1 pymysql +deepface +exifread + diff --git a/intel_extension_for_transformers/neural_chat/requirements_hpu.txt b/intel_extension_for_transformers/neural_chat/requirements_hpu.txt index dc95bb6a447..a5262435694 100644 --- a/intel_extension_for_transformers/neural_chat/requirements_hpu.txt +++ b/intel_extension_for_transformers/neural_chat/requirements_hpu.txt @@ -35,3 +35,8 @@ tiktoken==0.4.0 lm_eval spacy neural-compressor==2.3.1 +intel_extension_for_pytorch +pymysql +deepface +exifread + diff --git a/intel_extension_for_transformers/neural_chat/server/README.md b/intel_extension_for_transformers/neural_chat/server/README.md index afad6950e06..4e4105e8c7b 100644 --- a/intel_extension_for_transformers/neural_chat/server/README.md +++ b/intel_extension_for_transformers/neural_chat/server/README.md @@ -10,7 +10,7 @@ neuralchat_server help ### Start the server - Command Line (Recommended) -NeuralChat provides a default chatbot configuration in `./conf/neuralchat.yaml`. User could customize the behavior of this chatbot by modifying the value of these fields in the configuration file to specify which LLM model and plugins to be used. +NeuralChat provides a default chatbot configuration in `./config/neuralchat.yaml`. User could customize the behavior of this chatbot by modifying the value of these fields in the configuration file to specify which LLM model and plugins to be used. | Fields | Sub Fields | Default Values | Possible Values | | ------------------------- | ------------------------ | --------------------------------------- | --------------------------------- | @@ -42,18 +42,27 @@ NeuralChat provides a default chatbot configuration in `./conf/neuralchat.yaml`. | | args.process | false | true, false | | cache | enable | false | true, false | | | args.config_dir | "../../pipeline/plugins/caching/cache_config.yaml" | A valid directory path | -| | args.embedding_model_dir | "hkunlp/instructor-large" | A valid directory path | +| | args.embedding_model_dir | "hkunlp/instructor-large" | A valid directory path | | safety_checker | enable | false | true, false | -| tasks_list | | ['textchat', 'retrieval'] | List of task names, including 'textchat', 'voicechat', 'retrieval', 'text2image', 'finetune' | +| ner | enable | false | true, false | +| | args.model_path | "meta-llama/Llama-2-7b-chat-hf" | A valid directory path of llm model | +| | args.spacy_model | "en_core_web_lg" | A valid name of downloaded spacy model | +| | args.bf16 | false | true, false | +| ner_int | enable | false | true, false | +| | args.model_path | "meta-llama/Llama-2-7b-chat-hf" | A valid directory path of llm model | +| | args.spacy_model | "en_core_web_lg" | A valid name of downloaded spacy model | +| | args.compute_dtype | "fp32" | "fp32", "int8" | +| | args.weight_dtype | "int8" | "int8", "int4" | +| tasks_list | | ['textchat', 'retrieval'] | List of task names, including 'textchat', 'voicechat', 'retrieval', 'text2image', 'finetune', 'photoai' | -First set the service-related configuration parameters, similar to `./conf/neuralchat.yaml`. Set `tasks_list`, which represents the supported tasks included in the service to be started. +First set the service-related configuration parameters, similar to `./config/neuralchat.yaml`. Set `tasks_list`, which represents the supported tasks included in the service to be started. **Note:** If the service can be started normally in the container, but the client access IP is unreachable, you can try to replace the `host` address in the configuration file with the local IP address. Then start the service: ```bash -neuralchat_server start --config_file ./server/conf/neuralchat.yaml +neuralchat_server start --config_file ./server/config/neuralchat.yaml ``` - Python API @@ -62,7 +71,7 @@ from neuralchat.server.neuralchat_server import NeuralChatServerExecutor server_executor = NeuralChatServerExecutor() server_executor( - config_file="./conf/neuralchat.yaml", + config_file="./config/neuralchat.yaml", log_file="./log/neuralchat.log") ``` diff --git a/intel_extension_for_transformers/neural_chat/server/config/neuralchat.yaml b/intel_extension_for_transformers/neural_chat/server/config/neuralchat.yaml index a4e952df9b3..cfb07f273a6 100644 --- a/intel_extension_for_transformers/neural_chat/server/config/neuralchat.yaml +++ b/intel_extension_for_transformers/neural_chat/server/config/neuralchat.yaml @@ -77,6 +77,7 @@ safety_checker: ner: enable: false args: + device: "cpu" model_path: "meta-llama/Llama-2-7b-chat-hf" spacy_model: "en_core_web_lg" bf16: False @@ -84,10 +85,11 @@ ner: ner_int: enable: false args: + device: "cpu" model_path: "meta-llama/Llama-2-7b-chat-hf" spacy_model: "en_core_web_lg" compute_dtype: "fp32" weight_dtype: "int8" -# task choices = ['textchat', 'voicechat', 'retrieval', 'text2image', 'finetune'] +# task choices = ['textchat', 'voicechat', 'retrieval', 'text2image', 'finetune'， 'photoai'] tasks_list: ['textchat', 'retrieval'] diff --git a/intel_extension_for_transformers/neural_chat/server/restful/api.py b/intel_extension_for_transformers/neural_chat/server/restful/api.py index e7b40c48714..38940ed43e7 100644 --- a/intel_extension_for_transformers/neural_chat/server/restful/api.py +++ b/intel_extension_for_transformers/neural_chat/server/restful/api.py @@ -44,6 +44,7 @@ from .retrieval_api import router as retrieval_router from .text2image_api import router as text2image_router from .finetune_api import router as finetune_router +from .photoai_api import router as photoai_router _router = APIRouter() @@ -53,7 +54,8 @@ 'voicechat': voicechat_router, 'retrieval': retrieval_router, 'text2image': text2image_router, - 'finetune': finetune_router + 'finetune': finetune_router, + 'photoai': photoai_router } def setup_router(api_list, chatbot): diff --git a/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py b/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py new file mode 100644 index 00000000000..31ec69a2722 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/server/restful/photoai_api.py @@ -0,0 +1,491 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +import base64 +import asyncio +from typing import Optional +from fastapi.routing import APIRouter +from fastapi import APIRouter +from ...cli.log import logger +from ...config import GenerationConfig +from ...utils.database.mysqldb import MysqlDb +from fastapi import Request, BackgroundTasks, status, UploadFile, File +from fastapi.responses import JSONResponse, Response, StreamingResponse +from .photoai_services import * +from .photoai_utils import ( + byte64_to_image, + image_to_byte64, + generate_random_name +) +from .voicechat_api import ( + handle_talkingbot_asr as talkingbot_asr, + create_speaker_embedding as talkingbot_embd +) +from intel_extension_for_transformers.neural_chat.pipeline.plugins.ner.ner import NamedEntityRecognition + + +class PhotoAIAPIRouter(APIRouter): + + def __init__(self) -> None: + super().__init__() + self.chatbot = None + + def set_chatbot(self, chatbot) -> None: + self.chatbot = chatbot + + def get_chatbot(self): + if self.chatbot is None: + logger.error("Chatbot instance is not found.") + raise RuntimeError("Chatbot instance has not been set.") + return self.chatbot + + async def handle_voice_chat_request(self, prompt: str, audio_output_path: Optional[str]=None) -> str: + chatbot = self.get_chatbot() + try: + config = GenerationConfig(audio_output_path=audio_output_path) + result = chatbot.chat_stream(query=prompt, config=config) + def audio_file_generate(result): + for path in result: + with open(path,mode="rb") as file: + bytes = file.read() + data = base64.b64encode(bytes) + yield f"data: {data}\n\n" + yield f"data: [DONE]\n\n" + return StreamingResponse(audio_file_generate(result), media_type="text/event-stream") + except Exception as e: + raise Exception(e) + + +router = PhotoAIAPIRouter() + + +@router.post("/v1/aiphotos/uploadImages") +async def handle_ai_photos_upload_images(request: Request, background_tasks: BackgroundTasks): + user_id = request.client.host + logger.info(f' user id is: {user_id}') + res = check_user_ip(user_id) + logger.info(" "+str(res)) + + params = await request.json() + image_list = params['image_list'] + + image_path = IMAGE_ROOT_PATH+'/user'+str(user_id) + os.makedirs(image_path, exist_ok=True) + mysql_db = MysqlDb() + + return_list = [] + image_obj_list = [] + + for image in image_list: + img_b64 = image['imgSrc'].split(',')[1] + + img_obj = byte64_to_image(str.encode(img_b64)) + tmp_name = generate_random_name() + img_name = tmp_name+'.jpg' + img_path = image_path+'/'+ img_name + # save exif info from origin image + exif = img_obj.info.get('exif', b"") + + # save image info into db + empty_tags = '{}' + insert_sql = f"INSERT INTO image_info VALUES(null, '{user_id}', '{img_path}', null, '', \ + 'None', 'None', 'None', 'None', true, '{empty_tags}', 'processing', 'active');" + try: + with mysql_db.transaction(): + mysql_db.insert(sql=insert_sql, params=None) + except Exception as e: + logger.error(" "+str(e)) + return JSONResponse(content=f'Database insert failed for image {img_path}', status_code=500) + + # get image id + fetch_sql = f"SELECT * FROM image_info WHERE image_path='{img_path}';" + try: + result = mysql_db.fetch_one(sql=fetch_sql) + except Exception as e: + logger.info(" "+str(e)) + return JSONResponse(content=f'Database select failed for image {img_path}', status_code=500) + img_id = result['image_id'] + frontend_path = format_image_path(user_id, img_name) + item = {'img_id': img_id, 'img_path': frontend_path} + logger.info(f' Image id is {img_id}, image path is {frontend_path}') + return_list.append(item) + obj_item = {"img_obj": img_obj, "exif": exif, "img_path": img_path, "img_id": img_id} + image_obj_list.append(obj_item) + mysql_db._close() + background_tasks.add_task(process_images_in_background, user_id, image_obj_list) + logger.info(' Finish image uploading and saving') + return return_list + + +@router.post("/v1/aiphotos/getAllImages") +def handle_ai_photos_get_all_images(request: Request): + user_id = request.client.host + logger.info(f' user id is: {user_id}') + check_user_ip(user_id) + origin = request.headers.get("Origin") + logger.info(f' origin: {origin}') + + try: + result_list = [] + mysql_db = MysqlDb() + image_list = mysql_db.fetch_all( + sql=f'''SELECT image_id, image_path FROM image_info + WHERE user_id="{user_id}" AND exist_status="active";''') + for image in image_list: + image_name = image['image_path'].split('/')[-1] + result_list.append({"image_id": image['image_id'], "image_path": format_image_path(user_id, image_name)}) + except Exception as e: + return JSONResponse(content=e, status_code=500) + else: + mysql_db._close() + logger.info(f' all images of user {user_id}: {result_list}') + return result_list + + +@router.post("/v1/aiphotos/getTypeList") +def handle_ai_photos_get_type_list(request: Request): + user_id = request.client.host + logger.info(f' user id is: {user_id}') + check_user_ip(user_id) + + type_result_dict = {"type_list": {}, "prompt_list": {}} + + # address + address_result = get_type_obj_from_attr('address', user_id) + type_result_dict['type_list']['address'] = address_result + + # time + time_result = get_type_obj_from_attr('time', user_id) + type_result_dict['type_list']['time'] = time_result + + # person + person_result = get_face_list_by_user_id(user_id) + type_result_dict['type_list']['person'] = person_result + + # other + other_time_result = get_images_by_type(user_id, type="time", subtype="None") + other_add_result = get_images_by_type(user_id, type="address", subtype="default") + logger.info(f' other time result: {other_time_result}') + logger.info(f' other address result: {other_add_result}') + for time_res in other_time_result: + if time_res in other_add_result: + continue + other_add_result.append(time_res) + logger.info(f' final other result: {other_add_result}') + type_result_dict['type_list']['other'] = other_add_result + + # prompt list + address_list = get_address_list(user_id) + type_result_dict['prompt_list']['address'] = address_list + type_result_dict['prompt_list']['time'] = list(time_result.keys()) + type_result_dict['prompt_list']['person'] = list(person_result.keys()) + + # process status + type_result_dict["process_status"] = get_process_status(user_id) + return type_result_dict + + +@router.post("/v1/aiphotos/getImageByType") +async def handle_ai_photos_get_image_by_type(request: Request): + user_id = request.client.host + logger.info(f' user id is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + type = params['type'] + subtype = params['subtype'] + + try: + result = get_images_by_type(user_id, type, subtype) + except Exception as e: + return Response(content=str(e), status_code=status.HTTP_400_BAD_REQUEST) + return result + + +@router.post("/v1/aiphotos/getImageDetail") +async def handle_ai_photos_get_image_detail(request: Request): + user_id = request.client.host + logger.info(f' user id is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + image_id = params['image_id'] + logger.info(f' Getting image detail of image {image_id} by user {user_id}') + + try: + mysql_db = MysqlDb() + image_info = mysql_db.fetch_one( + sql=f'''SELECT * FROM image_info WHERE + image_id={image_id} AND user_id="{user_id}" AND exist_status="active";''', + params=None) + except Exception as e: + logger.error(" "+str(e)) + return JSONResponse(content=f'Exception {e} occurred when selecting image {image_id} from MySQL.') + finally: + mysql_db._close() + + if image_info: + image_detail = format_image_info(image_info) + logger.info(f' Image detail of image {image_id} is: {image_detail}') + return image_detail + else: + return JSONResponse( + content=f"No image id: {image_id} for user {user_id}", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + +@router.post("/v1/aiphotos/deleteImage") +async def handel_ai_photos_delete_Image(request: Request): + params = await request.json() + image_id = params['image_id'] + logger.info(f' Getting image detail of image {image_id}') + + user_id = request.client.host + logger.info(f' user id is: {user_id}') + check_user_ip(user_id) + + try: + delete_single_image(user_id, image_id) + except Exception as e: + logger.error(" "+str(e)) + return Response(content=e, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) + + logger.info(f' Image {image_id} successfully deleted.') + return "Succeed" + + +@router.post("/v1/aiphotos/deleteUser") +def handle_ai_photos_delete_user(request: Request): + user_id = request.client.host + logger.info(f' user ip is: {user_id}') + check_user_ip(user_id) + + try: + delete_user_infos(user_id) + except Exception as e: + logger.error(" "+str(e)) + raise Exception(e) + + return "Succeed" + + +@router.post("/v1/aiphotos/updateLabel") +async def handle_ai_photos_update_label(request: Request): + # check request user + user_id = request.client.host + logger.info(f' user id is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + label_list = params['label_list'] + + try: + mysql_db = MysqlDb() + for label_obj in label_list: + label = label_obj['label'] + label_from = label_obj['from'] + label_to = label_obj['to'] + if label == 'person': + with mysql_db.transaction(): + mysql_db.update( + sql=f'''UPDATE face_info SET face_tag="{label_to}" + WHERE face_tag="{label_from}"''', + params=None) + mysql_db.update( + sql=f"""UPDATE image_face SET face_tag='{label_to}' + WHERE user_id='{user_id}' and face_tag='{label_from}';""", + params=None) + continue + if label == 'address': + update_sql = f"""UPDATE image_info SET address='{label_to}' + WHERE user_id='{user_id}' and address='{label_from}';""" + elif label == 'time': + update_sql = f"""UPDATE image_info SET captured_time='{label_to}' + WHERE user_id='{user_id}' and captured_time='{label_from}';""" + else: + return JSONResponse( + content=f"Illegal label name: {label}", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + with mysql_db.transaction(): + mysql_db.update(sql=update_sql, params=None) + logger.info(f' Label {label} updated from {label_from} to {label_to}.') + except Exception as e: + return JSONResponse(content=e, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) + else: + mysql_db._close() + logger.info(' Image Labels updated successfully.') + + return "Succeed" + + +@router.post("/v1/aiphotos/updateTags") +async def handel_ai_photos_update_tags(request: Request): + # check request user + user_id = request.client.host + logger.info(f' user ip is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + image_list = params['image_list'] + + try: + for image in image_list: + update_image_tags(image) + + except Exception as e: + logger.error(" "+str(e)) + return Response(content=str(e), status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) + else: + logger.info(' Image tags updated successfully.') + + return "Succeed" + + +@router.post("/v1/aiphotos/updateCaption") +async def handel_ai_photos_update_caption(request: Request): + # check request user + user_id = request.client.host + logger.info(f' user ip is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + image_list = params['image_list'] + + for image in image_list: + try: + update_image_attr(image, 'caption') + except Exception as e: + logger.error(" "+str(e)) + return Response(content=str(e), status_code=status.HTTP_400_BAD_REQUEST) + + return "Succeed" + + +@router.post("/v1/aiphotos/chatWithImage") +async def handle_ai_photos_chat_to_image(request: Request): + user_id = request.client.host + logger.info(f' user ip is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + query = params['query'] + logger.info(f' generating chat to image for user {user_id} with query: {query}') + + try: + start_time = time.time() + ner_obj = NamedEntityRecognition(model_path="mosaicml/mpt-7b-chat", bf16=False) + result = ner_obj.inference(query=query) + end_time = time.time() + print(" NER inference cost {} seconds.".format(end_time - start_time)) + except Exception as e: + logger.error(" "+str(e)) + raise Exception(e) + logger.info(f' NER result: {result}') + + try: + result_image_list = get_image_list_by_ner_query(result, user_id, query) + except Exception as e: + logger.error(" "+str(e)) + raise Exception(e) + return "No query result" if result_image_list==[] else result_image_list + + +@router.post("/v1/aiphotos/image2Image") +async def handle_image_to_image(request: Request): + user_id = request.client.host + logger.info(f' user ip is: {user_id}') + check_user_ip(user_id) + + params = await request.json() + query = params['query'] + image_list = params['ImageList'] + logger.info(f' user: {user_id}, image to image query command: {query}') + + generated_images = [] + steps=25 + strength=0.75 + seed=42 + guidance_scale=7.5 + for img_info in image_list: + img_id = img_info["imgId"] + img_path = img_info["imgSrc"] + userid, img_name = img_path.split('/')[-2], img_path.split('/')[-1] + image_path = IMAGE_ROOT_PATH+'/'+userid+'/'+img_name + logger.info(f' current image id: {img_id}, image path: {image_path}') + + img_b64 = image_to_byte64(image_path) + data = {"source_img": img_b64.decode(), "prompt": query, "steps": steps, + "guidance_scale": guidance_scale, "seed": seed, "strength": strength, + "token": "intel_sd_bf16_112233"} + start_time = time.time() + img_str = stable_defusion_func(data) + logger.info(" elapsed time: ", str(time.time() - start_time)) + generated_images.append({"imgId": img_id, "imgSrc": "data:image/jpeg;base64,"+img_str}) + + return generated_images + + +# ================== For streaming ================== +@router.post("/v1/aiphotos/talkingbot/asr") +async def handle_talkingbot_asr(file: UploadFile = File(...)): + keyword_list = { + "intel": "Intel", + " i ": " I ", + "shanghai": "Shanghai", + "china": "China", + "beijing": "Beijing" + } + # get asr result from neural chat + asr_result = talkingbot_asr(file=file) + res = await asyncio.gather(asr_result) + res = res[0]['asr_result'] + # substitude keywords manually + result_list = [] + words = res.split(" ") + for word in words: + if word in keyword_list.keys(): + word = keyword_list[word] + result_list.append(word) + asr_result = ' '.join(result_list) + final_result = asr_result[0].upper() + asr_result[1:] + '.' + + return {"asr_result": final_result} + + +@router.post("/v1/aiphotos/talkingbot/create_embed") +async def handle_talkingbot_create_embedding(file: UploadFile = File(...)): + result = talkingbot_embd(file=file) + res = await asyncio.gather(result) + final_result = res['spk_id'] + return {"voice_id": final_result} + + +@router.post("/v1/aiphotos/talkingbot/llm_tts") +async def handle_talkingbot_llm_tts(request: Request): + data = await request.json() + text = data["text"] + voice = data["voice"] + knowledge_id = data["knowledge_id"] + audio_output_path = data["audio_output_path"] if "audio_output_path" in data else "output_audio" + + logger.info(f'Received prompt: {text}, and use voice: {voice} knowledge_id: {knowledge_id}') + + return await router.handle_voice_chat_request(text, audio_output_path) + diff --git a/intel_extension_for_transformers/neural_chat/server/restful/photoai_services.py b/intel_extension_for_transformers/neural_chat/server/restful/photoai_services.py new file mode 100644 index 00000000000..7a67d8613da --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/server/restful/photoai_services.py @@ -0,0 +1,778 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import json +import requests +import datetime +from deepface import DeepFace +from typing import List, Dict +from .photoai_utils import ( + find_GPS_image, + get_address_from_gps, + generate_caption, + transfer_xywh +) +from ...cli.log import logger +from ...utils.database.mysqldb import MysqlDb +from datetime import timedelta, timezone + + +IMAGE_ROOT_PATH = "/home/nfs_images" + + +def check_user_ip(user_ip: str) -> bool: + mysql_db = MysqlDb() + user_list = mysql_db.fetch_one(sql=f'select * from user_info where user_id = "{user_ip}";') + logger.info(f'[Check IP] user list: {user_list}') + if user_list == None: + logger.info(f'[Check IP] no current user, add into db.') + cur_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + with mysql_db.transaction(): + mysql_db.insert(sql=f"insert into user_info values('{user_ip}', '{cur_time}', null, 1);", params=None) + mysql_db._close() + return True + + +def check_image_status(image_id: str): + mysql_db = MysqlDb() + image = mysql_db.fetch_one( + sql=f'select * from image_info where image_id="{image_id}" and exist_status="active"', + params=None + ) + if image==None: + raise ValueError(f'No image {image_id} saved in MySQL DB.') + mysql_db._close() + return image + + +def update_image_tags(image): + image_id = image['image_id'] + tags = image['tags'] + image_info = check_image_status(image_id) + update_sql = 'UPDATE image_info SET ' + update_sql_list = [] + old_tags = eval(image_info['other_tags']) + logger.info(f'[Update Tags] old_tags: {old_tags}') + tag_name_list = [] + for key, value in tags.items(): + if key == 'time' and value != image_info['captured_time']: + update_sql_list.append(f' captured_time="{value}" ') + tag_name_list.append('time') + elif key == 'latitude' and value != image_info['latitude']: + update_sql_list.append(f' latitude="{value}" ') + tag_name_list.append('latitude') + elif key == 'longitude' and value != image_info['longitude']: + update_sql_list.append(f' longitude="{value}" ') + tag_name_list.append('longitude') + elif key == 'altitude' and value != image_info['altitude']: + update_sql_list.append(f' altitude="{value}" ') + tag_name_list.append('altitude') + elif key == 'location' and value != image_info['address']: + update_sql_list.append(f' address="{value}" ') + tag_name_list.append('location') + + for tag_name in tag_name_list: + tags.pop(tag_name) + old_tags.update(tags) + new_tags = str(old_tags) + update_sql_list.append(f' other_tags="{new_tags}" ') + update_sql_tmp = ','.join(update_sql_list) + final_sql = update_sql+update_sql_tmp+f' where image_id={image_id}' + logger.info(f'[Update Tags] update sql: {final_sql}') + mysql_db = MysqlDb() + with mysql_db.transaction(): + mysql_db.update(sql=final_sql, params=None) + mysql_db._close() + + +def update_image_attr(image, attr): + image_id = image['image_id'] + check_image_status(image_id) + + new_attr = image[attr] + try: + mysql_db = MysqlDb() + if attr=='checked': + new_checked = 1 if new_attr else 0 + with mysql_db.transaction(): + mysql_db.update( + sql=f"UPDATE image_info SET {attr}={new_checked} WHERE image_id={image_id}", + params=None + ) + else: + with mysql_db.transaction(): + mysql_db.update( + sql=f'UPDATE image_info SET {attr}="{new_attr}" WHERE image_id={image_id}', + params=None + ) + except Exception as e: + logger.error(e) + else: + mysql_db._close() + logger.info(f'Image {attr} updated successfully.') + + +def format_image_path(user_id: str, image_name: str) -> str: + server_ip = os.getenv("IMAGE_SERVER_IP") + if not server_ip: + raise Exception("Please configure SERVER IP to environment variables.") + image_path = "https://"+server_ip+"/ai_photos/user"+user_id+'/'+image_name + return image_path + + +def format_image_info(image_info: dict) -> dict: + image_item = {} + image_item['image_id'] = image_info['image_id'] + image_item['user_id'] = image_info['user_id'] + image_name = image_info['image_path'].split('/')[-1] + image_item['image_path'] = format_image_path(image_info['user_id'], image_name) + image_item['caption'] = image_info['caption'] + tag_list = {} + if image_info['captured_time']: + tag_list['time'] = datetime.datetime.date(image_info['captured_time']) + if image_info['address'] != 'None': + tag_list['location'] = image_info['address'] + other_tags = eval(image_info['other_tags']) + tag_list.update(other_tags) + image_item['tag_list'] = tag_list + return image_item + + +def delete_single_image(user_id, image_id): + logger.info(f'[Delete] Deleting image {image_id}') + mysql_db = MysqlDb() + image_path = mysql_db.fetch_one( + sql=f'SELECT image_path FROM image_info WHERE image_id={image_id}', + params=None + ) + if image_path==None: + info = f'[Delete] Image {image_id} does not exist in MySQL.' + logger.error(info) + raise Exception(info) + image_path = image_path['image_path'] + + # delete local image + os.remove(image_path) + logger.info(f'[Delete] Image {image_path} successfully deleted.') + + # update db info, set image status as 'deleted' + try: + with mysql_db.transaction(): + mysql_db.update( + sql=f"UPDATE image_info SET exist_status='deleted' WHERE image_id={image_id} ;", + params=None + ) + except Exception as e: + logger.error(e) + raise Exception(e) + finally: + mysql_db._close() + logger.info(f'[Delete] Image {image_id} deleted successfully.') + + +def process_images_in_background( user_id: str, image_obj_list: List[Dict]): + try: + logger.info(f'[backgroud] ======= processing image list for user {user_id} in background =======') + for i in range(len(image_obj_list)): + # save image into local path + image_id = image_obj_list[i]['img_id'] + image_path = image_obj_list[i]['img_path'] + image_obj = image_obj_list[i]['img_obj'] + image_exif = image_obj_list[i]['exif'] + image_obj.save(image_path, exif=image_exif) + logger.info(f'[backgroud] Image saved into local path {image_path}') + # process image and generate infos + try: + process_single_image(image_id, image_path, user_id) + except Exception as e: + logger.error("[backgroud] "+str(e)) + logger.error(f'[backgroud] error occurred, delete image.') + delete_single_image(user_id, image_id) + + except Exception as e: + logger.error(e) + raise ValueError(str(e)) + else: + logger.info('[backgroud] Background images process finished.') + + +def process_single_image(img_id, img_path, user_id): + logger.info(f'[background - single] ----- processing image {img_path} in background -----') + + # generate gps info + result_gps = find_GPS_image(img_path) + captured_time = result_gps['date_information'] + gps_info = result_gps['GPS_information'] + latitude, longitude, altitude = None, None, None + if 'GPSLatitude' in gps_info: + latitude = gps_info['GPSLatitude'] + if 'GPSLongitude' in gps_info: + longitude = gps_info['GPSLongitude'] + if 'GPSAltitude' in gps_info: + altitude = gps_info['GPSAltitude'] + logger.info(f'[background - single] Image is captured at: {captured_time},' + + 'latitude: {latitude}, longitude: {longitude}, altitude: {altitude}') + if latitude: + update_image_attr(image={"image_id": img_id, "latitude": latitude}, attr='latitude') + if longitude: + update_image_attr(image={"image_id": img_id, "longitude": longitude}, attr='longitude') + if altitude: + update_image_attr(image={"image_id": img_id, "altitude": altitude}, attr='altitude') + if captured_time: + update_image_attr(image={"image_id": img_id, "captured_time": captured_time}, attr='captured_time') + else: + SHA_TZ = timezone( + timedelta(hours=8), + name='Asia/Shanghai' + ) + utc_now = datetime.datetime.utcnow().replace(tzinfo=timezone.utc) + beijing_time = utc_now.astimezone(SHA_TZ) + update_image_attr(image={"image_id": img_id, "captured_time": beijing_time}, attr='captured_time') + + # generate address info + api_key = os.environ.get("GOOGLE_API_KEY") + if not api_key: + raise Exception("Please configure environment variable of GOOGLE_API_KEY.") + address = get_address_from_gps(latitude, longitude, api_key) + if address: + logger.info(f'[background - single] Image address: {address}') + address_components = [] + if address.get('country', None): + address_components.append(address['country']) + if address.get('administrative_area_level_1', None): + address_components.append(address['administrative_area_level_1']) + if address.get('locality', None): + address_components.append(address['locality']) + if address.get('sublocality', None): + address_components.append(address['sublocality']) + formatted_address = ', '.join(address_components) + update_image_attr(image={"image_id": img_id, "address": formatted_address}, attr='address') + else: + address = None + logger.info(f'[background - single] Can not get address from image.') + + # generate caption info + logger.info(f'[background - single] Generating caption of image {img_path}') + try: + result_caption = generate_caption(img_path) + except Exception as e: + logger.error("[background - single] "+str(e)) + if result_caption: + logger.info(f'[background - single] Image caption: {result_caption}') + update_image_attr(image={"image_id": img_id, "caption": result_caption}, attr='caption') + else: + logger.info(f'[background - single] Can not generate caption for image.') + + # process faces for image + db_path = IMAGE_ROOT_PATH+"/user"+user_id + try: + process_face_for_single_image(image_id=img_id, image_path=img_path, db_path=db_path, user_id=user_id) + except Exception as e: + logger.error(f'[background - single] Error occurred while processing face.') + raise Exception('[background - single]', str(e)) + logger.info(f'[background - single] Face process done for image {img_id}') + + # update image status + try: + mysql_db = MysqlDb() + with mysql_db.transaction(): + mysql_db.update(sql=f"UPDATE image_info SET process_status='ready' WHERE image_id={img_id}", params=None) + except Exception as e: + logger.error("[background - single] "+str(e)) + finally: + mysql_db._close() + logger.info(f"[background - single] ----- finish image {img_path} processing -----") + + +def process_face_for_single_image(image_id, image_path, db_path, user_id): + logger.info(f'[background - face] ### processing face for {image_path} in background ###') + + # 1. check whether image contains faces + try: + face_objs = DeepFace.represent(img_path=image_path, model_name='Facenet512') + except: + # no face in this image, finish process + logger.info(f"[background - face] Image {image_id} does not contains faces") + logger.info(f"[background - face] Image {image_id} face process finished.") + return None + face_cnt = len(face_objs) + logger.info(f'[background - face] Found {face_cnt} faces in image {image_id}') + face_xywh_list = [] + for face_obj in face_objs: + xywh = face_obj['facial_area'] + transferred_xywh = transfer_xywh(xywh) + face_xywh_list.append(transferred_xywh) + logger.info(f'[background - face] face xywh list of image {image_id} is: {face_xywh_list}') + + # 2. check same faces in db + import os + pkl_path = db_path+'/representations_facenet512.pkl' + if os.path.exists(pkl_path): + logger.info(f'[background - face] pkl file already exists, delete it.') + os.remove(pkl_path) + dfs = DeepFace.find(img_path=image_path, db_path=db_path, model_name='Facenet512', enforce_detection=False) + logger.info(f'[background - face] Finding match faces in image database.') + assert face_cnt == len(dfs) + logger.info(f'[background - face] dfs: {dfs}') + mysql_db = MysqlDb() + for df in dfs: + logger.info(f'[background - face] current df: {df}') + # no face matched for current face of image, add new faces later + if len(df) <= 1: + logger.info(f'[background - face] length of current df less than 1, continue') + continue + # findd ref image + ref_image_path = None + ref_image_list = df['identity'] + for ref_image_name in ref_image_list: + logger.info(f'[background - face] current ref_image_name: {ref_image_name}') + if ref_image_name!=image_path: + ref_image_path = ref_image_name + break + # no ref image found + if not ref_image_path: + logger.info(f'[background - face] no other reference image found, continue') + continue + # find faces in img2: one or many + find_face_sql = f""" + SELECT face_id, face_tag, xywh FROM image_face WHERE + image_path='{ref_image_path}' AND user_id='{user_id}'; + """ + try: + img_face_list = mysql_db.fetch_all(sql=find_face_sql) + except Exception as e: + logger.error("[background - face] "+str(e)) + raise Exception(f"Exception ocurred while selecting info from image_face: {e}") + logger.info(f"[background - face] reference image and faces: {img_face_list}") + # wrong ref image found + if img_face_list == (): + logger.error(f"img_face_list is {img_face_list}, wrong ref image found") + continue + # verify face xywh of ref image + obj = DeepFace.verify(img1_path=image_path, img2_path=ref_image_path, model_name="Facenet512") + ref_xywh = transfer_xywh(obj['facial_areas']['img2']) + image_xywh = transfer_xywh(obj['facial_areas']['img1']) + face_id = -1 + face_tag = None + # find corresponding face_id and face_tag + for img_face in img_face_list: + if img_face['xywh'] == ref_xywh: + face_id = img_face['face_id'] + face_tag = img_face['face_tag'] + if face_id == -1 and face_tag == None: + raise Exception(f'Error occurred when verifing faces for reference image: Inconsistent face infomation.') + # insert into image_face + insert_img_face_sql = f"""INSERT INTO image_face + VALUES(null, {image_id}, '{image_path}', {face_id}, '{image_xywh}', '{user_id}', '{face_tag}');""" + try: + with mysql_db.transaction(): + mysql_db.insert(sql=insert_img_face_sql, params=None) + except Exception as e: + logger.error("[background - face] "+str(e)) + raise Exception(f"Exception ocurred while inserting info into image_face: {e}") + # current face matched and saved into db, delete from face_xywh_list + logger.info(f'[background - face] image_face data inserted: {insert_img_face_sql}') + if image_xywh in face_xywh_list: + face_xywh_list.remove(image_xywh) + logger.info(f'[background - face] current face_xywh_list: {face_xywh_list}') + + # all faces matched in db, no faces left + if len(face_xywh_list) == 0: + logger.info(f"[background - face] Image {image_id} face process finished.") + return None + + # 3. add new faces for current image (no reference in db) + logger.info(f'[background - face] Adding new faces for image {image_id}') + for cur_xywh in face_xywh_list: + face_cnt = mysql_db.fetch_one(sql="SELECT COUNT(*) AS cnt FROM face_info;")['cnt'] + tag = 'person'+str(face_cnt+1) + face_sql = f"INSERT INTO face_info VALUES(null, '{tag}');" + try: + with mysql_db.transaction(): + mysql_db.insert(sql=face_sql, params=None) + except Exception as e: + logger.error("[background - face] "+str(e)) + raise Exception(f"Exception ocurred while inserting new face into face_info: {e}") + logger.info(f"[background - face] face {tag} inserted into db.") + face_id = mysql_db.fetch_one(f"SELECT * FROM face_info WHERE face_tag='{tag}';")['face_id'] + logger.info(f"[background - face] new face id is: {face_id}") + img_face_sql = f"""INSERT INTO image_face VALUES + (null, {image_id}, '{image_path}', {face_id}, '{cur_xywh}', '{user_id}', '{tag}');""" + try: + with mysql_db.transaction(): + mysql_db.insert(sql=img_face_sql, params=None) + except Exception as e: + logger.error("[background - face] "+str(e)) + raise Exception(f"Exception ocurred while inserting new face into image_face: {e}") + logger.info(f"[background - face] img_face {img_face_sql} inserted into db.") + mysql_db._close() + logger.info(f"[background - face] Image {image_id} face process finished.") + + +def get_type_obj_from_attr(attr, user_id): + logger.info(f'Geting image type of {attr}') + + if attr == 'time': + select_sql = f'''SELECT DATE(captured_time) AS date FROM image_info + WHERE user_id = "{user_id}" AND exist_status="active" GROUP BY date ORDER BY date;''' + elif attr == 'address': + select_sql = f'''SELECT address FROM image_info + WHERE user_id="{user_id}" AND exist_status="active" GROUP BY address;''' + else: + return {} + + mysql_db = MysqlDb() + select_list = mysql_db.fetch_all(sql=select_sql) + select_result = {} + for item in select_list: + logger.info(f"current item of {attr} is: {item}") + if attr == 'time': + item = item['date'] + if item == None: + continue + example_image_path = mysql_db.fetch_one( + sql=f'''SELECT image_path FROM image_info + WHERE DATEDIFF(captured_time, "{item}") = 0 and user_id="{user_id}" + and exist_status="active" LIMIT 1;''', + params=None)['image_path'] + elif attr == 'address': + item = item['address'] + if item == None or item == 'None' or item == 'null': + continue + example_image_path = mysql_db.fetch_one( + sql=f'''SELECT image_path FROM image_info WHERE + address="{item}" and user_id="{user_id}" and exist_status="active" LIMIT 1;''', + params=None)['image_path'] + + image_name = example_image_path.split('/')[-1] + image_path = format_image_path(user_id, image_name) + select_result[item] = image_path + + mysql_db._close() + + # return time result directly + if attr == 'time': + logger.info(f'type list: {select_result}') + return select_result + + # check whether address simplification is needed + simplify_flag = True + cur_country = None + address_list = list(select_result.keys()) + for address in address_list: + country = address.split(', ')[0] + if not cur_country: + cur_country = country + else: + if country != cur_country: + simplify_flag = False + break + + # simplify address name dynamically + if simplify_flag: + logger.info(f'address need to be simplified') + new_result = {} + for key, value in select_result.items(): + new_key = ', '.join(key.split(', ')[1:]) + new_result[new_key] = value + logger.info(f'type list: {new_result}') + return new_result + else: + logger.info(f'type list: {select_result}') + return select_result + + +def get_address_list(user_id) -> list[str]: + logger.info(f'Getting address list of user {user_id}') + mysql_db = MysqlDb() + select_sql = f'''SELECT address FROM image_info WHERE + user_id="{user_id}" AND exist_status="active" GROUP BY address;''' + select_list = mysql_db.fetch_all(sql=select_sql) + result_list = [] + for item in select_list: + address = item['address'] + if address == None or address == 'None' or address == 'null': + continue + add_list = address.split(', ') + for add in add_list: + if add not in result_list: + result_list.append(add) + logger.info(f'address list of user {user_id} is {result_list}') + return result_list + + +def get_process_status(user_id): + logger.info(f'Geting process status of user {user_id}') + mysql_db = MysqlDb() + total_cnt = mysql_db.fetch_one( + sql=f"""SELECT COUNT(*) AS cnt FROM image_info WHERE + user_id='{user_id}' AND exist_status='active';""")['cnt'] + processing_cnt = mysql_db.fetch_one( + sql=f"""SELECT COUNT(*) AS cnt FROM image_info WHERE + user_id='{user_id}' AND exist_status='active' AND process_status='processing';""")['cnt'] + mysql_db._close() + result = {} + result['total_image'] = total_cnt + result['processing_image'] = processing_cnt + result['status'] = "done" if processing_cnt ==0 else 'processing' + return result + + +def get_images_by_type(user_id, type, subtype) -> List: + logger.info(f'Getting image by type {type} - {subtype}') + + if type == 'address': + if subtype == 'default': + subtype = 'None' + sql=f"""SELECT image_id, image_path FROM image_info WHERE + user_id='{user_id}' AND exist_status='active' AND address LIKE '%{subtype}%';""" + + elif type == 'time': + if subtype == 'None': + sql = f'''SELECT image_id, image_path FROM image_info + WHERE captured_time is null AND user_id="{user_id}" AND exist_status="active";''' + else: + sql = f'''SELECT image_id, image_path FROM image_info + WHERE DATE(captured_time)="{subtype}" AND user_id="{user_id}" AND exist_status="active";''' + + elif type == 'person': + sql = f"""SELECT image_info.image_id, image_info.image_path FROM image_face + INNER JOIN image_info ON image_info.image_id=image_face.image_id + WHERE image_info.user_id='{user_id}' AND image_info.exist_status='active' + AND image_face.face_tag='{subtype}'""" + + logger.info(f'sql: {sql}') + mysql_db = MysqlDb() + images = mysql_db.fetch_all(sql=sql, params=None) + mysql_db._close() + logger.info(f"image list: {images}") + if len(images) == 0: + logger.error(f'no label {subtype} in {type}') + return [] + else: + result = [] + for image in images: + image_name = image['image_path'].split('/')[-1] + image_path = format_image_path(user_id, image_name) + obj = {"image_id": image['image_id'], "image_path": image_path} + result.append(obj) + return result + + +def get_face_list_by_user_id(user_id: str) -> List[Dict]: + logger.info(f'getting face list of user {user_id}') + group_by_face_sql = f'''SELECT group_concat(image_face.image_path) AS image_path, + group_concat(image_face.face_tag) AS face_tag FROM image_face + INNER JOIN image_info ON image_info.image_id=image_face.image_id + WHERE image_info.user_id = "{user_id}" AND image_info.exist_status="active" GROUP BY face_id;''' + try: + mysql_db = MysqlDb() + query_list = mysql_db.fetch_all(sql=group_by_face_sql) + except Exception as e: + logger.error(e) + raise Exception(e) + finally: + mysql_db._close() + logger.info(f'query result list: {query_list}') + response_person = {} + for item in query_list: + logger.info(f'current item: {item}') + image_name = item['image_path'].split('/')[-1] + face_tag = item['face_tag'] + if ',' in face_tag: + face_tag = face_tag.split(',')[0] + response_person[face_tag] = format_image_path(user_id, image_name) + logger.info(f'person list: {response_person}') + return response_person + + +def get_image_list_by_ner_query(ner_result: Dict, user_id: str, query: str) -> List[Dict]: + + logger.info(f'[NER query] start query from ner results') + query_sql = "SELECT image_info.image_id, image_info.image_path FROM image_info " + query_flag = False + mysql_db = MysqlDb() + + # get person name query + face_list = mysql_db.fetch_all( + sql=f"""select image_face.face_tag from image_face inner join image_info + on image_info.image_id=image_face.image_id where + image_info.user_id='{user_id}' AND exist_status='active';""", + params=None) + logger.info(f"[NER query] face list is: {face_list}") + if face_list: + sql_conditions = [] + for face_tag in face_list: + face_tag = face_tag['face_tag'] + if face_tag in query: + logger.info(f'[NER query] other face detected in db: [{face_tag}]') + sql_conditions.append(f' image_face.face_tag LIKE "%{face_tag}%" ') + if sql_conditions != []: + query_flag = True + sql = 'OR'.join(sql_conditions) + query_sql += "INNER JOIN image_face ON image_info.image_id=image_face.image_id WHERE " + query_sql += '('+sql+')' + else: + logger.info(f'[NER query] no person name in ner query') + else: + logger.info(f'[NER query] no person name in ner query') + + # get location query + location_list = get_address_list(user_id) + logger.info(f"[NER query] location list is: {location_list}") + if location_list: + sql_conditions = [] + for db_loc in location_list: + if db_loc in query: + sql_conditions.append(f' image_info.address LIKE "%{db_loc}%" ') + if sql_conditions != []: + if not query_flag: + query_sql += " WHERE " + query_flag = True + sql = 'OR'.join(sql_conditions) + if query_sql[-1] == ')': + query_sql += ' AND ' + query_sql += '('+sql+')' + else: + logger.info(f'[NER query] no location in query') + + # get time query + if ner_result['time']: + time_points = ner_result['time'] + sql_conditions = [] + for loc in time_points: + sql_conditions.append(f' image_info.captured_time LIKE "%{loc}%" ') + if sql_conditions != []: + if not query_flag: + query_sql += " WHERE " + query_flag = True + sql = 'OR'.join(sql_conditions) + if query_sql[-1] == ')': + query_sql += ' AND ' + query_sql += '('+sql+')' + else: + logger.info(f'[NER query] no time in query') + + # get time period query + if ner_result['period']: + periods = ner_result['period'] + logger.info(f'[NER query] periods: {periods}') + sql_conditions = [] + for period in periods: + from_time = period['from'] + to_time = period['to'] + format = "%Y-%m-%d" + to_time = datetime.datetime.strptime(to_time, format) + new_to_time = to_time + datetime.timedelta(days=1) + sql_conditions.append( + f' image_info.captured_time BETWEEN "{from_time}" AND "{new_to_time.strftime(format)}" ' + ) + if sql_conditions != []: + if not query_flag: + query_sql += " WHERE " + query_flag = True + sql = 'OR'.join(sql_conditions) + if query_sql[-1] == ')': + query_sql += ' AND ' + query_sql += '('+sql+')' + else: + logger.info(f'[NER query] no time period in query') + + if not query_flag: + logger.info(f'[NER query] no compatible data for current query') + return [] + query_sql += f' AND ( image_info.user_id="{user_id}" ) AND ( exist_status="active" ) ;' + logger.info(f'[NER query] query sql: {query_sql}') + + try: + query_result = mysql_db.fetch_all(sql=query_sql, params=None) + except Exception as e: + raise Exception("[NER query] "+str(e)) + result_image_list = [] + for res in query_result: + image_name = res['image_path'].split('/')[-1] + image_path = format_image_path(user_id, image_name) + item = {"image_id": res['image_id'], "imgSrc": image_path} + result_image_list.append(item) + logger.info(f'[NER query] result: {result_image_list}') + mysql_db._close() + return result_image_list + + +def delete_user_infos(user_id: str): + logger.info(f'[delete user] start query from ner results') + + try: + mysql_db = MysqlDb() + with mysql_db.transaction(): + # delete image_face and face_info + logger.info(f'[delete user] delete image_face and face_info of user {user_id}.') + mysql_db.delete( + sql=f"""DELETE image_face, face_info FROM image_face + INNER JOIN face_info ON image_face.face_id=face_info.face_id WHERE user_id='{user_id}'""", + params=None) + + # delete image_info + logger.info(f'[delete user] delete image_info of user {user_id}.') + mysql_db.delete(sql=f"DELETE FROM image_info WHERE user_id='{user_id}'", params=None) + + # delete user_info + logger.info(f'[delete user] delete user_info of user {user_id}.') + mysql_db.delete(sql=f"DELETE FROM user_info WHERE user_id='{user_id}'", params=None) + except Exception as e: + raise Exception(e) + finally: + mysql_db._close() + + # delete local images + try: + logger.info(f'[delete user] delete local images of user {user_id}.') + folder_path = IMAGE_ROOT_PATH+'/user'+str(user_id) + if not os.path.exists(folder_path): + logger.info(f'[delete user] no image folder for user {user_id}') + return + else: + if os.path.isdir(folder_path): + import shutil + shutil.rmtree(folder_path) + else: + os.remove(folder_path) + logger.info(f'[delete user] local images of user {user_id} is deleted.') + except Exception as e: + raise Exception(e) + + logger.info(f'[delete user] user {user_id} infomation all deleted.') + + +def forward_req_to_sd_inference_runner(inputs): + resp = requests.post("http://{}:{}".format("198.175.88.27", "80"), + data=json.dumps(inputs), timeout=200) + try: + img_str = json.loads(resp.text)["img_str"] + print("compute node: ", json.loads(resp.text)["ip"]) + except: + print('no inference result. please check server connection') + return None + + return img_str + + +def stable_defusion_func(inputs): + return forward_req_to_sd_inference_runner(inputs) + diff --git a/intel_extension_for_transformers/neural_chat/server/restful/photoai_utils.py b/intel_extension_for_transformers/neural_chat/server/restful/photoai_utils.py new file mode 100644 index 00000000000..bad067dfce9 --- /dev/null +++ b/intel_extension_for_transformers/neural_chat/server/restful/photoai_utils.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +import exifread +import time +import requests +import base64 +import datetime; +import random; +from io import BytesIO +from PIL import Image +from transformers import BlipProcessor, BlipForConditionalGeneration + + +def latitude_and_longitude_convert_to_decimal_system(*arg): + """ + Convert latitude&longitude into decimal system + """ + return float(arg[0]) + ((float(arg[1]) + (float(arg[2].split('/')[0]) / float(arg[2].split('/')[-1]) / 60)) / 60) + + +def find_GPS_image(pic_path): + """ + generate GPS and timestamp from image + """ + GPS = {} + date = '' + with open(pic_path, 'rb') as f: + tags = exifread.process_file(f) + print(f'====== image metadata ======') + for tag, value in tags.items(): + if re.match('GPS GPSLatitudeRef', tag): + GPS['GPSLatitudeRef'] = str(value) + elif re.match('GPS GPSLongitudeRef', tag): + GPS['GPSLongitudeRef'] = str(value) + elif re.match('GPS GPSAltitudeRef', tag): + GPS['GPSAltitudeRef'] = str(value) + elif re.match('GPS GPSLatitude', tag): + try: + match_result = re.match('\[(\w*),(\w*),(\w.*)/(\w.*)\]', str(value)).groups() + GPS['GPSLatitude'] = int(match_result[0]), int(match_result[1]), int(match_result[2]) + except: + deg, min, sec = [x.replace(' ', '') for x in str(value)[1:-1].split(',')] + GPS['GPSLatitude'] = latitude_and_longitude_convert_to_decimal_system(deg, min, sec) + elif re.match('GPS GPSLongitude', tag): + try: + match_result = re.match('\[(\w*),(\w*),(\w.*)/(\w.*)\]', str(value)).groups() + GPS['GPSLongitude'] = int(match_result[0]), int(match_result[1]), int(match_result[2]) + except: + deg, min, sec = [x.replace(' ', '') for x in str(value)[1:-1].split(',')] + GPS['GPSLongitude'] = latitude_and_longitude_convert_to_decimal_system(deg, min, sec) + elif re.match('GPS GPSAltitude', tag): + GPS['GPSAltitude'] = str(value) + elif re.match('Image DateTime', tag): + date = str(value) + return {'GPS_information': GPS, 'date_information': date} + + +def get_address_from_gps(latitude, longitude, api_key): + base_url = "https://maps.googleapis.com/maps/api/geocode/json" + params = { + 'latlng': f"{latitude},{longitude}", + 'key': api_key + } + start_time = time.time() + response = requests.get(base_url, params=params) + data = response.json() + if data['status'] == 'OK': + address_components = data['results'][0]['address_components'] + result = {} + for component in address_components: + if 'country' in component['types']: + result['country'] = component['long_name'].replace(' ', '').capitalize() + elif 'administrative_area_level_1' in component['types']: + result['administrative_area_level_1'] = component['long_name'].replace(' ', '').capitalize() + elif 'locality' in component['types']: + result['locality'] = component['long_name'].replace(' ', '').capitalize() + elif 'sublocality' in component['types']: + result['sublocality'] = component['long_name'].replace(' ', '').capitalize() + print("Generate address elapsed time: ", time.time() - start_time) + return result + else: + return None + + +def infer_image(pic_path, processor, model): + raw_image = Image.open(pic_path).convert('RGB') + text = f"You take a photo of" + inputs = processor(raw_image, text, return_tensors="pt") + out = model.generate(**inputs, max_new_tokens=50) + result_str = processor.decode(out[0], skip_special_tokens=True) + return result_str + + +def generate_caption(img_path): + processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-large") + model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-large") + start_time = time.time() + result_str = infer_image(img_path, processor, model) + print("Generate caption elapsed time: ", time.time() - start_time) + return result_str + + +def image_to_byte64(img_path): + image = Image.open(img_path) + img_bytes = BytesIO() + image = image.convert("RGB") + image.save(img_bytes, format="JPEG") + img_bytes = img_bytes.getvalue() + img_b64 = base64.b64encode(img_bytes) + return img_b64 + + +def byte64_to_image(img_b64): + img_bytes = base64.b64decode(img_b64) + bytes_stream = BytesIO(img_bytes) + img = Image.open(bytes_stream) + return img + + +def generate_random_name(): + nowTime=datetime.datetime.now().strftime("%Y%m%dT%H%M%S%f") + randomNum=random.randint(0,100) + if randomNum<=10: + randomNum=str(0)+str(randomNum) + uniqueNum=str(nowTime)+str(randomNum) + return uniqueNum + + +def transfer_xywh(facial_area: dict): + items = ['x', 'y', 'w', 'h'] + result = '' + for item in items: + result += str(facial_area[item]) + '_' + return result + diff --git a/requirements.txt b/requirements.txt index b2b10389d7e..5546ce2ce92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ setuptools>=65 setuptools_scm[toml]>=6.2 --extra-index-url https://download.pytorch.org/whl/cpu torch==2.1.0+cpu -accelerate +accelerate \ No newline at end of file