From 5f45020971bc0e756214e860be04aa6381452fe8 Mon Sep 17 00:00:00 2001 From: Karen Shaw Date: Wed, 7 Aug 2024 17:01:47 +0000 Subject: [PATCH] Tweak prompt for length and no relevant results found --- chat/src/handlers/chat.py | 10 ++++++++-- chat/src/helpers/prompts.py | 2 +- chat/test/helpers/test_metrics.py | 4 ++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/chat/src/handlers/chat.py b/chat/src/handlers/chat.py index 8ec897ed..3630bfab 100644 --- a/chat/src/handlers/chat.py +++ b/chat/src/handlers/chat.py @@ -13,7 +13,8 @@ RESPONSE_TYPES = { "base": ["answer", "ref"], "debug": ["answer", "attributes", "azure_endpoint", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "temperature", "text_key", "token_counts"], - "log": ["answer", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "size", "source_documents", "temperature", "token_counts"] + "log": ["answer", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "size", "source_documents", "temperature", "token_counts"], + "error": ["question", "error", "source_documents"] } def handler(event, context): @@ -37,7 +38,12 @@ def handler(event, context): config.setup_llm_request() response = Response(config) final_response = response.prepare_response() - config.socket.send(reshape_response(final_response, 'debug' if config.debug_mode else 'base')) + if "error" in final_response: + logging.error(f'Error: {final_response["error"]}') + config.socket.send({"type": "error", "message": "Internal Server Error"}) + return {"statusCode": 500, "body": "Internal Server Error"} + else: + config.socket.send(reshape_response(final_response, 'debug' if config.debug_mode else 'base')) log_group = os.getenv('METRICS_LOG_GROUP') log_stream = context.log_stream_name diff --git a/chat/src/helpers/prompts.py b/chat/src/helpers/prompts.py index 9d660427..440be17d 100644 --- a/chat/src/helpers/prompts.py +++ b/chat/src/helpers/prompts.py @@ -2,7 +2,7 @@ def prompt_template() -> str: - return """Please provide a brief answer to the question based on the documents provided. Include specific details from the documents that support your answer. Keep your answer concise. Each document is identified by a 'title' and a unique 'source' UUID: + return """Please provide a brief answer to the question based on the documents provided. Include specific details from the documents that support your answer. Keep your answer concise and keep reading time under 45 seconds. Each document is identified by a 'title' and a unique 'source' UUID. If the documents do not answer the question, please respond that you can't seem to find enough information about [keyword] that in the Digital Collection and suggest they search NUL's catalog and construct a search link using this format: https://search.library.northwestern.edu/discovery/search?field=any&query=any,contains,keyword&institution=01NWU&vid=01NWU_INST:NULVNEW&search_scope=MyInst_and_CI&tab=Everything&mode=Basic&displayMode=full&bulkSize=10&highlight=true&dum=true&displayField=all&pcAvailabiltyMode=true&facet=rtype,exclude,reviews,lk: Documents: {context} diff --git a/chat/test/helpers/test_metrics.py b/chat/test/helpers/test_metrics.py index de147ab6..308748a3 100644 --- a/chat/test/helpers/test_metrics.py +++ b/chat/test/helpers/test_metrics.py @@ -99,10 +99,10 @@ def test_token_usage(self): expected_result = { "answer": 12, - "prompt": 314, + "prompt": 462, "question": 5, "source_documents": 527, - "total": 858 + "total": 1006 } self.assertEqual(result, expected_result)