[Security solution] Improve AI connector error handling

x-pack/packages/kbn-elastic-assistant/impl/assistant/api.tsx

@@ -61,19 +61,24 @@ export const fetchConnectorExecuteAction = async ({
       ? `/internal/elastic_assistant/actions/connector/${apiConfig?.connectorId}/_execute`
       : `/api/actions/connector/${apiConfig?.connectorId}/_execute`;
 
-    const response = await http.fetch<{ connector_id: string; status: string; data: string }>(
-      path,
-      {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(requestBody),
-        signal,
-      }
-    );
+    const response = await http.fetch<{
+      connector_id: string;
+      status: string;
+      data: string;
+      service_message?: string;
+    }>(path, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify(requestBody),
+      signal,
+    });
 
     if (response.status !== 'ok' || !response.data) {
+      if (response.service_message) {
+        return `${API_ERROR} \n\n${response.service_message}`;
+      }
       return API_ERROR;
     }
 

x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.test.tsx

@@ -26,7 +26,7 @@ beforeAll(() => {
 describe('actionTypeRegistry.get() works', () => {
   test('connector type static data is as expected', () => {
     expect(actionTypeModel.id).toEqual(ACTION_TYPE_ID);
-    expect(actionTypeModel.selectMessage).toBe('Send a request to AWS Bedrock systems.');
+    expect(actionTypeModel.selectMessage).toBe('Send a request to AWS Bedrock.');
     expect(actionTypeModel.actionTypeTitle).toBe('AWS Bedrock');
   });
 });

x-pack/plugins/stack_connectors/public/connector_types/bedrock/bedrock.tsx

@@ -21,7 +21,7 @@ export function getConnectorType(): BedrockConnector {
     id: BEDROCK_CONNECTOR_ID,
     iconClass: lazy(() => import('./logo')),
     selectMessage: i18n.translate('xpack.stackConnectors.components.bedrock.selectMessageText', {
-      defaultMessage: 'Send a request to AWS Bedrock systems.',
+      defaultMessage: 'Send a request to AWS Bedrock.',
     }),
     actionTypeTitle: BEDROCK_TITLE,
     validateParams: async (

x-pack/plugins/stack_connectors/public/connector_types/openai/openai.test.tsx

@@ -26,7 +26,9 @@ beforeAll(() => {
 describe('actionTypeRegistry.get() works', () => {
   test('connector type static data is as expected', () => {
     expect(actionTypeModel.id).toEqual(ACTION_TYPE_ID);
-    expect(actionTypeModel.selectMessage).toBe('Send a request to OpenAI systems.');
+    expect(actionTypeModel.selectMessage).toBe(
+      'Send a request to an OpenAI or Azure OpenAI service.'
+    );
     expect(actionTypeModel.actionTypeTitle).toBe('OpenAI');
   });
 });

x-pack/plugins/stack_connectors/public/connector_types/openai/openai.tsx

@@ -21,7 +21,7 @@ export function getConnectorType(): OpenAIConnector {
     id: OPENAI_CONNECTOR_ID,
     iconClass: lazy(() => import('./logo')),
     selectMessage: i18n.translate('xpack.stackConnectors.components.genAi.selectMessageText', {
-      defaultMessage: 'Send a request to OpenAI systems.',
+      defaultMessage: 'Send a request to an OpenAI or Azure OpenAI service.',
     }),
     actionTypeTitle: OPENAI_TITLE,
     validateParams: async (

x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.test.ts

@@ -16,6 +16,7 @@ import {
   DEFAULT_BEDROCK_URL,
 } from '../../../common/bedrock/constants';
 import { DEFAULT_BODY } from '../../../public/connector_types/bedrock/constants';
+import { AxiosError } from 'axios';
 
 jest.mock('aws4', () => ({
   sign: () => ({ signed: true }),
@@ -151,5 +152,55 @@ describe('BedrockConnector', () => {
         await expect(connector.invokeAI(aiAssistantBody)).rejects.toThrow('API Error');
       });
     });
+    describe('getResponseErrorMessage', () => {
+      it('returns an unknown error message', () => {
+        // @ts-expect-error expects an axios error as the parameter
+        expect(connector.getResponseErrorMessage({})).toEqual(
+          `Unexpected API Error:  - Unknown error`
+        );
+      });
+
+      it('returns the error.message', () => {
+        // @ts-expect-error expects an axios error as the parameter
+        expect(connector.getResponseErrorMessage({ message: 'a message' })).toEqual(
+          `Unexpected API Error:  - a message`
+        );
+      });
+
+      it('returns the error.response.data.error.message', () => {
+        const err = {
+          response: {
+            headers: {},
+            status: 404,
+            statusText: 'Resource Not Found',
+            data: {
+              message: 'Resource not found',
+            },
+          },
+        } as AxiosError<{ message?: string }>;
+        expect(
+          // @ts-expect-error expects an axios error as the parameter
+          connector.getResponseErrorMessage(err)
+        ).toEqual(`API Error: Resource Not Found - Resource not found`);
+      });
+
+      it('returns auhtorization error', () => {
+        const err = {
+          response: {
+            headers: {},
+            status: 401,
+            statusText: 'Auth error',
+            data: {
+              message: 'The api key was invalid.',
+            },
+          },
+        } as AxiosError<{ message?: string }>;
+
+        // @ts-expect-error expects an axios error as the parameter
+        expect(connector.getResponseErrorMessage(err)).toEqual(
+          `Unauthorized API Error - The api key was invalid.`
+        );
+      });
+    });
   });
 });

x-pack/plugins/stack_connectors/server/connector_types/bedrock/bedrock.ts

@@ -63,15 +63,17 @@ export class BedrockConnector extends SubActionConnector<Config, Secrets> {
     });
   }
 
-  protected getResponseErrorMessage(error: AxiosError<{ error?: { message?: string } }>): string {
+  protected getResponseErrorMessage(error: AxiosError<{ message?: string }>): string {
     if (!error.response?.status) {
-      return `Unexpected API Error: ${error.code} - ${error.message}`;
+      return `Unexpected API Error: ${error.code ?? ''} - ${error.message ?? 'Unknown error'}`;
     }
     if (error.response.status === 401) {
-      return 'Unauthorized API Error';
+      return `Unauthorized API Error${
+        error.response?.data?.message ? ` - ${error.response.data.message}` : ''
+      }`;
     }
-    return `API Error: ${error.response?.status} - ${error.response?.statusText}${
-      error.response?.data?.error?.message ? ` - ${error.response.data.error?.message}` : ''
+    return `API Error: ${error.response?.statusText}${
+      error.response?.data?.message ? ` - ${error.response.data.message}` : ''
     }`;
   }
 

x-pack/plugins/stack_connectors/server/connector_types/openai/openai.test.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { AxiosError } from 'axios';
 import { OpenAIConnector } from './openai';
 import { actionsConfigMock } from '@kbn/actions-plugin/server/actions_config.mock';
 import {
@@ -282,6 +283,60 @@ describe('OpenAIConnector', () => {
         await expect(connector.invokeAI(sampleOpenAiBody)).rejects.toThrow('API Error');
       });
     });
+    describe('getResponseErrorMessage', () => {
+      it('returns an unknown error message', () => {
+        // @ts-expect-error expects an axios error as the parameter
+        expect(connector.getResponseErrorMessage({})).toEqual(
+          `Unexpected API Error:  - Unknown error`
+        );
+      });
+
+      it('returns the error.message', () => {
+        // @ts-expect-error expects an axios error as the parameter
+        expect(connector.getResponseErrorMessage({ message: 'a message' })).toEqual(
+          `Unexpected API Error:  - a message`
+        );
+      });
+
+      it('returns the error.response.data.error.message', () => {
+        const err = {
+          response: {
+            headers: {},
+            status: 404,
+            statusText: 'Resource Not Found',
+            data: {
+              error: {
+                message: 'Resource not found',
+              },
+            },
+          },
+        } as AxiosError<{ error?: { message?: string } }>;
+        expect(
+          // @ts-expect-error expects an axios error as the parameter
+          connector.getResponseErrorMessage(err)
+        ).toEqual(`API Error: Resource Not Found - Resource not found`);
+      });
+
+      it('returns auhtorization error', () => {
+        const err = {
+          response: {
+            headers: {},
+            status: 401,
+            statusText: 'Auth error',
+            data: {
+              error: {
+                message: 'The api key was invalid.',
+              },
+            },
+          },
+        } as AxiosError<{ error?: { message?: string } }>;
+
+        // @ts-expect-error expects an axios error as the parameter
+        expect(connector.getResponseErrorMessage(err)).toEqual(
+          `Unauthorized API Error - The api key was invalid.`
+        );
+      });
+    });
   });
 
   describe('AzureAI', () => {

x-pack/plugins/stack_connectors/server/connector_types/openai/openai.ts

@@ -86,12 +86,14 @@ export class OpenAIConnector extends SubActionConnector<Config, Secrets> {
 
   protected getResponseErrorMessage(error: AxiosError<{ error?: { message?: string } }>): string {
     if (!error.response?.status) {
-      return `Unexpected API Error: ${error.code} - ${error.message}`;
+      return `Unexpected API Error: ${error.code ?? ''} - ${error.message ?? 'Unknown error'}`;
     }
     if (error.response.status === 401) {
-      return 'Unauthorized API Error';
+      return `Unauthorized API Error${
+        error.response?.data?.error?.message ? ` - ${error.response.data.error?.message}` : ''
+      }`;
     }
-    return `API Error: ${error.response?.status} - ${error.response?.statusText}${
+    return `API Error: ${error.response?.statusText}${
       error.response?.data?.error?.message ? ` - ${error.response.data.error?.message}` : ''
     }`;
   }
@@ -193,8 +195,6 @@ export class OpenAIConnector extends SubActionConnector<Config, Secrets> {
       return result;
     }
 
-    // TO DO: Pass actual error
-    // tracked here https://github.com/elastic/security-team/issues/7373
-    return 'An error occurred sending your message. If the problem persists, please test the connector configuration.';
+    return 'An error occurred sending your message. If the problem persists, please test the connector configuration. \n\nAPI Error: The response from OpenAI was in an unrecognized format.';
   }
 }

x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/bedrock.ts

@@ -444,6 +444,35 @@ export default function bedrockTest({ getService }: FtrProviderContext) {
             retry: false,
           });
         });
+
+        it('should return an error when error happens', async () => {
+          const DEFAULT_BODY = {
+            prompt: `Hello world!`,
+            max_tokens_to_sample: 300,
+            stop_sequences: ['\n\nHuman:'],
+          };
+          const { body } = await supertest
+            .post(`/api/actions/connector/${bedrockActionId}/_execute`)
+            .set('kbn-xsrf', 'foo')
+            .send({
+              params: {
+                subAction: 'test',
+                subActionParams: {
+                  body: JSON.stringify(DEFAULT_BODY),
+                },
+              },
+            })
+            .expect(200);
+
+          expect(body).to.eql({
+            status: 'error',
+            connector_id: bedrockActionId,
+            message: 'an error occurred while running the action',
+            retry: true,
+            service_message:
+              'Status code: 422. Message: API Error: Unprocessable Entity - Malformed input request: extraneous key [ooooo] is not permitted, please reformat your input and try again.',
+          });
+        });
       });
     });
   });

x-pack/test/alerting_api_integration/security_and_spaces/group2/tests/actions/connector_types/openai.ts

@@ -46,7 +46,7 @@ export default function genAiTest({ getService }: FtrProviderContext) {
     return body.id;
   };
 
-  describe('GenAi', () => {
+  describe('OpenAI', () => {
     after(() => {
       objectRemover.removeAll();
     });
@@ -463,6 +463,30 @@ export default function genAiTest({ getService }: FtrProviderContext) {
               retry: false,
             });
           });
+
+          it('should return a error when error happens', async () => {
+            const { body } = await supertest
+              .post(`/api/actions/connector/${genAiActionId}/_execute`)
+              .set('kbn-xsrf', 'foo')
+              .send({
+                params: {
+                  subAction: 'test',
+                  subActionParams: {
+                    body: '{"model":"gpt-3.5-turbo","messages":[{"role":"user","content":"Hello world"}]}',
+                  },
+                },
+              })
+              .expect(200);
+
+            expect(body).to.eql({
+              status: 'error',
+              connector_id: genAiActionId,
+              message: 'an error occurred while running the action',
+              retry: true,
+              service_message:
+                'Status code: 422. Message: API Error: Unprocessable Entity - The model `bad model` does not exist',
+            });
+          });
         });
       });
     });

x-pack/test/observability_ai_assistant_api_integration/tests/chat/chat.spec.ts

@@ -179,9 +179,9 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         connectorId,
         functions: [],
       });
-
+      console.log('ERRRR', response.body.message);
       expect(response.body.message).to.contain(
-        `400 - Bad Request - This model's maximum context length is 8192 tokens. However, your messages resulted in 11036 tokens. Please reduce the length of the messages.`
+        `an error occurred while running the action - Status code: 400. Message: API Error: Bad Request - This model's maximum context length is 8192 tokens. However, your messages resulted in 11036 tokens. Please reduce the length of the messages.`
       );
     });