[Feat] AI Assistant [2] (#2777)

examples/demo-app/esbuild.config.mjs

@@ -58,7 +58,8 @@ const config = {
     'process.env.FoursquareClientId': JSON.stringify(process.env.FoursquareClientId || ''),
     'process.env.FoursquareDomain': JSON.stringify(process.env.FoursquareDomain || ''),
     'process.env.FoursquareAPIURL': JSON.stringify(process.env.FoursquareAPIURL || ''),
-    'process.env.FoursquareUserMapsURL': JSON.stringify(process.env.FoursquareUserMapsURL || '')
+    'process.env.FoursquareUserMapsURL': JSON.stringify(process.env.FoursquareUserMapsURL || ''),
+    'process.env.OpenAIToken': JSON.stringify(process.env.OpenAIToken || '')
   },
   plugins: [
     // automatically injected kepler.gl package version into the bundle

src/ai-assistant/package.json

@@ -35,8 +35,10 @@
     "@kepler.gl/layers": "3.1.0-alpha.1",
     "@kepler.gl/types": "3.1.0-alpha.1",
     "@kepler.gl/utils": "3.1.0-alpha.1",
+    "color-interpolate": "^1.0.5",
+    "echarts": "^5.5.1",
     "global": "^4.3.0",
-    "react-ai-assist": "0.0.8"
+    "react-ai-assist": "0.0.20"
   },
   "nyc": {
     "sourceMap": false,

src/ai-assistant/src/components/ai-assistant-component.tsx

@@ -3,13 +3,18 @@
 
 import React, {useEffect} from 'react';
 import styled, {withTheme} from 'styled-components';
-import {AiAssistant, MessageModel, useAssistant} from 'react-ai-assist';
+import {
+  AiAssistant,
+  MessageModel,
+  useAssistant,
+  histogramFunctionDefinition
+} from 'react-ai-assist';
 import 'react-ai-assist/dist/index.css';
 
 import {textColorLT} from '@kepler.gl/styles';
-import {ActionHandler, addDataToMap, loadFiles, mapStyleChange} from '@kepler.gl/actions';
+import {ActionHandler} from '@kepler.gl/actions';
 import {MapStyle} from '@kepler.gl/reducers';
-import {Loader} from '@loaders.gl/loader-utils';
+import {VisState} from '@kepler.gl/schemas';
 
 import {basemapFunctionDefinition} from '../tools/basemap-functions';
 import {loadUrlFunctionDefinition} from '../tools/loadurl-function';
@@ -21,25 +26,24 @@ import {
   ASSISTANT_NAME,
   ASSISTANT_VERSION,
   INSTRUCTIONS,
+  PROMPT_IDEAS,
   WELCOME_MESSAGE
 } from '../constants';
+import {filterFunctionDefinition} from '../tools/filter-function';
+import {addLayerFunctionDefinition} from '../tools/layer-creation-function';
+import {updateLayerColorFunctionDefinition} from '../tools/layer-style-function';
+import {SelectedKeplerGlActions} from './ai-assistant-manager';
+import {getDatasetContext, getValuesFromDataset, highlightRows} from '../tools/utils';
 
 export type AiAssistantComponentProps = {
   theme: any;
   aiAssistant: AiAssistantState;
   updateAiAssistantMessages: ActionHandler<typeof updateAiAssistantMessages>;
   setStartScreenCapture: ActionHandler<typeof setStartScreenCapture>;
   setScreenCaptured: ActionHandler<typeof setScreenCaptured>;
-  keplerGlActions: {
-    mapStyleChange: ActionHandler<typeof mapStyleChange>;
-    loadFiles: ActionHandler<typeof loadFiles>;
-    addDataToMap: ActionHandler<typeof addDataToMap>;
-  };
+  keplerGlActions: SelectedKeplerGlActions;
   mapStyle: MapStyle;
-  visState: {
-    loaders: Loader[];
-    loadOptions: object;
-  };
+  visState: VisState;
 };
 
 const StyledAiAssistantComponent = styled.div`
@@ -70,6 +74,33 @@ function AiAssistantComponentFactory() {
         addDataToMap: keplerGlActions.addDataToMap,
         loaders: visState.loaders,
         loadOptions: visState.loadOptions
+      }),
+      addLayerFunctionDefinition({
+        addLayer: keplerGlActions.addLayer,
+        datasets: visState.datasets
+      }),
+      updateLayerColorFunctionDefinition({
+        layerVisualChannelConfigChange: keplerGlActions.layerVisualChannelConfigChange,
+        layers: visState.layers
+      }),
+      filterFunctionDefinition({
+        datasets: visState.datasets,
+        filters: visState.filters,
+        createOrUpdateFilter: keplerGlActions.createOrUpdateFilter,
+        setFilter: keplerGlActions.setFilter,
+        setFilterPlot: keplerGlActions.setFilterPlot
+      }),
+      histogramFunctionDefinition({
+        getValues: (datasetName: string, variableName: string): number[] =>
+          getValuesFromDataset(visState.datasets, datasetName, variableName),
+        onSelected: (datasetName: string, selectedRowIndices: number[]) =>
+          highlightRows(
+            visState.datasets,
+            visState.layers,
+            datasetName,
+            selectedRowIndices,
+            keplerGlActions.layerSetIsValid
+          )
       })
     ];
 
@@ -87,12 +118,25 @@ function AiAssistantComponentFactory() {
       functions
     };
 
-    const {initializeAssistant} = useAssistant(assistantProps);
+    const {initializeAssistant, addAdditionalContext} = useAssistant(assistantProps);
+
+    const initializeAssistantWithContext = async () => {
+      await initializeAssistant();
+      const context = getDatasetContext(visState.datasets, visState.layers);
+      addAdditionalContext({context});
+    };
 
     useEffect(() => {
-      initializeAssistant();
+      initializeAssistantWithContext();
+      // re-initialize assistant when datasets, filters or layers change
       // eslint-disable-next-line react-hooks/exhaustive-deps
-    }, []);
+    }, [visState.datasets, visState.filters, visState.layers]);
+
+    const onRestartAssistant = () => {
+      // clean up aiAssistant state
+      updateAiAssistantMessages([]);
+      initializeAssistantWithContext();
+    };
 
     const onMessagesUpdated = (messages: MessageModel[]) => {
       updateAiAssistantMessages(messages);
@@ -121,9 +165,11 @@ function AiAssistantComponentFactory() {
           onScreenshotClick={onScreenshotClick}
           screenCapturedBase64={aiAssistant.screenshotToAsk.screenCaptured}
           onRemoveScreenshot={onRemoveScreenshot}
+          onRestartChat={onRestartAssistant}
           fontSize={'text-tiny'}
           botMessageClassName={''}
           githubIssueLink={'https://github.com/keplergl/kepler.gl/issues'}
+          ideas={PROMPT_IDEAS}
         />
       </StyledAiAssistantComponent>
     );

src/ai-assistant/src/components/ai-assistant-manager.tsx

@@ -5,8 +5,19 @@ import React, {useCallback} from 'react';
 import styled from 'styled-components';
 import {injectIntl, IntlShape} from 'react-intl';
 
-import {MapStyle} from '@kepler.gl/reducers';
-import {ActionHandler, mapStyleChange, loadFiles, addDataToMap} from '@kepler.gl/actions';
+import {MapStyle, mapStyleLens, visStateLens} from '@kepler.gl/reducers';
+import {
+  ActionHandler,
+  mapStyleChange,
+  loadFiles,
+  addDataToMap,
+  addLayer,
+  createOrUpdateFilter,
+  setFilter,
+  setFilterPlot,
+  layerSetIsValid,
+  layerVisualChannelConfigChange
+} from '@kepler.gl/actions';
 import {withState, SidePanelTitleFactory, Icons} from '@kepler.gl/components';
 import {VisState} from '@kepler.gl/schemas';
 
@@ -20,18 +31,26 @@ import {
 import AiAssistantConfigFactory from './ai-assistant-config';
 import AiAssistantComponentFactory from './ai-assistant-component';
 
+export type SelectedKeplerGlActions = {
+  mapStyleChange: ActionHandler<typeof mapStyleChange>;
+  loadFiles: ActionHandler<typeof loadFiles>;
+  addDataToMap: ActionHandler<typeof addDataToMap>;
+  addLayer: ActionHandler<typeof addLayer>;
+  layerVisualChannelConfigChange: ActionHandler<typeof layerVisualChannelConfigChange>;
+  createOrUpdateFilter: ActionHandler<typeof createOrUpdateFilter>;
+  setFilter: ActionHandler<typeof setFilter>;
+  setFilterPlot: ActionHandler<typeof setFilterPlot>;
+  layerSetIsValid: ActionHandler<typeof layerSetIsValid>;
+};
+
 export type AiAssistantManagerState = {
   aiAssistantActions: {
     updateAiAssistantConfig: ActionHandler<typeof updateAiAssistantConfig>;
     updateAiAssistantMessages: ActionHandler<typeof updateAiAssistantMessages>;
     setStartScreenCapture: ActionHandler<typeof setStartScreenCapture>;
     setScreenCaptured: ActionHandler<typeof setScreenCaptured>;
   };
-  keplerGlActions: {
-    mapStyleChange: ActionHandler<typeof mapStyleChange>;
-    loadFiles: ActionHandler<typeof loadFiles>;
-    addDataToMap: ActionHandler<typeof addDataToMap>;
-  };
+  keplerGlActions: SelectedKeplerGlActions;
   aiAssistant: AiAssistantState;
   mapStyle: MapStyle;
   visState: VisState;
@@ -50,6 +69,8 @@ const StyledAiAssistantPanelContainer = styled.div`
   justify-content: space-between;
   overflow: hidden;
   height: 100%;
+  min-width: ${props => props.theme.aiAssistantPanelWidth}px;
+  max-width: ${props => props.theme.aiAssistantPanelWidth}px;
   & > * {
     /* all children should allow input */
     pointer-events: all;
@@ -68,7 +89,6 @@ const StyledAiAssistantPanel = styled.div`
 const StyledAiAssistantPanelHeader = styled.div`
   padding: 16px 16px 4px 16px;
   border-bottom: 1px solid ${props => props.theme.borderColor};
-  min-width: 345px;
   color: ${props => props.theme.subtextColorActive};
 `;
 
@@ -142,22 +162,26 @@ function AiAssistantManagerFactory(
   };
 
   return withState(
-    [],
+    [visStateLens, mapStyleLens],
     state => {
       // todo: find a better way to get the state key
       const stateKey = Object.keys(state)[0];
-      const mapKey = Object.keys(state[stateKey].keplerGl)[0];
       return {
-        aiAssistant: state[stateKey].aiAssistant,
-        mapStyle: state[stateKey].keplerGl[mapKey].mapStyle,
-        visState: {
-          loaders: state[stateKey].keplerGl[mapKey].visState.loaders,
-          loadOptions: state[stateKey].keplerGl[mapKey].visState.loadOptions
-        }
+        aiAssistant: state[stateKey].aiAssistant
       };
     },
     {
-      keplerGlActions: {mapStyleChange, loadFiles, addDataToMap},
+      keplerGlActions: {
+        mapStyleChange,
+        loadFiles,
+        addDataToMap,
+        addLayer,
+        createOrUpdateFilter,
+        setFilter,
+        setFilterPlot,
+        layerSetIsValid,
+        layerVisualChannelConfigChange
+      },
       aiAssistantActions: {
         updateAiAssistantConfig,
         updateAiAssistantMessages,

src/ai-assistant/src/constants.ts

@@ -1,14 +1,52 @@
-// SPDX-License-Identifier: MIT
-// Copyright contributors to the kepler.gl project
-
-export const TASK_LIST = '1. Change the basemap style.\n2. Load data from url.';
+export const TASK_LIST =
+  '1. Show dataset/layer/variable info.\n2. Change the basemap style.\n3. Load data from url.\n4. Create a map layer using variable.\n5. Filter the data of a variable.\n6. Create a histogram.';
 
 export const WELCOME_MESSAGE = `Hi, I am Kepler.gl AI Assistant!\nHere are some tasks I can help you with:\n\n${TASK_LIST}`;
 
-export const INSTRUCTIONS = `You are a helpful assistant that can answer questions and help with tasks. If you can use tools, please use them. If the parameters of functions are not provided, please ask user to specify them. Otherwise, just answer the question with plain textdirectly. Do not include any programming code in your response. You can do the following tasks: ${TASK_LIST}`;
+export const INSTRUCTIONS = `You are a Kepler.gl AI Assistant that can answer questions and help with tasks of mapping and spatial data analysis.
+
+When responding to user queries:
+1. Analyze if the task requires one or multiple function calls
+2. For each required function:
+   - Identify the appropriate function to call
+   - Determine all required parameters
+   - If parameters are missing, ask the user to provide them
+   - Execute functions in a sequential order
+
+You can execute multiple functions to complete complex tasks, but execute them one at a time in a logical sequence. Always validate the success of each function call before proceeding to the next one.
+
+Remember to:
+- Return function calls in a structured format that can be parsed and executed
+- Wait for confirmation of each function's completion before proceeding
+- Prompt user to proceed to the next function if needed
+- Provide clear feedback about what action is being taken
+- Do not include raw programming code in responses to users`;
+
+export const PROMPT_IDEAS = [
+  {
+    title: 'Show Metadata ',
+    description: 'list dataset info'
+  },
+  {
+    title: 'Change Basemap ',
+    description: 'use Positron style'
+  },
+  {
+    title: 'Load Data',
+    description: 'load data from https://geodacenter.github.io/data-and-lab/data/lehd.geojson'
+  },
+  {
+    title: 'Create a Map Layer',
+    description: 'update its color inspired by Van Gogh Starry Night'
+  },
+  {
+    title: 'Data Insight',
+    description: 'create a histogram'
+  }
+];
 
 export const ASSISTANT_NAME = 'kepler-gl-ai-assistant';
 
 export const ASSISTANT_DESCRIPTION = 'A Kepler.gl AI Assistant';
 
-export const ASSISTANT_VERSION = '0.0.1-1';
+export const ASSISTANT_VERSION = '0.0.1-9';

src/ai-assistant/src/reducers/index.ts

@@ -19,6 +19,7 @@ export type AiAssistantConfig = {
   temperature: number;
   topP: number;
 };
+
 // Initial state for the reducer
 const initialConfig: AiAssistantConfig = {
   isReady: false,
@@ -50,36 +51,39 @@ const initialState: AiAssistantState = {
 
 export const aiAssistantReducer = handleActions<AiAssistantState, any>(
   {
-    [UPDATE_AI_ASSISTANT_CONFIG]: updateAiAssistantConfig,
-    [UPDATE_AI_ASSISTANT_MESSAGES]: updateAiAssistantMessages,
-    [SET_START_SCREEN_CAPTURE]: setStartScreenCapture,
-    [SET_SCREEN_CAPTURED]: setScreenCaptured
+    [UPDATE_AI_ASSISTANT_CONFIG]: updateAiAssistantConfigHandler,
+    [UPDATE_AI_ASSISTANT_MESSAGES]: updateAiAssistantMessagesHandler,
+    [SET_START_SCREEN_CAPTURE]: setStartScreenCaptureHandler,
+    [SET_SCREEN_CAPTURED]: setScreenCapturedHandler
   },
   initialState
 );
 
-function updateAiAssistantConfig(state: AiAssistantState, action: Action<AiAssistantConfig>) {
+function updateAiAssistantConfigHandler(
+  state: AiAssistantState,
+  action: Action<AiAssistantConfig>
+) {
   return {
     ...state,
     config: {...state.config, ...action.payload}
   };
 }
 
-function updateAiAssistantMessages(state: AiAssistantState, action: Action<MessageModel[]>) {
+function updateAiAssistantMessagesHandler(state: AiAssistantState, action: Action<MessageModel[]>) {
   return {
     ...state,
     messages: action.payload
   };
 }
 
-function setStartScreenCapture(state: AiAssistantState, action: Action<boolean>) {
+function setStartScreenCaptureHandler(state: AiAssistantState, action: Action<boolean>) {
   return {
     ...state,
     screenshotToAsk: {startScreenCapture: action.payload, screenCaptured: ''}
   };
 }
 
-function setScreenCaptured(state: AiAssistantState, action: Action<string>) {
+function setScreenCapturedHandler(state: AiAssistantState, action: Action<string>) {
   return {
     ...state,
     screenshotToAsk: {...state.screenshotToAsk, screenCaptured: action.payload}