diff --git a/api/src/controllers/model.js b/api/src/controllers/model.js index 20b7aca8..a417b735 100644 --- a/api/src/controllers/model.js +++ b/api/src/controllers/model.js @@ -1,17 +1,24 @@ import { createClient } from "redis"; -import { indexNewModel, stopIndexingModels } from "../libs/composedb.js"; +import { createNewModel, indexNewModel, stopIndexingModels } from "../libs/composedb.js"; export const info = async (req, res) => { const runtimeDefinition = req.app.get("runtimeDefinition"); const modelFragments = req.app.get("modelFragments"); - res.json({ runtimeDefinition, modelFragments, }); }; +export const create = async (req, res, next) => { + const indexResult = await createNewModel( + req.body.schema + ); + res.json(indexResult); +}; + export const deploy = async (req, res, next) => { + const pubClient = createClient({ url: process.env.REDIS_CONNECTION_STRING, }); @@ -19,7 +26,6 @@ export const deploy = async (req, res, next) => { const indexResult = await indexNewModel( req.app, req.params.id, - req.headers.authorization, ); if (indexResult) { pubClient.publish("newModel", req.params.id); @@ -35,7 +41,6 @@ export const remove = async (req, res, next) => { const indexResult = await stopIndexingModels( req.app, req.params.id, - req.headers.authorization, ); if (indexResult) { pubClient.publish("newModel", req.params.id); diff --git a/api/src/libs/composedb.js b/api/src/libs/composedb.js index 0d571fd7..6b88fa5d 100644 --- a/api/src/libs/composedb.js +++ b/api/src/libs/composedb.js @@ -413,6 +413,7 @@ let defaultRuntime = { }, }; + export const jsonSchemaToGraphQLFragment = (schema, prefix = false) => { function resolveRef(ref, defs) { const refPath = ref.replace(/^#\/\$defs\//, ""); @@ -488,56 +489,53 @@ export const jsonSchemaToGraphQLFragment = (schema, prefix = false) => { return `... on ${schema.name} {\n${finalFragment}\n}`; }; -export const indexNewModel = async (app, modelId, ceramicAdminPrivateKey) => { - const indexerCeramic = new CeramicClient(process.env.CERAMIC_HOST); - if (!ceramicAdminPrivateKey) { - return false; +export const createNewModel = async (graphQLSchema) => { + await authenticateAdmin() + try { + const response = await Composite.create({ ceramic, schema: graphQLSchema, index: false }) + return {status: true, models: response.modelIDs} + } catch (e) { + return {status: false, error: e.message} } - const key = fromString(ceramicAdminPrivateKey, "base16"); - const did = new DID({ - resolver: getResolver(), - provider: new Ed25519Provider(key), - }); - await did.authenticate(); - if (!did.authenticated) { - return false; +} + +export const indexNewModel = async (app, modelId) => { + + const modelName = await ceramic.loadStream(modelId); + const protectedModelNames = Object.keys(defaultRuntime.models) + if (protectedModelNames.includes(modelName)) { + console.log(`Model name is protected`) + return false + } + const indexedModelList =await ceramic.admin.getIndexedModels() + if (indexedModelList.includes(modelId)) { + console.log(`Model is already indexed`) + return false } - indexerCeramic.did = did; + await authenticateAdmin() await Composite.fromModels({ - ceramic: indexerCeramic, + ceramic, models: [modelId], index: true, }); await setIndexedModelParams(app); - return true; }; export const stopIndexingModels = async ( app, modelId, - ceramicAdminPrivateKey, ) => { - const indexerCeramic = new CeramicClient(process.env.CERAMIC_HOST); - if (!ceramicAdminPrivateKey) { - return false; + await authenticateAdmin() + const modelName = await ceramic.loadStream(modelId); + const protectedModelNames = Object.keys(defaultRuntime.models) + if (protectedModelNames.includes(modelName)) { + console.log(`Model name is protected`) + return false } - const key = fromString(ceramicAdminPrivateKey, "base16"); - const did = new DID({ - resolver: getResolver(), - provider: new Ed25519Provider(key), - }); - await did.authenticate(); - if (!did.authenticated) { - return false; - } - indexerCeramic.did = did; - - const models = await indexerCeramic.admin.stopIndexingModels([modelId]); - + const models = await ceramic.admin.stopIndexingModels([modelId]); await setIndexedModelParams(app); - return models; }; export const setIndexedModelParams = async (app) => { diff --git a/api/src/packages/api.js b/api/src/packages/api.js index dcb6a596..a8ef402a 100644 --- a/api/src/packages/api.js +++ b/api/src/packages/api.js @@ -659,22 +659,30 @@ app.post( app.get("/model/info", modelController.info); app.post( - "/model/index/:id", + "/model/:id", validator.params( Joi.object({ id: Joi.custom(isStreamID, "Model ID").required(), }), ), + authCheckMiddleware, modelController.deploy, ); +app.post( + "/model", + authCheckMiddleware, + modelController.create, +); + app.delete( - "/model/index/:id", + "/model/:id", validator.params( Joi.object({ id: Joi.custom(isStreamID, "Model ID").required(), }), ), + authCheckMiddleware, modelController.remove, ); diff --git a/sdk/js/README.md b/sdk/js/README.md index e305b244..26998cb4 100644 --- a/sdk/js/README.md +++ b/sdk/js/README.md @@ -59,8 +59,59 @@ const webPage = await indexClient.crawlWebPage("http://www.paulgraham.com/publis await indexClient.addItemToIndex(index.id, webPage.id); ``` + +### Using Custom Schemas +If you want to use your own schema, you can do so by creating and deploying a custom model. Below are the methods and examples of how to use them. + +#### Creating a Custom Model +Use the createModel method to create a custom model using a GraphQL schema. + +```typescript + +const modelResponse = await indexClient.createModel(` + type CustomObject { + title: String! @string(maxLength: 50) + } + + type YourModel @createModel(accountRelation: LIST, description: "Full schema for models") { + id: ID! + booleanValue: Boolean! + intValue: Int! + floatValue: Float! + did: DID! + streamId: StreamID! + commitId: CommitID! + cid: CID! + chainId: ChainID! + accountId: AccountID! + uri: URI! @string(maxLength: 2000) + date: Date! + dateTime: DateTime! + time: Time! + localDate: LocalDate! + localTime: LocalTime! + timeZone: TimeZone! + utcOffset: UTCOffset! + duration: Duration! + stringValue: String! @string(maxLength: 10) + objectArray: [CustomObject!] @list(maxLength: 30) + singleObject: CustomObject + } +`); + +``` + +#### Deploying a Custom Model +After creating a custom model, use the deployModel method to deploy it. + +```typescript +await indexClient.deployModel(modelResponse.models[0]); +``` + +## Interact with your index Your index is now ready for interaction! To start a conversation and interact with the data, follow these steps: + ```typescript // Create a conversation const conversationParams = { diff --git a/sdk/js/package.json b/sdk/js/package.json index 91e5ff29..4e79b4e3 100644 --- a/sdk/js/package.json +++ b/sdk/js/package.json @@ -1,6 +1,6 @@ { "name": "@indexnetwork/sdk", - "version": "0.1.22", + "version": "0.1.24", "main": "dist/indexclient.cjs.js", "module": "dist/indexclient.es.js", "types": "dist/index.d.ts", diff --git a/sdk/js/src/index.ts b/sdk/js/src/index.ts index 948c1bad..43df6113 100644 --- a/sdk/js/src/index.ts +++ b/sdk/js/src/index.ts @@ -302,6 +302,26 @@ export default class IndexClient { }); } + public async createModel(graphQLSchema: string): ApiResponse { + return this.request(`/model`, { + method: "POST", + body: JSON.stringify({ + schema: graphQLSchema + }), + }); + } + + public async deployModel(modelId: string): ApiResponse { + return this.request(`/model/${modelId}`, { + method: "POST", + }); + } + public async removeModel(modelId: string): ApiResponse { + return this.request(`/model/${modelId}`, { + method: "DELETE", + }); + } + public async getConversation(conversationId: string): ApiResponse { return this.request(`/conversations/${conversationId}`, { method: "GET", diff --git a/web-app/src/components/sections/IndexConversation/TabContainer/IndexItemsTab.tsx b/web-app/src/components/sections/IndexConversation/TabContainer/IndexItemsTab.tsx index 0aefd7bf..b0fb6b44 100644 --- a/web-app/src/components/sections/IndexConversation/TabContainer/IndexItemsTab.tsx +++ b/web-app/src/components/sections/IndexConversation/TabContainer/IndexItemsTab.tsx @@ -7,7 +7,7 @@ import LinkInput from "@/components/site/input/LinkInput"; import { useApi } from "@/context/APIContext"; import { useRole } from "@/hooks/useRole"; import { ITEM_ADDED, trackEvent } from "@/services/tracker"; -import { addItem, removeItem } from "@/store/api/index"; +import { addItem, fetchIndexItems, removeItem } from "@/store/api/index"; import { selectIndex, setAddItemLoading } from "@/store/slices/indexSlice"; import { useAppDispatch, useAppSelector } from "@/store/store"; import { IndexItem } from "@/types/entity"; @@ -19,8 +19,7 @@ import { useIndexConversation } from "../IndexConversationContext"; const CONCURRENCY_LIMIT = 10; export default function IndexItemsTabSection() { - const { setItemsState, searchLoading, fetchIndexItems, fetchMoreIndexItems } = - useIndexConversation(); + const { setItemsState, searchLoading } = useIndexConversation(); const { isCreator } = useRole(); const { data: viewedIndex, @@ -29,11 +28,24 @@ export default function IndexItemsTabSection() { addItemLoading, } = useAppSelector(selectIndex); const dispatch = useAppDispatch(); - const { api, ready: apiReady } = useApi(); + const [search, setSearch] = useState(""); - const [addedItem, setAddedItem] = useState(null); - const [progress, setProgress] = useState({ current: 0, total: 0 }); + + const loadMoreItems = useCallback(async () => { + if (!viewedIndex || !api) { + return; + } + + await dispatch( + fetchIndexItems({ + indexID: viewedIndex.id, + api, + resetCursor: false, + params: { query: search }, + }), + ); + }, [dispatch, viewedIndex, api, search]); // useEffect(() => { // if (addedItem) { @@ -109,7 +121,7 @@ export default function IndexItemsTabSection() { const updatedItems = [...urls, ...indexIds]; dispatch(setAddItemLoading(true)); - setProgress({ current: 0, total: updatedItems.length }); + // setProgress({ current: 0, total: updatedItems.length }); await processUrlsInBatches(updatedItems, async (item: any) => { try { @@ -152,7 +164,7 @@ export default function IndexItemsTabSection() { dispatch(setAddItemLoading(false)); } }, - [api, viewedIndex, apiReady, fetchIndexItems], + [api, viewedIndex, apiReady], ); // const handleRemove = useCallback( @@ -209,7 +221,7 @@ export default function IndexItemsTabSection() { @@ -221,10 +233,9 @@ export default function IndexItemsTabSection() { search={search} hasMore={!!items.cursor} removeItem={handleRemoveItem} - loadMore={() => - viewedIndex && - fetchMoreIndexItems(viewedIndex?.id, { resetCursor: false }) - } + loadMore={() => { + loadMoreItems(); + }} /> diff --git a/web-app/src/store/api/index.ts b/web-app/src/store/api/index.ts index 51770473..232fbb93 100644 --- a/web-app/src/store/api/index.ts +++ b/web-app/src/store/api/index.ts @@ -18,6 +18,12 @@ type CreateIndexPayload = { type FetchIndexItemsPayload = { indexID: string; api: ApiService; + resetCursor?: boolean; + params?: { + limit?: number; + cursor?: string; + query?: string; + }; }; type AddItemPayload = { @@ -88,14 +94,35 @@ export const fetchIndex = createAsyncThunk( export const fetchIndexItems = createAsyncThunk( "index/fetchIndexItems", - async ({ indexID, api }: FetchIndexItemsPayload, { rejectWithValue }) => { + async ( + { indexID, api, resetCursor = true, params }: FetchIndexItemsPayload, + { getState, rejectWithValue }, + ) => { try { + const { index } = getState() as any; const itemParams: GetItemQueryParams = {}; + + if (!resetCursor && index.items.cursor) { + itemParams.cursor = index.items.cursor; + } + + if (params?.query) { + itemParams.query = params.query; + } + const items = await api.getItems(indexID, { queryParams: itemParams, }); - return items; + + return { + items: + resetCursor || itemParams.query + ? items.items + : [...(index.items.data || []), ...items.items], + cursor: items.endCursor || index.items.cursor, + }; } catch (err: any) { + console.log("33", err); return rejectWithValue(err.response.data); } }, diff --git a/web-app/src/store/slices/indexSlice.ts b/web-app/src/store/slices/indexSlice.ts index a6a1998f..8a5b82fd 100644 --- a/web-app/src/store/slices/indexSlice.ts +++ b/web-app/src/store/slices/indexSlice.ts @@ -50,14 +50,14 @@ const indexSlice = createSlice({ state.error = action.payload as any; }) .addCase(fetchIndexItems.pending, (state) => { - state.loading = true; - state.items.data = null; - state.items.cursor = null; + if (!state.items.data) { + state.loading = true; + } }) .addCase(fetchIndexItems.fulfilled, (state, action) => { state.loading = false; state.items.data = action.payload.items; - state.items.cursor = action.payload.endCursor; + state.items.cursor = action.payload.cursor; }) .addCase(fetchIndexItems.rejected, (state, action) => { state.loading = false;