Merge pull request #135 from indexnetwork/dev

load more index items

api/src/controllers/model.js

@@ -1,25 +1,31 @@
 import { createClient } from "redis";
-import { indexNewModel, stopIndexingModels } from "../libs/composedb.js";
+import { createNewModel, indexNewModel, stopIndexingModels } from "../libs/composedb.js";
 
 export const info = async (req, res) => {
   const runtimeDefinition = req.app.get("runtimeDefinition");
   const modelFragments = req.app.get("modelFragments");
-
   res.json({
     runtimeDefinition,
     modelFragments,
   });
 };
 
+export const create = async (req, res, next) => {
+  const indexResult = await createNewModel(
+    req.body.schema
+  );
+  res.json(indexResult);
+};
+
 export const deploy = async (req, res, next) => {
+
   const pubClient = createClient({
     url: process.env.REDIS_CONNECTION_STRING,
   });
   await pubClient.connect();
   const indexResult = await indexNewModel(
     req.app,
     req.params.id,
-    req.headers.authorization,
   );
   if (indexResult) {
     pubClient.publish("newModel", req.params.id);
@@ -35,7 +41,6 @@ export const remove = async (req, res, next) => {
   const indexResult = await stopIndexingModels(
     req.app,
     req.params.id,
-    req.headers.authorization,
   );
   if (indexResult) {
     pubClient.publish("newModel", req.params.id);

api/src/libs/composedb.js

@@ -413,6 +413,7 @@ let defaultRuntime = {
   },
 };
 
+
 export const jsonSchemaToGraphQLFragment = (schema, prefix = false) => {
   function resolveRef(ref, defs) {
     const refPath = ref.replace(/^#\/\$defs\//, "");
@@ -488,56 +489,53 @@ export const jsonSchemaToGraphQLFragment = (schema, prefix = false) => {
   return `... on ${schema.name} {\n${finalFragment}\n}`;
 };
 
-export const indexNewModel = async (app, modelId, ceramicAdminPrivateKey) => {
-  const indexerCeramic = new CeramicClient(process.env.CERAMIC_HOST);
-  if (!ceramicAdminPrivateKey) {
-    return false;
+export const createNewModel = async (graphQLSchema) => {
+  await authenticateAdmin()
+  try {
+    const response = await Composite.create({ ceramic, schema: graphQLSchema, index: false })
+    return {status: true, models: response.modelIDs}
+  } catch (e) {
+    return {status: false, error: e.message}
   }
-  const key = fromString(ceramicAdminPrivateKey, "base16");
-  const did = new DID({
-    resolver: getResolver(),
-    provider: new Ed25519Provider(key),
-  });
-  await did.authenticate();
-  if (!did.authenticated) {
-    return false;
+}
+
+export const indexNewModel = async (app, modelId) => {
+
+  const modelName = await ceramic.loadStream(modelId);
+  const protectedModelNames = Object.keys(defaultRuntime.models)
+  if (protectedModelNames.includes(modelName)) {
+    console.log(`Model name is protected`)
+    return false
+  }
+  const indexedModelList =await ceramic.admin.getIndexedModels()
+  if (indexedModelList.includes(modelId)) {
+    console.log(`Model is already indexed`)
+    return false
   }
-  indexerCeramic.did = did;
 
+  await authenticateAdmin()
   await Composite.fromModels({
-    ceramic: indexerCeramic,
+    ceramic,
     models: [modelId],
     index: true,
   });
   await setIndexedModelParams(app);
-
   return true;
 };
 
 export const stopIndexingModels = async (
   app,
   modelId,
-  ceramicAdminPrivateKey,
 ) => {
-  const indexerCeramic = new CeramicClient(process.env.CERAMIC_HOST);
-  if (!ceramicAdminPrivateKey) {
-    return false;
+  await authenticateAdmin()
+  const modelName = await ceramic.loadStream(modelId);
+  const protectedModelNames = Object.keys(defaultRuntime.models)
+  if (protectedModelNames.includes(modelName)) {
+    console.log(`Model name is protected`)
+    return false
   }
-  const key = fromString(ceramicAdminPrivateKey, "base16");
-  const did = new DID({
-    resolver: getResolver(),
-    provider: new Ed25519Provider(key),
-  });
-  await did.authenticate();
-  if (!did.authenticated) {
-    return false;
-  }
-  indexerCeramic.did = did;
-
-  const models = await indexerCeramic.admin.stopIndexingModels([modelId]);
-
+  const models = await ceramic.admin.stopIndexingModels([modelId]);
   await setIndexedModelParams(app);
-
   return models;
 };
 export const setIndexedModelParams = async (app) => {

api/src/packages/api.js

@@ -659,22 +659,30 @@ app.post(
 
 app.get("/model/info", modelController.info);
 app.post(
-  "/model/index/:id",
+  "/model/:id",
   validator.params(
     Joi.object({
       id: Joi.custom(isStreamID, "Model ID").required(),
     }),
   ),
+  authCheckMiddleware,
   modelController.deploy,
 );
 
+app.post(
+  "/model",
+  authCheckMiddleware,
+  modelController.create,
+);
+
 app.delete(
-  "/model/index/:id",
+  "/model/:id",
   validator.params(
     Joi.object({
       id: Joi.custom(isStreamID, "Model ID").required(),
     }),
   ),
+  authCheckMiddleware,
   modelController.remove,
 );
 

sdk/js/README.md

@@ -59,8 +59,59 @@ const webPage = await indexClient.crawlWebPage("http://www.paulgraham.com/publis
 await indexClient.addItemToIndex(index.id, webPage.id);
 ```
 
+
+### Using Custom Schemas
+If you want to use your own schema, you can do so by creating and deploying a custom model. Below are the methods and examples of how to use them.
+
+#### Creating a Custom Model
+Use the createModel method to create a custom model using a GraphQL schema.
+
+```typescript
+
+const modelResponse = await indexClient.createModel(`
+  type CustomObject {
+    title: String! @string(maxLength: 50)
+  }
+
+  type YourModel @createModel(accountRelation: LIST, description: "Full schema for models") {
+    id: ID!
+    booleanValue: Boolean!
+    intValue: Int!
+    floatValue: Float!
+    did: DID!
+    streamId: StreamID!
+    commitId: CommitID!
+    cid: CID!
+    chainId: ChainID!
+    accountId: AccountID!
+    uri: URI! @string(maxLength: 2000)
+    date: Date!
+    dateTime: DateTime!
+    time: Time!
+    localDate: LocalDate!
+    localTime: LocalTime!
+    timeZone: TimeZone!
+    utcOffset: UTCOffset!
+    duration: Duration!
+    stringValue: String! @string(maxLength: 10)
+    objectArray: [CustomObject!] @list(maxLength: 30)
+    singleObject: CustomObject
+  }
+`);
+
+```
+
+#### Deploying a Custom Model
+After creating a custom model, use the deployModel method to deploy it.
+
+```typescript
+await indexClient.deployModel(modelResponse.models[0]);
+```
+
+## Interact with your index
 Your index is now ready for interaction! To start a conversation and interact with the data, follow these steps:
 
+
 ```typescript
 // Create a conversation
 const conversationParams = {

sdk/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indexnetwork/sdk",
-  "version": "0.1.22",
+  "version": "0.1.24",
   "main": "dist/indexclient.cjs.js",
   "module": "dist/indexclient.es.js",
   "types": "dist/index.d.ts",

sdk/js/src/index.ts

@@ -302,6 +302,26 @@ export default class IndexClient {
     });
   }
 
+  public async createModel(graphQLSchema: string): ApiResponse<any> {
+    return this.request(`/model`, {
+      method: "POST",
+      body: JSON.stringify({
+        schema: graphQLSchema
+      }),
+    });
+  }
+
+  public async deployModel(modelId: string): ApiResponse<any> {
+    return this.request(`/model/${modelId}`, {
+      method: "POST",
+    });
+  }
+  public async removeModel(modelId: string): ApiResponse<any> {
+    return this.request(`/model/${modelId}`, {
+      method: "DELETE",
+    });
+  }
+
   public async getConversation(conversationId: string): ApiResponse<any> {
     return this.request(`/conversations/${conversationId}`, {
       method: "GET",

web-app/src/components/sections/IndexConversation/TabContainer/IndexItemsTab.tsx

@@ -7,7 +7,7 @@ import LinkInput from "@/components/site/input/LinkInput";
 import { useApi } from "@/context/APIContext";
 import { useRole } from "@/hooks/useRole";
 import { ITEM_ADDED, trackEvent } from "@/services/tracker";
-import { addItem, removeItem } from "@/store/api/index";
+import { addItem, fetchIndexItems, removeItem } from "@/store/api/index";
 import { selectIndex, setAddItemLoading } from "@/store/slices/indexSlice";
 import { useAppDispatch, useAppSelector } from "@/store/store";
 import { IndexItem } from "@/types/entity";
@@ -19,8 +19,7 @@ import { useIndexConversation } from "../IndexConversationContext";
 const CONCURRENCY_LIMIT = 10;
 
 export default function IndexItemsTabSection() {
-  const { setItemsState, searchLoading, fetchIndexItems, fetchMoreIndexItems } =
-    useIndexConversation();
+  const { setItemsState, searchLoading } = useIndexConversation();
   const { isCreator } = useRole();
   const {
     data: viewedIndex,
@@ -29,11 +28,24 @@ export default function IndexItemsTabSection() {
     addItemLoading,
   } = useAppSelector(selectIndex);
   const dispatch = useAppDispatch();
-
   const { api, ready: apiReady } = useApi();
+
   const [search, setSearch] = useState("");
-  const [addedItem, setAddedItem] = useState(null);
-  const [progress, setProgress] = useState({ current: 0, total: 0 });
+
+  const loadMoreItems = useCallback(async () => {
+    if (!viewedIndex || !api) {
+      return;
+    }
+
+    await dispatch(
+      fetchIndexItems({
+        indexID: viewedIndex.id,
+        api,
+        resetCursor: false,
+        params: { query: search },
+      }),
+    );
+  }, [dispatch, viewedIndex, api, search]);
 
   // useEffect(() => {
   //   if (addedItem) {
@@ -109,7 +121,7 @@ export default function IndexItemsTabSection() {
       const updatedItems = [...urls, ...indexIds];
 
       dispatch(setAddItemLoading(true));
-      setProgress({ current: 0, total: updatedItems.length });
+      // setProgress({ current: 0, total: updatedItems.length });
 
       await processUrlsInBatches(updatedItems, async (item: any) => {
         try {
@@ -152,7 +164,7 @@ export default function IndexItemsTabSection() {
         dispatch(setAddItemLoading(false));
       }
     },
-    [api, viewedIndex, apiReady, fetchIndexItems],
+    [api, viewedIndex, apiReady],
   );
 
   // const handleRemove = useCallback(
@@ -209,7 +221,7 @@ export default function IndexItemsTabSection() {
               <LinkInput
                 loading={addItemLoading}
                 onItemAdd={handleAddItem}
-                progress={progress}
+                progress={items.progress}
               />
             </Col>
           </FlexRow>
@@ -221,10 +233,9 @@ export default function IndexItemsTabSection() {
             search={search}
             hasMore={!!items.cursor}
             removeItem={handleRemoveItem}
-            loadMore={() =>
-              viewedIndex &&
-              fetchMoreIndexItems(viewedIndex?.id, { resetCursor: false })
-            }
+            loadMore={() => {
+              loadMoreItems();
+            }}
           />
         </div>
       </Flex>