put import functionality to cli + fixes

apps/edb-cli/package.json

@@ -28,6 +28,7 @@
     "@slub/edb-core-utils": "workspace:*",
     "@slub/edb-graph-traversal": "workspace:*",
     "@slub/remote-query-implementations": "workspace:*",
+    "@slub/prisma-db-impl": "workspace:*",
     "@slub/sparql-schema": "workspace:*",
     "@slub/edb-authorities": "workspace:*",
     "@slub/edb-data-mapping": "workspace:*",

apps/edb-cli/src/dataStore.ts

@@ -4,15 +4,55 @@ import {
   getSPARQLFlavour,
 } from "@slub/remote-query-implementations";
 import { initSPARQLDataStoreFromConfig } from "@slub/sparql-db-impl";
+import { extendSchemaShortcut } from "@slub/json-schema-utils";
+import { primaryFields, schema } from "@slub/exhibition-schema";
+import { JSONSchema7 } from "json-schema";
+import { initPrismaStore } from "@slub/prisma-db-impl";
+import {
+  typeIRItoTypeName,
+  typeNameToTypeIRI,
+} from "@slub/edb-api/src/dataStore";
+import { PrismaClient } from "@prisma/edb-exhibition-client";
+
+const initPrisma = async () => {
+  const rootSchema = extendSchemaShortcut(schema as JSONSchema7, "type", "id");
+  const prisma = new PrismaClient();
+  //bun only runs if we call it here: why??
+  //find first object that can be counted:
+  for (const key of Object.keys(prisma)) {
+    if (prisma[key]?.count) {
+      const c = await prisma[key].count();
+      console.log(c);
+      break;
+    }
+  }
+  return initPrismaStore(prisma, rootSchema, primaryFields, {
+    jsonldContext: config.defaultJsonldContext,
+    defaultPrefix: config.defaultPrefix,
+    typeIRItoTypeName: typeIRItoTypeName,
+    typeNameToTypeIRI: typeNameToTypeIRI,
+  });
+};
 
 export const provider = getProviderOrDefault(config.sparqlEndpoint);
 if (!provider) {
   throw new Error("No provider found for the given SPARQL endpoint");
 }
 export const crudFunctions = provider(config.sparqlEndpoint);
 
-export const dataStore = initSPARQLDataStoreFromConfig(
-  config,
-  crudFunctions,
-  getSPARQLFlavour(config.sparqlEndpoint),
-);
+export const dataStore =
+  process.env.DATABASE_PROVIDER === "sparql"
+    ? initSPARQLDataStoreFromConfig(
+        config,
+        crudFunctions,
+        getSPARQLFlavour(config.sparqlEndpoint),
+      )
+    : await initPrisma();
+
+export const importStores = {
+  oxigraph: initSPARQLDataStoreFromConfig(
+    config,
+    crudFunctions,
+    getSPARQLFlavour(config.sparqlEndpoint),
+  ),
+};

apps/edb-cli/src/index.ts

@@ -1,10 +1,15 @@
 import { run, subcommands } from "cmd-ts";
-import { dataStore } from "./dataStore";
+import { dataStore, importStores } from "./dataStore";
 import { flatImportHandler } from "./flatImportHandler";
 import { availableFlatMappings } from "@slub/exhibition-schema";
 import { makeEdbCli } from "@slub/edb-cli-creator";
 
-const cli = makeEdbCli(dataStore, [], availableFlatMappings, flatImportHandler);
+const cli = makeEdbCli(
+  dataStore,
+  importStores,
+  availableFlatMappings,
+  flatImportHandler,
+);
 
 run(
   subcommands({

packages/edb-cli-creator/src/makeEdbCli.ts

@@ -26,7 +26,7 @@ export type FlatImportHandler = (option: {
 }) => Promise<void>;
 export const makeEdbCli = (
   dataStore: AbstractDatastore,
-  importStores: AbstractDatastore[],
+  importStores: Record<string, AbstractDatastore>,
   flatMappings?: AvailableFlatMappings,
   flatImportHandler?: FlatImportHandler,
 ) => {
@@ -100,6 +100,7 @@ export const makeEdbCli = (
         console.log(formatJSONResult(item, pretty, noJsonld));
         return Promise.resolve();
       });
+      process.exit(0);
     },
   });
 
@@ -155,5 +156,49 @@ export const makeEdbCli = (
       handler: flatImportHandler,
     });
 
-  return { list, get, flatImport };
+  const importCommand = command({
+    name: "import",
+    description: "Recursively import data from another data store",
+    args: {
+      typeName: positional({
+        type: string,
+        displayName: "type",
+        description: "The Type of the document",
+      }),
+      importStoreName: option({
+        type: oneOf(Object.keys(importStores)),
+        long: "importFrom",
+        short: "i",
+        description: `the store where data should be imported (${Object.keys(importStores).join(" , ")})`,
+      }),
+      entityIRI: option({
+        type: optional(string),
+        long: "entityIRI",
+        short: "e",
+        description: "only import a single entity identified by the entityIRI",
+      }),
+      limit: option({
+        type: optional(number),
+        description: "The number of documents to import",
+        defaultValue: () => 10000,
+        long: "limit",
+        short: "l",
+      }),
+    },
+    handler: async ({ entityIRI, typeName, importStoreName, limit }) => {
+      const importStore = importStores[importStoreName];
+      if (!importStore) {
+        throw new Error(`import store ${importStoreName} not found in config`);
+        process.exit(1);
+      }
+      if (entityIRI) {
+        await dataStore.importDocument(typeName, entityIRI, importStore);
+      } else {
+        await dataStore.importDocuments(typeName, importStore, limit || 10);
+      }
+      process.exit(0);
+    },
+  });
+
+  return { list, get, flatImport, import: importCommand };
 };

packages/prisma-db-impl/src/import/startBulkImport.ts

@@ -26,7 +26,18 @@ export const startBulkImport = async (
   const bar = new cliProgress.SingleBar({}, cliProgress.Presets.shades_classic);
   bar.start(amount, 0);
   for await (let doc of docs) {
-    await importDocument(typeName, doc, importStore, prisma, visited, errored);
+    try {
+      await importDocument(
+        typeName,
+        doc,
+        importStore,
+        prisma,
+        visited,
+        errored,
+      );
+    } catch (e) {
+      console.error(e);
+    }
     bar.increment();
   }
   bar.stop();