2640 - Add esingest package with base covid opensource dataset ingest

.vscode/launch.json

@@ -23,6 +23,16 @@
       "args": [
         "--spring.profiles.active=default,local"
       ]
+    },
+    {
+      "type": "java",
+      "name": "ElasticIngestApplication",
+      "request": "launch",
+      "mainClass": "software.uncharted.terarium.esingest.ElasticIngestApplication",
+      "projectName": "es-ingest",
+      "args": [
+        "--spring.profiles.active=default,local"
+      ]
     }
   ]
 }

packages/es-ingest/.gitignore

@@ -0,0 +1,37 @@
+HELP.md
+.gradle
+build/
+!gradle/wrapper/gradle-wrapper.jar
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+bin/
+!**/src/main/**/bin/
+!**/src/test/**/bin/
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+out/
+!**/src/main/**/out/
+!**/src/test/**/out/
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+
+### VS Code ###
+.vscode/

packages/es-ingest/README.md

@@ -0,0 +1,88 @@
+# Terarium Elasticsearch Ingest
+
+This package is designed to quickly import source documents along with their embeddings for knn semantic search in Elasticsearch.
+
+## How to setup an ingest:
+
+### Create the input class definitions:
+
+An ingest requires an input `InputDocument` class that implements the `IInputDocument` interface and an `InputEmbeddingChunk` class that implements the `IInputEmbeddingChunk` interface.
+
+```java
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ExampleDocument implements IInputDocument {
+	UUID id;
+  String title;
+  String body;
+}
+
+@Data
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class ExampleEmbedding implements IInputEmbeddingChunk {
+	private UUID id;
+	private UUID embeddingChunkId;
+	private long[] spans;
+	private String title;
+	private double[] embedding;
+}
+```
+
+### Create an `IElasticIngest` implementation:
+
+Each ingest will require some logic to convert the `input` types to output types, this is done by implementing the `IElasticIngest` interface:
+
+```java
+public class ExampleIngest implements IElasticIngest<ExampleDocument, Document, ExampleEmbedding, EmbeddingChunk> {
+
+	ObjectMapper mapper = new ObjectMapper();
+
+	public Document processDocument(ExampleDocument input) {
+		Document doc = new Document();
+		doc.setId(input.getId());
+		doc.setTitle(input.getTitle());
+		doc.setFullText(input.getBody());
+		return doc;
+	}
+
+	public EmbeddingChunk processEmbedding(ExampleEmbedding input) {
+		Embedding embedding = new Embedding();
+		embedding.setEmbeddingId(input.getEmbeddingChunkId());
+		embedding.setSpans(input.getSpans());
+		embedding.setVector(input.getEmbedding());
+		EmbeddingChunk chunk = new EmbeddingChunk();
+		chunk.setId(input.getId());
+		chunk.setEmbedding(embedding);
+		return chunk;
+	}
+
+	public ExampleDocument deserializeDocument(String line) {
+		try {
+			return mapper.readValue(line, CovidDocument.class);
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+	public ExampleEmbedding deserializeEmbedding(String line) {
+		try {
+			return mapper.readValue(line, CovidEmbedding.class);
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+}
+```
+
+### Configuring the ingest in `application.properties`:
+
+Add an ingest entry to the `application.properties`:
+
+```
+terarium.esingest.ingestParams[0].name="A sample ingest"
+terarium.esingest.ingestParams[0].inputDir=/path/to/source/dir
+terarium.esingest.ingestParams[0].outputIndexRoot=example
+terarium.esingest.ingestParams[0].ingestClass=software.uncharted.terarium.esingest.ingests.CovidIngest
+terarium.esingest.ingestParams[0].clearBeforeIngest=true
+```

packages/es-ingest/build.gradle

@@ -0,0 +1,48 @@
+plugins {
+    id 'java'
+    id 'org.springframework.boot' version '3.1.5'
+    id 'io.spring.dependency-management' version '1.1.4'
+}
+
+group = 'software.uncharted'
+version = '1.0.0-SNAPSHOT'
+sourceCompatibility = '17'
+
+apply plugin: 'idea'
+
+configurations {
+    compileOnly {
+        extendsFrom annotationProcessor
+    }
+}
+
+project.ext {
+    artifactName = 'es-ingest'
+    description = 'imports models into es'
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    implementation 'org.springframework:spring-web'
+    implementation 'co.elastic.clients:elasticsearch-java:8.8.1'
+    implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.1'
+    implementation 'org.springframework.boot:spring-boot-starter'
+    implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.2'
+    compileOnly 'org.projectlombok:lombok'
+    developmentOnly 'org.springframework.boot:spring-boot-devtools'
+    annotationProcessor 'org.projectlombok:lombok'
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+    testAnnotationProcessor 'org.projectlombok:lombok'
+    testCompileOnly 'org.projectlombok:lombok'
+}
+
+tasks.named('test') {
+    useJUnitPlatform()
+}
+
+dependencyLocking {
+    lockAllConfigurations()
+}

packages/es-ingest/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists