SO-4122: index schema migration support

commons/com.b2international.index.test.tools/src/com/b2international/index/BaseIndexTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2011-2022 B2i Healthcare Pte Ltd, http://b2i.sg
+ * Copyright 2011-2023 B2i Healthcare Pte Ltd, http://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -131,7 +131,7 @@ protected final <T> Stream<Hits<T>> stream(final Query<T> query) {
 
 	protected void assertDocEquals(Object expected, Object actual) {
 		assertNotNull("Actual document is missing from index", actual);
-		for (Field actualField : index.getIndex().admin().mappings().getMapping(actual.getClass()).getFields()) {
+		for (Field actualField : index.getIndex().admin().getIndexMapping().getMapping(actual.getClass()).getFields()) {
 			Field existingField = null;
 			try {
 				existingField = Reflections.getField(expected.getClass(), actualField.getName());

commons/com.b2international.index.test.tools/src/com/b2international/index/IndexResource.java

@@ -120,7 +120,7 @@ protected void before() throws Throwable {
 		// then mapping changes
 		revisionIndex.admin().updateMappings(new Mappings(types));
 
-		// then settings changes
+		// then update settings changes for existing indices (TODO move this into create? or updateMappings?)
 		revisionIndex.admin().updateSettings(indexSettings.get());
 
 		// then make sure we have all indexes ready for tests

commons/com.b2international.index.test.tools/src/com/b2international/index/revision/BaseRevisionIndexTest.java

@@ -182,7 +182,7 @@ protected final <T> Hits<T> searchRaw(final Query<T> query) {
 
 	protected void assertDocEquals(Object expected, Object actual) {
 		assertNotNull("Actual document is missing from index", actual);
-		for (Field f : index.getIndex().admin().mappings().getMapping(expected.getClass()).getFields()) {
+		for (Field f : index.getIndex().admin().getIndexMapping().getMapping(expected.getClass()).getFields()) {
 			if (Revision.Fields.CREATED.equals(f.getName()) 
 					|| Revision.Fields.REVISED.equals(f.getName())
 					|| WithScore.SCORE.equals(f.getName())

commons/com.b2international.index.tests/src/com/b2international/index/BoolQueryTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 B2i Healthcare Pte Ltd, http://b2i.sg
+ * Copyright 2022-2023 B2i Healthcare Pte Ltd, http://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -101,7 +101,7 @@ public void mergeSingleAndMultiTermFilterClauses() throws Exception {
 			.build();
 
 		IndexAdmin indexAdmin = index().admin();
-		DocumentMapping mapping = indexAdmin.mappings().getMapping(Data.class);
+		DocumentMapping mapping = indexAdmin.getIndexMapping().getMapping(Data.class);
 		Map<String, Object> settings = indexAdmin.settings();
 
 		// Single filter matching "id1" should be preserved
@@ -123,7 +123,7 @@ public void mergeDisjunctTermFilterClauses() throws Exception {
 			.build();
 
 		IndexAdmin indexAdmin = index().admin();
-		DocumentMapping mapping = indexAdmin.mappings().getMapping(Data.class);
+		DocumentMapping mapping = indexAdmin.getIndexMapping().getMapping(Data.class);
 		Map<String, Object> settings = indexAdmin.settings();
 
 		QueryBuilder esQueryBuilder = new EsQueryBuilder(mapping, settings, LOG).build(actual);
@@ -150,7 +150,7 @@ public void mergeDisjunctTermFilterClausesOnCollection() throws Exception {
 			.build();
 
 		IndexAdmin indexAdmin = index().admin();
-		DocumentMapping mapping = indexAdmin.mappings().getMapping(Data.class);
+		DocumentMapping mapping = indexAdmin.getIndexMapping().getMapping(Data.class);
 		Map<String, Object> settings = indexAdmin.settings();
 
 		QueryBuilder esQueryBuilder = new EsQueryBuilder(mapping, settings, LOG).build(actual);

commons/com.b2international.index.tests/src/com/b2international/index/NestedDocumentIndexTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2011-2021 B2i Healthcare Pte Ltd, http://b2i.sg
+ * Copyright 2011-2023 B2i Healthcare Pte Ltd, http://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,8 @@
 
 import org.junit.Test;
 
-import static com.b2international.index.Fixtures.*;
+import com.b2international.index.Fixtures.NestedData;
+import com.b2international.index.Fixtures.ParentData;
 import com.b2international.index.query.Expressions;
 import com.b2international.index.query.Query;
 import com.google.common.collect.ImmutableList;
@@ -38,28 +39,12 @@ protected final Collection<Class<?>> getTypes() {
 	}
 
 	@Test
-	public void indexNestedDocument() throws Exception {
+	public void indexDocumentWithNestedPart() throws Exception {
 		final ParentData data = new ParentData(KEY1, "field1", new NestedData("field2"));
 		indexDocument(data);
 		assertEquals(data, getDocument(ParentData.class, KEY1));
 	}
 
-	@Test
-	public void deleteDocumentWithNestedDocShouldDeleteNested() throws Exception {
-		indexNestedDocument();
-		deleteDocument(ParentData.class, KEY1);
-
-		// query to get parent document, should be none
-		final Query<ParentData> parentDataQuery = Query.select(ParentData.class).where(Expressions.matchAll()).build();
-		final Iterable<ParentData> parentDocs = search(parentDataQuery);
-		assertThat(parentDocs).isEmpty();
-
-		// query to get nested child document, should be none
-		final Query<NestedData> nestedDataQuery = Query.select(NestedData.class).parent(ParentData.class).where(Expressions.matchAll()).build();
-		final Iterable<NestedData> nestedDocs = search(nestedDataQuery);
-		assertThat(nestedDocs).isEmpty();
-	}
-
 	@Test
 	public void searchNestedDocument() throws Exception {
 		final ParentData data = new ParentData(KEY1, "field1", new NestedData("field2"));
@@ -70,6 +55,6 @@ public void searchNestedDocument() throws Exception {
 		final Iterable<ParentData> matches = search(query);
 		assertThat(matches).hasSize(1);
 		assertThat(matches).containsOnly(data);
-	}
+	}                                                                                                                                            
 
 }

commons/com.b2international.index.tests/src/com/b2international/index/ShardConfigurationTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2022 B2i Healthcare Pte Ltd, http://b2i.sg
+ * Copyright 2022-2023 B2i Healthcare Pte Ltd, http://b2i.sg
  * 
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -52,7 +52,7 @@ protected Map<String, Object> getIndexSettings() {
 
 	@Test
 	public void shardReplicaConfigTest() throws Exception {
-		String typeIndex = index().admin().getTypeIndex(index().admin().mappings().getMapping(ShardConfig.class));
+		String typeIndex = index().admin().getIndexMapping().getTypeIndex(ShardConfig.class);
 		Settings settings = index().admin().client().indices().settings(new GetSettingsRequest().indices(typeIndex)).getIndexToSettings().get(typeIndex).getAsSettings("index");
 		assertThat(settings.get(IndexClientFactory.NUMBER_OF_SHARDS)).isEqualTo("2");
 	}

commons/com.b2international.index.tests/src/com/b2international/index/mapping/MappingMigrationTest.java

@@ -29,10 +29,14 @@
 import com.b2international.index.*;
 import com.b2international.index.admin.IndexAdmin;
 import com.b2international.index.mapping.FieldAlias.FieldAliasType;
+import com.b2international.index.migrate.DocumentMappingMigrationStrategy;
+import com.b2international.index.migrate.DocumentMappingMigrator;
+import com.b2international.index.migrate.SchemaRevision;
 import com.b2international.index.query.Expressions;
 import com.b2international.index.query.Query;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 
 /**
  * @since 7.16.1
@@ -41,14 +45,14 @@
 public class MappingMigrationTest extends BaseIndexTest {
 
 	@Doc(type = "schema") // this will ensure the same index name will be used for all subclasses and versions
-	static class Schema {
+	static class SchemaDoc {
 
 		@ID
 		private String id;
 		private String field;
 
 		@JsonCreator
-		public Schema(@JsonProperty("id") String id, @JsonProperty("field") String field) {
+		public SchemaDoc(@JsonProperty("id") String id, @JsonProperty("field") String field) {
 			this.id = id;
 			this.field = field;
 		}
@@ -64,7 +68,29 @@ public String getField() {
 	}
 
 	@Doc(type = "schema")
-	static class SchemaWithNewField extends Schema {
+	static class SchemaWithoutSchemaRevision extends SchemaDoc {
+
+		private String newField;
+
+		@JsonCreator
+		public SchemaWithoutSchemaRevision(@JsonProperty("id") String id, @JsonProperty("field") String field, @JsonProperty("newField") String newField) {
+			super(id, field);
+			this.newField = newField;
+		}
+
+		public String getNewField() {
+			return newField;
+		}
+
+	}
+
+	@Doc(
+		type = "schema",
+		revisions = {
+			@SchemaRevision(version = 2, description = "add newField field", strategy = DocumentMappingMigrationStrategy.NO_REINDEX)
+		}
+	)
+	static class SchemaWithNewField extends SchemaDoc {
 
 		private String newField;
 
@@ -80,7 +106,12 @@ public String getNewField() {
 
 	}
 
-	@Doc(type = "schema")
+	@Doc(
+		type = "schema",
+		revisions = {
+			@SchemaRevision(version = 2, description = "add new field alias `field.text`", strategy = DocumentMappingMigrationStrategy.REINDEX_INPLACE)
+		}
+	)
 	static class SchemaWithNewTextField {
 
 		@ID
@@ -109,7 +140,12 @@ public String getField() {
 
 	}
 
-	@Doc(type = "schema")
+	@Doc(
+		type = "schema",
+		revisions = {
+			@SchemaRevision(version = 2, description = "add new field alias `field.exact`", strategy = DocumentMappingMigrationStrategy.REINDEX_INPLACE)
+		}
+	)
 	static class SchemaWithNewKeywordField {
 
 		@ID
@@ -138,7 +174,12 @@ public String getField() {
 
 	}
 
-	@Doc(type = "schema")
+	@Doc(
+		type = "schema",
+		revisions = {
+			@SchemaRevision(version = 3, description = "add new field aliases `field.exact` and `field.text`", strategy = DocumentMappingMigrationStrategy.REINDEX_INPLACE)
+		}
+	)
 	static class SchemaWithTextFieldAndNewKeywordField {
 
 		@ID
@@ -167,9 +208,52 @@ public String getField() {
 		}
 
 	}
+
+	@Doc(
+		type = "schema",
+		revisions = {
+			@SchemaRevision(version = 2, migrator = SchemaDocRenamedField.DocumentMigratorVersion1_2.class, strategy = DocumentMappingMigrationStrategy.REINDEX_SCRIPT)
+		}
+	) 
+	static class SchemaDocRenamedField {
+
+		public static final class DocumentMigratorVersion1_2 implements DocumentMappingMigrator {
+
+			@Override
+			public void init(Searcher searcher) {
+			}
+
+			@Override
+			public ObjectNode migrate(ObjectNode oldDocument) {
+				// simply convert the existing field fieldValue to field2 fieldValue
+				oldDocument.set("field2", oldDocument.remove("field"));
+				return oldDocument;
+			}
+
+		}
+
+		@ID
+		private String id;
+		private String field2;
+
+		@JsonCreator
+		public SchemaDocRenamedField(@JsonProperty("id") String id, @JsonProperty("field2") String field2) {
+			this.id = id;
+			this.field2 = field2;
+		}
+
+		public String getId() {
+			return id;
+		}
+
+		public String getField2() {
+			return field2;
+		}
+
+	}
 
-	private Schema existingDoc1;
-	private Schema existingDoc2;
+	private SchemaDoc existingDoc1;
+	private SchemaDoc existingDoc2;
 
 	@Override
 	protected Collection<Class<?>> getTypes() {
@@ -178,20 +262,28 @@ protected Collection<Class<?>> getTypes() {
 
 	@Before
 	public void setup() {
-		Mappings mappings = new Mappings(Schema.class);
+		Mappings mappings = new Mappings(SchemaDoc.class);
 		// enable class overrides for test(s) in this class
 		mappings.enableRuntimeMappingOverrides = true;
 		// make sure we always start with the basic Schema
 		admin().updateMappings(mappings);
 		admin().create();
 		// index two documents with the existing schema
-		existingDoc1 = new Schema(KEY1, "Existing Field One");
-		existingDoc2 = new Schema(KEY2, "Existing Field Two");
+		existingDoc1 = new SchemaDoc(KEY1, "Existing Field One");
+		existingDoc2 = new SchemaDoc(KEY2, "Existing Field Two");
 		indexDocuments(existingDoc1, existingDoc2);
 	}
 
+	@Test(expected = IndexException.class)
+	public void migrate01_NoSchemaRevisionRegistered() throws Exception {
+		// update Mappings to new schema
+		admin().updateMappings(new Mappings(SchemaWithoutSchemaRevision.class));
+		// then recreate indices using the new mappings (and to perform potential migration as well)
+		admin().create();
+	}
+
 	@Test
-	public void migrate01_NewField() throws Exception {
+	public void migrate02_NewField() throws Exception {
 		// update Mappings to new schema
 		admin().updateMappings(new Mappings(SchemaWithNewField.class));
 		// then recreate indices using the new mappings (and to perform potential migration as well)
@@ -202,7 +294,7 @@ public void migrate01_NewField() throws Exception {
 	}
 
 	@Test
-	public void migrate02_NewTextField() throws Exception {
+	public void migrate03_NewTextField() throws Exception {
 		// update Mappings to new schema
 		admin().updateMappings(new Mappings(SchemaWithNewTextField.class));
 		// then recreate indices using the new mappings (and to perform potential migration as well)
@@ -218,7 +310,7 @@ public void migrate02_NewTextField() throws Exception {
 	}
 
 	@Test
-	public void migrate03_NewKeywordField() throws Exception {
+	public void migrate04_NewKeywordField() throws Exception {
 		// update Mappings to new schema
 		admin().updateMappings(new Mappings(SchemaWithNewKeywordField.class));
 		// then recreate indices using the new mappings (and to perform potential migration as well)
@@ -234,7 +326,7 @@ public void migrate03_NewKeywordField() throws Exception {
 	}
 
 	@Test
-	public void migrate04_NewKeywordFieldOnExistingKeywordAndTextField() throws Exception {
+	public void migrate05_NewKeywordFieldOnExistingKeywordAndTextField() throws Exception {
 		// update Mapping to perform the first migration
 		admin().updateMappings(new Mappings(SchemaWithNewTextField.class));
 		admin().create();
@@ -259,6 +351,17 @@ public void migrate04_NewKeywordFieldOnExistingKeywordAndTextField() throws Exce
 		).hasSize(1);
 	}
 
+	@Test
+	public void migrate06_RenameFieldWithMigratorScript() throws Exception {
+		// update Mapping to perform the migration using script and reindex
+		admin().updateMappings(new Mappings(SchemaDocRenamedField.class));
+		admin().create();
+
+		// confirm that the two documents still exists and they have their values set correctly in the new field
+		assertDocEquals(new SchemaDocRenamedField(existingDoc1.getId(), existingDoc1.getField()), getDocument(SchemaDocRenamedField.class, KEY1));
+		assertDocEquals(new SchemaDocRenamedField(existingDoc2.getId(), existingDoc2.getField()), getDocument(SchemaDocRenamedField.class, KEY2));
+	}
+
 	@After
 	public void teardown() {
 		// delete the indexes completely