Refactor engine package structure (opensearch-project#1913)

Refactors package structure for engines. First, it renames the
package from util to engine. Next, it moves actual utils to the util
package. Then, it puts the engines in separate sub-packages. Purpose for
this is to separate between engines.

Signed-off-by: John Mazanec <jmazane@amazon.com>

CHANGELOG.md

@@ -22,3 +22,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Maintenance
 ### Refactoring
 * Clean up parsing for query [#1824](https://github.com/opensearch-project/k-NN/pull/1824)
+* Refactor engine package structure [#1913](https://github.com/opensearch-project/k-NN/pull/1913)

qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/FaissSQIT.java

@@ -20,10 +20,10 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.knn.KNNResult;
 import org.opensearch.knn.common.KNNConstants;
-import org.opensearch.knn.index.KNNMethod;
+import org.opensearch.knn.index.engine.KNNMethod;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.query.KNNQueryBuilder;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.io.IOException;
 import java.util.Arrays;

src/main/java/org/opensearch/knn/index/KNNIndexShard.java

@@ -23,7 +23,7 @@
 import org.opensearch.knn.index.memory.NativeMemoryCacheManager;
 import org.opensearch.knn.index.memory.NativeMemoryEntryContext;
 import org.opensearch.knn.index.memory.NativeMemoryLoadStrategy;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -37,7 +37,7 @@
 import static org.opensearch.knn.common.KNNConstants.MODEL_ID;
 import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE;
 import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD;
-import static org.opensearch.knn.index.IndexUtil.getParametersAtLoading;
+import static org.opensearch.knn.index.util.IndexUtil.getParametersAtLoading;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFilePrefix;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileSuffix;
 

src/main/java/org/opensearch/knn/index/codec/BasePerFieldKnnVectorsFormat.java

@@ -13,7 +13,7 @@
 import org.opensearch.knn.index.codec.params.KNNScalarQuantizedVectorsFormatParams;
 import org.opensearch.knn.index.codec.params.KNNVectorsFormatParams;
 import org.opensearch.knn.index.mapper.KNNVectorFieldMapper;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Optional;
 import java.util.function.Function;

src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80CompoundFormat.java

@@ -12,7 +12,7 @@
 import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.io.IOException;
 import java.util.HashSet;

src/main/java/org/opensearch/knn/index/codec/KNN80Codec/KNN80DocValuesConsumer.java

@@ -14,7 +14,7 @@
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.DeprecationHandler;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
-import org.opensearch.knn.index.IndexUtil;
+import org.opensearch.knn.index.util.IndexUtil;
 import org.opensearch.knn.index.KNNSettings;
 import org.opensearch.knn.index.VectorDataType;
 import org.opensearch.knn.index.codec.transfer.VectorTransfer;
@@ -23,7 +23,7 @@
 import org.opensearch.knn.jni.JNIService;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.index.codec.util.KNNCodecUtil;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 import org.opensearch.knn.indices.Model;
 import org.opensearch.knn.indices.ModelCache;
 import org.opensearch.knn.plugin.stats.KNNCounter;
@@ -60,7 +60,7 @@
 import static org.opensearch.knn.common.KNNConstants.PARAMETERS;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.buildEngineFileName;
 import static org.opensearch.knn.index.codec.util.KNNCodecUtil.calculateArraySize;
-import static org.opensearch.knn.index.util.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX;
+import static org.opensearch.knn.index.engine.faiss.Faiss.FAISS_BINARY_INDEX_DESCRIPTION_PREFIX;
 
 /**
  * This class writes the KNN docvalues to the segments

src/main/java/org/opensearch/knn/index/codec/KNN950Codec/KNN950PerFieldKnnVectorsFormat.java

@@ -8,7 +8,7 @@
 import org.apache.lucene.backward_codecs.lucene95.Lucene95HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Optional;
 

src/main/java/org/opensearch/knn/index/codec/KNN990Codec/KNN990PerFieldKnnVectorsFormat.java

@@ -9,7 +9,7 @@
 import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.knn.index.codec.BasePerFieldKnnVectorsFormat;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.engine.KNNEngine;
 
 import java.util.Optional;
 

src/main/java/org/opensearch/knn/index/codec/params/KNNScalarQuantizedVectorsFormatParams.java

@@ -6,7 +6,7 @@
 package org.opensearch.knn.index.codec.params;
 
 import lombok.Getter;
-import org.opensearch.knn.index.MethodComponentContext;
+import org.opensearch.knn.index.engine.MethodComponentContext;
 
 import java.util.Map;
 

src/main/java/org/opensearch/knn/index/util/AbstractKNNLibrary.java → src/main/java/org/opensearch/knn/index/engine/AbstractKNNLibrary.java

@@ -3,14 +3,12 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import org.opensearch.common.ValidationException;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.training.VectorSpaceInfo;
 
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/DefaultHnswContext.java → src/main/java/org/opensearch/knn/index/engine/DefaultHnswContext.java

@@ -1,18 +1,12 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import com.google.common.collect.ImmutableMap;
-import org.opensearch.knn.index.Parameter;
+import org.opensearch.knn.index.engine.model.QueryContext;
 import org.opensearch.knn.index.query.request.MethodParameter;
 
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/DefaultIVFContext.java → src/main/java/org/opensearch/knn/index/engine/DefaultIVFContext.java

@@ -1,18 +1,12 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import com.google.common.collect.ImmutableMap;
-import org.opensearch.knn.index.Parameter;
+import org.opensearch.knn.index.engine.model.QueryContext;
 import org.opensearch.knn.index.query.request.MethodParameter;
 
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/EngineSpecificMethodContext.java → src/main/java/org/opensearch/knn/index/engine/EngineSpecificMethodContext.java

@@ -1,17 +1,11 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
-import org.opensearch.knn.index.Parameter;
+import org.opensearch.knn.index.engine.model.QueryContext;
 
 import java.util.Collections;
 import java.util.Map;

src/main/java/org/opensearch/knn/index/util/JVMLibrary.java → src/main/java/org/opensearch/knn/index/engine/JVMLibrary.java

@@ -3,10 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
-
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
+package org.opensearch.knn.index.engine;
 
 import java.util.Map;
 
@@ -23,7 +20,7 @@ public abstract class JVMLibrary extends AbstractKNNLibrary {
      * @param methods Map of k-NN methods that the library supports
      * @param version String representing version of library
      */
-    JVMLibrary(Map<String, KNNMethod> methods, Map<String, EngineSpecificMethodContext> engineMethodMetadataMap, String version) {
+    public JVMLibrary(Map<String, KNNMethod> methods, Map<String, EngineSpecificMethodContext> engineMethodMetadataMap, String version) {
         super(methods, engineMethodMetadataMap, version);
     }
 

src/main/java/org/opensearch/knn/index/util/KNNEngine.java → src/main/java/org/opensearch/knn/index/engine/KNNEngine.java

@@ -3,13 +3,14 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import com.google.common.collect.ImmutableSet;
 import org.opensearch.common.ValidationException;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.index.SpaceType;
+import org.opensearch.knn.index.engine.faiss.Faiss;
+import org.opensearch.knn.index.engine.lucene.Lucene;
+import org.opensearch.knn.index.engine.nmslib.Nmslib;
 import org.opensearch.knn.training.VectorSpaceInfo;
 
 import java.util.List;

src/main/java/org/opensearch/knn/index/util/KNNLibrary.java → src/main/java/org/opensearch/knn/index/engine/KNNLibrary.java

@@ -1,19 +1,11 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import org.opensearch.common.ValidationException;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.training.VectorSpaceInfo;
 

src/main/java/org/opensearch/knn/index/KNNMethod.java → src/main/java/org/opensearch/knn/index/engine/KNNMethod.java

@@ -1,20 +1,15 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import org.opensearch.common.ValidationException;
 import org.opensearch.knn.common.KNNConstants;
+import org.opensearch.knn.index.SpaceType;
 import org.opensearch.knn.training.VectorSpaceInfo;
 
 import java.util.ArrayList;

src/main/java/org/opensearch/knn/index/KNNMethodContext.java → src/main/java/org/opensearch/knn/index/engine/KNNMethodContext.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
@@ -20,7 +14,7 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.knn.index.util.KNNEngine;
+import org.opensearch.knn.index.SpaceType;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.mapper.MapperParsingException;

src/main/java/org/opensearch/knn/index/MethodComponent.java → src/main/java/org/opensearch/knn/index/engine/MethodComponent.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.Getter;
 import org.opensearch.Version;
@@ -25,7 +19,7 @@
 import java.util.List;
 import java.util.ArrayList;
 
-import static org.opensearch.knn.validation.ParameterValidator.validateParameters;
+import static org.opensearch.knn.index.engine.validation.ParameterValidator.validateParameters;
 
 /**
  * MethodComponent defines the structure of an individual component that can make up an index

src/main/java/org/opensearch/knn/index/MethodComponentContext.java → src/main/java/org/opensearch/knn/index/engine/MethodComponentContext.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;

src/main/java/org/opensearch/knn/index/util/NativeLibrary.java → src/main/java/org/opensearch/knn/index/engine/NativeLibrary.java

@@ -3,12 +3,10 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.knn.index.util;
+package org.opensearch.knn.index.engine;
 
 import lombok.Getter;
 import org.opensearch.knn.common.KNNConstants;
-import org.opensearch.knn.index.KNNMethod;
-import org.opensearch.knn.index.KNNMethodContext;
 import org.opensearch.knn.index.SpaceType;
 
 import java.util.Map;
@@ -20,7 +18,7 @@
  * Abstract implementation of KNNLibrary. It contains several default methods and fields that
  * are common across different underlying libraries.
  */
-abstract class NativeLibrary extends AbstractKNNLibrary {
+public abstract class NativeLibrary extends AbstractKNNLibrary {
     private final Map<SpaceType, Function<Float, Float>> scoreTranslation;
     @Getter
     private final String extension;
@@ -34,7 +32,7 @@ abstract class NativeLibrary extends AbstractKNNLibrary {
      * @param version String representation of version of the library
      * @param extension String representing the extension that library files should use
      */
-    NativeLibrary(
+    public NativeLibrary(
         Map<String, KNNMethod> methods,
         Map<String, EngineSpecificMethodContext> engineMethods,
         Map<SpaceType, Function<Float, Float>> scoreTranslation,

src/main/java/org/opensearch/knn/index/Parameter.java → src/main/java/org/opensearch/knn/index/engine/Parameter.java

@@ -1,15 +1,9 @@
 /*
+ * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- *
- * Modifications Copyright OpenSearch Contributors. See
- * GitHub history for details.
  */
 
-package org.opensearch.knn.index;
+package org.opensearch.knn.index.engine;
 
 import org.opensearch.common.ValidationException;
 import org.opensearch.knn.training.VectorSpaceInfo;