forked from opensearch-project/k-NN
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Refactor method structure and definitions (opensearch-project#1920)
* Remove getMethod from KNNLibrary interface Removes the getMethod from KNNLibrary interface. Users should not directly retrieve methods from the engines as this creates additional complexity. Instead, they should get everything they need about a method from the KNNLibrary itself. This change will allow us to better maintain the different methods. * Move EngineSpecificMethodContext into KNNMethod Moving the EngineSpecificMethodContext into the KNNMethod class. EngineSpecificMethodContext is mainly used during search to provide parameter validation/configure dynamic updates. Moving into KNNMethod will encapsulate the structure of the method into one class. * Change KNNMethod to interface Changes KNNMethod to interface and creates methods per engine/algo combination. For one, this will make code much more maintainable as we wont have to deal with big maps and builders. For the other, we can implement more complex logic between engines. Signed-off-by: John Mazanec <jmazane@amazon.com> * Add simple encoder interface Adds simple encoder interface to clean up definitions of the encoders. --------- Signed-off-by: John Mazanec <jmazane@amazon.com>
- Loading branch information
1 parent
ec6451c
commit adaf150
Showing
36 changed files
with
1,031 additions
and
795 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
119 changes: 119 additions & 0 deletions
119
src/main/java/org/opensearch/knn/index/engine/AbstractKNNMethod.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.engine; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import org.opensearch.common.ValidationException; | ||
| import org.opensearch.knn.common.KNNConstants; | ||
| import org.opensearch.knn.index.SpaceType; | ||
| import org.opensearch.knn.training.VectorSpaceInfo; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Locale; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Abstract class for KNN methods. This class provides the common functionality for all KNN methods. | ||
| * It defines the common attributes and methods that all KNN methods should implement. | ||
| */ | ||
| @AllArgsConstructor | ||
| public abstract class AbstractKNNMethod implements KNNMethod { | ||
|
|
||
| protected final MethodComponent methodComponent; | ||
| protected final Set<SpaceType> spaces; | ||
| protected final EngineSpecificMethodContext engineSpecificMethodContext; | ||
|
|
||
| @Override | ||
| public boolean isSpaceTypeSupported(SpaceType space) { | ||
| return spaces.contains(space); | ||
| } | ||
|
|
||
| @Override | ||
| public ValidationException validate(KNNMethodContext knnMethodContext) { | ||
| List<String> errorMessages = new ArrayList<>(); | ||
| if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { | ||
| errorMessages.add( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", | ||
| this.methodComponent.getName(), | ||
| knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), | ||
| knnMethodContext.getSpaceType().getValue() | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| ValidationException methodValidation = methodComponent.validate(knnMethodContext.getMethodComponentContext()); | ||
| if (methodValidation != null) { | ||
| errorMessages.addAll(methodValidation.validationErrors()); | ||
| } | ||
|
|
||
| if (errorMessages.isEmpty()) { | ||
| return null; | ||
| } | ||
|
|
||
| ValidationException validationException = new ValidationException(); | ||
| validationException.addValidationErrors(errorMessages); | ||
| return validationException; | ||
| } | ||
|
|
||
| @Override | ||
| public ValidationException validateWithData(KNNMethodContext knnMethodContext, VectorSpaceInfo vectorSpaceInfo) { | ||
| List<String> errorMessages = new ArrayList<>(); | ||
| if (!isSpaceTypeSupported(knnMethodContext.getSpaceType())) { | ||
| errorMessages.add( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "\"%s\" with \"%s\" configuration does not support space type: " + "\"%s\".", | ||
| this.methodComponent.getName(), | ||
| knnMethodContext.getKnnEngine().getName().toLowerCase(Locale.ROOT), | ||
| knnMethodContext.getSpaceType().getValue() | ||
| ) | ||
| ); | ||
| } | ||
|
|
||
| ValidationException methodValidation = methodComponent.validateWithData( | ||
| knnMethodContext.getMethodComponentContext(), | ||
| vectorSpaceInfo | ||
| ); | ||
| if (methodValidation != null) { | ||
| errorMessages.addAll(methodValidation.validationErrors()); | ||
| } | ||
|
|
||
| if (errorMessages.isEmpty()) { | ||
| return null; | ||
| } | ||
|
|
||
| ValidationException validationException = new ValidationException(); | ||
| validationException.addValidationErrors(errorMessages); | ||
| return validationException; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean isTrainingRequired(KNNMethodContext knnMethodContext) { | ||
| return methodComponent.isTrainingRequired(knnMethodContext.getMethodComponentContext()); | ||
| } | ||
|
|
||
| @Override | ||
| public int estimateOverheadInKB(KNNMethodContext knnMethodContext, int dimension) { | ||
| return methodComponent.estimateOverheadInKB(knnMethodContext.getMethodComponentContext(), dimension); | ||
| } | ||
|
|
||
| @Override | ||
| public Map<String, Object> getAsMap(KNNMethodContext knnMethodContext) { | ||
| Map<String, Object> parameterMap = new HashMap<>(methodComponent.getAsMap(knnMethodContext.getMethodComponentContext())); | ||
| parameterMap.put(KNNConstants.SPACE_TYPE, knnMethodContext.getSpaceType().getValue()); | ||
| return parameterMap; | ||
| } | ||
|
|
||
| @Override | ||
| public EngineSpecificMethodContext getEngineSpecificMethodContext() { | ||
| return engineSpecificMethodContext; | ||
| } | ||
| } |
27 changes: 27 additions & 0 deletions
27
src/main/java/org/opensearch/knn/index/engine/Encoder.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.knn.index.engine; | ||
|
|
||
| /** | ||
| * Interface representing an encoder. An encoder generally refers to a vector quantizer. | ||
| */ | ||
| public interface Encoder { | ||
| /** | ||
| * The name of the encoder does not have to be unique. Howevwer, when using within a method, there cannot be | ||
| * 2 encoders with the same name. | ||
| * | ||
| * @return Name of the encoder | ||
| */ | ||
| default String getName() { | ||
| return getMethodComponent().getName(); | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @return Method component associated with the encoder | ||
| */ | ||
| MethodComponent getMethodComponent(); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.