forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introducing ZStd compression codec plugin (opensearch-project#9658) (o…
…pensearch-project#9832) * introducing zstd compression codec plugin * Moving zstd compression codec as a plugin * introducing zstd compression codec plugin * Adding checks to EngineConfig and fixing tests * incorporating review comments * fixing tests * introducing zstd compression codec plugin * addressing review comments * nit fixes * implementing codec aliases * addressing review comments * review comments * moving codec aliases to custom codec * adding zstd default codec for backward compatibility * renaming to deprecated codec * incorporating review comments * nit fixes --------- (cherry picked from commit 76f1b52) Signed-off-by: Sarthak Aggarwal <sarthagg@amazon.com> Signed-off-by: Prabhakar Sithanandam <backslasht@users.noreply.github.com> Signed-off-by: Andrew Ross <andrross@amazon.com> Co-authored-by: Sarthak Aggarwal <sarthagg@amazon.com> Co-authored-by: Prabhakar Sithanandam <backslasht@users.noreply.github.com>
- Loading branch information
3 people
authored
Sep 6, 2023
1 parent
655d223
commit 90310c3
Showing
30 changed files
with
724 additions
and
147 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| * | ||
| * Modifications Copyright OpenSearch Contributors. See | ||
| * GitHub history for details. | ||
| */ | ||
|
|
||
| apply plugin: 'opensearch.opensearchplugin' | ||
| apply plugin: 'opensearch.internal-cluster-test' | ||
|
|
||
| opensearchplugin { | ||
| name 'custom-codecs' | ||
| description 'A plugin that implements custom compression codecs.' | ||
| classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' | ||
| licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') | ||
| noticeFile rootProject.file('NOTICE.txt') | ||
| } | ||
|
|
||
| dependencies { | ||
| api "com.github.luben:zstd-jni:1.5.5-5" | ||
| } | ||
|
|
||
| testingConventions.enabled = false; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
...ustom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.codec.CodecServiceFactory; | ||
| import org.opensearch.index.engine.EngineConfig; | ||
| import org.opensearch.plugins.EnginePlugin; | ||
| import org.opensearch.plugins.Plugin; | ||
|
|
||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * A plugin that implements custom codecs. Supports these codecs: | ||
| * <ul> | ||
| * <li>ZSTD | ||
| * <li>ZSTDNODICT | ||
| * </ul> | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public final class CustomCodecPlugin extends Plugin implements EnginePlugin { | ||
|
|
||
| /** | ||
| * Creates a new instance | ||
| */ | ||
| public CustomCodecPlugin() {} | ||
|
|
||
| /** | ||
| * @param indexSettings is the default indexSettings | ||
| * @return the engine factory | ||
| */ | ||
| @Override | ||
| public Optional<CodecServiceFactory> getCustomCodecServiceFactory(final IndexSettings indexSettings) { | ||
| String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); | ||
| if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { | ||
| return Optional.of(new CustomCodecServiceFactory()); | ||
| } | ||
| return Optional.empty(); | ||
| } | ||
| } |
72 changes: 72 additions & 0 deletions
72
...stom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.codecs.Codec; | ||
| import org.opensearch.common.collect.MapBuilder; | ||
| import org.opensearch.index.IndexSettings; | ||
| import org.opensearch.index.codec.CodecService; | ||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Map; | ||
| import java.util.stream.Stream; | ||
|
|
||
| import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; | ||
|
|
||
| /** | ||
| * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. | ||
| */ | ||
| public class CustomCodecService extends CodecService { | ||
| private final Map<String, Codec> codecs; | ||
| /** | ||
| * ZStandard codec | ||
| */ | ||
| public static final String ZSTD_CODEC = "zstd"; | ||
| /** | ||
| * ZStandard without dictionary codec | ||
| */ | ||
| public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; | ||
|
|
||
| /** | ||
| * Creates a new CustomCodecService. | ||
| * | ||
| * @param mapperService The mapper service. | ||
| * @param indexSettings The index settings. | ||
| * @param logger The logger. | ||
| */ | ||
| public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { | ||
| super(mapperService, indexSettings, logger); | ||
| int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); | ||
| final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder(); | ||
| if (mapperService == null) { | ||
| codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); | ||
| codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); | ||
| } else { | ||
| codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); | ||
| codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); | ||
| } | ||
| this.codecs = codecs.immutableMap(); | ||
| } | ||
|
|
||
| @Override | ||
| public Codec codec(String name) { | ||
| Codec codec = codecs.get(name); | ||
| if (codec == null) { | ||
| return super.codec(name); | ||
| } | ||
| return codec; | ||
| } | ||
|
|
||
| @Override | ||
| public String[] availableCodecs() { | ||
| return Stream.concat(Arrays.stream(super.availableCodecs()), codecs.keySet().stream()).toArray(String[]::new); | ||
| } | ||
| } |
27 changes: 27 additions & 0 deletions
27
...decs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.opensearch.index.codec.CodecService; | ||
| import org.opensearch.index.codec.CodecServiceConfig; | ||
| import org.opensearch.index.codec.CodecServiceFactory; | ||
|
|
||
| /** | ||
| * A factory for creating new {@link CodecService} instance | ||
| */ | ||
| public class CustomCodecServiceFactory implements CodecServiceFactory { | ||
|
|
||
| /** Creates a new instance. */ | ||
| public CustomCodecServiceFactory() {} | ||
|
|
||
| @Override | ||
| public CodecService createCodecService(CodecServiceConfig config) { | ||
| return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.