From 41a660d99ae7403a916d6e811c480b4e31f04bfe Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:12:42 -0700 Subject: [PATCH 01/14] Mute flaky test ReadContextListenerTests.testReadContextListenerFailure (#9842) (#9848) (cherry picked from commit e4a1841e23dadd3351216383fdc16bf56a0b783d) Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] --- .../blobstore/stream/read/listener/ReadContextListenerTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index 21b7b47390a9b..0db768a54b4b9 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -77,6 +77,7 @@ public void testReadContextListener() throws InterruptedException, IOException { assertEquals(NUMBER_OF_PARTS * PART_SIZE, Files.size(fileLocation)); } + @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9776") public void testReadContextListenerFailure() throws Exception { Path fileLocation = path.resolve(UUID.randomUUID().toString()); List blobPartStreams = initializeBlobPartStreams(); From cace166aa86da4644daa193029dcb0942f74e5f4 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 21:43:02 -0400 Subject: [PATCH 02/14] Introducing ZStd compression codec plugin (#9658) (#9832) (#9844) * introducing zstd compression codec plugin * Moving zstd compression codec as a plugin * introducing zstd compression codec plugin * Adding checks to EngineConfig and fixing tests * incorporating review comments * fixing tests * introducing zstd compression codec plugin * addressing review comments * nit fixes * implementing codec aliases * addressing review comments * review comments * moving codec aliases to custom codec * adding zstd default codec for backward compatibility * renaming to deprecated codec * incorporating review comments * nit fixes --------- (cherry picked from commit 76f1b52647cf19bd19e3ce9d0759953b5d376593) (cherry picked from commit 90310c319336820732a8eeb6d81776fbfa6e6d48) Signed-off-by: Sarthak Aggarwal Signed-off-by: Prabhakar Sithanandam Signed-off-by: Andrew Ross Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: Sarthak Aggarwal Co-authored-by: Prabhakar Sithanandam --- modules/reindex/build.gradle | 2 + plugins/custom-codecs/build.gradle | 27 ++ .../codec/customcodecs/CustomCodecPlugin.java | 47 ++++ .../customcodecs/CustomCodecService.java | 72 +++++ .../CustomCodecServiceFactory.java | 27 ++ .../customcodecs/ZstdDeprecatedCodec.java | 61 +++++ .../plugin-metadata/plugin-security.policy | 11 + .../codec/customcodecs/CustomCodecTests.java | 250 ++++++++++++++++++ 8 files changed, 497 insertions(+) create mode 100644 plugins/custom-codecs/build.gradle create mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java create mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java create mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java create mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java create mode 100644 plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy create mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 37526a924da73..bb04fffc4f03a 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -69,6 +69,7 @@ dependencies { testImplementation project(':modules:transport-netty4') // for parent/child testing testImplementation project(':modules:parent-join') + testImplementation project(':plugins:custom-codecs') } restResources { @@ -95,4 +96,5 @@ forbiddenPatterns { tasks.named("bundlePlugin").configure { dependsOn("copyParentJoinMetadata") dependsOn("copyTransportNetty4Metadata") + dependsOn("copyCustomCodecsMetadata") } diff --git a/plugins/custom-codecs/build.gradle b/plugins/custom-codecs/build.gradle new file mode 100644 index 0000000000000..253822e88b817 --- /dev/null +++ b/plugins/custom-codecs/build.gradle @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +apply plugin: 'opensearch.opensearchplugin' +apply plugin: 'opensearch.internal-cluster-test' + +opensearchplugin { + name 'custom-codecs' + description 'A plugin that implements custom compression codecs.' + classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' + licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') + noticeFile rootProject.file('NOTICE.txt') +} + +dependencies { + api "com.github.luben:zstd-jni:1.5.5-5" +} + +testingConventions.enabled = false; diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java new file mode 100644 index 0000000000000..91a13a1d924a2 --- /dev/null +++ b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.index.IndexSettings; +import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.plugins.EnginePlugin; +import org.opensearch.plugins.Plugin; + +import java.util.Optional; + +/** + * A plugin that implements custom codecs. Supports these codecs: + *
    + *
  • ZSTD + *
  • ZSTDNODICT + *
+ * + * @opensearch.internal + */ +public final class CustomCodecPlugin extends Plugin implements EnginePlugin { + + /** + * Creates a new instance + */ + public CustomCodecPlugin() {} + + /** + * @param indexSettings is the default indexSettings + * @return the engine factory + */ + @Override + public Optional getCustomCodecServiceFactory(final IndexSettings indexSettings) { + String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); + if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { + return Optional.of(new CustomCodecServiceFactory()); + } + return Optional.empty(); + } +} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java new file mode 100644 index 0000000000000..de0eb2b3286d3 --- /dev/null +++ b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -0,0 +1,72 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.Codec; +import org.opensearch.common.collect.MapBuilder; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.codec.CodecService; +import org.opensearch.index.mapper.MapperService; + +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Stream; + +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + +/** + * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. + */ +public class CustomCodecService extends CodecService { + private final Map codecs; + /** + * ZStandard codec + */ + public static final String ZSTD_CODEC = "zstd"; + /** + * ZStandard without dictionary codec + */ + public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; + + /** + * Creates a new CustomCodecService. + * + * @param mapperService The mapper service. + * @param indexSettings The index settings. + * @param logger The logger. + */ + public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { + super(mapperService, indexSettings, logger); + int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + final MapBuilder codecs = MapBuilder.newMapBuilder(); + if (mapperService == null) { + codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); + } else { + codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); + codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); + } + this.codecs = codecs.immutableMap(); + } + + @Override + public Codec codec(String name) { + Codec codec = codecs.get(name); + if (codec == null) { + return super.codec(name); + } + return codec; + } + + @Override + public String[] availableCodecs() { + return Stream.concat(Arrays.stream(super.availableCodecs()), codecs.keySet().stream()).toArray(String[]::new); + } +} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java new file mode 100644 index 0000000000000..d634616162684 --- /dev/null +++ b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.index.codec.CodecService; +import org.opensearch.index.codec.CodecServiceConfig; +import org.opensearch.index.codec.CodecServiceFactory; + +/** + * A factory for creating new {@link CodecService} instance + */ +public class CustomCodecServiceFactory implements CodecServiceFactory { + + /** Creates a new instance. */ + public CustomCodecServiceFactory() {} + + @Override + public CodecService createCodecService(CodecServiceConfig config) { + return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger()); + } +} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java new file mode 100644 index 0000000000000..02fa386db97b3 --- /dev/null +++ b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +/** + * ZstdDeprecatedCodec provides ZSTD compressor using the zstd-jni library. + * Added to support backward compatibility for indices created with Lucene95CustomCodec as codec name. + */ +@Deprecated(since = "2.10") +public class ZstdDeprecatedCodec extends Lucene95CustomCodec implements CodecSettings { + + /** + * Creates a new ZstdDefaultCodec instance with the default compression level. + */ + public ZstdDeprecatedCodec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new ZstdDefaultCodec instance. + * + * @param compressionLevel The compression level. + */ + public ZstdDeprecatedCodec(int compressionLevel) { + super(Mode.ZSTD_DEPRECATED, compressionLevel); + } + + /** + * Creates a new ZstdDefaultCodec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public ZstdDeprecatedCodec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.ZSTD_DEPRECATED, compressionLevel, mapperService, logger); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } +} diff --git a/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy b/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..8161010cfa897 --- /dev/null +++ b/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,11 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +grant codeBase "${codebase.zstd-jni}" { + permission java.lang.RuntimePermission "loadLibrary.*"; +}; diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java new file mode 100644 index 0000000000000..5365b9e222d9a --- /dev/null +++ b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java @@ -0,0 +1,250 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.LogManager; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; +import org.apache.lucene.codecs.lucene95.Lucene95Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; +import org.opensearch.common.settings.IndexScopedSettings; +import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.codec.CodecService; +import org.opensearch.index.codec.CodecServiceConfig; +import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.indices.mapper.MapperRegistry; +import org.opensearch.plugins.MapperPlugin; +import org.opensearch.test.IndexSettingsModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; + +import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; +import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + +@SuppressCodecs("*") // we test against default codec so never get a random one here! +public class CustomCodecTests extends OpenSearchTestCase { + + private CustomCodecPlugin plugin; + + @Before + public void setup() { + plugin = new CustomCodecPlugin(); + } + + public void testZstd() throws Exception { + Codec codec = createCodecService(false).codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDict() throws Exception { + Codec codec = createCodecService(false).codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdDeprecatedCodec() { + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> createCodecService(false).codec("ZSTD_DEPRECATED") + ); + assertTrue(e.getMessage().startsWith("failed to find codec")); + } + + public void testZstdWithCompressionLevel() throws Exception { + int randomCompressionLevel = randomIntBetween(1, 6); + Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictWithCompressionLevel() throws Exception { + int randomCompressionLevel = randomIntBetween(1, 6); + Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + } + + public void testBestCompressionWithCompressionLevel() { + final Settings zstdSettings = Settings.builder() + .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(ZSTD_CODEC, ZSTD_NO_DICT_CODEC)) + .build(); + + // able to validate zstd + final IndexScopedSettings zstdIndexScopedSettings = new IndexScopedSettings( + zstdSettings, + IndexScopedSettings.BUILT_IN_INDEX_SETTINGS + ); + zstdIndexScopedSettings.validate(zstdSettings, true); + } + + public void testLuceneCodecsWithCompressionLevel() { + final Settings customCodecSettings = Settings.builder() + .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) + .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom("zstd", "zstd_no_dict")) + .build(); + + final IndexScopedSettings customCodecIndexScopedSettings = new IndexScopedSettings( + customCodecSettings, + IndexScopedSettings.BUILT_IN_INDEX_SETTINGS + ); + customCodecIndexScopedSettings.validate(customCodecSettings, true); + } + + public void testZstandardCompressionLevelSupport() throws Exception { + CodecService codecService = createCodecService(false); + CodecSettings zstdCodec = (CodecSettings) codecService.codec("zstd"); + CodecSettings zstdNoDictCodec = (CodecSettings) codecService.codec("zstd_no_dict"); + assertTrue(zstdCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); + assertTrue(zstdNoDictCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); + } + + public void testDefaultMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("default"); + assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); + } + + public void testBestCompressionMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("best_compression"); + assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); + } + + public void testZstdMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("zstd"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testZstdNoDictMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("zstd_no_dict"); + assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); + Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + // write some docs with it, inspect .si to see this was the used compression + private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { + SegmentReader sr = getSegmentReader(actual); + String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); + assertNotNull(v); + assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); + } + + private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expected, Codec actual) throws Exception { + SegmentReader sr = getSegmentReader(actual); + String v = sr.getSegmentInfo().info.getAttribute(Lucene95CustomStoredFieldsFormat.MODE_KEY); + assertNotNull(v); + assertEquals(expected, Lucene95CustomCodec.Mode.valueOf(v)); + } + + private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + if (isMapperServiceNull) { + return new CustomCodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); + } + return buildCodecService(nodeSettings); + } + + private CodecService createCodecService(int randomCompressionLevel, String codec) throws IOException { + Settings nodeSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .put("index.codec", codec) + .put("index.codec.compression_level", randomCompressionLevel) + .build(); + return buildCodecService(nodeSettings); + } + + private CodecService buildCodecService(Settings nodeSettings) throws IOException { + + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); + SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); + IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); + MapperService service = new MapperService( + indexSettings, + indexAnalyzers, + xContentRegistry(), + similarityService, + mapperRegistry, + () -> null, + () -> false, + null + ); + + Optional customCodecServiceFactory = plugin.getCustomCodecServiceFactory(indexSettings); + if (customCodecServiceFactory.isPresent()) { + return customCodecServiceFactory.get().createCodecService(new CodecServiceConfig(indexSettings, service, logger)); + } + return new CustomCodecService(service, indexSettings, LogManager.getLogger("test")); + } + + private SegmentReader getSegmentReader(Codec codec) throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(null); + iwc.setCodec(codec); + IndexWriter iw = new IndexWriter(dir, iwc); + iw.addDocument(new Document()); + iw.commit(); + iw.close(); + DirectoryReader ir = DirectoryReader.open(dir); + SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); + ir.close(); + dir.close(); + return sr; + } + +} From 7e9a87695f7aba5b7bbc9bbac822d00fd73a5776 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Fri, 8 Sep 2023 14:36:01 -0700 Subject: [PATCH 03/14] Remove custom-codecs plugin (#9947) This plugin has been moved to its own [repository][1]. [1]: https://github.com/opensearch-project/custom-codecs Signed-off-by: Andrew Ross --- modules/reindex/build.gradle | 2 - plugins/custom-codecs/build.gradle | 27 -- .../codec/customcodecs/CustomCodecPlugin.java | 47 ---- .../customcodecs/CustomCodecService.java | 72 ----- .../CustomCodecServiceFactory.java | 27 -- .../customcodecs/ZstdDeprecatedCodec.java | 61 ----- .../plugin-metadata/plugin-security.policy | 11 - .../codec/customcodecs/CustomCodecTests.java | 250 ------------------ 8 files changed, 497 deletions(-) delete mode 100644 plugins/custom-codecs/build.gradle delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java delete mode 100644 plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java delete mode 100644 plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy delete mode 100644 plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index bb04fffc4f03a..37526a924da73 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -69,7 +69,6 @@ dependencies { testImplementation project(':modules:transport-netty4') // for parent/child testing testImplementation project(':modules:parent-join') - testImplementation project(':plugins:custom-codecs') } restResources { @@ -96,5 +95,4 @@ forbiddenPatterns { tasks.named("bundlePlugin").configure { dependsOn("copyParentJoinMetadata") dependsOn("copyTransportNetty4Metadata") - dependsOn("copyCustomCodecsMetadata") } diff --git a/plugins/custom-codecs/build.gradle b/plugins/custom-codecs/build.gradle deleted file mode 100644 index 253822e88b817..0000000000000 --- a/plugins/custom-codecs/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -apply plugin: 'opensearch.opensearchplugin' -apply plugin: 'opensearch.internal-cluster-test' - -opensearchplugin { - name 'custom-codecs' - description 'A plugin that implements custom compression codecs.' - classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin' - licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt') - noticeFile rootProject.file('NOTICE.txt') -} - -dependencies { - api "com.github.luben:zstd-jni:1.5.5-5" -} - -testingConventions.enabled = false; diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java deleted file mode 100644 index 91a13a1d924a2..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.CodecServiceFactory; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.plugins.EnginePlugin; -import org.opensearch.plugins.Plugin; - -import java.util.Optional; - -/** - * A plugin that implements custom codecs. Supports these codecs: - *
    - *
  • ZSTD - *
  • ZSTDNODICT - *
- * - * @opensearch.internal - */ -public final class CustomCodecPlugin extends Plugin implements EnginePlugin { - - /** - * Creates a new instance - */ - public CustomCodecPlugin() {} - - /** - * @param indexSettings is the default indexSettings - * @return the engine factory - */ - @Override - public Optional getCustomCodecServiceFactory(final IndexSettings indexSettings) { - String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); - if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { - return Optional.of(new CustomCodecServiceFactory()); - } - return Optional.empty(); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java deleted file mode 100644 index de0eb2b3286d3..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.apache.lucene.codecs.Codec; -import org.opensearch.common.collect.MapBuilder; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.mapper.MapperService; - -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Stream; - -import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; - -/** - * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. - */ -public class CustomCodecService extends CodecService { - private final Map codecs; - /** - * ZStandard codec - */ - public static final String ZSTD_CODEC = "zstd"; - /** - * ZStandard without dictionary codec - */ - public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; - - /** - * Creates a new CustomCodecService. - * - * @param mapperService The mapper service. - * @param indexSettings The index settings. - * @param logger The logger. - */ - public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { - super(mapperService, indexSettings, logger); - int compressionLevel = indexSettings.getValue(INDEX_CODEC_COMPRESSION_LEVEL_SETTING); - final MapBuilder codecs = MapBuilder.newMapBuilder(); - if (mapperService == null) { - codecs.put(ZSTD_CODEC, new ZstdCodec(compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(compressionLevel)); - } else { - codecs.put(ZSTD_CODEC, new ZstdCodec(mapperService, logger, compressionLevel)); - codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDictCodec(mapperService, logger, compressionLevel)); - } - this.codecs = codecs.immutableMap(); - } - - @Override - public Codec codec(String name) { - Codec codec = codecs.get(name); - if (codec == null) { - return super.codec(name); - } - return codec; - } - - @Override - public String[] availableCodecs() { - return Stream.concat(Arrays.stream(super.availableCodecs()), codecs.keySet().stream()).toArray(String[]::new); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java deleted file mode 100644 index d634616162684..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.codec.CodecServiceConfig; -import org.opensearch.index.codec.CodecServiceFactory; - -/** - * A factory for creating new {@link CodecService} instance - */ -public class CustomCodecServiceFactory implements CodecServiceFactory { - - /** Creates a new instance. */ - public CustomCodecServiceFactory() {} - - @Override - public CodecService createCodecService(CodecServiceConfig config) { - return new CustomCodecService(config.getMapperService(), config.getIndexSettings(), config.getLogger()); - } -} diff --git a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java b/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java deleted file mode 100644 index 02fa386db97b3..0000000000000 --- a/plugins/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/ZstdDeprecatedCodec.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.Logger; -import org.opensearch.common.settings.Setting; -import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; - -/** - * ZstdDeprecatedCodec provides ZSTD compressor using the zstd-jni library. - * Added to support backward compatibility for indices created with Lucene95CustomCodec as codec name. - */ -@Deprecated(since = "2.10") -public class ZstdDeprecatedCodec extends Lucene95CustomCodec implements CodecSettings { - - /** - * Creates a new ZstdDefaultCodec instance with the default compression level. - */ - public ZstdDeprecatedCodec() { - this(DEFAULT_COMPRESSION_LEVEL); - } - - /** - * Creates a new ZstdDefaultCodec instance. - * - * @param compressionLevel The compression level. - */ - public ZstdDeprecatedCodec(int compressionLevel) { - super(Mode.ZSTD_DEPRECATED, compressionLevel); - } - - /** - * Creates a new ZstdDefaultCodec instance. - * - * @param mapperService The mapper service. - * @param logger The logger. - * @param compressionLevel The compression level. - */ - public ZstdDeprecatedCodec(MapperService mapperService, Logger logger, int compressionLevel) { - super(Mode.ZSTD_DEPRECATED, compressionLevel, mapperService, logger); - } - - /** The name for this codec. */ - @Override - public String toString() { - return getClass().getSimpleName(); - } - - @Override - public boolean supports(Setting setting) { - return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); - } -} diff --git a/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy b/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy deleted file mode 100644 index 8161010cfa897..0000000000000 --- a/plugins/custom-codecs/src/main/plugin-metadata/plugin-security.policy +++ /dev/null @@ -1,11 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -grant codeBase "${codebase.zstd-jni}" { - permission java.lang.RuntimePermission "loadLibrary.*"; -}; diff --git a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java b/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java deleted file mode 100644 index 5365b9e222d9a..0000000000000 --- a/plugins/custom-codecs/src/test/java/org/opensearch/index/codec/customcodecs/CustomCodecTests.java +++ /dev/null @@ -1,250 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.logging.log4j.LogManager; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene95.Lucene95Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.store.Directory; -import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; -import org.opensearch.common.settings.IndexScopedSettings; -import org.opensearch.common.settings.Settings; -import org.opensearch.env.Environment; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.analysis.IndexAnalyzers; -import org.opensearch.index.codec.CodecService; -import org.opensearch.index.codec.CodecServiceConfig; -import org.opensearch.index.codec.CodecServiceFactory; -import org.opensearch.index.codec.CodecSettings; -import org.opensearch.index.engine.EngineConfig; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.indices.mapper.MapperRegistry; -import org.opensearch.plugins.MapperPlugin; -import org.opensearch.test.IndexSettingsModule; -import org.opensearch.test.OpenSearchTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.util.Collections; -import java.util.Optional; - -import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; -import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; -import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; - -@SuppressCodecs("*") // we test against default codec so never get a random one here! -public class CustomCodecTests extends OpenSearchTestCase { - - private CustomCodecPlugin plugin; - - @Before - public void setup() { - plugin = new CustomCodecPlugin(); - } - - public void testZstd() throws Exception { - Codec codec = createCodecService(false).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDict() throws Exception { - Codec codec = createCodecService(false).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdDeprecatedCodec() { - final IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> createCodecService(false).codec("ZSTD_DEPRECATED") - ); - assertTrue(e.getMessage().startsWith("failed to find codec")); - } - - public void testZstdWithCompressionLevel() throws Exception { - int randomCompressionLevel = randomIntBetween(1, 6); - Codec codec = createCodecService(randomCompressionLevel, "zstd").codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDictWithCompressionLevel() throws Exception { - int randomCompressionLevel = randomIntBetween(1, 6); - Codec codec = createCodecService(randomCompressionLevel, "zstd_no_dict").codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); - } - - public void testBestCompressionWithCompressionLevel() { - final Settings zstdSettings = Settings.builder() - .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom(ZSTD_CODEC, ZSTD_NO_DICT_CODEC)) - .build(); - - // able to validate zstd - final IndexScopedSettings zstdIndexScopedSettings = new IndexScopedSettings( - zstdSettings, - IndexScopedSettings.BUILT_IN_INDEX_SETTINGS - ); - zstdIndexScopedSettings.validate(zstdSettings, true); - } - - public void testLuceneCodecsWithCompressionLevel() { - final Settings customCodecSettings = Settings.builder() - .put(INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getKey(), randomIntBetween(1, 6)) - .put(EngineConfig.INDEX_CODEC_SETTING.getKey(), randomFrom("zstd", "zstd_no_dict")) - .build(); - - final IndexScopedSettings customCodecIndexScopedSettings = new IndexScopedSettings( - customCodecSettings, - IndexScopedSettings.BUILT_IN_INDEX_SETTINGS - ); - customCodecIndexScopedSettings.validate(customCodecSettings, true); - } - - public void testZstandardCompressionLevelSupport() throws Exception { - CodecService codecService = createCodecService(false); - CodecSettings zstdCodec = (CodecSettings) codecService.codec("zstd"); - CodecSettings zstdNoDictCodec = (CodecSettings) codecService.codec("zstd_no_dict"); - assertTrue(zstdCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); - assertTrue(zstdNoDictCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); - } - - public void testDefaultMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("default"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec); - } - - public void testBestCompressionMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec); - } - - public void testZstdMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("zstd"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - public void testZstdNoDictMapperServiceNull() throws Exception { - Codec codec = createCodecService(true).codec("zstd_no_dict"); - assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, codec); - Lucene95CustomStoredFieldsFormat storedFieldsFormat = (Lucene95CustomStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene95CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); - } - - // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception { - SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene95Codec.Mode.valueOf(v)); - } - - private void assertStoredFieldsCompressionEquals(Lucene95CustomCodec.Mode expected, Codec actual) throws Exception { - SegmentReader sr = getSegmentReader(actual); - String v = sr.getSegmentInfo().info.getAttribute(Lucene95CustomStoredFieldsFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene95CustomCodec.Mode.valueOf(v)); - } - - private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { - Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); - if (isMapperServiceNull) { - return new CustomCodecService(null, IndexSettingsModule.newIndexSettings("_na", nodeSettings), LogManager.getLogger("test")); - } - return buildCodecService(nodeSettings); - } - - private CodecService createCodecService(int randomCompressionLevel, String codec) throws IOException { - Settings nodeSettings = Settings.builder() - .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) - .put("index.codec", codec) - .put("index.codec.compression_level", randomCompressionLevel) - .build(); - return buildCodecService(nodeSettings); - } - - private CodecService buildCodecService(Settings nodeSettings) throws IOException { - - IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("_na", nodeSettings); - SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); - IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; - MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); - MapperService service = new MapperService( - indexSettings, - indexAnalyzers, - xContentRegistry(), - similarityService, - mapperRegistry, - () -> null, - () -> false, - null - ); - - Optional customCodecServiceFactory = plugin.getCustomCodecServiceFactory(indexSettings); - if (customCodecServiceFactory.isPresent()) { - return customCodecServiceFactory.get().createCodecService(new CodecServiceConfig(indexSettings, service, logger)); - } - return new CustomCodecService(service, indexSettings, LogManager.getLogger("test")); - } - - private SegmentReader getSegmentReader(Codec codec) throws IOException { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(codec); - IndexWriter iw = new IndexWriter(dir, iwc); - iw.addDocument(new Document()); - iw.commit(); - iw.close(); - DirectoryReader ir = DirectoryReader.open(dir); - SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - ir.close(); - dir.close(); - return sr; - } - -} From f40d280aab5470f6ad21d4b346f053e739bdbd33 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Fri, 11 Aug 2023 16:56:20 -0400 Subject: [PATCH 04/14] Implement on behalf of token passing for extensions (#8679) Implement on behalf of token passing for extensions Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Signed-off-by: Ryan Liang Co-authored-by: Ryan Liang Co-authored-by: Peter Nied --- CHANGELOG.md | 7 + .../identity/shiro/ShiroTokenManager.java | 15 ++- .../identity/shiro/AuthTokenHandlerTests.java | 29 ++++- .../extensions/ExtensionsManager.java | 11 +- .../extensions/NoopExtensionsManager.java | 4 +- .../extensions/rest/ExtensionRestRequest.java | 8 +- .../rest/RestActionsRequestHandler.java | 12 +- .../rest/RestSendToExtensionAction.java | 26 ++-- .../identity/noop/NoopTokenManager.java | 13 +- .../opensearch/identity/tokens/AuthToken.java | 1 + .../identity/tokens/BasicAuthToken.java | 9 ++ .../identity/tokens/BearerAuthToken.java | 5 + .../identity/tokens/OnBehalfOfClaims.java | 87 +++++++++++++ .../identity/tokens/TokenManager.java | 16 ++- .../main/java/org/opensearch/node/Node.java | 3 +- .../extensions/ExtensionsManagerTests.java | 35 ++--- .../rest/ExtensionRestRequestTests.java | 18 ++- .../rest/RestSendToExtensionActionTests.java | 122 +++++++++++++++--- 18 files changed, 355 insertions(+), 66 deletions(-) create mode 100644 server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ea905dffb845..3b8d8c7d92bea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) +- Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847)) +- Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636)) +- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) +- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) +- Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107)) +- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) + ### Dependencies - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 345b98d5cb423..60239d609a08a 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -15,8 +15,11 @@ import org.apache.shiro.authc.UsernamePasswordToken; import org.opensearch.common.Randomness; import org.opensearch.identity.IdentityService; +import org.opensearch.identity.Subject; +import org.opensearch.identity.noop.NoopSubject; import org.opensearch.identity.tokens.AuthToken; import org.opensearch.identity.tokens.BasicAuthToken; +import org.opensearch.identity.tokens.OnBehalfOfClaims; import org.opensearch.identity.tokens.TokenManager; import java.util.Arrays; @@ -54,15 +57,16 @@ public Optional translateAuthToken(org.opensearch.identity. final BasicAuthToken basicAuthToken = (BasicAuthToken) authenticationToken; return Optional.of(new UsernamePasswordToken(basicAuthToken.getUser(), basicAuthToken.getPassword())); } - return Optional.empty(); } @Override - public AuthToken issueToken(String audience) { + public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) { String password = generatePassword(); - final byte[] rawEncoded = Base64.getEncoder().encode((audience + ":" + password).getBytes(UTF_8)); + final byte[] rawEncoded = Base64.getEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new + // ShiroSubject w/ + // audience as name final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; BasicAuthToken token = new BasicAuthToken(header); @@ -71,6 +75,11 @@ public AuthToken issueToken(String audience) { return token; } + @Override + public Subject authenticateToken(AuthToken authToken) { + return new NoopSubject(); + } + public boolean validateToken(AuthToken token) { if (token instanceof BasicAuthToken) { final BasicAuthToken basicAuthToken = (BasicAuthToken) token; diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java index 24a06bd9ac71a..5898ea3dad855 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java @@ -10,10 +10,14 @@ import org.apache.shiro.authc.AuthenticationToken; import org.apache.shiro.authc.UsernamePasswordToken; +import org.junit.Before; +import org.opensearch.identity.Subject; +import org.opensearch.identity.noop.NoopSubject; import org.opensearch.identity.noop.NoopTokenManager; import org.opensearch.identity.tokens.AuthToken; import org.opensearch.identity.tokens.BasicAuthToken; import org.opensearch.identity.tokens.BearerAuthToken; +import org.opensearch.identity.tokens.OnBehalfOfClaims; import org.opensearch.test.OpenSearchTestCase; import org.junit.Before; @@ -34,16 +38,15 @@ public class AuthTokenHandlerTests extends OpenSearchTestCase { private ShiroTokenManager shiroAuthTokenHandler; - private NoopTokenManager noopTokenManager; @Before public void testSetup() { shiroAuthTokenHandler = new ShiroTokenManager(); - noopTokenManager = new NoopTokenManager(); } public void testShouldExtractBasicAuthTokenSuccessfully() { final BasicAuthToken authToken = new BasicAuthToken("Basic YWRtaW46YWRtaW4="); // admin:admin + assertEquals(authToken.asAuthHeaderValue(), "YWRtaW46YWRtaW4="); final AuthenticationToken translatedToken = shiroAuthTokenHandler.translateAuthToken(authToken).get(); assertThat(translatedToken, is(instanceOf(UsernamePasswordToken.class))); @@ -109,7 +112,7 @@ public void testShoudPassMapLookupWithToken() { assertTrue(authToken.getPassword().equals(shiroAuthTokenHandler.getShiroTokenPasswordMap().get(authToken))); } - public void testShouldPassThrougbResetToken(AuthToken token) { + public void testShouldPassThroughResetToken() { final BearerAuthToken bearerAuthToken = new BearerAuthToken("header.payload.signature"); shiroAuthTokenHandler.resetToken(bearerAuthToken); } @@ -124,6 +127,7 @@ public void testVerifyBearerTokenObject() { assertEquals(testGoodToken.getPayload(), "payload"); assertEquals(testGoodToken.getSignature(), "signature"); assertEquals(testGoodToken.toString(), "Bearer auth token with header=header, payload=payload, signature=signature"); + assertEquals(testGoodToken.asAuthHeaderValue(), "header.payload.signature"); } public void testGeneratedPasswordContents() { @@ -147,4 +151,23 @@ public void testGeneratedPasswordContents() { validator.validate(data); } + public void testIssueOnBehalfOfTokenFromClaims() { + Subject subject = new NoopSubject(); + OnBehalfOfClaims claims = new OnBehalfOfClaims("test", "test"); + BasicAuthToken authToken = (BasicAuthToken) shiroAuthTokenHandler.issueOnBehalfOfToken(subject, claims); + assertTrue(authToken instanceof BasicAuthToken); + UsernamePasswordToken translatedToken = (UsernamePasswordToken) shiroAuthTokenHandler.translateAuthToken(authToken).get(); + assertEquals(authToken.getPassword(), new String(translatedToken.getPassword())); + assertTrue(shiroAuthTokenHandler.getShiroTokenPasswordMap().containsKey(authToken)); + assertEquals(shiroAuthTokenHandler.getShiroTokenPasswordMap().get(authToken), new String(translatedToken.getPassword())); + } + + public void testTokenNoopIssuance() { + NoopTokenManager tokenManager = new NoopTokenManager(); + OnBehalfOfClaims claims = new OnBehalfOfClaims("test", "test"); + Subject subject = new NoopSubject(); + AuthToken token = tokenManager.issueOnBehalfOfToken(subject, claims); + assertTrue(token instanceof AuthToken); + } + } diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 8073fb9cb7f56..e4112758d466f 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -40,6 +40,7 @@ import org.opensearch.extensions.rest.RestActionsRequestHandler; import org.opensearch.extensions.settings.CustomSettingsRequestHandler; import org.opensearch.extensions.settings.RegisterCustomSettingsRequest; +import org.opensearch.identity.IdentityService; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.TransportException; @@ -141,9 +142,15 @@ public void initializeServicesAndRestHandler( TransportService transportService, ClusterService clusterService, Settings initialEnvironmentSettings, - NodeClient client + NodeClient client, + IdentityService identityService ) { - this.restActionsRequestHandler = new RestActionsRequestHandler(actionModule.getRestController(), extensionIdMap, transportService); + this.restActionsRequestHandler = new RestActionsRequestHandler( + actionModule.getRestController(), + extensionIdMap, + transportService, + identityService + ); this.customSettingsRequestHandler = new CustomSettingsRequestHandler(settingsModule); this.transportService = transportService; this.clusterService = clusterService; diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index 327dce187059c..34aa3ec1367a7 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -16,6 +16,7 @@ import org.opensearch.extensions.action.ExtensionActionRequest; import org.opensearch.extensions.action.ExtensionActionResponse; import org.opensearch.extensions.action.RemoteExtensionActionResponse; +import org.opensearch.identity.IdentityService; import org.opensearch.transport.TransportService; import java.io.IOException; @@ -40,7 +41,8 @@ public void initializeServicesAndRestHandler( TransportService transportService, ClusterService clusterService, Settings initialEnvironmentSettings, - NodeClient client + NodeClient client, + IdentityService identityService ) { // no-op } diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java index 65478e85081fa..968aadcfbb7b9 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import static java.util.Objects.requireNonNull; /** * Request to execute REST actions on extension node. @@ -86,7 +87,7 @@ public ExtensionRestRequest( this.headers = headers; this.mediaType = mediaType; this.content = content; - this.principalIdentifierToken = principalIdentifier; + this.principalIdentifierToken = requireNonNull(principalIdentifier); this.httpVersion = httpVersion; } @@ -280,7 +281,7 @@ public boolean isContentConsumed() { } /** - * Gets a parser for the contents of this request if there is content and an xContentType. + * Gets a parser for the contents of this request if there is content, an xContentType, and a principal identifier. * * @param xContentRegistry The extension's xContentRegistry * @return A parser for the given content and content type. @@ -291,6 +292,9 @@ public final XContentParser contentParser(NamedXContentRegistry xContentRegistry if (!hasContent() || getXContentType() == null) { throw new OpenSearchParseException("There is no request body or the ContentType is invalid."); } + if (getRequestIssuerIdentity() == null) { + throw new OpenSearchParseException("There is no request body or the requester identity is invalid."); + } return getXContentType().xContent().createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java b/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java index b6d628ae9253f..8385cdf7b2a34 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java @@ -8,16 +8,16 @@ package org.opensearch.extensions.rest; +import java.util.Map; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.core.transport.TransportResponse; import org.opensearch.extensions.AcknowledgedResponse; import org.opensearch.extensions.DiscoveryExtensionNode; +import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestController; import org.opensearch.rest.RestHandler; import org.opensearch.transport.TransportService; -import java.util.Map; - /** * Handles requests to register extension REST actions. * @@ -28,6 +28,7 @@ public class RestActionsRequestHandler { private final RestController restController; private final Map extensionIdMap; private final TransportService transportService; + private final IdentityService identityService; /** * Instantiates a new REST Actions Request Handler using the Node's RestController. @@ -39,11 +40,13 @@ public class RestActionsRequestHandler { public RestActionsRequestHandler( RestController restController, Map extensionIdMap, - TransportService transportService + TransportService transportService, + IdentityService identityService ) { this.restController = restController; this.extensionIdMap = extensionIdMap; this.transportService = transportService; + this.identityService = identityService; } /** @@ -62,7 +65,8 @@ public TransportResponse handleRegisterRestActionsRequest( restActionsRequest, discoveryExtensionNode, transportService, - dynamicActionRegistry + dynamicActionRegistry, + identityService ); restController.registerHandler(handler); return new AcknowledgedResponse(true); diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index d33ddd3848fe4..33f44a913dd8a 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -19,6 +19,10 @@ import org.opensearch.extensions.DiscoveryExtensionNode; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.http.HttpRequest; +import org.opensearch.identity.IdentityService; +import org.opensearch.identity.Subject; +import org.opensearch.identity.tokens.OnBehalfOfClaims; +import org.opensearch.identity.tokens.TokenManager; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.NamedRoute; @@ -31,7 +35,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.security.Principal; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -54,19 +57,13 @@ public class RestSendToExtensionAction extends BaseRestHandler { private static final String SEND_TO_EXTENSION_ACTION = "send_to_extension_action"; private static final Logger logger = LogManager.getLogger(RestSendToExtensionAction.class); - // To replace with user identity see https://github.com/opensearch-project/OpenSearch/pull/4247 - private static final Principal DEFAULT_PRINCIPAL = new Principal() { - @Override - public String getName() { - return "OpenSearchUser"; - } - }; private final List routes; private final List deprecatedRoutes; private final String pathPrefix; private final DiscoveryExtensionNode discoveryExtensionNode; private final TransportService transportService; + private final IdentityService identityService; private static final Set allowList = Set.of("Content-Type"); private static final Set denyList = Set.of("Authorization", "Proxy-Authorization"); @@ -82,7 +79,8 @@ public RestSendToExtensionAction( RegisterRestActionsRequest restActionsRequest, DiscoveryExtensionNode discoveryExtensionNode, TransportService transportService, - DynamicActionRegistry dynamicActionRegistry + DynamicActionRegistry dynamicActionRegistry, + IdentityService identityService ) { this.pathPrefix = "/_extensions/_" + restActionsRequest.getUniqueId(); RestRequest.Method method; @@ -147,6 +145,7 @@ public RestSendToExtensionAction( this.discoveryExtensionNode = discoveryExtensionNode; this.transportService = transportService; + this.identityService = identityService; } @Override @@ -240,12 +239,15 @@ public String executor() { }; try { + // Will be replaced with ExtensionTokenProcessor and PrincipalIdentifierToken classes from feature/identity - final String extensionTokenProcessor = "placeholder_token_processor"; - final String requestIssuerIdentity = "placeholder_request_issuer_identity"; Map> filteredHeaders = filterHeaders(headers, allowList, denyList); + TokenManager tokenManager = identityService.getTokenManager(); + Subject subject = this.identityService.getSubject(); + OnBehalfOfClaims claims = new OnBehalfOfClaims(discoveryExtensionNode.getId(), subject.getPrincipal().getName()); + transportService.sendRequest( discoveryExtensionNode, ExtensionsManager.REQUEST_REST_EXECUTE_ON_EXTENSION_ACTION, @@ -259,7 +261,7 @@ public String executor() { filteredHeaders, contentType, content, - requestIssuerIdentity, + tokenManager.issueOnBehalfOfToken(subject, claims).asAuthHeaderValue(), httpVersion ), restExecuteOnExtensionResponseHandler diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java index a55f28e02a8aa..1255e822cea6e 100644 --- a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java +++ b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java @@ -11,7 +11,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.identity.IdentityService; +import org.opensearch.identity.Subject; import org.opensearch.identity.tokens.AuthToken; +import org.opensearch.identity.tokens.OnBehalfOfClaims; import org.opensearch.identity.tokens.TokenManager; /** @@ -26,8 +28,17 @@ public class NoopTokenManager implements TokenManager { * @return a new Noop Token */ @Override - public AuthToken issueToken(String audience) { + public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims) { return new AuthToken() { + @Override + public String asAuthHeaderValue() { + return "noopToken"; + } }; } + + @Override + public Subject authenticateToken(AuthToken authToken) { + return null; + } } diff --git a/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java b/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java index 6e113f6eaa96a..c929e7421b3d8 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java +++ b/server/src/main/java/org/opensearch/identity/tokens/AuthToken.java @@ -15,4 +15,5 @@ */ public interface AuthToken { + String asAuthHeaderValue(); } diff --git a/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java b/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java index 9cd6cb6b6208a..71b8fe504a5d1 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java +++ b/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java @@ -51,4 +51,13 @@ public void revoke() { this.password = ""; this.user = ""; } + + @Override + public String asAuthHeaderValue() { + if (user == null || password == null) { + return null; + } + String usernamepassword = user + ":" + password; + return Base64.getEncoder().encodeToString(usernamepassword.getBytes(StandardCharsets.UTF_8)); + } } diff --git a/server/src/main/java/org/opensearch/identity/tokens/BearerAuthToken.java b/server/src/main/java/org/opensearch/identity/tokens/BearerAuthToken.java index eac164af1c5d3..217538c7b001b 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/BearerAuthToken.java +++ b/server/src/main/java/org/opensearch/identity/tokens/BearerAuthToken.java @@ -58,4 +58,9 @@ public String getTokenIdentifier() { public String toString() { return "Bearer auth token with header=" + header + ", payload=" + payload + ", signature=" + signature; } + + @Override + public String asAuthHeaderValue() { + return completeToken; + } } diff --git a/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java b/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java new file mode 100644 index 0000000000000..3fef248ee6d3a --- /dev/null +++ b/server/src/main/java/org/opensearch/identity/tokens/OnBehalfOfClaims.java @@ -0,0 +1,87 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.identity.tokens; + +/** + * This class represents the claims of an OnBehalfOf token. + */ +public class OnBehalfOfClaims { + + private final String audience; + private final String subject; + private final Long expiration; + private final Long not_before; + private final Long issued_at; + + /** + * Constructor for OnBehalfOfClaims + * @param aud the Audience for the token + * @param subject the subject of the token + * @param expiration the expiration time in seconds for the token + * @param not_before the not_before time in seconds for the token + * @param issued_at the issued_at time in seconds for the token + */ + public OnBehalfOfClaims(String aud, String subject, Long expiration, Long not_before, Long issued_at) { + this.audience = aud; + this.subject = subject; + this.expiration = expiration; + this.not_before = not_before; + this.issued_at = issued_at; + } + + /** + * A constructor that sets a default issued at time of the current time + * @param aud the Audience for the token + * @param subject the subject of the token + * @param expiration the expiration time in seconds for the token + * @param not_before the not_before time in seconds for the token + */ + public OnBehalfOfClaims(String aud, String subject, Long expiration, Long not_before) { + this(aud, subject, expiration, not_before, System.currentTimeMillis() / 1000); + } + + /** + * A constructor which sets a default not before time of the current time + * @param aud the Audience for the token + * @param subject the subject of the token + * @param expiration the expiration time in seconds for the token + */ + public OnBehalfOfClaims(String aud, String subject, Long expiration) { + this(aud, subject, expiration, System.currentTimeMillis() / 1000); + } + + /** + * A constructor which sets the default expiration time of 5 minutes from the current time + * @param aud the Audience for the token + * @param subject the subject of the token + */ + public OnBehalfOfClaims(String aud, String subject) { + this(aud, subject, System.currentTimeMillis() / 1000 + 300); + } + + public String getAudience() { + return audience; + } + + public String getSubject() { + return subject; + } + + public Long getExpiration() { + return expiration; + } + + public Long getNot_before() { + return not_before; + } + + public Long getIssued_at() { + return issued_at; + } +} diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java index 029ce430e7532..4f6ddeb48dea3 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java +++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java @@ -8,15 +8,25 @@ package org.opensearch.identity.tokens; +import org.opensearch.identity.Subject; + /** * This interface defines the expected methods of a token manager */ public interface TokenManager { /** - * Create a new auth token - * @param audience: The audience for the token + * Create a new on behalf of token + * + * @param claims: A list of claims for the token to be generated with * @return A new auth token */ - public AuthToken issueToken(String audience); + public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims); + + /** + * Authenticates a provided authToken + * @param authToken: The authToken to authenticate + * @return The authenticated subject + */ + public Subject authenticateToken(AuthToken authToken); } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 1b22c919c313c..41d8c18ae6a58 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -915,7 +915,8 @@ protected Node( transportService, clusterService, environment.settings(), - client + client, + identityService ); final PersistedStateRegistry persistedStateRegistry = new PersistedStateRegistry(); final GatewayMetaState gatewayMetaState = new GatewayMetaState(); diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 75327d71d54ca..808616a3f8b8a 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -8,8 +8,19 @@ package org.opensearch.extensions; +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; +import org.junit.After; +import org.junit.Before; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionModule; @@ -57,23 +68,9 @@ import org.opensearch.transport.TransportService; import org.opensearch.transport.nio.MockNioTransport; import org.opensearch.usage.UsageService; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; -import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; @@ -82,6 +79,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static org.opensearch.test.ClusterServiceUtils.createClusterService; public class ExtensionsManagerTests extends OpenSearchTestCase { private TransportService transportService; @@ -94,6 +92,8 @@ public class ExtensionsManagerTests extends OpenSearchTestCase { private Setting customSetting = Setting.simpleString("custom_extension_setting", "none", Property.ExtensionScope); private NodeClient client; private MockNioTransport transport; + private IdentityService identityService; + private final ThreadPool threadPool = new TestThreadPool(ExtensionsManagerTests.class.getSimpleName()); private final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) @@ -166,6 +166,7 @@ public List> getExtensionSettings() { Collections.emptyList() ); client = new NoOpNodeClient(this.getTestName()); + identityService = new IdentityService(Settings.EMPTY, List.of()); } @Override @@ -772,7 +773,8 @@ public void testRegisterHandler() throws Exception { mockTransportService, clusterService, settings, - client + client, + identityService ); verify(mockTransportService, times(9)).registerRequestHandler(anyString(), anyString(), anyBoolean(), anyBoolean(), any(), any()); @@ -889,7 +891,8 @@ private void initialize(ExtensionsManager extensionsManager) { transportService, clusterService, settings, - client + client, + identityService ); } } diff --git a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java index 9b875ce89d587..b60795a73238b 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java @@ -8,6 +8,9 @@ package org.opensearch.extensions.rest; + +import org.opensearch.common.settings.Settings; +import org.opensearch.core.rest.RestStatus; import org.opensearch.OpenSearchParseException; import org.opensearch.common.io.stream.BytesStreamOutput; import org.opensearch.core.common.bytes.BytesArray; @@ -21,6 +24,11 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpRequest; +import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; +import org.opensearch.identity.IdentityService; +import org.opensearch.identity.Subject; +import org.opensearch.identity.tokens.OnBehalfOfClaims; +import org.opensearch.identity.tokens.TokenManager; import org.opensearch.rest.BytesRestResponse; import org.opensearch.rest.RestRequest.Method; import org.opensearch.test.OpenSearchTestCase; @@ -46,10 +54,10 @@ public class ExtensionRestRequestTests extends OpenSearchTestCase { String extensionUniqueId1; Principal userPrincipal; HttpRequest.HttpVersion expectedHttpVersion; - // Will be replaced with ExtensionTokenProcessor and PrincipalIdentifierToken classes from feature/identity String extensionTokenProcessor; String expectedRequestIssuerIdentity; NamedWriteableRegistry registry; + private IdentityService identityService; public void setUp() throws Exception { super.setUp(); @@ -67,7 +75,13 @@ public void setUp() throws Exception { userPrincipal = () -> "user1"; expectedHttpVersion = HttpRequest.HttpVersion.HTTP_1_1; extensionTokenProcessor = "placeholder_extension_token_processor"; - expectedRequestIssuerIdentity = "placeholder_request_issuer_identity"; + identityService = new IdentityService(Settings.EMPTY, List.of()); + TokenManager tokenManager = identityService.getTokenManager(); + Subject subject = this.identityService.getSubject(); + OnBehalfOfClaims claims = new OnBehalfOfClaims("testID", subject.getPrincipal().getName()); + expectedRequestIssuerIdentity = identityService.getTokenManager() + .issueOnBehalfOfToken(identityService.getSubject(), claims) + .asAuthHeaderValue(); } public void testExtensionRestRequest() throws Exception { diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index ea6c44d8c36ba..d1e4df509baab 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -8,6 +8,18 @@ package org.opensearch.extensions.rest; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.junit.After; +import org.junit.Before; import org.opensearch.Version; import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; @@ -66,6 +78,7 @@ public class RestSendToExtensionActionTests extends OpenSearchTestCase { private DiscoveryExtensionNode discoveryExtensionNode; private ActionModule actionModule; private DynamicActionRegistry dynamicActionRegistry; + private IdentityService identityService; private final ThreadPool threadPool = new TestThreadPool(RestSendToExtensionActionTests.class.getSimpleName()); @Before @@ -123,6 +136,7 @@ public void setup() throws Exception { new IdentityService(Settings.EMPTY, new ArrayList<>()), new ExtensionsManager(Set.of()) ); + identityService = new IdentityService(Settings.EMPTY, new ArrayList<>()); dynamicActionRegistry = actionModule.getDynamicActionRegistry(); } @@ -144,7 +158,8 @@ public void testRestSendToExtensionAction() throws Exception { registerRestActionRequest, discoveryExtensionNode, transportService, - dynamicActionRegistry + dynamicActionRegistry, + identityService ); assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); @@ -176,7 +191,8 @@ public void testRestSendToExtensionActionWithNamedRoute() throws Exception { registerRestActionRequest, discoveryExtensionNode, transportService, - dynamicActionRegistry + dynamicActionRegistry, + identityService ); assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); @@ -221,7 +237,8 @@ public void testRestSendToExtensionActionWithNamedRouteAndLegacyActionName() thr registerRestActionRequest, discoveryExtensionNode, transportService, - dynamicActionRegistry + dynamicActionRegistry, + identityService ); assertEquals("send_to_extension_action", restSendToExtensionAction.getName()); @@ -273,7 +290,13 @@ public void testRestSendToExtensionActionWithoutUniqueNameShouldFail() { ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -285,7 +308,13 @@ public void testRestSendToExtensionMultipleNamedRoutesWithSameName() throws Exce ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -297,7 +326,13 @@ public void testRestSendToExtensionMultipleNamedRoutesWithSameLegacyActionName() ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -309,7 +344,13 @@ public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPath() throws ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -321,7 +362,13 @@ public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPathWithDiffer ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -333,7 +380,13 @@ public void testRestSendToExtensionMultipleRoutesWithSameMethodAndPathWithPathPa ); try { - new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry); + new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ); } catch (IllegalArgumentException e) { fail("IllegalArgumentException should not be thrown for different paths"); } @@ -355,7 +408,13 @@ public void testRestSendToExtensionWithNamedRouteCollidingWithDynamicTransportAc expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -369,7 +428,13 @@ public void testRestSendToExtensionWithNamedRouteCollidingWithNativeTransportAct ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -383,7 +448,8 @@ public void testRestSendToExtensionActionFilterHeaders() throws Exception { registerRestActionRequest, discoveryExtensionNode, transportService, - dynamicActionRegistry + dynamicActionRegistry, + identityService ); Map> headers = new HashMap<>(); @@ -409,7 +475,13 @@ public void testRestSendToExtensionActionBadMethod() throws Exception { ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -421,7 +493,13 @@ public void testRestSendToExtensionActionBadDeprecatedMethod() throws Exception ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -433,7 +511,13 @@ public void testRestSendToExtensionActionMissingUri() throws Exception { ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } @@ -445,7 +529,13 @@ public void testRestSendToExtensionActionMissingDeprecatedUri() throws Exception ); expectThrows( IllegalArgumentException.class, - () -> new RestSendToExtensionAction(registerRestActionRequest, discoveryExtensionNode, transportService, dynamicActionRegistry) + () -> new RestSendToExtensionAction( + registerRestActionRequest, + discoveryExtensionNode, + transportService, + dynamicActionRegistry, + identityService + ) ); } } From 9136aea11ee8ee0908c01a944c41a9f77a48d34e Mon Sep 17 00:00:00 2001 From: Stephen Crawford Date: Thu, 14 Sep 2023 13:29:00 -0400 Subject: [PATCH 05/14] Just need to wait for service account merge Signed-off-by: Stephen Crawford --- .../opensearch/common/settings/Settings.java | 52 ++++++++++- .../InitializeExtensionSecurityRequest.java | 63 +++++++++++++ .../InitializeExtensionSecurityResponse.java | 88 +++++++++++++++++++ .../extensions/ExtensionsManager.java | 66 ++++++++++++++ .../rest/RestInitializeExtensionAction.java | 61 ++++++++++++- .../identity/tokens/TokenManager.java | 2 + .../main/java/org/opensearch/node/Node.java | 1 + 7 files changed, 328 insertions(+), 5 deletions(-) create mode 100644 server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java create mode 100644 server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 4df257612b91a..230eb5499935c 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -270,6 +270,21 @@ public String get(String setting, String defaultValue) { return retVal == null ? defaultValue : retVal; } + /** + * Returns a setting value based on the setting key. + */ + public Settings getNestedSettings(String key) { + return (Settings) settings.get(key); + } + + /** + * Returns a setting value based on the setting key. + */ + public List getNestedListOfSettings(String key) { + return (List) settings.get(key); + } + + /** * Returns the setting value (as float) associated with the setting key. If it does not exists, * returns the default value provided. @@ -666,6 +681,7 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder fromXContent(parser, keyBuilder, builder, allowNullValues); } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { List list = new ArrayList<>(); + List listOfObjects = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { list.add(parser.text()); @@ -674,12 +690,19 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder } else if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { list.add(String.valueOf(parser.text())); } else { - throw new IllegalStateException("only value lists are allowed in serialized settings"); + listOfObjects.add(fromXContent(parser, true, false)); + //throw new IllegalStateException("only value lists are allowed in serialized settings"); } } String key = keyBuilder.toString(); validateValue(key, list, parser, allowNullValues); builder.putList(key, list); + if (!listOfObjects.isEmpty()) { + builder.putListOfObjects(key, listOfObjects); + } + if (!list.isEmpty() && !listOfObjects.isEmpty()) { + throw new IllegalStateException("list cannot contain both values and objects"); + } } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { String key = keyBuilder.toString(); validateValue(key, null, parser, allowNullValues); @@ -786,6 +809,20 @@ public String get(String key) { return Settings.toString(map.get(key)); } + /** + * Returns a setting value based on the setting key. + */ + public Settings getNestedSettings(String key) { + return (Settings) map.get(key); + } + + /** + * Returns a setting value based on the setting key. + */ + public List getNestedListOfSettings(String key) { + return (List) map.get(key); + } + /** Return the current secure settings, or {@code null} if none have been set. */ public SecureSettings getSecureSettings() { return secureSettings.get(); @@ -1023,6 +1060,19 @@ public Builder putList(String setting, List values) { return this; } + /** + * Sets the setting with the provided setting key and a list of values. + * + * @param setting The setting key + * @param values The values + * @return The builder + */ + public Builder putListOfObjects(String setting, List values) { + remove(setting); + map.put(setting, new ArrayList<>(values)); + return this; + } + /** * Sets all the provided settings including secure settings */ diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java new file mode 100644 index 0000000000000..c18d8144098f1 --- /dev/null +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java @@ -0,0 +1,63 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.discovery; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.transport.TransportRequest; + +import java.io.IOException; +import java.util.Objects; + +/** + * InitializeExtensionRequest to initialize plugin + * + * @opensearch.internal + */ +public class InitializeExtensionSecurityRequest extends TransportRequest { + + private final String serviceAccountToken; + + public InitializeExtensionSecurityRequest(String serviceAccountToken) { + this.serviceAccountToken = serviceAccountToken; + } + + public InitializeExtensionSecurityRequest(StreamInput in) throws IOException { + super(in); + serviceAccountToken = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(serviceAccountToken); + } + + public String getServiceAccountToken() { + return serviceAccountToken; + } + + @Override + public String toString() { + return "InitializeExtensionsRequest{" + "serviceAccountToken= " + serviceAccountToken + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InitializeExtensionSecurityRequest that = (InitializeExtensionSecurityRequest) o; + return Objects.equals(serviceAccountToken, that.serviceAccountToken); + } + + @Override + public int hashCode() { + return Objects.hash(serviceAccountToken); + } +} diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java new file mode 100644 index 0000000000000..bcad61f3d59ba --- /dev/null +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.discovery; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.transport.TransportResponse; + +import java.io.IOException; +import java.util.Objects; + +/** + * PluginResponse to intialize plugin + * + * @opensearch.internal + */ +public class InitializeExtensionSecurityResponse extends TransportResponse { + private String name; + + public InitializeExtensionSecurityResponse(String name) { + this.name = name; + } + + public InitializeExtensionSecurityResponse(StreamInput in) throws IOException { + name = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + } + + /** + * @return the node that is currently leading, according to the responding node. + */ + + public String getName() { + return this.name; + } + + @Override + public String toString() { + return "InitializeExtensionResponse{" + "name = " + name + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InitializeExtensionSecurityResponse that = (InitializeExtensionSecurityResponse) o; + return Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } +} diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index e4112758d466f..2b074d98d78a5 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -8,6 +8,9 @@ package org.opensearch.extensions; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -28,6 +31,8 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.discovery.InitializeExtensionRequest; import org.opensearch.discovery.InitializeExtensionResponse; +import org.opensearch.discovery.InitializeExtensionSecurityRequest; +import org.opensearch.discovery.InitializeExtensionSecurityResponse; import org.opensearch.env.EnvironmentSettingsResponse; import org.opensearch.extensions.ExtensionsSettings.Extension; import org.opensearch.extensions.action.ExtensionActionRequest; @@ -41,6 +46,7 @@ import org.opensearch.extensions.settings.CustomSettingsRequestHandler; import org.opensearch.extensions.settings.RegisterCustomSettingsRequest; import org.opensearch.identity.IdentityService; +import org.opensearch.identity.tokens.AuthToken; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.ConnectTransportException; import org.opensearch.transport.TransportException; @@ -77,6 +83,7 @@ public class ExtensionsManager { public static final String REQUEST_EXTENSION_HANDLE_TRANSPORT_ACTION = "internal:extensions/handle-transportaction"; public static final String REQUEST_EXTENSION_HANDLE_REMOTE_TRANSPORT_ACTION = "internal:extensions/handle-remote-transportaction"; public static final String TRANSPORT_ACTION_REQUEST_FROM_EXTENSION = "internal:extensions/request-transportaction-from-extension"; + public static final String REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS = "internal:discovery/registersecuritysettings"; public static final int EXTENSION_REQUEST_WAIT_TIMEOUT = 10; private static final Logger logger = LogManager.getLogger(ExtensionsManager.class); @@ -89,6 +96,7 @@ public static enum OpenSearchRequestType { REQUEST_OPENSEARCH_NAMED_WRITEABLE_REGISTRY } + private ExtensionTransportActionsHandler extensionTransportActionsHandler; private Map extensionSettingsMap; private Map initializedExtensions; @@ -101,6 +109,7 @@ public static enum OpenSearchRequestType { private Settings environmentSettings; private AddSettingsUpdateConsumerRequestHandler addSettingsUpdateConsumerRequestHandler; private NodeClient client; + private IdentityService identityService; /** * Instantiate a new ExtensionsManager object to handle requests and responses from extensions. This is called during Node bootstrap. @@ -402,10 +411,62 @@ protected void doRun() throws Exception { new InitializeExtensionRequest(transportService.getLocalNode(), extension), initializeExtensionResponseHandler ); + initializeExtensionSecurity(extension); } }); } + private void initializeExtensionSecurity(DiscoveryExtensionNode extension) { + final CompletableFuture inProgressFuture = new CompletableFuture<>(); + final TransportResponseHandler initializeExtensionSecurityResponseHandler = + new TransportResponseHandler() { + + @Override + public InitializeExtensionSecurityResponse read(StreamInput in) throws IOException { + return new InitializeExtensionSecurityResponse(in); + } + + @Override + public void handleResponse(InitializeExtensionSecurityResponse response) { + System.out.println("Registered security settings for " + response.getName()); + inProgressFuture.complete(response); + } + + @Override + public void handleException(TransportException exp) { + logger.error(new ParameterizedMessage("Extension initialization failed"), exp); + inProgressFuture.completeExceptionally(exp); + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + }; + try { + logger.info("Sending extension request type: " + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS); + AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getId()); + transportService.sendRequest( + extension, + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS, + new InitializeExtensionSecurityRequest(serviceAccountToken.asAuthHeaderValue()), + initializeExtensionSecurityResponseHandler + ); + + inProgressFuture.orTimeout(EXTENSION_REQUEST_WAIT_TIMEOUT, TimeUnit.SECONDS).join(); + } catch (CompletionException | ConnectTransportException e) { + if (e.getCause() instanceof TimeoutException || e instanceof ConnectTransportException) { + logger.info("No response from extension to request.", e); + } else if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } else if (e.getCause() instanceof Error) { + throw (Error) e.getCause(); + } else { + throw new RuntimeException(e.getCause()); + } + } + } + /** * Handles an {@link ExtensionRequest}. * @@ -498,6 +559,11 @@ void setCustomSettingsRequestHandler(CustomSettingsRequestHandler customSettings this.customSettingsRequestHandler = customSettingsRequestHandler; } + + public void setIdentityService(IdentityService identityService) { + this.identityService = identityService; + } + AddSettingsUpdateConsumerRequestHandler getAddSettingsUpdateConsumerRequestHandler() { return addSettingsUpdateConsumerRequestHandler; } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 4b622b841a040..147ef7a6609c6 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -62,6 +62,53 @@ public RestInitializeExtensionAction(ExtensionsManager extensionsManager) { this.extensionsManager = extensionsManager; } + private static Map unflattenMap(Map flatMap) { + Map unflattenedMap = new HashMap<>(); + + for (Map.Entry entry : flatMap.entrySet()) { + String[] keys = entry.getKey().split("\\."); + putNested(unflattenedMap, keys, entry.getValue()); + } + + return unflattenedMap; + } + + private static void putNested(Map map, String[] keys, Object value) { + for (int i = 0; i < keys.length; i++) { + String key = keys[i]; + + if (i == keys.length - 1) { + map.put(key, value); + } else if (keys[i + 1].matches("\\d+")) { + int index = Integer.parseInt(keys[++i]); + + List> list; + if (map.containsKey(key)) { + list = (List>) map.get(key); + } else { + list = new ArrayList<>(); + map.put(key, list); + } + + while (list.size() <= index) { + list.add(new HashMap<>()); + } + + map = list.get(index); + } else { + Map nestedMap; + if (map.containsKey(key)) { + nestedMap = (Map) map.get(key); + } else { + nestedMap = new HashMap<>(); + map.put(key, nestedMap); + } + + map = nestedMap; + } + } + } + @Override public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { String name = null; @@ -124,13 +171,19 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client } } - Map additionalSettingsMap = extensionMap.entrySet() + Map additionalSettingsMap = extensionMap.entrySet() .stream() - .filter(kv -> additionalSettingsKeys.contains(kv.getKey())) - .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); + .filter(kv -> additionalSettingsKeys.stream().anyMatch(k -> { + if (k.endsWith(".")) { + return kv.getKey().startsWith(k); + } else { + return kv.getKey().equals(k); + } + })).collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); Settings.Builder output = Settings.builder(); - output.loadFromMap(additionalSettingsMap); + Map unflattenedMap = unflattenMap(additionalSettingsMap); + output.loadFromMap(unflattenedMap); extAdditionalSettings.applySettings(output.build()); // Create extension read from initialization request diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java index 4f6ddeb48dea3..555ca32ba37b2 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java +++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java @@ -29,4 +29,6 @@ public interface TokenManager { * @return The authenticated subject */ public Subject authenticateToken(AuthToken authToken); + + } diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 41d8c18ae6a58..3ba8d8b6fd579 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -497,6 +497,7 @@ protected Node( additionalSettings.addAll(extAwarePlugin.getExtensionSettings()); } this.extensionsManager = new ExtensionsManager(additionalSettings); + this.extensionsManager.setIdentityService(identityService); } else { this.extensionsManager = new NoopExtensionsManager(); } From cec24b07e34a6b5dde4b04d4b0bff18ffe2a5cb4 Mon Sep 17 00:00:00 2001 From: Stephen Crawford Date: Thu, 14 Sep 2023 13:56:08 -0400 Subject: [PATCH 06/14] Add service account changes minus tests Signed-off-by: Stephen Crawford --- .../identity/shiro/ShiroTokenManager.java | 30 +++++-- .../extensions/ExtensionsManager.java | 57 ++++++++++++- .../extensions/NoopExtensionsManager.java | 3 +- .../action/IssueServiceAccountRequest.java | 69 ++++++++++++++++ .../action/IssueServiceAccountResponse.java | 81 +++++++++++++++++++ .../rest/RestInitializeExtensionAction.java | 1 + .../identity/noop/NoopTokenManager.java | 16 ++++ .../identity/tokens/TokenManager.java | 8 ++ .../main/java/org/opensearch/node/Node.java | 2 +- .../opensearch/action/ActionModuleTests.java | 2 +- .../extensions/ExtensionsManagerTests.java | 36 ++++----- .../RestInitializeExtensionActionTests.java | 14 ++-- .../rest/RestSendToExtensionActionTests.java | 2 +- 13 files changed, 285 insertions(+), 36 deletions(-) create mode 100644 server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java create mode 100644 server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 60239d609a08a..0c8deddb5351d 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -64,9 +64,10 @@ public Optional translateAuthToken(org.opensearch.identity. public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) { String password = generatePassword(); - final byte[] rawEncoded = Base64.getEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new - // ShiroSubject w/ - // audience as name + final byte[] rawEncoded = Base64.getUrlEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new + // ShiroSubject w/ + // audience as + // name final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; BasicAuthToken token = new BasicAuthToken(header); @@ -75,6 +76,19 @@ public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) return token; } + @Override + public AuthToken issueServiceAccountToken(String audience) { + + String password = generatePassword(); + final byte[] rawEncoded = Base64.getUrlEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); // Make a new + final String usernamePassword = new String(rawEncoded, UTF_8); + final String header = "Basic " + usernamePassword; + + BasicAuthToken token = new BasicAuthToken(header); + shiroTokenPasswordMap.put(token, password); + return token; + } + @Override public Subject authenticateToken(AuthToken authToken) { return new NoopSubject(); @@ -84,7 +98,7 @@ public boolean validateToken(AuthToken token) { if (token instanceof BasicAuthToken) { final BasicAuthToken basicAuthToken = (BasicAuthToken) token; return basicAuthToken.getUser().equals(SecurityUtils.getSubject().toString()) - && basicAuthToken.getPassword().equals(shiroTokenPasswordMap.get(basicAuthToken)); + && basicAuthToken.getPassword().equals(shiroTokenPasswordMap.get(basicAuthToken)); } return false; } @@ -126,10 +140,10 @@ public String generatePassword() { CharacterRule specialCharacterRule = new CharacterRule(EnglishCharacterData.Special, 1); List rules = Arrays.asList( - lowercaseCharacterRule, - uppercaseCharacterRule, - numericCharacterRule, - specialCharacterRule + lowercaseCharacterRule, + uppercaseCharacterRule, + numericCharacterRule, + specialCharacterRule ); PasswordGenerator passwordGenerator = new PasswordGenerator(); diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 2b074d98d78a5..64bdb9bd036b7 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -52,6 +52,8 @@ import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; +import org.opensearch.extensions.action.IssueServiceAccountRequest; +import org.opensearch.extensions.action.IssueServiceAccountResponse; import java.io.IOException; import java.net.InetAddress; @@ -81,6 +83,7 @@ public class ExtensionsManager { public static final String REQUEST_EXTENSION_REGISTER_TRANSPORT_ACTIONS = "internal:discovery/registertransportactions"; public static final String REQUEST_REST_EXECUTE_ON_EXTENSION_ACTION = "internal:extensions/restexecuteonextensiontaction"; public static final String REQUEST_EXTENSION_HANDLE_TRANSPORT_ACTION = "internal:extensions/handle-transportaction"; + public static final String REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT = "internal:extensions/issue-service-account"; public static final String REQUEST_EXTENSION_HANDLE_REMOTE_TRANSPORT_ACTION = "internal:extensions/handle-remote-transportaction"; public static final String TRANSPORT_ACTION_REQUEST_FROM_EXTENSION = "internal:extensions/request-transportaction-from-extension"; public static final String REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS = "internal:discovery/registersecuritysettings"; @@ -117,7 +120,7 @@ public static enum OpenSearchRequestType { * @param additionalSettings Additional settings to read in from extension initialization request * @throws IOException If the extensions discovery file is not properly retrieved. */ - public ExtensionsManager(Set> additionalSettings) throws IOException { + public ExtensionsManager(Set> additionalSettings, IdentityService identityService) throws IOException { logger.info("ExtensionsManager initialized"); this.initializedExtensions = new HashMap(); this.extensionIdMap = new HashMap(); @@ -132,6 +135,7 @@ public ExtensionsManager(Set> additionalSettings) throws IOException } this.client = null; this.extensionTransportActionsHandler = null; + this.identityService = identityService; } /** @@ -503,6 +507,57 @@ TransportResponse handleExtensionRequest(ExtensionRequest extensionRequest) thro } } + /** + * A separate transport action handler used to issue service accounts to extensions during initialization + * @param extension The extension to be issued a service account + */ + public void issueServiceAccount(Extension extension) { + DiscoveryExtensionNode discoveryExtensionNode = extensionIdMap.get(extension.getUniqueId()); + AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getUniqueId()); + String authTokenAsString = serviceAccountToken.asAuthHeaderValue(); + final CompletableFuture inProgressFuture = new CompletableFuture<>(); + final TransportResponseHandler issueServiceAccountResponseHandler = new TransportResponseHandler< + IssueServiceAccountResponse>() { + + @Override + public IssueServiceAccountResponse read(StreamInput in) throws IOException { + return new IssueServiceAccountResponse(in); + } + + @Override + public void handleResponse(IssueServiceAccountResponse response) { + for (DiscoveryExtensionNode extension : extensionIdMap.values()) { + if (extension.getName().equals(response.getName()) + && (serviceAccountToken.equals(response.getServiceAccountString()))) { + logger.info("Successfully issued service account token to extension: " + extension.getName()); + break; + } + } + inProgressFuture.complete(response); + } + + @Override + public void handleException(TransportException exp) { + logger.error(new ParameterizedMessage("Issuance of service account token failed"), exp); + inProgressFuture.completeExceptionally(exp); + } + + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + }; + + transportService.sendRequest( + discoveryExtensionNode, + REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT, + new IssueServiceAccountRequest(authTokenAsString), + issueServiceAccountResponseHandler + ); + } + + + static String getRequestExtensionActionName() { return REQUEST_EXTENSION_ACTION_NAME; } diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index 34aa3ec1367a7..16114622d3b1a 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -8,6 +8,7 @@ package org.opensearch.extensions; +import java.util.List; import org.opensearch.action.ActionModule; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; @@ -31,7 +32,7 @@ public class NoopExtensionsManager extends ExtensionsManager { public NoopExtensionsManager() throws IOException { - super(Set.of()); + super(Set.of(), new IdentityService(Settings.EMPTY, List.of())); } @Override diff --git a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java new file mode 100644 index 0000000000000..fc6bbbc2151f7 --- /dev/null +++ b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.extensions.action; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.transport.TransportRequest; + +import java.io.IOException; +import java.util.Objects; + +/** + * This class represents a Transport Request for issuing a service account to an extension. + */ +public class IssueServiceAccountRequest extends TransportRequest { + + private final String serviceAccountToken; + + /** + * This takes a string representing a service account token + * @param serviceAccountToken The string making up the service account token + */ + public IssueServiceAccountRequest(String serviceAccountToken) { + this.serviceAccountToken = serviceAccountToken; + } + + /** + * This takes a stream representing a service account token + * @param in The stream passing the token + */ + public IssueServiceAccountRequest(StreamInput in) throws IOException { + super(in); + this.serviceAccountToken = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(serviceAccountToken); + } + + public String getServiceAccountToken() { + return this.serviceAccountToken; + } + + @Override + public String toString() { + return "IssueServiceAccountRequest {" + "serviceAccountToken=" + serviceAccountToken + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IssueServiceAccountRequest that = (IssueServiceAccountRequest) o; + return Objects.equals(serviceAccountToken, that.serviceAccountToken); + } + + @Override + public int hashCode() { + return Objects.hash(serviceAccountToken); + } +} diff --git a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java new file mode 100644 index 0000000000000..ed481092b717b --- /dev/null +++ b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java @@ -0,0 +1,81 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.extensions.action; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.transport.TransportResponse; + +import java.io.IOException; +import java.util.Objects; + +/** + * This class represents a Transport Request for issuing a service account to an extension. + */ +public class IssueServiceAccountResponse extends TransportResponse { + + private String name; + private String serviceAccountToken; + + /** + * This takes in a name for the extension and the service account token string + * @param name The name of the extension + * @param serviceAccountToken A string encapsulating the service account token + */ + public IssueServiceAccountResponse(String name, String serviceAccountToken) { + this.name = name; + this.serviceAccountToken = serviceAccountToken; + } + + /** + * This takes in a stream containing for the extension and the service account token + * @param in the stream containing the extension name and the service account token + */ + public IssueServiceAccountResponse(StreamInput in) throws IOException { + this.name = in.readString(); + this.serviceAccountToken = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeString(serviceAccountToken); + } + + /** + * @return the node that is currently leading, according to the responding node. + */ + + public String getName() { + return this.name; + } + + public String getServiceAccountString() { + return this.serviceAccountToken; + } + + @Override + public String toString() { + return "IssueServiceAccountResponse{" + "name = " + name + " , " + "received service account = " + serviceAccountToken + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IssueServiceAccountResponse that = (IssueServiceAccountResponse) o; + return Objects.equals(name, that.name) && Objects.equals(serviceAccountToken, that.serviceAccountToken); + } + + @Override + public int hashCode() { + return Objects.hash(name, serviceAccountToken); + } + +} diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index 147ef7a6609c6..d1893fc94e273 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -214,6 +214,7 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client try { extensionsManager.loadExtension(extension); extensionsManager.initialize(); + extensionsManager.issueServiceAccount(extension); } catch (CompletionException e) { Throwable cause = e.getCause(); if (cause instanceof TimeoutException) { diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java index 1255e822cea6e..4a304e4e3d4b6 100644 --- a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java +++ b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java @@ -37,6 +37,22 @@ public String asAuthHeaderValue() { }; } + /** + * Issue a new Noop Token + * @return a new Noop Token + */ + @Override + public AuthToken issueServiceAccountToken(final String audience) { + return new AuthToken() { + @Override + public String asAuthHeaderValue() { + return "noopToken"; + } + }; + } + + + @Override public Subject authenticateToken(AuthToken authToken) { return null; diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java index 555ca32ba37b2..2d9da5ebad865 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java +++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java @@ -23,6 +23,14 @@ public interface TokenManager { */ public AuthToken issueOnBehalfOfToken(final Subject subject, final OnBehalfOfClaims claims); + /** + * Create a new service account token + * + * @param audience: A string representing the unique id of the extension for which a service account token should be generated + * @return a new auth token + */ + public AuthToken issueServiceAccountToken(final String audience); + /** * Authenticates a provided authToken * @param authToken: The authToken to authenticate diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 3ba8d8b6fd579..b874857675c95 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -496,7 +496,7 @@ protected Node( for (ExtensionAwarePlugin extAwarePlugin : extensionAwarePlugins) { additionalSettings.addAll(extAwarePlugin.getExtensionSettings()); } - this.extensionsManager = new ExtensionsManager(additionalSettings); + this.extensionsManager = new ExtensionsManager(additionalSettings, identityService); this.extensionsManager.setIdentityService(identityService); } else { this.extensionsManager = new NoopExtensionsManager(); diff --git a/server/src/test/java/org/opensearch/action/ActionModuleTests.java b/server/src/test/java/org/opensearch/action/ActionModuleTests.java index deb0bb58a5c4d..8479f011adf48 100644 --- a/server/src/test/java/org/opensearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/opensearch/action/ActionModuleTests.java @@ -143,7 +143,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() throws IOException { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of()) + new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) ); actionModule.initRestHandlers(null); // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 808616a3f8b8a..7f7e0d007934a 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -182,7 +182,7 @@ public void testLoadExtensions() throws Exception { Set> additionalSettings = extAwarePlugin.getExtensionSettings().stream().collect(Collectors.toSet()); ExtensionScopedSettings extensionScopedSettings = new ExtensionScopedSettings(additionalSettings); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); ExtensionDependency dependentExtension = new ExtensionDependency("uniqueid0", Version.fromString("2.0.0")); Extension firstExtension = new Extension( @@ -278,7 +278,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { null, null ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); extensionsManager.loadExtension(firstExtension); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(secondExtension)); assertEquals( @@ -319,7 +319,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { public void testMissingRequiredFieldsWhileLoadingExtension() throws Exception { Extension firstExtension = new Extension("firstExtension", "uniqueid1", "127.0.0.0", "9300", "0.0.7", "3.0.0", "", null, null); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(firstExtension)); assertEquals("Required field [minimum opensearch version] is missing in the request", exception.getMessage()); @@ -376,7 +376,7 @@ public void testExtensionDependency() throws Exception { } public void testInitialize() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); @@ -418,7 +418,7 @@ public void testInitialize() throws Exception { public void testHandleRegisterRestActionsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -432,7 +432,7 @@ public void testHandleRegisterRestActionsRequest() throws Exception { } public void testHandleRegisterSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -448,7 +448,7 @@ public void testHandleRegisterSettingsRequest() throws Exception { } public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -463,7 +463,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Excep } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -478,7 +478,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() th } public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET", "PUT /bar", "POST /baz"); @@ -492,7 +492,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exceptio } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); @@ -506,7 +506,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throw } public void testHandleExtensionRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); ExtensionRequest clusterStateRequest = new ExtensionRequest(ExtensionRequestProto.RequestType.REQUEST_EXTENSION_CLUSTER_STATE); @@ -660,7 +660,7 @@ public void testEnvironmentSettingsDefaultValue() throws Exception { } public void testAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); List> componentSettings = List.of( @@ -704,7 +704,7 @@ public void testAddSettingsUpdateConsumerRequest() throws Exception { } public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); List> componentSettings = List.of( @@ -724,7 +724,7 @@ public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { } public void testUpdateSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); Setting componentSetting = Setting.boolSetting("falseSetting", false, Property.Dynamic); @@ -753,7 +753,7 @@ public void testUpdateSettingsRequest() throws Exception { public void testRegisterHandler() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); TransportService mockTransportService = spy( new TransportService( @@ -781,7 +781,7 @@ public void testRegisterHandler() throws Exception { } public void testIncompatibleExtensionRegistration() throws IOException { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); Extension firstExtension = new Extension( "firstExtension", "uniqueid1", @@ -822,7 +822,7 @@ public List> getExtensionSettings() { extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( @@ -860,7 +860,7 @@ public void testAdditionalExtensionSettingsForExtensionWithoutCustomSettingSet() extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 87767978147cd..8f8bc3834183d 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -21,6 +21,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.extensions.ExtensionsSettings; +import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; @@ -46,6 +47,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -118,7 +120,7 @@ public void testRestInitializeExtensionActionResponse() throws Exception { } public void testRestInitializeExtensionActionFailure() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(extensionsManager); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"\",\"hostAddress\":\"127.0.0.1\"," @@ -151,13 +153,14 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting), new IdentityService(Settings.EMPTY, List.of())); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: when(spy.getAdditionalSettings()).thenCallRealMethod(); Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); - Mockito.doNothing().when(spy).initialize(); + doNothing().when(spy).issueServiceAccount(any()); + doNothing().when(spy).initialize(); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" @@ -198,13 +201,14 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting), new IdentityService(Settings.EMPTY, List.of())); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: when(spy.getAdditionalSettings()).thenCallRealMethod(); Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); - Mockito.doNothing().when(spy).initialize(); + doNothing().when(spy).issueServiceAccount(any()); + doNothing().when(spy).initialize(); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index d1e4df509baab..8a831f8a0bcf8 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -134,7 +134,7 @@ public void setup() throws Exception { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of()) + new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) ); identityService = new IdentityService(Settings.EMPTY, new ArrayList<>()); dynamicActionRegistry = actionModule.getDynamicActionRegistry(); From 3b237370edb2f78249a36ea32c5a6b326a255f2a Mon Sep 17 00:00:00 2001 From: Ryan Liang Date: Mon, 18 Sep 2023 11:35:45 -0700 Subject: [PATCH 07/14] Grant getClassLoader Signed-off-by: Ryan Liang --- .../main/resources/org/opensearch/bootstrap/security.policy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy index 77cd0ab05278e..66cc95c28916a 100644 --- a/server/src/main/resources/org/opensearch/bootstrap/security.policy +++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy @@ -37,6 +37,10 @@ //// SecurityManager impl: //// Must have all permissions to properly perform access checks +grant { + permission java.lang.RuntimePermission "getClassLoader"; +}; + grant codeBase "${codebase.opensearch-secure-sm}" { permission java.security.AllPermission; }; @@ -48,6 +52,8 @@ grant codeBase "${codebase.opensearch}" { permission java.lang.RuntimePermission "setContextClassLoader"; // needed for SPI class loading permission java.lang.RuntimePermission "accessDeclaredMembers"; + + permission java.lang.RuntimePermission "getClassLoader"; }; //// Very special jar permissions: From 903f18050537f39d38af8a52e9f67047c5c177e3 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Thu, 28 Sep 2023 15:21:21 -0400 Subject: [PATCH 08/14] Fixes spotless errors and replaces sysout with logger statement Signed-off-by: Darshit Chanpura --- .../identity/shiro/ShiroTokenManager.java | 10 +- .../identity/shiro/AuthTokenHandlerTests.java | 1 - .../opensearch/common/settings/Settings.java | 15 ++- .../InitializeExtensionSecurityRequest.java | 76 +++++++-------- .../InitializeExtensionSecurityResponse.java | 64 ++++++------- .../extensions/ExtensionsManager.java | 74 +++++++------- .../extensions/NoopExtensionsManager.java | 2 +- .../action/IssueServiceAccountRequest.java | 78 +++++++-------- .../action/IssueServiceAccountResponse.java | 96 +++++++++---------- .../extensions/rest/ExtensionRestRequest.java | 1 + .../rest/RestActionsRequestHandler.java | 3 +- .../rest/RestInitializeExtensionAction.java | 15 +-- .../identity/noop/NoopTokenManager.java | 2 - .../identity/tokens/TokenManager.java | 1 - .../extensions/ExtensionsManagerTests.java | 26 ++--- .../rest/ExtensionRestRequestTests.java | 5 +- .../RestInitializeExtensionActionTests.java | 10 +- .../rest/RestSendToExtensionActionTests.java | 12 --- 18 files changed, 239 insertions(+), 252 deletions(-) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 0c8deddb5351d..73cf9506aef2c 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -98,7 +98,7 @@ public boolean validateToken(AuthToken token) { if (token instanceof BasicAuthToken) { final BasicAuthToken basicAuthToken = (BasicAuthToken) token; return basicAuthToken.getUser().equals(SecurityUtils.getSubject().toString()) - && basicAuthToken.getPassword().equals(shiroTokenPasswordMap.get(basicAuthToken)); + && basicAuthToken.getPassword().equals(shiroTokenPasswordMap.get(basicAuthToken)); } return false; } @@ -140,10 +140,10 @@ public String generatePassword() { CharacterRule specialCharacterRule = new CharacterRule(EnglishCharacterData.Special, 1); List rules = Arrays.asList( - lowercaseCharacterRule, - uppercaseCharacterRule, - numericCharacterRule, - specialCharacterRule + lowercaseCharacterRule, + uppercaseCharacterRule, + numericCharacterRule, + specialCharacterRule ); PasswordGenerator passwordGenerator = new PasswordGenerator(); diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java index 5898ea3dad855..7c7414843d86f 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java @@ -10,7 +10,6 @@ import org.apache.shiro.authc.AuthenticationToken; import org.apache.shiro.authc.UsernamePasswordToken; -import org.junit.Before; import org.opensearch.identity.Subject; import org.opensearch.identity.noop.NoopSubject; import org.opensearch.identity.noop.NoopTokenManager; diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index 230eb5499935c..d94b5bcd4754d 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -284,7 +284,6 @@ public List getNestedListOfSettings(String key) { return (List) settings.get(key); } - /** * Returns the setting value (as float) associated with the setting key. If it does not exists, * returns the default value provided. @@ -691,18 +690,18 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder list.add(String.valueOf(parser.text())); } else { listOfObjects.add(fromXContent(parser, true, false)); - //throw new IllegalStateException("only value lists are allowed in serialized settings"); + // throw new IllegalStateException("only value lists are allowed in serialized settings"); } } String key = keyBuilder.toString(); validateValue(key, list, parser, allowNullValues); builder.putList(key, list); - if (!listOfObjects.isEmpty()) { - builder.putListOfObjects(key, listOfObjects); - } - if (!list.isEmpty() && !listOfObjects.isEmpty()) { - throw new IllegalStateException("list cannot contain both values and objects"); - } + if (!listOfObjects.isEmpty()) { + builder.putListOfObjects(key, listOfObjects); + } + if (!list.isEmpty() && !listOfObjects.isEmpty()) { + throw new IllegalStateException("list cannot contain both values and objects"); + } } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { String key = keyBuilder.toString(); validateValue(key, null, parser, allowNullValues); diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java index c18d8144098f1..530d69e418b25 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java @@ -22,42 +22,42 @@ */ public class InitializeExtensionSecurityRequest extends TransportRequest { - private final String serviceAccountToken; - - public InitializeExtensionSecurityRequest(String serviceAccountToken) { - this.serviceAccountToken = serviceAccountToken; - } - - public InitializeExtensionSecurityRequest(StreamInput in) throws IOException { - super(in); - serviceAccountToken = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(serviceAccountToken); - } - - public String getServiceAccountToken() { - return serviceAccountToken; - } - - @Override - public String toString() { - return "InitializeExtensionsRequest{" + "serviceAccountToken= " + serviceAccountToken + "}"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InitializeExtensionSecurityRequest that = (InitializeExtensionSecurityRequest) o; - return Objects.equals(serviceAccountToken, that.serviceAccountToken); - } - - @Override - public int hashCode() { - return Objects.hash(serviceAccountToken); - } + private final String serviceAccountToken; + + public InitializeExtensionSecurityRequest(String serviceAccountToken) { + this.serviceAccountToken = serviceAccountToken; + } + + public InitializeExtensionSecurityRequest(StreamInput in) throws IOException { + super(in); + serviceAccountToken = in.readString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(serviceAccountToken); + } + + public String getServiceAccountToken() { + return serviceAccountToken; + } + + @Override + public String toString() { + return "InitializeExtensionsRequest{" + "serviceAccountToken= " + serviceAccountToken + "}"; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InitializeExtensionSecurityRequest that = (InitializeExtensionSecurityRequest) o; + return Objects.equals(serviceAccountToken, that.serviceAccountToken); + } + + @Override + public int hashCode() { + return Objects.hash(serviceAccountToken); + } } diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java index bcad61f3d59ba..31d1675ead2ec 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java @@ -45,44 +45,44 @@ * @opensearch.internal */ public class InitializeExtensionSecurityResponse extends TransportResponse { - private String name; + private String name; - public InitializeExtensionSecurityResponse(String name) { - this.name = name; - } + public InitializeExtensionSecurityResponse(String name) { + this.name = name; + } - public InitializeExtensionSecurityResponse(StreamInput in) throws IOException { - name = in.readString(); - } + public InitializeExtensionSecurityResponse(StreamInput in) throws IOException { + name = in.readString(); + } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name); - } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + } - /** - * @return the node that is currently leading, according to the responding node. - */ + /** + * @return the node that is currently leading, according to the responding node. + */ - public String getName() { - return this.name; - } + public String getName() { + return this.name; + } - @Override - public String toString() { - return "InitializeExtensionResponse{" + "name = " + name + "}"; - } + @Override + public String toString() { + return "InitializeExtensionResponse{" + "name = " + name + "}"; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InitializeExtensionSecurityResponse that = (InitializeExtensionSecurityResponse) o; - return Objects.equals(name, that.name); - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + InitializeExtensionSecurityResponse that = (InitializeExtensionSecurityResponse) o; + return Objects.equals(name, that.name); + } - @Override - public int hashCode() { - return Objects.hash(name); - } + @Override + public int hashCode() { + return Objects.hash(name); + } } diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 64bdb9bd036b7..a219374578106 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -8,9 +8,6 @@ package org.opensearch.extensions; -import java.util.concurrent.CompletionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; @@ -38,6 +35,8 @@ import org.opensearch.extensions.action.ExtensionActionRequest; import org.opensearch.extensions.action.ExtensionActionResponse; import org.opensearch.extensions.action.ExtensionTransportActionsHandler; +import org.opensearch.extensions.action.IssueServiceAccountRequest; +import org.opensearch.extensions.action.IssueServiceAccountResponse; import org.opensearch.extensions.action.RegisterTransportActionsRequest; import org.opensearch.extensions.action.RemoteExtensionActionResponse; import org.opensearch.extensions.action.TransportActionRequestFromExtension; @@ -52,8 +51,6 @@ import org.opensearch.transport.TransportException; import org.opensearch.transport.TransportResponseHandler; import org.opensearch.transport.TransportService; -import org.opensearch.extensions.action.IssueServiceAccountRequest; -import org.opensearch.extensions.action.IssueServiceAccountResponse; import java.io.IOException; import java.net.InetAddress; @@ -63,6 +60,9 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; /** @@ -99,7 +99,6 @@ public static enum OpenSearchRequestType { REQUEST_OPENSEARCH_NAMED_WRITEABLE_REGISTRY } - private ExtensionTransportActionsHandler extensionTransportActionsHandler; private Map extensionSettingsMap; private Map initializedExtensions; @@ -423,38 +422,38 @@ protected void doRun() throws Exception { private void initializeExtensionSecurity(DiscoveryExtensionNode extension) { final CompletableFuture inProgressFuture = new CompletableFuture<>(); final TransportResponseHandler initializeExtensionSecurityResponseHandler = - new TransportResponseHandler() { + new TransportResponseHandler() { - @Override - public InitializeExtensionSecurityResponse read(StreamInput in) throws IOException { - return new InitializeExtensionSecurityResponse(in); - } + @Override + public InitializeExtensionSecurityResponse read(StreamInput in) throws IOException { + return new InitializeExtensionSecurityResponse(in); + } - @Override - public void handleResponse(InitializeExtensionSecurityResponse response) { - System.out.println("Registered security settings for " + response.getName()); - inProgressFuture.complete(response); - } + @Override + public void handleResponse(InitializeExtensionSecurityResponse response) { + logger.info("Registered security settings for " + response.getName()); + inProgressFuture.complete(response); + } - @Override - public void handleException(TransportException exp) { - logger.error(new ParameterizedMessage("Extension initialization failed"), exp); - inProgressFuture.completeExceptionally(exp); - } + @Override + public void handleException(TransportException exp) { + logger.error(new ParameterizedMessage("Extension initialization failed"), exp); + inProgressFuture.completeExceptionally(exp); + } - @Override - public String executor() { - return ThreadPool.Names.GENERIC; - } - }; + @Override + public String executor() { + return ThreadPool.Names.GENERIC; + } + }; try { logger.info("Sending extension request type: " + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS); AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getId()); transportService.sendRequest( - extension, - REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS, - new InitializeExtensionSecurityRequest(serviceAccountToken.asAuthHeaderValue()), - initializeExtensionSecurityResponseHandler + extension, + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS, + new InitializeExtensionSecurityRequest(serviceAccountToken.asAuthHeaderValue()), + initializeExtensionSecurityResponseHandler ); inProgressFuture.orTimeout(EXTENSION_REQUEST_WAIT_TIMEOUT, TimeUnit.SECONDS).join(); @@ -517,7 +516,7 @@ public void issueServiceAccount(Extension extension) { String authTokenAsString = serviceAccountToken.asAuthHeaderValue(); final CompletableFuture inProgressFuture = new CompletableFuture<>(); final TransportResponseHandler issueServiceAccountResponseHandler = new TransportResponseHandler< - IssueServiceAccountResponse>() { + IssueServiceAccountResponse>() { @Override public IssueServiceAccountResponse read(StreamInput in) throws IOException { @@ -528,7 +527,7 @@ public IssueServiceAccountResponse read(StreamInput in) throws IOException { public void handleResponse(IssueServiceAccountResponse response) { for (DiscoveryExtensionNode extension : extensionIdMap.values()) { if (extension.getName().equals(response.getName()) - && (serviceAccountToken.equals(response.getServiceAccountString()))) { + && (serviceAccountToken.equals(response.getServiceAccountString()))) { logger.info("Successfully issued service account token to extension: " + extension.getName()); break; } @@ -549,15 +548,13 @@ public String executor() { }; transportService.sendRequest( - discoveryExtensionNode, - REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT, - new IssueServiceAccountRequest(authTokenAsString), - issueServiceAccountResponseHandler + discoveryExtensionNode, + REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT, + new IssueServiceAccountRequest(authTokenAsString), + issueServiceAccountResponseHandler ); } - - static String getRequestExtensionActionName() { return REQUEST_EXTENSION_ACTION_NAME; } @@ -614,7 +611,6 @@ void setCustomSettingsRequestHandler(CustomSettingsRequestHandler customSettings this.customSettingsRequestHandler = customSettingsRequestHandler; } - public void setIdentityService(IdentityService identityService) { this.identityService = identityService; } diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index 16114622d3b1a..81b1b91b11481 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -8,7 +8,6 @@ package org.opensearch.extensions; -import java.util.List; import org.opensearch.action.ActionModule; import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.service.ClusterService; @@ -21,6 +20,7 @@ import org.opensearch.transport.TransportService; import java.io.IOException; +import java.util.List; import java.util.Optional; import java.util.Set; diff --git a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java index fc6bbbc2151f7..7d152a33e800a 100644 --- a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java +++ b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java @@ -20,50 +20,50 @@ */ public class IssueServiceAccountRequest extends TransportRequest { - private final String serviceAccountToken; + private final String serviceAccountToken; - /** - * This takes a string representing a service account token - * @param serviceAccountToken The string making up the service account token - */ - public IssueServiceAccountRequest(String serviceAccountToken) { - this.serviceAccountToken = serviceAccountToken; - } + /** + * This takes a string representing a service account token + * @param serviceAccountToken The string making up the service account token + */ + public IssueServiceAccountRequest(String serviceAccountToken) { + this.serviceAccountToken = serviceAccountToken; + } - /** - * This takes a stream representing a service account token - * @param in The stream passing the token - */ - public IssueServiceAccountRequest(StreamInput in) throws IOException { - super(in); - this.serviceAccountToken = in.readString(); - } + /** + * This takes a stream representing a service account token + * @param in The stream passing the token + */ + public IssueServiceAccountRequest(StreamInput in) throws IOException { + super(in); + this.serviceAccountToken = in.readString(); + } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(serviceAccountToken); - } + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(serviceAccountToken); + } - public String getServiceAccountToken() { - return this.serviceAccountToken; - } + public String getServiceAccountToken() { + return this.serviceAccountToken; + } - @Override - public String toString() { - return "IssueServiceAccountRequest {" + "serviceAccountToken=" + serviceAccountToken + "}"; - } + @Override + public String toString() { + return "IssueServiceAccountRequest {" + "serviceAccountToken=" + serviceAccountToken + "}"; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IssueServiceAccountRequest that = (IssueServiceAccountRequest) o; - return Objects.equals(serviceAccountToken, that.serviceAccountToken); - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IssueServiceAccountRequest that = (IssueServiceAccountRequest) o; + return Objects.equals(serviceAccountToken, that.serviceAccountToken); + } - @Override - public int hashCode() { - return Objects.hash(serviceAccountToken); - } + @Override + public int hashCode() { + return Objects.hash(serviceAccountToken); + } } diff --git a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java index ed481092b717b..78e41f1806e9e 100644 --- a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java +++ b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java @@ -20,62 +20,62 @@ */ public class IssueServiceAccountResponse extends TransportResponse { - private String name; - private String serviceAccountToken; + private String name; + private String serviceAccountToken; - /** - * This takes in a name for the extension and the service account token string - * @param name The name of the extension - * @param serviceAccountToken A string encapsulating the service account token - */ - public IssueServiceAccountResponse(String name, String serviceAccountToken) { - this.name = name; - this.serviceAccountToken = serviceAccountToken; - } + /** + * This takes in a name for the extension and the service account token string + * @param name The name of the extension + * @param serviceAccountToken A string encapsulating the service account token + */ + public IssueServiceAccountResponse(String name, String serviceAccountToken) { + this.name = name; + this.serviceAccountToken = serviceAccountToken; + } - /** - * This takes in a stream containing for the extension and the service account token - * @param in the stream containing the extension name and the service account token - */ - public IssueServiceAccountResponse(StreamInput in) throws IOException { - this.name = in.readString(); - this.serviceAccountToken = in.readString(); - } + /** + * This takes in a stream containing for the extension and the service account token + * @param in the stream containing the extension name and the service account token + */ + public IssueServiceAccountResponse(StreamInput in) throws IOException { + this.name = in.readString(); + this.serviceAccountToken = in.readString(); + } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name); - out.writeString(serviceAccountToken); - } + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeString(serviceAccountToken); + } - /** - * @return the node that is currently leading, according to the responding node. - */ + /** + * @return the node that is currently leading, according to the responding node. + */ - public String getName() { - return this.name; - } + public String getName() { + return this.name; + } - public String getServiceAccountString() { - return this.serviceAccountToken; - } + public String getServiceAccountString() { + return this.serviceAccountToken; + } - @Override - public String toString() { - return "IssueServiceAccountResponse{" + "name = " + name + " , " + "received service account = " + serviceAccountToken + "}"; - } + @Override + public String toString() { + return "IssueServiceAccountResponse{" + "name = " + name + " , " + "received service account = " + serviceAccountToken + "}"; + } - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IssueServiceAccountResponse that = (IssueServiceAccountResponse) o; - return Objects.equals(name, that.name) && Objects.equals(serviceAccountToken, that.serviceAccountToken); - } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IssueServiceAccountResponse that = (IssueServiceAccountResponse) o; + return Objects.equals(name, that.name) && Objects.equals(serviceAccountToken, that.serviceAccountToken); + } - @Override - public int hashCode() { - return Objects.hash(name, serviceAccountToken); - } + @Override + public int hashCode() { + return Objects.hash(name, serviceAccountToken); + } } diff --git a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java index 968aadcfbb7b9..89df1e4fbde35 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java +++ b/server/src/main/java/org/opensearch/extensions/rest/ExtensionRestRequest.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; + import static java.util.Objects.requireNonNull; /** diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java b/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java index 8385cdf7b2a34..97851cbd394a0 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestActionsRequestHandler.java @@ -8,7 +8,6 @@ package org.opensearch.extensions.rest; -import java.util.Map; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.core.transport.TransportResponse; import org.opensearch.extensions.AcknowledgedResponse; @@ -18,6 +17,8 @@ import org.opensearch.rest.RestHandler; import org.opensearch.transport.TransportService; +import java.util.Map; + /** * Handles requests to register extension REST actions. * diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java index d1893fc94e273..86c306e2637fa 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestInitializeExtensionAction.java @@ -173,13 +173,14 @@ public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client Map additionalSettingsMap = extensionMap.entrySet() .stream() - .filter(kv -> additionalSettingsKeys.stream().anyMatch(k -> { - if (k.endsWith(".")) { - return kv.getKey().startsWith(k); - } else { - return kv.getKey().equals(k); - } - })).collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); + .filter(kv -> additionalSettingsKeys.stream().anyMatch(k -> { + if (k.endsWith(".")) { + return kv.getKey().startsWith(k); + } else { + return kv.getKey().equals(k); + } + })) + .collect(Collectors.toMap(map -> map.getKey(), map -> map.getValue())); Settings.Builder output = Settings.builder(); Map unflattenedMap = unflattenMap(additionalSettingsMap); diff --git a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java index 4a304e4e3d4b6..1dc3a58916b5c 100644 --- a/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java +++ b/server/src/main/java/org/opensearch/identity/noop/NoopTokenManager.java @@ -51,8 +51,6 @@ public String asAuthHeaderValue() { }; } - - @Override public Subject authenticateToken(AuthToken authToken) { return null; diff --git a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java index 2d9da5ebad865..cc4ce012fbba2 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java +++ b/server/src/main/java/org/opensearch/identity/tokens/TokenManager.java @@ -38,5 +38,4 @@ public interface TokenManager { */ public Subject authenticateToken(AuthToken authToken); - } diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 7f7e0d007934a..7e181d7524b12 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -8,19 +8,8 @@ package org.opensearch.extensions; -import java.io.IOException; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; -import org.junit.After; -import org.junit.Before; import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionModule; @@ -68,9 +57,23 @@ import org.opensearch.transport.TransportService; import org.opensearch.transport.nio.MockNioTransport; import org.opensearch.usage.UsageService; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.emptySet; +import static org.opensearch.test.ClusterServiceUtils.createClusterService; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; @@ -79,7 +82,6 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.opensearch.test.ClusterServiceUtils.createClusterService; public class ExtensionsManagerTests extends OpenSearchTestCase { private TransportService transportService; diff --git a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java index b60795a73238b..8b73f2e81972f 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/ExtensionRestRequestTests.java @@ -8,11 +8,9 @@ package org.opensearch.extensions.rest; - -import org.opensearch.common.settings.Settings; -import org.opensearch.core.rest.RestStatus; import org.opensearch.OpenSearchParseException; import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.settings.Settings; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.common.io.stream.BytesStreamInput; @@ -24,7 +22,6 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.http.HttpRequest; -import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput; import org.opensearch.identity.IdentityService; import org.opensearch.identity.Subject; import org.opensearch.identity.tokens.OnBehalfOfClaims; diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 8f8bc3834183d..2271b6eeac6fa 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -153,7 +153,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting), new IdentityService(Settings.EMPTY, List.of())); + ExtensionsManager extensionsManager = new ExtensionsManager( + Set.of(boolSetting, stringSetting, intSetting, listSetting), + new IdentityService(Settings.EMPTY, List.of()) + ); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: @@ -201,7 +204,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting), new IdentityService(Settings.EMPTY, List.of())); + ExtensionsManager extensionsManager = new ExtensionsManager( + Set.of(boolSetting, stringSetting, intSetting, listSetting), + new IdentityService(Settings.EMPTY, List.of()) + ); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index 8a831f8a0bcf8..d7a852be73650 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -8,18 +8,6 @@ package org.opensearch.extensions.rest; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import org.junit.After; -import org.junit.Before; import org.opensearch.Version; import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; From 3b6f0e118096acb28b1942db2409166790782734 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Thu, 28 Sep 2023 16:46:02 -0400 Subject: [PATCH 09/14] Cleans up unwanted changes Signed-off-by: Darshit Chanpura --- CHANGELOG.md | 7 --- .../identity/shiro/ShiroTokenManager.java | 11 ++-- .../opensearch/common/settings/Settings.java | 51 +------------------ .../InitializeExtensionSecurityResponse.java | 24 --------- .../extensions/ExtensionsManager.java | 4 +- .../extensions/NoopExtensionsManager.java | 3 +- .../rest/RestSendToExtensionAction.java | 1 - .../main/java/org/opensearch/node/Node.java | 3 +- .../opensearch/action/ActionModuleTests.java | 2 +- .../listener/ReadContextListenerTests.java | 1 - .../extensions/ExtensionsManagerTests.java | 36 ++++++------- .../RestInitializeExtensionActionTests.java | 16 ++---- .../rest/RestSendToExtensionActionTests.java | 2 +- 13 files changed, 35 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b8d8c7d92bea..0ea905dffb845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,13 +11,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) -- Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847)) -- Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636)) -- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151)) -- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854)) -- Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107)) -- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679)) - ### Dependencies - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575)) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 73cf9506aef2c..475642be239d1 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -57,6 +57,7 @@ public Optional translateAuthToken(org.opensearch.identity. final BasicAuthToken basicAuthToken = (BasicAuthToken) authenticationToken; return Optional.of(new UsernamePasswordToken(basicAuthToken.getUser(), basicAuthToken.getPassword())); } + return Optional.empty(); } @@ -64,10 +65,8 @@ public Optional translateAuthToken(org.opensearch.identity. public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) { String password = generatePassword(); - final byte[] rawEncoded = Base64.getUrlEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); // Make a new - // ShiroSubject w/ - // audience as - // name + final byte[] rawEncoded = Base64.getUrlEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); + // Make a new ShiroSubject w/ audience as name final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; BasicAuthToken token = new BasicAuthToken(header); @@ -80,12 +79,12 @@ public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) public AuthToken issueServiceAccountToken(String audience) { String password = generatePassword(); - final byte[] rawEncoded = Base64.getUrlEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); // Make a new + final byte[] rawEncoded = Base64.getUrlEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; - BasicAuthToken token = new BasicAuthToken(header); shiroTokenPasswordMap.put(token, password); + return token; } diff --git a/server/src/main/java/org/opensearch/common/settings/Settings.java b/server/src/main/java/org/opensearch/common/settings/Settings.java index d94b5bcd4754d..4df257612b91a 100644 --- a/server/src/main/java/org/opensearch/common/settings/Settings.java +++ b/server/src/main/java/org/opensearch/common/settings/Settings.java @@ -270,20 +270,6 @@ public String get(String setting, String defaultValue) { return retVal == null ? defaultValue : retVal; } - /** - * Returns a setting value based on the setting key. - */ - public Settings getNestedSettings(String key) { - return (Settings) settings.get(key); - } - - /** - * Returns a setting value based on the setting key. - */ - public List getNestedListOfSettings(String key) { - return (List) settings.get(key); - } - /** * Returns the setting value (as float) associated with the setting key. If it does not exists, * returns the default value provided. @@ -680,7 +666,6 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder fromXContent(parser, keyBuilder, builder, allowNullValues); } else if (parser.currentToken() == XContentParser.Token.START_ARRAY) { List list = new ArrayList<>(); - List listOfObjects = new ArrayList<>(); while (parser.nextToken() != XContentParser.Token.END_ARRAY) { if (parser.currentToken() == XContentParser.Token.VALUE_STRING) { list.add(parser.text()); @@ -689,19 +674,12 @@ private static void fromXContent(XContentParser parser, StringBuilder keyBuilder } else if (parser.currentToken() == XContentParser.Token.VALUE_BOOLEAN) { list.add(String.valueOf(parser.text())); } else { - listOfObjects.add(fromXContent(parser, true, false)); - // throw new IllegalStateException("only value lists are allowed in serialized settings"); + throw new IllegalStateException("only value lists are allowed in serialized settings"); } } String key = keyBuilder.toString(); validateValue(key, list, parser, allowNullValues); builder.putList(key, list); - if (!listOfObjects.isEmpty()) { - builder.putListOfObjects(key, listOfObjects); - } - if (!list.isEmpty() && !listOfObjects.isEmpty()) { - throw new IllegalStateException("list cannot contain both values and objects"); - } } else if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { String key = keyBuilder.toString(); validateValue(key, null, parser, allowNullValues); @@ -808,20 +786,6 @@ public String get(String key) { return Settings.toString(map.get(key)); } - /** - * Returns a setting value based on the setting key. - */ - public Settings getNestedSettings(String key) { - return (Settings) map.get(key); - } - - /** - * Returns a setting value based on the setting key. - */ - public List getNestedListOfSettings(String key) { - return (List) map.get(key); - } - /** Return the current secure settings, or {@code null} if none have been set. */ public SecureSettings getSecureSettings() { return secureSettings.get(); @@ -1059,19 +1023,6 @@ public Builder putList(String setting, List values) { return this; } - /** - * Sets the setting with the provided setting key and a list of values. - * - * @param setting The setting key - * @param values The values - * @return The builder - */ - public Builder putListOfObjects(String setting, List values) { - remove(setting); - map.put(setting, new ArrayList<>(values)); - return this; - } - /** * Sets all the provided settings including secure settings */ diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java index 31d1675ead2ec..14281b0c77d6d 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java @@ -6,30 +6,6 @@ * compatible open source license. */ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - package org.opensearch.discovery; import org.opensearch.core.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index a219374578106..58c29824369a4 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -119,7 +119,7 @@ public static enum OpenSearchRequestType { * @param additionalSettings Additional settings to read in from extension initialization request * @throws IOException If the extensions discovery file is not properly retrieved. */ - public ExtensionsManager(Set> additionalSettings, IdentityService identityService) throws IOException { + public ExtensionsManager(Set> additionalSettings) throws IOException { logger.info("ExtensionsManager initialized"); this.initializedExtensions = new HashMap(); this.extensionIdMap = new HashMap(); @@ -134,7 +134,6 @@ public ExtensionsManager(Set> additionalSettings, IdentityService ide } this.client = null; this.extensionTransportActionsHandler = null; - this.identityService = identityService; } /** @@ -157,6 +156,7 @@ public void initializeServicesAndRestHandler( NodeClient client, IdentityService identityService ) { + this.identityService = identityService; this.restActionsRequestHandler = new RestActionsRequestHandler( actionModule.getRestController(), extensionIdMap, diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index 81b1b91b11481..bda84b0d3ca9f 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -32,7 +32,7 @@ public class NoopExtensionsManager extends ExtensionsManager { public NoopExtensionsManager() throws IOException { - super(Set.of(), new IdentityService(Settings.EMPTY, List.of())); + super(Set.of()); } @Override @@ -45,6 +45,7 @@ public void initializeServicesAndRestHandler( NodeClient client, IdentityService identityService ) { + this.setIdentityService(new IdentityService(Settings.EMPTY, List.of())); // no-op } diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java index 33f44a913dd8a..b9c3e2dd211ac 100644 --- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java +++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java @@ -239,7 +239,6 @@ public String executor() { }; try { - // Will be replaced with ExtensionTokenProcessor and PrincipalIdentifierToken classes from feature/identity Map> filteredHeaders = filterHeaders(headers, allowList, denyList); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index b874857675c95..41d8c18ae6a58 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -496,8 +496,7 @@ protected Node( for (ExtensionAwarePlugin extAwarePlugin : extensionAwarePlugins) { additionalSettings.addAll(extAwarePlugin.getExtensionSettings()); } - this.extensionsManager = new ExtensionsManager(additionalSettings, identityService); - this.extensionsManager.setIdentityService(identityService); + this.extensionsManager = new ExtensionsManager(additionalSettings); } else { this.extensionsManager = new NoopExtensionsManager(); } diff --git a/server/src/test/java/org/opensearch/action/ActionModuleTests.java b/server/src/test/java/org/opensearch/action/ActionModuleTests.java index 8479f011adf48..deb0bb58a5c4d 100644 --- a/server/src/test/java/org/opensearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/opensearch/action/ActionModuleTests.java @@ -143,7 +143,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() throws IOException { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) + new ExtensionsManager(Set.of()) ); actionModule.initRestHandlers(null); // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java index 0db768a54b4b9..21b7b47390a9b 100644 --- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java +++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java @@ -77,7 +77,6 @@ public void testReadContextListener() throws InterruptedException, IOException { assertEquals(NUMBER_OF_PARTS * PART_SIZE, Files.size(fileLocation)); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9776") public void testReadContextListenerFailure() throws Exception { Path fileLocation = path.resolve(UUID.randomUUID().toString()); List blobPartStreams = initializeBlobPartStreams(); diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 7e181d7524b12..3596263a48f3c 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -184,7 +184,7 @@ public void testLoadExtensions() throws Exception { Set> additionalSettings = extAwarePlugin.getExtensionSettings().stream().collect(Collectors.toSet()); ExtensionScopedSettings extensionScopedSettings = new ExtensionScopedSettings(additionalSettings); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); ExtensionDependency dependentExtension = new ExtensionDependency("uniqueid0", Version.fromString("2.0.0")); Extension firstExtension = new Extension( @@ -280,7 +280,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { null, null ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); extensionsManager.loadExtension(firstExtension); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(secondExtension)); assertEquals( @@ -321,7 +321,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { public void testMissingRequiredFieldsWhileLoadingExtension() throws Exception { Extension firstExtension = new Extension("firstExtension", "uniqueid1", "127.0.0.0", "9300", "0.0.7", "3.0.0", "", null, null); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(firstExtension)); assertEquals("Required field [minimum opensearch version] is missing in the request", exception.getMessage()); @@ -378,7 +378,7 @@ public void testExtensionDependency() throws Exception { } public void testInitialize() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); @@ -420,7 +420,7 @@ public void testInitialize() throws Exception { public void testHandleRegisterRestActionsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -434,7 +434,7 @@ public void testHandleRegisterRestActionsRequest() throws Exception { } public void testHandleRegisterSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -450,7 +450,7 @@ public void testHandleRegisterSettingsRequest() throws Exception { } public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -465,7 +465,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Excep } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -480,7 +480,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() th } public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET", "PUT /bar", "POST /baz"); @@ -494,7 +494,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exceptio } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); @@ -508,7 +508,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throw } public void testHandleExtensionRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); ExtensionRequest clusterStateRequest = new ExtensionRequest(ExtensionRequestProto.RequestType.REQUEST_EXTENSION_CLUSTER_STATE); @@ -662,7 +662,7 @@ public void testEnvironmentSettingsDefaultValue() throws Exception { } public void testAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); List> componentSettings = List.of( @@ -706,7 +706,7 @@ public void testAddSettingsUpdateConsumerRequest() throws Exception { } public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); List> componentSettings = List.of( @@ -726,7 +726,7 @@ public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { } public void testUpdateSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); initialize(extensionsManager); Setting componentSetting = Setting.boolSetting("falseSetting", false, Property.Dynamic); @@ -755,7 +755,7 @@ public void testUpdateSettingsRequest() throws Exception { public void testRegisterHandler() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); TransportService mockTransportService = spy( new TransportService( @@ -783,7 +783,7 @@ public void testRegisterHandler() throws Exception { } public void testIncompatibleExtensionRegistration() throws IOException { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); Extension firstExtension = new Extension( "firstExtension", "uniqueid1", @@ -824,7 +824,7 @@ public List> getExtensionSettings() { extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( @@ -862,7 +862,7 @@ public void testAdditionalExtensionSettingsForExtensionWithoutCustomSettingSet() extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 2271b6eeac6fa..99642292d09da 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -21,7 +21,6 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.extensions.ExtensionsSettings; -import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; @@ -120,7 +119,7 @@ public void testRestInitializeExtensionActionResponse() throws Exception { } public void testRestInitializeExtensionActionFailure() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(extensionsManager); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"\",\"hostAddress\":\"127.0.0.1\"," @@ -153,10 +152,7 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager( - Set.of(boolSetting, stringSetting, intSetting, listSetting), - new IdentityService(Settings.EMPTY, List.of()) - ); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: @@ -204,17 +200,13 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager( - Set.of(boolSetting, stringSetting, intSetting, listSetting), - new IdentityService(Settings.EMPTY, List.of()) - ); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: when(spy.getAdditionalSettings()).thenCallRealMethod(); Mockito.doCallRealMethod().when(spy).loadExtension(any(ExtensionsSettings.Extension.class)); - doNothing().when(spy).issueServiceAccount(any()); - doNothing().when(spy).initialize(); + Mockito.doNothing().when(spy).initialize(); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(spy); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"ad-extension\",\"hostAddress\":\"127.0.0.1\"," + "\"port\":\"4532\",\"version\":\"1.0\",\"opensearchVersion\":\"" diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index d7a852be73650..c7eacc0da48d0 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -122,7 +122,7 @@ public void setup() throws Exception { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) + new ExtensionsManager(Set.of()) ); identityService = new IdentityService(Settings.EMPTY, new ArrayList<>()); dynamicActionRegistry = actionModule.getDynamicActionRegistry(); From 3c3b7c6e80985be2176ff1ca33f4137cab5ec3f4 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Fri, 29 Sep 2023 13:32:15 -0400 Subject: [PATCH 10/14] Adds this PR to change log Signed-off-by: Darshit Chanpura --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ea905dffb845..034eb8de65add 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) +- Adds changes related On-behalf-Of and Service Accounts feature ([#10258](https://github.com/opensearch-project/OpenSearch/pull/10258)) ### Dependencies - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) From a302af4ce81c484bfa0f9b759d9d8f13360146b8 Mon Sep 17 00:00:00 2001 From: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Date: Fri, 22 Sep 2023 10:21:11 -0400 Subject: [PATCH 11/14] Provide service accounts tokens to extensions (#9618) Provide service accounts tokens to extensions This change adds a new transport action which passes the extension a string representation of its service account auth token. This token is created by the TokenManager interface implementation. The token is expected to be an encoded basic auth credential string which can be used by the extension to interact with its own system index. Signed-off-by: Stephen Crawford Signed-off-by: Stephen Crawford <65832608+scrawfor99@users.noreply.github.com> Signed-off-by: Peter Nied Co-authored-by: Owais Kazi Co-authored-by: Peter Nied (cherry picked from commit 994e11518ca0754ed6275ae7f88744b913255323) --- .../identity/shiro/ShiroTokenManager.java | 8 +- .../identity/shiro/AuthTokenHandlerTests.java | 12 ++ .../discovery/InitializeExtensionRequest.java | 14 ++- .../extensions/ExtensionsManager.java | 119 ++---------------- .../extensions/NoopExtensionsManager.java | 2 +- .../identity/tokens/BasicAuthToken.java | 2 +- .../main/java/org/opensearch/node/Node.java | 2 +- .../opensearch/action/ActionModuleTests.java | 2 +- .../InitializeExtensionRequestTests.java | 9 +- .../extensions/ExtensionsManagerTests.java | 36 +++--- .../RestInitializeExtensionActionTests.java | 13 +- .../rest/RestSendToExtensionActionTests.java | 2 +- 12 files changed, 76 insertions(+), 145 deletions(-) diff --git a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java index 475642be239d1..ddfb99e626718 100644 --- a/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java +++ b/plugins/identity-shiro/src/main/java/org/opensearch/identity/shiro/ShiroTokenManager.java @@ -57,7 +57,6 @@ public Optional translateAuthToken(org.opensearch.identity. final BasicAuthToken basicAuthToken = (BasicAuthToken) authenticationToken; return Optional.of(new UsernamePasswordToken(basicAuthToken.getUser(), basicAuthToken.getPassword())); } - return Optional.empty(); } @@ -65,8 +64,9 @@ public Optional translateAuthToken(org.opensearch.identity. public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) { String password = generatePassword(); + // Make a new ShiroSubject audience as name final byte[] rawEncoded = Base64.getUrlEncoder().encode((claims.getAudience() + ":" + password).getBytes(UTF_8)); - // Make a new ShiroSubject w/ audience as name + final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; BasicAuthToken token = new BasicAuthToken(header); @@ -79,12 +79,12 @@ public AuthToken issueOnBehalfOfToken(Subject subject, OnBehalfOfClaims claims) public AuthToken issueServiceAccountToken(String audience) { String password = generatePassword(); - final byte[] rawEncoded = Base64.getUrlEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); + final byte[] rawEncoded = Base64.getUrlEncoder().withoutPadding().encode((audience + ":" + password).getBytes(UTF_8)); // Make a new final String usernamePassword = new String(rawEncoded, UTF_8); final String header = "Basic " + usernamePassword; + BasicAuthToken token = new BasicAuthToken(header); shiroTokenPasswordMap.put(token, password); - return token; } diff --git a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java index 7c7414843d86f..db77ced298991 100644 --- a/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java +++ b/plugins/identity-shiro/src/test/java/org/opensearch/identity/shiro/AuthTokenHandlerTests.java @@ -167,6 +167,18 @@ public void testTokenNoopIssuance() { Subject subject = new NoopSubject(); AuthToken token = tokenManager.issueOnBehalfOfToken(subject, claims); assertTrue(token instanceof AuthToken); + AuthToken serviceAccountToken = tokenManager.issueServiceAccountToken("test"); + assertTrue(serviceAccountToken instanceof AuthToken); + assertEquals(serviceAccountToken.asAuthHeaderValue(), "noopToken"); } + public void testShouldSucceedIssueServiceAccountToken() { + String audience = "testExtensionName"; + BasicAuthToken authToken = (BasicAuthToken) shiroAuthTokenHandler.issueServiceAccountToken(audience); + assertTrue(authToken instanceof BasicAuthToken); + UsernamePasswordToken translatedToken = (UsernamePasswordToken) shiroAuthTokenHandler.translateAuthToken(authToken).get(); + assertEquals(authToken.getPassword(), new String(translatedToken.getPassword())); + assertTrue(shiroAuthTokenHandler.getShiroTokenPasswordMap().containsKey(authToken)); + assertEquals(shiroAuthTokenHandler.getShiroTokenPasswordMap().get(authToken), new String(translatedToken.getPassword())); + } } diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java index 33cdad3045780..6e9fb8b7201a4 100644 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java +++ b/server/src/main/java/org/opensearch/discovery/InitializeExtensionRequest.java @@ -25,16 +25,19 @@ public class InitializeExtensionRequest extends TransportRequest { private final DiscoveryNode sourceNode; private final DiscoveryExtensionNode extension; + private final String serviceAccountHeader; - public InitializeExtensionRequest(DiscoveryNode sourceNode, DiscoveryExtensionNode extension) { + public InitializeExtensionRequest(DiscoveryNode sourceNode, DiscoveryExtensionNode extension, String serviceAccountHeader) { this.sourceNode = sourceNode; this.extension = extension; + this.serviceAccountHeader = serviceAccountHeader; } public InitializeExtensionRequest(StreamInput in) throws IOException { super(in); sourceNode = new DiscoveryNode(in); extension = new DiscoveryExtensionNode(in); + serviceAccountHeader = in.readString(); } @Override @@ -42,6 +45,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); sourceNode.writeTo(out); extension.writeTo(out); + out.writeString(serviceAccountHeader); } public DiscoveryNode getSourceNode() { @@ -52,6 +56,10 @@ public DiscoveryExtensionNode getExtension() { return extension; } + public String getServiceAccountHeader() { + return serviceAccountHeader; + } + @Override public String toString() { return "InitializeExtensionsRequest{" + "sourceNode=" + sourceNode + ", extension=" + extension + '}'; @@ -62,7 +70,9 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; InitializeExtensionRequest that = (InitializeExtensionRequest) o; - return Objects.equals(sourceNode, that.sourceNode) && Objects.equals(extension, that.extension); + return Objects.equals(sourceNode, that.sourceNode) + && Objects.equals(extension, that.extension) + && Objects.equals(serviceAccountHeader, that.getServiceAccountHeader()); } @Override diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java index 58c29824369a4..a6900db2553e9 100644 --- a/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/ExtensionsManager.java @@ -28,15 +28,11 @@ import org.opensearch.core.transport.TransportResponse; import org.opensearch.discovery.InitializeExtensionRequest; import org.opensearch.discovery.InitializeExtensionResponse; -import org.opensearch.discovery.InitializeExtensionSecurityRequest; -import org.opensearch.discovery.InitializeExtensionSecurityResponse; import org.opensearch.env.EnvironmentSettingsResponse; import org.opensearch.extensions.ExtensionsSettings.Extension; import org.opensearch.extensions.action.ExtensionActionRequest; import org.opensearch.extensions.action.ExtensionActionResponse; import org.opensearch.extensions.action.ExtensionTransportActionsHandler; -import org.opensearch.extensions.action.IssueServiceAccountRequest; -import org.opensearch.extensions.action.IssueServiceAccountResponse; import org.opensearch.extensions.action.RegisterTransportActionsRequest; import org.opensearch.extensions.action.RemoteExtensionActionResponse; import org.opensearch.extensions.action.TransportActionRequestFromExtension; @@ -60,9 +56,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; /** @@ -83,10 +76,8 @@ public class ExtensionsManager { public static final String REQUEST_EXTENSION_REGISTER_TRANSPORT_ACTIONS = "internal:discovery/registertransportactions"; public static final String REQUEST_REST_EXECUTE_ON_EXTENSION_ACTION = "internal:extensions/restexecuteonextensiontaction"; public static final String REQUEST_EXTENSION_HANDLE_TRANSPORT_ACTION = "internal:extensions/handle-transportaction"; - public static final String REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT = "internal:extensions/issue-service-account"; public static final String REQUEST_EXTENSION_HANDLE_REMOTE_TRANSPORT_ACTION = "internal:extensions/handle-remote-transportaction"; public static final String TRANSPORT_ACTION_REQUEST_FROM_EXTENSION = "internal:extensions/request-transportaction-from-extension"; - public static final String REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS = "internal:discovery/registersecuritysettings"; public static final int EXTENSION_REQUEST_WAIT_TIMEOUT = 10; private static final Logger logger = LogManager.getLogger(ExtensionsManager.class); @@ -119,7 +110,7 @@ public static enum OpenSearchRequestType { * @param additionalSettings Additional settings to read in from extension initialization request * @throws IOException If the extensions discovery file is not properly retrieved. */ - public ExtensionsManager(Set> additionalSettings) throws IOException { + public ExtensionsManager(Set> additionalSettings, IdentityService identityService) throws IOException { logger.info("ExtensionsManager initialized"); this.initializedExtensions = new HashMap(); this.extensionIdMap = new HashMap(); @@ -134,6 +125,7 @@ public ExtensionsManager(Set> additionalSettings) throws IOException } this.client = null; this.extensionTransportActionsHandler = null; + this.identityService = identityService; } /** @@ -156,7 +148,6 @@ public void initializeServicesAndRestHandler( NodeClient client, IdentityService identityService ) { - this.identityService = identityService; this.restActionsRequestHandler = new RestActionsRequestHandler( actionModule.getRestController(), extensionIdMap, @@ -411,65 +402,13 @@ protected void doRun() throws Exception { transportService.sendRequest( extension, REQUEST_EXTENSION_ACTION_NAME, - new InitializeExtensionRequest(transportService.getLocalNode(), extension), + new InitializeExtensionRequest(transportService.getLocalNode(), extension, issueServiceAccount(extension)), initializeExtensionResponseHandler ); - initializeExtensionSecurity(extension); } }); } - private void initializeExtensionSecurity(DiscoveryExtensionNode extension) { - final CompletableFuture inProgressFuture = new CompletableFuture<>(); - final TransportResponseHandler initializeExtensionSecurityResponseHandler = - new TransportResponseHandler() { - - @Override - public InitializeExtensionSecurityResponse read(StreamInput in) throws IOException { - return new InitializeExtensionSecurityResponse(in); - } - - @Override - public void handleResponse(InitializeExtensionSecurityResponse response) { - logger.info("Registered security settings for " + response.getName()); - inProgressFuture.complete(response); - } - - @Override - public void handleException(TransportException exp) { - logger.error(new ParameterizedMessage("Extension initialization failed"), exp); - inProgressFuture.completeExceptionally(exp); - } - - @Override - public String executor() { - return ThreadPool.Names.GENERIC; - } - }; - try { - logger.info("Sending extension request type: " + REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS); - AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getId()); - transportService.sendRequest( - extension, - REQUEST_EXTENSION_REGISTER_SECURITY_SETTINGS, - new InitializeExtensionSecurityRequest(serviceAccountToken.asAuthHeaderValue()), - initializeExtensionSecurityResponseHandler - ); - - inProgressFuture.orTimeout(EXTENSION_REQUEST_WAIT_TIMEOUT, TimeUnit.SECONDS).join(); - } catch (CompletionException | ConnectTransportException e) { - if (e.getCause() instanceof TimeoutException || e instanceof ConnectTransportException) { - logger.info("No response from extension to request.", e); - } else if (e.getCause() instanceof RuntimeException) { - throw (RuntimeException) e.getCause(); - } else if (e.getCause() instanceof Error) { - throw (Error) e.getCause(); - } else { - throw new RuntimeException(e.getCause()); - } - } - } - /** * Handles an {@link ExtensionRequest}. * @@ -507,52 +446,12 @@ TransportResponse handleExtensionRequest(ExtensionRequest extensionRequest) thro } /** - * A separate transport action handler used to issue service accounts to extensions during initialization + * A helper method called during initialization that issues a service accounts to extensions * @param extension The extension to be issued a service account */ - public void issueServiceAccount(Extension extension) { - DiscoveryExtensionNode discoveryExtensionNode = extensionIdMap.get(extension.getUniqueId()); - AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getUniqueId()); - String authTokenAsString = serviceAccountToken.asAuthHeaderValue(); - final CompletableFuture inProgressFuture = new CompletableFuture<>(); - final TransportResponseHandler issueServiceAccountResponseHandler = new TransportResponseHandler< - IssueServiceAccountResponse>() { - - @Override - public IssueServiceAccountResponse read(StreamInput in) throws IOException { - return new IssueServiceAccountResponse(in); - } - - @Override - public void handleResponse(IssueServiceAccountResponse response) { - for (DiscoveryExtensionNode extension : extensionIdMap.values()) { - if (extension.getName().equals(response.getName()) - && (serviceAccountToken.equals(response.getServiceAccountString()))) { - logger.info("Successfully issued service account token to extension: " + extension.getName()); - break; - } - } - inProgressFuture.complete(response); - } - - @Override - public void handleException(TransportException exp) { - logger.error(new ParameterizedMessage("Issuance of service account token failed"), exp); - inProgressFuture.completeExceptionally(exp); - } - - @Override - public String executor() { - return ThreadPool.Names.GENERIC; - } - }; - - transportService.sendRequest( - discoveryExtensionNode, - REQUEST_EXTENSION_ISSUE_SERVICE_ACCOUNT, - new IssueServiceAccountRequest(authTokenAsString), - issueServiceAccountResponseHandler - ); + public String issueServiceAccount(DiscoveryExtensionNode extension) { + AuthToken serviceAccountToken = identityService.getTokenManager().issueServiceAccountToken(extension.getId()); + return serviceAccountToken.asAuthHeaderValue(); } static String getRequestExtensionActionName() { @@ -611,10 +510,6 @@ void setCustomSettingsRequestHandler(CustomSettingsRequestHandler customSettings this.customSettingsRequestHandler = customSettingsRequestHandler; } - public void setIdentityService(IdentityService identityService) { - this.identityService = identityService; - } - AddSettingsUpdateConsumerRequestHandler getAddSettingsUpdateConsumerRequestHandler() { return addSettingsUpdateConsumerRequestHandler; } diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index bda84b0d3ca9f..6da4ea1fa4192 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -32,7 +32,7 @@ public class NoopExtensionsManager extends ExtensionsManager { public NoopExtensionsManager() throws IOException { - super(Set.of()); + super(Set.of(), new IdentityService(Settings.EMPTY, List.of())); } @Override diff --git a/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java b/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java index 71b8fe504a5d1..4ad0bbe67d2a1 100644 --- a/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java +++ b/server/src/main/java/org/opensearch/identity/tokens/BasicAuthToken.java @@ -23,7 +23,7 @@ public final class BasicAuthToken implements AuthToken { public BasicAuthToken(final String headerValue) { final String base64Encoded = headerValue.substring(TOKEN_IDENTIFIER.length()).trim(); - final byte[] rawDecoded = Base64.getDecoder().decode(base64Encoded); + final byte[] rawDecoded = Base64.getUrlDecoder().decode(base64Encoded); final String usernamepassword = new String(rawDecoded, StandardCharsets.UTF_8); final String[] tokenParts = usernamepassword.split(":", 2); diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 41d8c18ae6a58..b931e2e8ba75e 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -496,7 +496,7 @@ protected Node( for (ExtensionAwarePlugin extAwarePlugin : extensionAwarePlugins) { additionalSettings.addAll(extAwarePlugin.getExtensionSettings()); } - this.extensionsManager = new ExtensionsManager(additionalSettings); + this.extensionsManager = new ExtensionsManager(additionalSettings, identityService); } else { this.extensionsManager = new NoopExtensionsManager(); } diff --git a/server/src/test/java/org/opensearch/action/ActionModuleTests.java b/server/src/test/java/org/opensearch/action/ActionModuleTests.java index deb0bb58a5c4d..8479f011adf48 100644 --- a/server/src/test/java/org/opensearch/action/ActionModuleTests.java +++ b/server/src/test/java/org/opensearch/action/ActionModuleTests.java @@ -143,7 +143,7 @@ public void testSetupRestHandlerContainsKnownBuiltin() throws IOException { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of()) + new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) ); actionModule.initRestHandlers(null); // At this point the easiest way to confirm that a handler is loaded is to try to register another one on top of it and to fail diff --git a/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java b/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java index ca94a3b52a56c..daa42d3abbc50 100644 --- a/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java +++ b/server/src/test/java/org/opensearch/discovery/InitializeExtensionRequestTests.java @@ -27,6 +27,7 @@ public class InitializeExtensionRequestTests extends OpenSearchTestCase { public void testInitializeExtensionRequest() throws Exception { String expectedUniqueId = "test uniqueid"; Version expectedVersion = Version.fromString("2.0.0"); + String expectedServiceAccountHeader = "test"; ExtensionDependency expectedDependency = new ExtensionDependency(expectedUniqueId, expectedVersion); DiscoveryExtensionNode expectedExtensionNode = new DiscoveryExtensionNode( "firstExtension", @@ -46,9 +47,14 @@ public void testInitializeExtensionRequest() throws Exception { Version.CURRENT ); - InitializeExtensionRequest initializeExtensionRequest = new InitializeExtensionRequest(expectedSourceNode, expectedExtensionNode); + InitializeExtensionRequest initializeExtensionRequest = new InitializeExtensionRequest( + expectedSourceNode, + expectedExtensionNode, + expectedServiceAccountHeader + ); assertEquals(expectedExtensionNode, initializeExtensionRequest.getExtension()); assertEquals(expectedSourceNode, initializeExtensionRequest.getSourceNode()); + assertEquals(expectedServiceAccountHeader, initializeExtensionRequest.getServiceAccountHeader()); try (BytesStreamOutput out = new BytesStreamOutput()) { initializeExtensionRequest.writeTo(out); @@ -58,6 +64,7 @@ public void testInitializeExtensionRequest() throws Exception { assertEquals(expectedExtensionNode, initializeExtensionRequest.getExtension()); assertEquals(expectedSourceNode, initializeExtensionRequest.getSourceNode()); + assertEquals(expectedServiceAccountHeader, initializeExtensionRequest.getServiceAccountHeader()); } } } diff --git a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java index 3596263a48f3c..7e181d7524b12 100644 --- a/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java +++ b/server/src/test/java/org/opensearch/extensions/ExtensionsManagerTests.java @@ -184,7 +184,7 @@ public void testLoadExtensions() throws Exception { Set> additionalSettings = extAwarePlugin.getExtensionSettings().stream().collect(Collectors.toSet()); ExtensionScopedSettings extensionScopedSettings = new ExtensionScopedSettings(additionalSettings); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); ExtensionDependency dependentExtension = new ExtensionDependency("uniqueid0", Version.fromString("2.0.0")); Extension firstExtension = new Extension( @@ -280,7 +280,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { null, null ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); extensionsManager.loadExtension(firstExtension); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(secondExtension)); assertEquals( @@ -321,7 +321,7 @@ public void testNonUniqueLoadedExtensions() throws Exception { public void testMissingRequiredFieldsWhileLoadingExtension() throws Exception { Extension firstExtension = new Extension("firstExtension", "uniqueid1", "127.0.0.0", "9300", "0.0.7", "3.0.0", "", null, null); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); IOException exception = expectThrows(IOException.class, () -> extensionsManager.loadExtension(firstExtension)); assertEquals("Required field [minimum opensearch version] is missing in the request", exception.getMessage()); @@ -378,7 +378,7 @@ public void testExtensionDependency() throws Exception { } public void testInitialize() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); @@ -420,7 +420,7 @@ public void testInitialize() throws Exception { public void testHandleRegisterRestActionsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -434,7 +434,7 @@ public void testHandleRegisterRestActionsRequest() throws Exception { } public void testHandleRegisterSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -450,7 +450,7 @@ public void testHandleRegisterSettingsRequest() throws Exception { } public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -465,7 +465,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidMethod() throws Excep } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; @@ -480,7 +480,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedMethod() th } public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET", "PUT /bar", "POST /baz"); @@ -494,7 +494,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidUri() throws Exceptio } public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); String uniqueIdStr = "uniqueid1"; List actionsList = List.of("GET /foo", "PUT /bar", "POST /baz"); @@ -508,7 +508,7 @@ public void testHandleRegisterRestActionsRequestWithInvalidDeprecatedUri() throw } public void testHandleExtensionRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); ExtensionRequest clusterStateRequest = new ExtensionRequest(ExtensionRequestProto.RequestType.REQUEST_EXTENSION_CLUSTER_STATE); @@ -662,7 +662,7 @@ public void testEnvironmentSettingsDefaultValue() throws Exception { } public void testAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); List> componentSettings = List.of( @@ -706,7 +706,7 @@ public void testAddSettingsUpdateConsumerRequest() throws Exception { } public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); List> componentSettings = List.of( @@ -726,7 +726,7 @@ public void testHandleAddSettingsUpdateConsumerRequest() throws Exception { } public void testUpdateSettingsRequest() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); initialize(extensionsManager); Setting componentSetting = Setting.boolSetting("falseSetting", false, Property.Dynamic); @@ -755,7 +755,7 @@ public void testUpdateSettingsRequest() throws Exception { public void testRegisterHandler() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); TransportService mockTransportService = spy( new TransportService( @@ -783,7 +783,7 @@ public void testRegisterHandler() throws Exception { } public void testIncompatibleExtensionRegistration() throws IOException { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), identityService); Extension firstExtension = new Extension( "firstExtension", "uniqueid1", @@ -824,7 +824,7 @@ public List> getExtensionSettings() { extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( @@ -862,7 +862,7 @@ public void testAdditionalExtensionSettingsForExtensionWithoutCustomSettingSet() extensionScopedSettings ); - ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings); + ExtensionsManager extensionsManager = new ExtensionsManager(additionalSettings, identityService); extensionsManager.loadExtension(firstExtension); DiscoveryExtensionNode extension = new DiscoveryExtensionNode( diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java index 99642292d09da..7d12d0837a3f7 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestInitializeExtensionActionTests.java @@ -21,6 +21,7 @@ import org.opensearch.core.xcontent.MediaTypeRegistry; import org.opensearch.extensions.ExtensionsManager; import org.opensearch.extensions.ExtensionsSettings; +import org.opensearch.identity.IdentityService; import org.opensearch.rest.RestRequest; import org.opensearch.telemetry.tracing.noop.NoopTracer; import org.opensearch.test.OpenSearchTestCase; @@ -119,7 +120,7 @@ public void testRestInitializeExtensionActionResponse() throws Exception { } public void testRestInitializeExtensionActionFailure() throws Exception { - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of()); + ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())); RestInitializeExtensionAction restInitializeExtensionAction = new RestInitializeExtensionAction(extensionsManager); final String content = "{\"name\":\"ad-extension\",\"uniqueId\":\"\",\"hostAddress\":\"127.0.0.1\"," @@ -152,7 +153,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettings() th Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager extensionsManager = new ExtensionsManager( + Set.of(boolSetting, stringSetting, intSetting, listSetting), + new IdentityService(Settings.EMPTY, List.of()) + ); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: @@ -200,7 +204,10 @@ public void testRestInitializeExtensionActionResponseWithAdditionalSettingsUsing Function.identity(), Setting.Property.ExtensionScope ); - ExtensionsManager extensionsManager = new ExtensionsManager(Set.of(boolSetting, stringSetting, intSetting, listSetting)); + ExtensionsManager extensionsManager = new ExtensionsManager( + Set.of(boolSetting, stringSetting, intSetting, listSetting), + new IdentityService(Settings.EMPTY, List.of()) + ); ExtensionsManager spy = spy(extensionsManager); // optionally, you can stub out some methods: diff --git a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java index c7eacc0da48d0..d7a852be73650 100644 --- a/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java +++ b/server/src/test/java/org/opensearch/extensions/rest/RestSendToExtensionActionTests.java @@ -122,7 +122,7 @@ public void setup() throws Exception { usageService, null, new IdentityService(Settings.EMPTY, new ArrayList<>()), - new ExtensionsManager(Set.of()) + new ExtensionsManager(Set.of(), new IdentityService(Settings.EMPTY, List.of())) ); identityService = new IdentityService(Settings.EMPTY, new ArrayList<>()); dynamicActionRegistry = actionModule.getDynamicActionRegistry(); From 9c755a3b0603f345e99f481e16c552b03e64b300 Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Mon, 2 Oct 2023 16:01:18 -0400 Subject: [PATCH 12/14] Fixes CHANGELOG entry Signed-off-by: Darshit Chanpura --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c1062a7068f30..48045d810c0f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110)) - Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189)) - Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562)) -- Adds changes related On-behalf-Of and Service Accounts feature ([#10258](https://github.com/opensearch-project/OpenSearch/pull/10258)) +- Adds changes related to On-behalf-Of and Service Accounts feature ([#10258](https://github.com/opensearch-project/OpenSearch/pull/10258)) ### Dependencies - Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963)) From 961c56d27644480a4b5ed1da56dc597c21183c9d Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Mon, 2 Oct 2023 16:08:47 -0400 Subject: [PATCH 13/14] Removes unneeded files Signed-off-by: Darshit Chanpura --- .../InitializeExtensionSecurityRequest.java | 63 --------------- .../InitializeExtensionSecurityResponse.java | 64 --------------- .../action/IssueServiceAccountRequest.java | 69 ---------------- .../action/IssueServiceAccountResponse.java | 81 ------------------- 4 files changed, 277 deletions(-) delete mode 100644 server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java delete mode 100644 server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java delete mode 100644 server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java delete mode 100644 server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java deleted file mode 100644 index 530d69e418b25..0000000000000 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityRequest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.discovery; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.transport.TransportRequest; - -import java.io.IOException; -import java.util.Objects; - -/** - * InitializeExtensionRequest to initialize plugin - * - * @opensearch.internal - */ -public class InitializeExtensionSecurityRequest extends TransportRequest { - - private final String serviceAccountToken; - - public InitializeExtensionSecurityRequest(String serviceAccountToken) { - this.serviceAccountToken = serviceAccountToken; - } - - public InitializeExtensionSecurityRequest(StreamInput in) throws IOException { - super(in); - serviceAccountToken = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(serviceAccountToken); - } - - public String getServiceAccountToken() { - return serviceAccountToken; - } - - @Override - public String toString() { - return "InitializeExtensionsRequest{" + "serviceAccountToken= " + serviceAccountToken + "}"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InitializeExtensionSecurityRequest that = (InitializeExtensionSecurityRequest) o; - return Objects.equals(serviceAccountToken, that.serviceAccountToken); - } - - @Override - public int hashCode() { - return Objects.hash(serviceAccountToken); - } -} diff --git a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java b/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java deleted file mode 100644 index 14281b0c77d6d..0000000000000 --- a/server/src/main/java/org/opensearch/discovery/InitializeExtensionSecurityResponse.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.discovery; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.transport.TransportResponse; - -import java.io.IOException; -import java.util.Objects; - -/** - * PluginResponse to intialize plugin - * - * @opensearch.internal - */ -public class InitializeExtensionSecurityResponse extends TransportResponse { - private String name; - - public InitializeExtensionSecurityResponse(String name) { - this.name = name; - } - - public InitializeExtensionSecurityResponse(StreamInput in) throws IOException { - name = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name); - } - - /** - * @return the node that is currently leading, according to the responding node. - */ - - public String getName() { - return this.name; - } - - @Override - public String toString() { - return "InitializeExtensionResponse{" + "name = " + name + "}"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - InitializeExtensionSecurityResponse that = (InitializeExtensionSecurityResponse) o; - return Objects.equals(name, that.name); - } - - @Override - public int hashCode() { - return Objects.hash(name); - } -} diff --git a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java deleted file mode 100644 index 7d152a33e800a..0000000000000 --- a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountRequest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.extensions.action; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.transport.TransportRequest; - -import java.io.IOException; -import java.util.Objects; - -/** - * This class represents a Transport Request for issuing a service account to an extension. - */ -public class IssueServiceAccountRequest extends TransportRequest { - - private final String serviceAccountToken; - - /** - * This takes a string representing a service account token - * @param serviceAccountToken The string making up the service account token - */ - public IssueServiceAccountRequest(String serviceAccountToken) { - this.serviceAccountToken = serviceAccountToken; - } - - /** - * This takes a stream representing a service account token - * @param in The stream passing the token - */ - public IssueServiceAccountRequest(StreamInput in) throws IOException { - super(in); - this.serviceAccountToken = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(serviceAccountToken); - } - - public String getServiceAccountToken() { - return this.serviceAccountToken; - } - - @Override - public String toString() { - return "IssueServiceAccountRequest {" + "serviceAccountToken=" + serviceAccountToken + "}"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IssueServiceAccountRequest that = (IssueServiceAccountRequest) o; - return Objects.equals(serviceAccountToken, that.serviceAccountToken); - } - - @Override - public int hashCode() { - return Objects.hash(serviceAccountToken); - } -} diff --git a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java b/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java deleted file mode 100644 index 78e41f1806e9e..0000000000000 --- a/server/src/main/java/org/opensearch/extensions/action/IssueServiceAccountResponse.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.extensions.action; - -import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.transport.TransportResponse; - -import java.io.IOException; -import java.util.Objects; - -/** - * This class represents a Transport Request for issuing a service account to an extension. - */ -public class IssueServiceAccountResponse extends TransportResponse { - - private String name; - private String serviceAccountToken; - - /** - * This takes in a name for the extension and the service account token string - * @param name The name of the extension - * @param serviceAccountToken A string encapsulating the service account token - */ - public IssueServiceAccountResponse(String name, String serviceAccountToken) { - this.name = name; - this.serviceAccountToken = serviceAccountToken; - } - - /** - * This takes in a stream containing for the extension and the service account token - * @param in the stream containing the extension name and the service account token - */ - public IssueServiceAccountResponse(StreamInput in) throws IOException { - this.name = in.readString(); - this.serviceAccountToken = in.readString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name); - out.writeString(serviceAccountToken); - } - - /** - * @return the node that is currently leading, according to the responding node. - */ - - public String getName() { - return this.name; - } - - public String getServiceAccountString() { - return this.serviceAccountToken; - } - - @Override - public String toString() { - return "IssueServiceAccountResponse{" + "name = " + name + " , " + "received service account = " + serviceAccountToken + "}"; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - IssueServiceAccountResponse that = (IssueServiceAccountResponse) o; - return Objects.equals(name, that.name) && Objects.equals(serviceAccountToken, that.serviceAccountToken); - } - - @Override - public int hashCode() { - return Objects.hash(name, serviceAccountToken); - } - -} From 71e6c2d86e3e8532d9c729e7ca185b1949808a9c Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Mon, 2 Oct 2023 16:22:25 -0400 Subject: [PATCH 14/14] Fixes precommit failure Signed-off-by: Darshit Chanpura --- .../java/org/opensearch/extensions/NoopExtensionsManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java index 6da4ea1fa4192..81b1b91b11481 100644 --- a/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java +++ b/server/src/main/java/org/opensearch/extensions/NoopExtensionsManager.java @@ -45,7 +45,6 @@ public void initializeServicesAndRestHandler( NodeClient client, IdentityService identityService ) { - this.setIdentityService(new IdentityService(Settings.EMPTY, List.of())); // no-op }