From c7925957f91fc4ba5a2b834f0e368dd4864f8cf4 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 11 Dec 2024 17:30:43 +0200 Subject: [PATCH] Unify logsdb index settings providers (#118342) * Unify logsdb index settings providers * restore diff * rename method --- .../xpack/logsdb/LogsDBPlugin.java | 17 +- .../LogsdbIndexModeSettingsProvider.java | 175 +++++++- .../SyntheticSourceIndexSettingsProvider.java | 200 --------- .../LogsdbIndexModeSettingsProviderTests.java | 408 +++++++++++++++++ ...exSettingsProviderLegacyLicenseTests.java} | 15 +- ...heticSourceIndexSettingsProviderTests.java | 417 ------------------ 6 files changed, 588 insertions(+), 644 deletions(-) delete mode 100644 x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java rename x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/{SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java => LogsdbIndexSettingsProviderLegacyLicenseTests.java} (91%) delete mode 100644 x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java index 904b00e6d0450..a8085f3d50a82 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java @@ -43,7 +43,7 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin { public LogsDBPlugin(Settings settings) { this.settings = settings; this.licenseService = new SyntheticSourceLicenseService(settings); - this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(settings); + this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(licenseService, settings); } @Override @@ -67,16 +67,13 @@ public Collection createComponents(PluginServices services) { @Override public Collection getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) { - if (DiscoveryNode.isStateless(settings)) { - return List.of(logsdbIndexModeSettingsProvider); + if (DiscoveryNode.isStateless(settings) == false) { + logsdbIndexModeSettingsProvider.init( + parameters.mapperServiceFactory(), + () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion() + ); } - var syntheticSettingProvider = new SyntheticSourceIndexSettingsProvider( - licenseService, - parameters.mapperServiceFactory(), - logsdbIndexModeSettingsProvider, - () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion() - ); - return List.of(syntheticSettingProvider, logsdbIndexModeSettingsProvider); + return List.of(logsdbIndexModeSettingsProvider); } @Override diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index 481657eaf7225..977b0e1c57578 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -7,25 +7,45 @@ package org.elasticsearch.xpack.logsdb; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettingProvider; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import java.io.IOException; import java.time.Instant; import java.util.List; import java.util.Locale; +import java.util.function.Supplier; +import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED; final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { + private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); private static final String LOGS_PATTERN = "logs-*-*"; + + private final SyntheticSourceLicenseService syntheticSourceLicenseService; + private final SetOnce> mapperServiceFactory = new SetOnce<>(); + private final SetOnce> createdIndexVersion = new SetOnce<>(); + private volatile boolean isLogsdbEnabled; - LogsdbIndexModeSettingsProvider(final Settings settings) { + LogsdbIndexModeSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService, final Settings settings) { + this.syntheticSourceLicenseService = syntheticSourceLicenseService; this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings); } @@ -33,6 +53,21 @@ void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) { this.isLogsdbEnabled = isLogsdbEnabled; } + void init(CheckedFunction factory, Supplier indexVersion) { + mapperServiceFactory.set(factory); + createdIndexVersion.set(indexVersion); + } + + private boolean supportFallbackToStoredSource() { + return mapperServiceFactory.get() != null; + } + + @Override + public boolean overrulesTemplateAndRequestSettings() { + // Indicates that the provider value takes precedence over any user setting. + return true; + } + @Override public Settings getAdditionalIndexSettings( final String indexName, @@ -40,20 +75,42 @@ public Settings getAdditionalIndexSettings( IndexMode templateIndexMode, final Metadata metadata, final Instant resolvedAt, - final Settings settings, + Settings settings, final List combinedTemplateMappings ) { - return getLogsdbModeSetting(dataStreamName, settings); - } - - Settings getLogsdbModeSetting(final String dataStreamName, final Settings settings) { + Settings.Builder settingsBuilder = null; if (isLogsdbEnabled && dataStreamName != null && resolveIndexMode(settings.get(IndexSettings.MODE.getKey())) == null && matchesLogsPattern(dataStreamName)) { - return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build(); + settingsBuilder = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()); + if (supportFallbackToStoredSource()) { + settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).put(settings).build(); + } + } + + if (supportFallbackToStoredSource()) { + // This index name is used when validating component and index templates, we should skip this check in that case. + // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) + boolean isTemplateValidation = "validate-index-name".equals(indexName); + boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed( + templateIndexMode, + indexName, + dataStreamName + ); + if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, settings, combinedTemplateMappings) + && syntheticSourceLicenseService.fallbackToStoredSource( + isTemplateValidation, + legacyLicensedUsageOfSyntheticSourceAllowed + )) { + LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName); + if (settingsBuilder == null) { + settingsBuilder = Settings.builder(); + } + settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()); + } } - return Settings.EMPTY; + return settingsBuilder == null ? Settings.EMPTY : settingsBuilder.build(); } private static boolean matchesLogsPattern(final String name) { @@ -63,4 +120,106 @@ private static boolean matchesLogsPattern(final String name) { private IndexMode resolveIndexMode(final String mode) { return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null; } + + boolean newIndexHasSyntheticSourceUsage( + String indexName, + IndexMode templateIndexMode, + Settings indexTemplateAndCreateRequestSettings, + List combinedTemplateMappings + ) { + if ("validate-index-name".equals(indexName)) { + // This index name is used when validating component and index templates, we should skip this check in that case. + // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) + return false; + } + + try { + var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); + var indexMode = tmpIndexMetadata.getIndexMode(); + if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) + || indexMode == IndexMode.LOGSDB + || indexMode == IndexMode.TIME_SERIES) { + // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default + // could be wrong. However, it doesn't really matter, because if the _source.mode mapping attribute is set to stored, + // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set + // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to + // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. + var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); + return sourceMode == SourceFieldMapper.Mode.SYNTHETIC; + } + + // TODO: remove this when _source.mode attribute has been removed: + try (var mapperService = mapperServiceFactory.get().apply(tmpIndexMetadata)) { + // combinedTemplateMappings can be null when creating system indices + // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. + if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) { + combinedTemplateMappings = List.of(new CompressedXContent("{}")); + } + mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); + return mapperService.documentMapper().sourceMapper().isSynthetic(); + } + } catch (AssertionError | Exception e) { + // In case invalid mappings or setting are provided, then mapper service creation can fail. + // In that case it is ok to return false here. The index creation will fail anyway later, so no need to fallback to stored + // source. + LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e); + return false; + } + } + + // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used: + private IndexMetadata buildIndexMetadataForMapperService( + String indexName, + IndexMode templateIndexMode, + Settings indexTemplateAndCreateRequestSettings + ) { + var tmpIndexMetadata = IndexMetadata.builder(indexName); + + int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings); + int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt( + IndexMetadata.SETTING_NUMBER_OF_SHARDS, + dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1 + ); + int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); + var finalResolvedSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, createdIndexVersion.get().get()) + .put(indexTemplateAndCreateRequestSettings) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); + + if (templateIndexMode == IndexMode.TIME_SERIES) { + finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); + // Avoid failing because index.routing_path is missing (in case fields are marked as dimension) + finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path")); + } + + tmpIndexMetadata.settings(finalResolvedSettings); + return tmpIndexMetadata.build(); + } + + /** + * The GA-ed use cases in which synthetic source usage is allowed with gold or platinum license. + */ + private boolean isLegacyLicensedUsageOfSyntheticSourceAllowed(IndexMode templateIndexMode, String indexName, String dataStreamName) { + if (templateIndexMode == IndexMode.TIME_SERIES) { + return true; + } + + // To allow the following patterns: profiling-metrics and profiling-events + if (dataStreamName != null && dataStreamName.startsWith("profiling-")) { + return true; + } + // To allow the following patterns: .profiling-sq-executables, .profiling-sq-leafframes and .profiling-stacktraces + if (indexName.startsWith(".profiling-")) { + return true; + } + // To allow the following patterns: metrics-apm.transaction.*, metrics-apm.service_transaction.*, metrics-apm.service_summary.*, + // metrics-apm.service_destination.*, "metrics-apm.internal-* and metrics-apm.app.* + if (dataStreamName != null && dataStreamName.startsWith("metrics-apm.")) { + return true; + } + + return false; + } } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java deleted file mode 100644 index 462bad4b19551..0000000000000 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.logsdb; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.UUIDs; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.Strings; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettingProvider; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.SourceFieldMapper; - -import java.io.IOException; -import java.time.Instant; -import java.util.List; -import java.util.function.Supplier; - -import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH; - -/** - * An index setting provider that overwrites the source mode from synthetic to stored if synthetic source isn't allowed to be used. - */ -final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider { - - private static final Logger LOGGER = LogManager.getLogger(SyntheticSourceIndexSettingsProvider.class); - - private final SyntheticSourceLicenseService syntheticSourceLicenseService; - private final CheckedFunction mapperServiceFactory; - private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider; - private final Supplier createdIndexVersion; - - SyntheticSourceIndexSettingsProvider( - SyntheticSourceLicenseService syntheticSourceLicenseService, - CheckedFunction mapperServiceFactory, - LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider, - Supplier createdIndexVersion - ) { - this.syntheticSourceLicenseService = syntheticSourceLicenseService; - this.mapperServiceFactory = mapperServiceFactory; - this.logsdbIndexModeSettingsProvider = logsdbIndexModeSettingsProvider; - this.createdIndexVersion = createdIndexVersion; - } - - @Override - public boolean overrulesTemplateAndRequestSettings() { - // Indicates that the provider value takes precedence over any user setting. - return true; - } - - @Override - public Settings getAdditionalIndexSettings( - String indexName, - String dataStreamName, - IndexMode templateIndexMode, - Metadata metadata, - Instant resolvedAt, - Settings indexTemplateAndCreateRequestSettings, - List combinedTemplateMappings - ) { - var logsdbSettings = logsdbIndexModeSettingsProvider.getLogsdbModeSetting(dataStreamName, indexTemplateAndCreateRequestSettings); - if (logsdbSettings != Settings.EMPTY) { - indexTemplateAndCreateRequestSettings = Settings.builder() - .put(logsdbSettings) - .put(indexTemplateAndCreateRequestSettings) - .build(); - } - - // This index name is used when validating component and index templates, we should skip this check in that case. - // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) - boolean isTemplateValidation = "validate-index-name".equals(indexName); - boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed( - templateIndexMode, - indexName, - dataStreamName - ); - if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings, combinedTemplateMappings) - && syntheticSourceLicenseService.fallbackToStoredSource(isTemplateValidation, legacyLicensedUsageOfSyntheticSourceAllowed)) { - LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName); - return Settings.builder() - .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString()) - .build(); - } - return Settings.EMPTY; - } - - boolean newIndexHasSyntheticSourceUsage( - String indexName, - IndexMode templateIndexMode, - Settings indexTemplateAndCreateRequestSettings, - List combinedTemplateMappings - ) { - if ("validate-index-name".equals(indexName)) { - // This index name is used when validating component and index templates, we should skip this check in that case. - // (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method) - return false; - } - - try { - var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings); - var indexMode = tmpIndexMetadata.getIndexMode(); - if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings()) - || indexMode == IndexMode.LOGSDB - || indexMode == IndexMode.TIME_SERIES) { - // In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default - // could be wrong. However, it doesn't really matter, because if the _source.mode mapping attribute is set to stored, - // then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set - // to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to - // stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService. - var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings()); - return sourceMode == SourceFieldMapper.Mode.SYNTHETIC; - } - - // TODO: remove this when _source.mode attribute has been removed: - try (var mapperService = mapperServiceFactory.apply(tmpIndexMetadata)) { - // combinedTemplateMappings can be null when creating system indices - // combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping. - if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) { - combinedTemplateMappings = List.of(new CompressedXContent("{}")); - } - mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); - return mapperService.documentMapper().sourceMapper().isSynthetic(); - } - } catch (AssertionError | Exception e) { - // In case invalid mappings or setting are provided, then mapper service creation can fail. - // In that case it is ok to return false here. The index creation will fail anyway later, so no need to fallback to stored - // source. - LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e); - return false; - } - } - - // Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used: - private IndexMetadata buildIndexMetadataForMapperService( - String indexName, - IndexMode templateIndexMode, - Settings indexTemplateAndCreateRequestSettings - ) { - var tmpIndexMetadata = IndexMetadata.builder(indexName); - - int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings); - int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt( - IndexMetadata.SETTING_NUMBER_OF_SHARDS, - dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1 - ); - int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); - var finalResolvedSettings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, createdIndexVersion.get()) - .put(indexTemplateAndCreateRequestSettings) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas) - .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()); - - if (templateIndexMode == IndexMode.TIME_SERIES) { - finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES); - // Avoid failing because index.routing_path is missing (in case fields are marked as dimension) - finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path")); - } - - tmpIndexMetadata.settings(finalResolvedSettings); - return tmpIndexMetadata.build(); - } - - /** - * The GA-ed use cases in which synthetic source usage is allowed with gold or platinum license. - */ - boolean isLegacyLicensedUsageOfSyntheticSourceAllowed(IndexMode templateIndexMode, String indexName, String dataStreamName) { - if (templateIndexMode == IndexMode.TIME_SERIES) { - return true; - } - - // To allow the following patterns: profiling-metrics and profiling-events - if (dataStreamName != null && dataStreamName.startsWith("profiling-")) { - return true; - } - // To allow the following patterns: .profiling-sq-executables, .profiling-sq-leafframes and .profiling-stacktraces - if (indexName.startsWith(".profiling-")) { - return true; - } - // To allow the following patterns: metrics-apm.transaction.*, metrics-apm.service_transaction.*, metrics-apm.service_summary.*, - // metrics-apm.service_destination.*, "metrics-apm.internal-* and metrics-apm.app.* - if (dataStreamName != null && dataStreamName.startsWith("metrics-apm.")) { - return true; - } - - return false; - } -} diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index 5f23dbdca1143..de4f0960f50e7 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -9,19 +9,37 @@ import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.license.License; +import org.elasticsearch.license.LicenseService; +import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.elasticsearch.common.settings.Settings.builder; +import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { @@ -43,8 +61,39 @@ public class LogsdbIndexModeSettingsProviderTests extends ESTestCase { } """; + private SyntheticSourceLicenseService syntheticSourceLicenseService; + private final AtomicInteger newMapperServiceCounter = new AtomicInteger(); + + @Before + public void setup() throws Exception { + MockLicenseState licenseState = MockLicenseState.createMock(); + when(licenseState.isAllowed(any())).thenReturn(true); + var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + licenseService.setLicenseState(licenseState); + var mockLicenseService = mock(LicenseService.class); + License license = createEnterpriseLicense(); + when(mockLicenseService.getLicense()).thenReturn(license); + syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); + syntheticSourceLicenseService.setLicenseState(licenseState); + syntheticSourceLicenseService.setLicenseService(mockLicenseService); + } + + LogsdbIndexModeSettingsProvider withSyntheticSourceDemotionSupport(boolean enabled) { + newMapperServiceCounter.set(0); + var provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, + Settings.builder().put("cluster.logsdb.enabled", enabled).build() + ); + provider.init(im -> { + newMapperServiceCounter.incrementAndGet(); + return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); + }, IndexVersion::current); + return provider; + } + public void testLogsDbDisabled() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", false).build() ); @@ -63,6 +112,7 @@ public void testLogsDbDisabled() throws IOException { public void testOnIndexCreation() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -81,6 +131,7 @@ public void testOnIndexCreation() throws IOException { public void testOnExplicitStandardIndex() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -99,6 +150,7 @@ public void testOnExplicitStandardIndex() throws IOException { public void testOnExplicitTimeSeriesIndex() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -117,6 +169,7 @@ public void testOnExplicitTimeSeriesIndex() throws IOException { public void testNonLogsDataStream() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -135,6 +188,7 @@ public void testNonLogsDataStream() throws IOException { public void testWithoutLogsComponentTemplate() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -153,6 +207,7 @@ public void testWithoutLogsComponentTemplate() throws IOException { public void testWithLogsComponentTemplate() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -171,6 +226,7 @@ public void testWithLogsComponentTemplate() throws IOException { public void testWithMultipleComponentTemplates() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -189,6 +245,7 @@ public void testWithMultipleComponentTemplates() throws IOException { public void testWithCustomComponentTemplatesOnly() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -207,6 +264,7 @@ public void testWithCustomComponentTemplatesOnly() throws IOException { public void testNonMatchingTemplateIndexPattern() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -225,6 +283,7 @@ public void testNonMatchingTemplateIndexPattern() throws IOException { public void testCaseSensitivity() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -243,6 +302,7 @@ public void testCaseSensitivity() throws IOException { public void testMultipleHyphensInDataStreamName() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", true).build() ); @@ -261,6 +321,7 @@ public void testMultipleHyphensInDataStreamName() throws IOException { public void testBeforeAndAFterSettingUpdate() throws IOException { final LogsdbIndexModeSettingsProvider provider = new LogsdbIndexModeSettingsProvider( + syntheticSourceLicenseService, Settings.builder().put("cluster.logsdb.enabled", false).build() ); @@ -323,4 +384,351 @@ private void assertIndexMode(final Settings settings, final String expectedIndex assertEquals(expectedIndexMode, settings.get(IndexSettings.MODE.getKey())); } + public void testNewIndexHasSyntheticSourceUsage() throws IOException { + String dataStreamName = "logs-app1"; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + Settings settings = Settings.EMPTY; + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + { + String mapping = """ + { + "_doc": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertTrue(result); + assertThat(newMapperServiceCounter.get(), equalTo(1)); + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); + } + { + String mapping; + boolean withSourceMode = randomBoolean(); + if (withSourceMode) { + mapping = """ + { + "_doc": { + "_source": { + "mode": "stored" + }, + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + } else { + mapping = """ + { + "_doc": { + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + } + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertFalse(result); + assertThat(newMapperServiceCounter.get(), equalTo(2)); + if (withSourceMode) { + assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); + } + } + } + + public void testValidateIndexName() throws IOException { + String indexName = "validate-index-name"; + String mapping = """ + { + "_doc": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + Settings settings = Settings.EMPTY; + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertFalse(result); + } + + public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException { + String dataStreamName = "logs-app1"; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + String mapping = """ + { + "_doc": { + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + { + Settings settings = Settings.builder().put("index.mode", "logsdb").build(); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertTrue(result); + assertThat(newMapperServiceCounter.get(), equalTo(0)); + } + { + Settings settings = Settings.builder().put("index.mode", "logsdb").build(); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); + assertTrue(result); + assertThat(newMapperServiceCounter.get(), equalTo(0)); + } + { + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); + assertFalse(result); + assertThat(newMapperServiceCounter.get(), equalTo(1)); + } + { + boolean result = provider.newIndexHasSyntheticSourceUsage( + indexName, + null, + Settings.EMPTY, + List.of(new CompressedXContent(mapping)) + ); + assertFalse(result); + assertThat(newMapperServiceCounter.get(), equalTo(2)); + } + } + + public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { + String dataStreamName = "logs-app1"; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + String mapping = """ + { + "_doc": { + "properties": { + "my_field": { + "type": "keyword", + "time_series_dimension": true + } + } + } + } + """; + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + { + Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertTrue(result); + } + { + Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); + assertTrue(result); + } + { + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); + assertFalse(result); + } + { + boolean result = provider.newIndexHasSyntheticSourceUsage( + indexName, + null, + Settings.EMPTY, + List.of(new CompressedXContent(mapping)) + ); + assertFalse(result); + } + } + + public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOException { + String dataStreamName = "logs-app1"; + String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); + Settings settings = Settings.builder().put("index.soft_deletes.enabled", false).build(); + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + { + String mapping = """ + { + "_doc": { + "_source": { + "mode": "synthetic" + }, + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertFalse(result); + assertThat(newMapperServiceCounter.get(), equalTo(1)); + } + { + String mapping = """ + { + "_doc": { + "properties": { + "my_field": { + "type": "keyword" + } + } + } + } + """; + boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); + assertFalse(result); + assertThat(newMapperServiceCounter.get(), equalTo(2)); + } + } + + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException { + String dataStreamName = "logs-app1"; + Metadata.Builder mb = Metadata.builder( + DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple(dataStreamName, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ).getMetadata() + ); + Metadata metadata = mb.build(); + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(false); + Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) + .build(); + + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(0)); + assertThat(newMapperServiceCounter.get(), equalTo(0)); + + syntheticSourceLicenseService.setSyntheticSourceFallback(true); + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertThat(newMapperServiceCounter.get(), equalTo(0)); + + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + IndexMode.TIME_SERIES, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertThat(newMapperServiceCounter.get(), equalTo(0)); + + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + IndexMode.LOGSDB, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(1)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertThat(newMapperServiceCounter.get(), equalTo(0)); + } + + public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException { + syntheticSourceLicenseService.setSyntheticSourceFallback(true); + LogsdbIndexModeSettingsProvider provider = withSyntheticSourceDemotionSupport(true); + final Settings settings = Settings.EMPTY; + + String dataStreamName = "logs-app1"; + Metadata.Builder mb = Metadata.builder( + DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple(dataStreamName, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ).getMetadata() + ); + Metadata metadata = mb.build(); + Settings result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(0)); + + dataStreamName = "logs-app1-0"; + mb = Metadata.builder( + DataStreamTestHelper.getClusterStateWithDataStreams( + List.of(Tuple.tuple(dataStreamName, 1)), + List.of(), + Instant.now().toEpochMilli(), + builder().build(), + 1 + ).getMetadata() + ); + metadata = mb.build(); + + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + settings, + List.of() + ); + assertThat(result.size(), equalTo(2)); + assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); + assertEquals(IndexMode.LOGSDB, IndexSettings.MODE.get(result)); + + result = provider.getAdditionalIndexSettings( + DataStream.getDefaultBackingIndexName(dataStreamName, 2), + dataStreamName, + null, + metadata, + Instant.ofEpochMilli(1L), + builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.toString()).build(), + List.of() + ); + assertThat(result.size(), equalTo(0)); + } + } diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java similarity index 91% rename from x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java rename to x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java index c871a7d0216ed..8a4adf18b3e67 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderLegacyLicenseTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexSettingsProviderLegacyLicenseTests.java @@ -25,15 +25,14 @@ import java.time.ZoneOffset; import java.util.List; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceIndexSettingsProviderTests.getLogsdbIndexModeSettingsProvider; import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createGoldOrPlatinumLicense; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public class SyntheticSourceIndexSettingsProviderLegacyLicenseTests extends ESTestCase { +public class LogsdbIndexSettingsProviderLegacyLicenseTests extends ESTestCase { - private SyntheticSourceIndexSettingsProvider provider; + private LogsdbIndexModeSettingsProvider provider; @Before public void setup() throws Exception { @@ -50,10 +49,9 @@ public void setup() throws Exception { syntheticSourceLicenseService.setLicenseState(licenseState); syntheticSourceLicenseService.setLicenseService(mockLicenseService); - provider = new SyntheticSourceIndexSettingsProvider( - syntheticSourceLicenseService, + provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY); + provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - getLogsdbIndexModeSettingsProvider(false), IndexVersion::current ); } @@ -112,10 +110,9 @@ public void testGetAdditionalIndexSettingsTsdbAfterCutoffDate() throws Exception syntheticSourceLicenseService.setLicenseState(licenseState); syntheticSourceLicenseService.setLicenseService(mockLicenseService); - provider = new SyntheticSourceIndexSettingsProvider( - syntheticSourceLicenseService, + provider = new LogsdbIndexModeSettingsProvider(syntheticSourceLicenseService, Settings.EMPTY); + provider.init( im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - getLogsdbIndexModeSettingsProvider(false), IndexVersion::current ); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java deleted file mode 100644 index df1fb8f2d958c..0000000000000 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.logsdb; - -import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.DataStreamTestHelper; -import org.elasticsearch.cluster.metadata.Metadata; -import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.MapperTestUtils; -import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.license.License; -import org.elasticsearch.license.LicenseService; -import org.elasticsearch.license.MockLicenseState; -import org.elasticsearch.test.ESTestCase; -import org.junit.Before; - -import java.io.IOException; -import java.time.Instant; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.elasticsearch.common.settings.Settings.builder; -import static org.elasticsearch.xpack.logsdb.SyntheticSourceLicenseServiceTests.createEnterpriseLicense; -import static org.hamcrest.Matchers.equalTo; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SyntheticSourceIndexSettingsProviderTests extends ESTestCase { - - private SyntheticSourceLicenseService syntheticSourceLicenseService; - private SyntheticSourceIndexSettingsProvider provider; - private final AtomicInteger newMapperServiceCounter = new AtomicInteger(); - - static LogsdbIndexModeSettingsProvider getLogsdbIndexModeSettingsProvider(boolean enabled) { - return new LogsdbIndexModeSettingsProvider(Settings.builder().put("cluster.logsdb.enabled", enabled).build()); - } - - @Before - public void setup() throws Exception { - MockLicenseState licenseState = MockLicenseState.createMock(); - when(licenseState.isAllowed(any())).thenReturn(true); - var licenseService = new SyntheticSourceLicenseService(Settings.EMPTY); - licenseService.setLicenseState(licenseState); - var mockLicenseService = mock(LicenseService.class); - License license = createEnterpriseLicense(); - when(mockLicenseService.getLicense()).thenReturn(license); - syntheticSourceLicenseService = new SyntheticSourceLicenseService(Settings.EMPTY); - syntheticSourceLicenseService.setLicenseState(licenseState); - syntheticSourceLicenseService.setLicenseService(mockLicenseService); - - provider = new SyntheticSourceIndexSettingsProvider(syntheticSourceLicenseService, im -> { - newMapperServiceCounter.incrementAndGet(); - return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()); - }, getLogsdbIndexModeSettingsProvider(false), IndexVersion::current); - newMapperServiceCounter.set(0); - } - - public void testNewIndexHasSyntheticSourceUsage() throws IOException { - String dataStreamName = "logs-app1"; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - Settings settings = Settings.EMPTY; - { - String mapping = """ - { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertTrue(result); - assertThat(newMapperServiceCounter.get(), equalTo(1)); - assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); - } - { - String mapping; - boolean withSourceMode = randomBoolean(); - if (withSourceMode) { - mapping = """ - { - "_doc": { - "_source": { - "mode": "stored" - }, - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - } else { - mapping = """ - { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - } - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(2)); - if (withSourceMode) { - assertWarnings(SourceFieldMapper.DEPRECATION_WARNING); - } - } - } - - public void testValidateIndexName() throws IOException { - String indexName = "validate-index-name"; - String mapping = """ - { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - Settings settings = Settings.EMPTY; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertFalse(result); - } - - public void testNewIndexHasSyntheticSourceUsageLogsdbIndex() throws IOException { - String dataStreamName = "logs-app1"; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - String mapping = """ - { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - { - Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertTrue(result); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - } - { - Settings settings = Settings.builder().put("index.mode", "logsdb").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); - assertTrue(result); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - } - { - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); - assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(1)); - } - { - boolean result = provider.newIndexHasSyntheticSourceUsage( - indexName, - null, - Settings.EMPTY, - List.of(new CompressedXContent(mapping)) - ); - assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(2)); - } - } - - public void testNewIndexHasSyntheticSourceUsageTimeSeries() throws IOException { - String dataStreamName = "logs-app1"; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - String mapping = """ - { - "_doc": { - "properties": { - "my_field": { - "type": "keyword", - "time_series_dimension": true - } - } - } - } - """; - { - Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertTrue(result); - } - { - Settings settings = Settings.builder().put("index.mode", "time_series").put("index.routing_path", "my_field").build(); - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of()); - assertTrue(result); - } - { - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, Settings.EMPTY, List.of()); - assertFalse(result); - } - { - boolean result = provider.newIndexHasSyntheticSourceUsage( - indexName, - null, - Settings.EMPTY, - List.of(new CompressedXContent(mapping)) - ); - assertFalse(result); - } - } - - public void testNewIndexHasSyntheticSourceUsage_invalidSettings() throws IOException { - String dataStreamName = "logs-app1"; - String indexName = DataStream.getDefaultBackingIndexName(dataStreamName, 0); - Settings settings = Settings.builder().put("index.soft_deletes.enabled", false).build(); - { - String mapping = """ - { - "_doc": { - "_source": { - "mode": "synthetic" - }, - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(1)); - } - { - String mapping = """ - { - "_doc": { - "properties": { - "my_field": { - "type": "keyword" - } - } - } - } - """; - boolean result = provider.newIndexHasSyntheticSourceUsage(indexName, null, settings, List.of(new CompressedXContent(mapping))); - assertFalse(result); - assertThat(newMapperServiceCounter.get(), equalTo(2)); - } - } - - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSource() throws IOException { - String dataStreamName = "logs-app1"; - Metadata.Builder mb = Metadata.builder( - DataStreamTestHelper.getClusterStateWithDataStreams( - List.of(Tuple.tuple(dataStreamName, 1)), - List.of(), - Instant.now().toEpochMilli(), - builder().build(), - 1 - ).getMetadata() - ); - Metadata metadata = mb.build(); - - Settings settings = builder().put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC) - .build(); - - Settings result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - null, - metadata, - Instant.ofEpochMilli(1L), - settings, - List.of() - ); - assertThat(result.size(), equalTo(0)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - - syntheticSourceLicenseService.setSyntheticSourceFallback(true); - result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - null, - metadata, - Instant.ofEpochMilli(1L), - settings, - List.of() - ); - assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - - result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - IndexMode.TIME_SERIES, - metadata, - Instant.ofEpochMilli(1L), - settings, - List.of() - ); - assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - - result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - IndexMode.LOGSDB, - metadata, - Instant.ofEpochMilli(1L), - settings, - List.of() - ); - assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - } - - public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch() throws IOException { - syntheticSourceLicenseService.setSyntheticSourceFallback(true); - provider = new SyntheticSourceIndexSettingsProvider( - syntheticSourceLicenseService, - im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()), - getLogsdbIndexModeSettingsProvider(true), - IndexVersion::current - ); - final Settings settings = Settings.EMPTY; - - String dataStreamName = "logs-app1"; - Metadata.Builder mb = Metadata.builder( - DataStreamTestHelper.getClusterStateWithDataStreams( - List.of(Tuple.tuple(dataStreamName, 1)), - List.of(), - Instant.now().toEpochMilli(), - builder().build(), - 1 - ).getMetadata() - ); - Metadata metadata = mb.build(); - Settings result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - null, - metadata, - Instant.ofEpochMilli(1L), - settings, - List.of() - ); - assertThat(result.size(), equalTo(0)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - - dataStreamName = "logs-app1-0"; - mb = Metadata.builder( - DataStreamTestHelper.getClusterStateWithDataStreams( - List.of(Tuple.tuple(dataStreamName, 1)), - List.of(), - Instant.now().toEpochMilli(), - builder().build(), - 1 - ).getMetadata() - ); - metadata = mb.build(); - - result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - null, - metadata, - Instant.ofEpochMilli(1L), - settings, - List.of() - ); - assertThat(result.size(), equalTo(1)); - assertEquals(SourceFieldMapper.Mode.STORED, SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(result)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - - result = provider.getAdditionalIndexSettings( - DataStream.getDefaultBackingIndexName(dataStreamName, 2), - dataStreamName, - null, - metadata, - Instant.ofEpochMilli(1L), - builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.toString()).build(), - List.of() - ); - assertThat(result.size(), equalTo(0)); - assertThat(newMapperServiceCounter.get(), equalTo(0)); - } -}