-
Notifications
You must be signed in to change notification settings - Fork 138
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Manual Backport 2.17] Add feature flag for async query scheduler (#2989
) * Add feature flag for async query scheduler (#2973) * Add feature flag for async query scheduler Signed-off-by: Louis Chu <clingzhi@amazon.com> * Fix Jacoco verification Signed-off-by: Louis Chu <clingzhi@amazon.com> --------- Signed-off-by: Louis Chu <clingzhi@amazon.com> (cherry picked from commit da622eb) * Fix type mismatch Signed-off-by: Louis Chu <clingzhi@amazon.com> --------- Signed-off-by: Louis Chu <clingzhi@amazon.com>
- Loading branch information
Showing
11 changed files
with
241 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
...ain/java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.spark.config; | ||
|
||
import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED; | ||
import static org.opensearch.sql.spark.data.constants.SparkConstants.FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL; | ||
|
||
import lombok.RequiredArgsConstructor; | ||
import org.opensearch.sql.common.setting.Settings; | ||
import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; | ||
import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; | ||
import org.opensearch.sql.spark.parameter.GeneralSparkParameterComposer; | ||
import org.opensearch.sql.spark.parameter.SparkSubmitParameters; | ||
|
||
@RequiredArgsConstructor | ||
public class OpenSearchAsyncQuerySchedulerConfigComposer implements GeneralSparkParameterComposer { | ||
private final Settings settings; | ||
|
||
@Override | ||
public void compose( | ||
SparkSubmitParameters sparkSubmitParameters, | ||
DispatchQueryRequest dispatchQueryRequest, | ||
AsyncQueryRequestContext context) { | ||
Boolean externalSchedulerEnabled = | ||
settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED); | ||
String externalSchedulerInterval = | ||
settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL); | ||
sparkSubmitParameters.setConfigItem( | ||
FLINT_JOB_EXTERNAL_SCHEDULER_ENABLED, String.valueOf(externalSchedulerEnabled)); | ||
sparkSubmitParameters.setConfigItem( | ||
FLINT_JOB_EXTERNAL_SCHEDULER_INTERVAL, externalSchedulerInterval); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
...java/org/opensearch/sql/spark/config/OpenSearchAsyncQuerySchedulerConfigComposerTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
package org.opensearch.sql.spark.config; | ||
|
||
import static org.mockito.Mockito.verify; | ||
import static org.mockito.Mockito.when; | ||
|
||
import org.junit.jupiter.api.BeforeEach; | ||
import org.junit.jupiter.api.Test; | ||
import org.junit.jupiter.api.extension.ExtendWith; | ||
import org.mockito.Mock; | ||
import org.mockito.junit.jupiter.MockitoExtension; | ||
import org.opensearch.sql.common.setting.Settings; | ||
import org.opensearch.sql.spark.asyncquery.model.AsyncQueryRequestContext; | ||
import org.opensearch.sql.spark.dispatcher.model.DispatchQueryRequest; | ||
import org.opensearch.sql.spark.parameter.SparkSubmitParameters; | ||
|
||
@ExtendWith(MockitoExtension.class) | ||
public class OpenSearchAsyncQuerySchedulerConfigComposerTest { | ||
|
||
@Mock private Settings settings; | ||
@Mock private SparkSubmitParameters sparkSubmitParameters; | ||
@Mock private DispatchQueryRequest dispatchQueryRequest; | ||
@Mock private AsyncQueryRequestContext context; | ||
|
||
private OpenSearchAsyncQuerySchedulerConfigComposer composer; | ||
|
||
@BeforeEach | ||
public void setUp() { | ||
composer = new OpenSearchAsyncQuerySchedulerConfigComposer(settings); | ||
} | ||
|
||
@Test | ||
public void testCompose() { | ||
when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) | ||
.thenReturn(true); | ||
when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)) | ||
.thenReturn("10 minutes"); | ||
|
||
composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); | ||
|
||
verify(sparkSubmitParameters) | ||
.setConfigItem("spark.flint.job.externalScheduler.enabled", "true"); | ||
verify(sparkSubmitParameters) | ||
.setConfigItem("spark.flint.job.externalScheduler.interval", "10 minutes"); | ||
} | ||
|
||
@Test | ||
public void testComposeWithDisabledScheduler() { | ||
when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) | ||
.thenReturn(false); | ||
|
||
composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); | ||
|
||
verify(sparkSubmitParameters) | ||
.setConfigItem("spark.flint.job.externalScheduler.enabled", "false"); | ||
} | ||
|
||
@Test | ||
public void testComposeWithMissingInterval() { | ||
when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_ENABLED)) | ||
.thenReturn(true); | ||
when(settings.getSettingValue(Settings.Key.ASYNC_QUERY_EXTERNAL_SCHEDULER_INTERVAL)) | ||
.thenReturn(""); | ||
|
||
composer.compose(sparkSubmitParameters, dispatchQueryRequest, context); | ||
|
||
verify(sparkSubmitParameters).setConfigItem("spark.flint.job.externalScheduler.interval", ""); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters