forked from yugabyte/yugabyte-db
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[PLAT-2540] Allow for configuring WsClient used for HA using RuntimeC…
…onfiguration Summary: At high level idea is to use runtime config to set **HA** specific `wsClient ` config. This can be done by setting newly added config object `yb.ha.ws` in GLOBAL scope using existing runtime config API to set key to HOCON or JSON object. (See `PlatformInstanceClientFactoryTest.setWsConfig()` for how to do this.) This config object uses same schema and everything that the default configuration of ws client via `play.ws.ssl` and supports all the same features set (thru reuse) # Solved problem of dynamically refreshing wsclient (close current and create new with updated config) whenever config changed. # Added ability to trigger such refresh by adding API to notify on a key about config change - so far this is a strawman implementation but can be improved easily if/when more use cases present. # Given that we had to solve #1, it made sense to allow for HA specific wsClient configuration - this way we can have separate configuration for any other wsClients (better security posture) # This should also allow for HA specific higher timeouts. # Note that you can use "+=" to append certificate to existing list of certificates (This is a generic feature implemented for runtime config) # You can also use `${yb.some.config.key}` to refer to other config. Such substitution (aka config-resolution) should now work. **WS Config Reference:** https://github.com/playframework/play-ws/blob/main/play-ws-standalone/src/main/resources/reference.conf Test Plan: Added unit test (`PlatformInstanceClientFactoryTest`) for case of specific interest - ability to set certs to establish trust between HA nodes. Itests: https://jenkins.dev.yugabyte.com/job/dev-itest-pipeline/846 https://jenkins.dev.yugabyte.com/job/dev-itest-pipeline/847 Reviewers: amalyshev, spotachev Reviewed By: amalyshev, spotachev Subscribers: jenkins-bot, yugaware Differential Revision: https://phabricator.dev.yugabyte.com/D14399
- Loading branch information
Showing
15 changed files
with
389 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
52 changes: 52 additions & 0 deletions
52
managed/src/main/java/com/yugabyte/yw/common/CustomWsClientFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| /* | ||
| * Copyright 2021 YugaByte, Inc. and Contributors | ||
| * | ||
| * Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you | ||
| * may not use this file except in compliance with the License. You | ||
| * may obtain a copy of the License at | ||
| * | ||
| * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt | ||
| */ | ||
|
|
||
| package com.yugabyte.yw.common; | ||
|
|
||
| import akka.stream.Materializer; | ||
| import com.google.inject.Inject; | ||
| import com.google.inject.Singleton; | ||
| import com.typesafe.config.Config; | ||
| import java.util.concurrent.CompletableFuture; | ||
| import play.Environment; | ||
| import play.inject.ApplicationLifecycle; | ||
| import play.libs.ws.WSClient; | ||
| import play.libs.ws.ahc.AhcWSClient; | ||
| import play.libs.ws.ahc.AhcWSClientConfigFactory; | ||
|
|
||
| @Singleton | ||
| public class CustomWsClientFactory { | ||
|
|
||
| private final ApplicationLifecycle lifecycle; | ||
| private final Materializer materializer; | ||
| private final Environment environment; | ||
|
|
||
| @Inject | ||
| public CustomWsClientFactory( | ||
| ApplicationLifecycle lifecycle, Materializer materializer, Environment environment) { | ||
| this.lifecycle = lifecycle; | ||
| this.materializer = materializer; | ||
| this.environment = environment; | ||
| } | ||
|
|
||
| public WSClient forCustomConfig(Config customConfig) { | ||
| AhcWSClient customeWsClient = | ||
| AhcWSClient.create( | ||
| AhcWSClientConfigFactory.forConfig(customConfig, environment.classLoader()), | ||
| null, // no HTTP caching | ||
| materializer); | ||
| lifecycle.addStopHook( | ||
| () -> { | ||
| customeWsClient.close(); | ||
| return CompletableFuture.completedFuture(null); | ||
| }); | ||
| return customeWsClient; | ||
| } | ||
| } |
23 changes: 0 additions & 23 deletions
23
managed/src/main/java/com/yugabyte/yw/common/PlatformInstanceClientFactory.java
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
80 changes: 80 additions & 0 deletions
80
managed/src/main/java/com/yugabyte/yw/common/ha/PlatformInstanceClientFactory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| /* | ||
| * Copyright 2022 YugaByte, Inc. and Contributors | ||
| * | ||
| * Licensed under the Polyform Free Trial License 1.0.0 (the "License"); you | ||
| * may not use this file except in compliance with the License. You | ||
| * may obtain a copy of the License at | ||
| * | ||
| * http://github.com/YugaByte/yugabyte-db/blob/master/licenses/POLYFORM-FREE-TRIAL-LICENSE-1.0.0.txt | ||
| */ | ||
|
|
||
| package com.yugabyte.yw.common.ha; | ||
|
|
||
| import com.google.inject.Inject; | ||
| import com.google.inject.Singleton; | ||
| import com.typesafe.config.Config; | ||
| import com.typesafe.config.ConfigFactory; | ||
| import com.typesafe.config.ConfigRenderOptions; | ||
| import com.typesafe.config.ConfigValue; | ||
| import com.yugabyte.yw.common.ApiHelper; | ||
| import com.yugabyte.yw.common.CustomWsClientFactory; | ||
| import com.yugabyte.yw.common.config.RuntimeConfigFactory; | ||
| import java.io.IOException; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import play.libs.ws.WSClient; | ||
|
|
||
| @Singleton | ||
| @Slf4j | ||
| public class PlatformInstanceClientFactory { | ||
|
|
||
| private final CustomWsClientFactory customWsClientFactory; | ||
| private final RuntimeConfigFactory runtimeConfigFactory; | ||
| private final ApiHelper fallbackApiHelper; | ||
| private WSClient customWsClient = null; | ||
|
|
||
| @Inject | ||
| public PlatformInstanceClientFactory( | ||
| ApiHelper defaultApiHelper, | ||
| CustomWsClientFactory customWsClientFactory, | ||
| RuntimeConfigFactory runtimeConfigFactory) { | ||
| this.fallbackApiHelper = defaultApiHelper; | ||
| this.customWsClientFactory = customWsClientFactory; | ||
| this.runtimeConfigFactory = runtimeConfigFactory; | ||
| } | ||
|
|
||
| public synchronized void refreshWsClient(String haWsConfigPath) { | ||
| ConfigValue haWsOverrides = runtimeConfigFactory.globalRuntimeConf().getValue(haWsConfigPath); | ||
| log.info( | ||
| "Creating ws client with config override: {}", | ||
| haWsOverrides.render(ConfigRenderOptions.concise())); | ||
| Config customWsConfig = | ||
| ConfigFactory.empty() | ||
| .withValue("play.ws", haWsOverrides) | ||
| .withFallback(runtimeConfigFactory.staticApplicationConf()) | ||
| .withOnlyPath("play.ws"); | ||
| // Enable trace level logging to debug actual config value being resolved: | ||
| log.trace("Creating ws client with config: {}", customWsConfig.root().render()); | ||
| closePreviousClient(customWsClient); | ||
| customWsClient = customWsClientFactory.forCustomConfig(customWsConfig); | ||
| } | ||
|
|
||
| private void closePreviousClient(WSClient previousWsClient) { | ||
| if (previousWsClient != null) { | ||
| try { | ||
| previousWsClient.close(); | ||
| } catch (IOException e) { | ||
| log.warn("Exception while closing wsClient. Ignored", e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public PlatformInstanceClient getClient(String clusterKey, String remoteAddress) { | ||
| if (customWsClient == null) { | ||
| log.info("Using fallbackApiHelper"); | ||
| return new PlatformInstanceClient(fallbackApiHelper, clusterKey, remoteAddress); | ||
| } else { | ||
| log.info("Using customApiHelper"); | ||
| return new PlatformInstanceClient(new ApiHelper(customWsClient), clusterKey, remoteAddress); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.