diff --git a/.env b/.env index 8735f8708eba..b6ad3b7d07ef 100644 --- a/.env +++ b/.env @@ -47,9 +47,9 @@ SUBMITTER_NUM_THREADS=10 # Job container images # Usually you should not need to set these, they have defaults already set -JOB_POD_SOCAT_IMAGE= -JOB_POD_BUSYBOX_IMAGE= -JOB_POD_CURL_IMAGE= +JOB_KUBE_SOCAT_IMAGE= +JOB_KUBE_BUSYBOX_IMAGE= +JOB_KUBE_CURL_IMAGE= # Miscellaneous TRACKING_STRATEGY=segment @@ -73,10 +73,10 @@ S3_PATH_STYLE_ACCESS= GCS_LOG_BUCKET= # Docker Resource Limits -JOB_POD_MAIN_CONTAINER_CPU_REQUEST= -JOB_POD_MAIN_CONTAINER_CPU_LIMIT= -JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST= -JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT= +JOB_MAIN_CONTAINER_CPU_REQUEST= +JOB_MAIN_CONTAINER_CPU_LIMIT= +JOB_MAIN_CONTAINER_MEMORY_REQUEST= +JOB_MAIN_CONTAINER_MEMORY_LIMIT= # Max attempts per sync and max retries per attempt SYNC_JOB_MAX_ATTEMPTS=3 diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java index 572bef00ff60..7ee283c1821a 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java @@ -85,29 +85,29 @@ public interface Configs { int getSyncJobMaxTimeoutDays(); - List getJobPodTolerations(); + List getJobKubeTolerations(); - Map getJobPodNodeSelectors(); + Map getJobKubeNodeSelectors(); - String getJobPodMainContainerImagePullPolicy(); + String getJobKubeMainContainerImagePullPolicy(); - String getJobPodMainContainerImagePullSecret(); + String getJobKubeMainContainerImagePullSecret(); - String getJobPodSocatImage(); + String getJobKubeSocatImage(); - String getJobPodBusyboxImage(); + String getJobKubeBusyboxImage(); - String getJobPodCurlImage(); + String getJobKubeCurlImage(); - String getJobPodKubeNamespace(); + String getJobKubeNamespace(); - String getJobPodMainContainerCpuRequest(); + String getJobMainContainerCpuRequest(); - String getJobPodMainContainerCpuLimit(); + String getJobMainContainerCpuLimit(); - String getJobPodMainContainerMemoryRequest(); + String getJobMainContainerMemoryRequest(); - String getJobPodMainContainerMemoryLimit(); + String getJobMainContainerMemoryLimit(); // Logging/Monitoring/Tracking LogConfigs getLogConfigs(); diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java index 6869bf6dbf3e..4ac1df231abc 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -56,12 +56,12 @@ public class EnvConfigs implements Configs { public static final String LOG_LEVEL = "LOG_LEVEL"; public static final String S3_PATH_STYLE_ACCESS = "S3_PATH_STYLE_ACCESS"; public static final String WEBAPP_URL = "WEBAPP_URL"; - public static final String JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY = "JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY"; - public static final String JOB_POD_TOLERATIONS = "JOB_POD_TOLERATIONS"; - public static final String JOB_POD_NODE_SELECTORS = "JOB_POD_NODE_SELECTORS"; - public static final String JOB_POD_SOCAT_IMAGE = "JOB_POD_SOCAT_IMAGE"; - public static final String JOB_POD_BUSYBOX_IMAGE = "JOB_POD_BUSYBOX_IMAGE"; - public static final String JOB_POD_CURL_IMAGE = "JOB_POD_CURL_IMAGE"; + public static final String JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY = "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY"; + public static final String JOB_KUBE_TOLERATIONS = "JOB_KUBE_TOLERATIONS"; + public static final String JOB_KUBE_NODE_SELECTORS = "JOB_KUBE_NODE_SELECTORS"; + public static final String JOB_KUBE_SOCAT_IMAGE = "JOB_KUBE_SOCAT_IMAGE"; + public static final String JOB_KUBE_BUSYBOX_IMAGE = "JOB_KUBE_BUSYBOX_IMAGE"; + public static final String JOB_KUBE_CURL_IMAGE = "JOB_KUBE_CURL_IMAGE"; public static final String SYNC_JOB_MAX_ATTEMPTS = "SYNC_JOB_MAX_ATTEMPTS"; public static final String SYNC_JOB_MAX_TIMEOUT_DAYS = "SYNC_JOB_MAX_TIMEOUT_DAYS"; private static final String MINIMUM_WORKSPACE_RETENTION_DAYS = "MINIMUM_WORKSPACE_RETENTION_DAYS"; @@ -73,14 +73,14 @@ public class EnvConfigs implements Configs { public static final String MAX_SYNC_WORKERS = "MAX_SYNC_WORKERS"; private static final String TEMPORAL_HOST = "TEMPORAL_HOST"; private static final String TEMPORAL_WORKER_PORTS = "TEMPORAL_WORKER_PORTS"; - private static final String JOB_POD_KUBE_NAMESPACE = "JOB_POD_KUBE_NAMESPACE"; + private static final String JOB_KUBE_NAMESPACE = "JOB_KUBE_NAMESPACE"; private static final String SUBMITTER_NUM_THREADS = "SUBMITTER_NUM_THREADS"; - public static final String JOB_POD_MAIN_CONTAINER_CPU_REQUEST = "JOB_POD_MAIN_CONTAINER_CPU_REQUEST"; - public static final String JOB_POD_MAIN_CONTAINER_CPU_LIMIT = "JOB_POD_MAIN_CONTAINER_CPU_LIMIT"; - public static final String JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST = "JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST"; - public static final String JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT = "JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT"; + public static final String JOB_MAIN_CONTAINER_CPU_REQUEST = "JOB_MAIN_CONTAINER_CPU_REQUEST"; + public static final String JOB_MAIN_CONTAINER_CPU_LIMIT = "JOB_MAIN_CONTAINER_CPU_LIMIT"; + public static final String JOB_MAIN_CONTAINER_MEMORY_REQUEST = "JOB_MAIN_CONTAINER_MEMORY_REQUEST"; + public static final String JOB_MAIN_CONTAINER_MEMORY_LIMIT = "JOB_MAIN_CONTAINER_MEMORY_LIMIT"; private static final String SECRET_PERSISTENCE = "SECRET_PERSISTENCE"; - public static final String JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET = "JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET"; + public static final String JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET = "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET"; private static final String PUBLISH_METRICS = "PUBLISH_METRICS"; private static final String CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION = "CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION"; private static final String CONFIGS_DATABASE_INITIALIZATION_TIMEOUT_MS = "CONFIGS_DATABASE_INITIALIZATION_TIMEOUT_MS"; @@ -100,15 +100,15 @@ public class EnvConfigs implements Configs { // defaults private static final String DEFAULT_SPEC_CACHE_BUCKET = "io-airbyte-cloud-spec-cache"; - public static final String DEFAULT_JOB_POD_KUBE_NAMESPACE = "default"; - private static final String DEFAULT_JOB_POD_CPU_REQUIREMENT = null; - private static final String DEFAULT_JOB_POD_MEMORY_REQUIREMENT = null; - private static final String DEFAULT_JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY = "IfNotPresent"; + public static final String DEFAULT_JOB_KUBE_NAMESPACE = "default"; + private static final String DEFAULT_JOB_CPU_REQUIREMENT = null; + private static final String DEFAULT_JOB_MEMORY_REQUIREMENT = null; + private static final String DEFAULT_JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY = "IfNotPresent"; private static final String SECRET_STORE_GCP_PROJECT_ID = "SECRET_STORE_GCP_PROJECT_ID"; private static final String SECRET_STORE_GCP_CREDENTIALS = "SECRET_STORE_GCP_CREDENTIALS"; - private static final String DEFAULT_JOB_POD_SOCAT_IMAGE = "alpine/socat:1.7.4.1-r1"; - private static final String DEFAULT_JOB_POD_BUSYBOX_IMAGE = "busybox:1.28"; - private static final String DEFAULT_JOB_POD_CURL_IMAGE = "curlimages/curl:7.77.0"; + private static final String DEFAULT_JOB_KUBE_SOCAT_IMAGE = "alpine/socat:1.7.4.1-r1"; + private static final String DEFAULT_JOB_KUBE_BUSYBOX_IMAGE = "busybox:1.28"; + private static final String DEFAULT_JOB_KUBE_CURL_IMAGE = "curlimages/curl:7.77.0"; private static final long DEFAULT_MINIMUM_WORKSPACE_RETENTION_DAYS = 1; private static final long DEFAULT_MAXIMUM_WORKSPACE_RETENTION_DAYS = 60; private static final long DEFAULT_MAXIMUM_WORKSPACE_SIZE_MB = 5000; @@ -371,11 +371,11 @@ public int getSyncJobMaxTimeoutDays() { *

* key=airbyte-server,operator=Exists,effect=NoSchedule;key=airbyte-server,operator=Equals,value=true,effect=NoSchedule * - * @return list of WorkerPodToleration parsed from env + * @return list of WorkerKubeToleration parsed from env */ @Override - public List getJobPodTolerations() { - final String tolerationsStr = getEnvOrDefault(JOB_POD_TOLERATIONS, ""); + public List getJobKubeTolerations() { + final String tolerationsStr = getEnvOrDefault(JOB_KUBE_TOLERATIONS, ""); final Stream tolerations = Strings.isNullOrEmpty(tolerationsStr) ? Stream.of() : Splitter.on(";") @@ -419,17 +419,17 @@ private TolerationPOJO parseToleration(final String tolerationStr) { * @return map containing kv pairs of node selectors */ @Override - public Map getJobPodNodeSelectors() { + public Map getJobKubeNodeSelectors() { return Splitter.on(",") - .splitToStream(getEnvOrDefault(JOB_POD_NODE_SELECTORS, "")) + .splitToStream(getEnvOrDefault(JOB_KUBE_NODE_SELECTORS, "")) .filter(s -> !Strings.isNullOrEmpty(s) && s.contains("=")) .map(s -> s.split("=")) .collect(Collectors.toMap(s -> s[0], s -> s[1])); } @Override - public String getJobPodMainContainerImagePullPolicy() { - return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY, DEFAULT_JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY); + public String getJobKubeMainContainerImagePullPolicy() { + return getEnvOrDefault(JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY, DEFAULT_JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY); } /** @@ -438,48 +438,48 @@ public String getJobPodMainContainerImagePullPolicy() { * no-op value. */ @Override - public String getJobPodMainContainerImagePullSecret() { - return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET, ""); + public String getJobKubeMainContainerImagePullSecret() { + return getEnvOrDefault(JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET, ""); } @Override - public String getJobPodSocatImage() { - return getEnvOrDefault(JOB_POD_SOCAT_IMAGE, DEFAULT_JOB_POD_SOCAT_IMAGE); + public String getJobKubeSocatImage() { + return getEnvOrDefault(JOB_KUBE_SOCAT_IMAGE, DEFAULT_JOB_KUBE_SOCAT_IMAGE); } @Override - public String getJobPodBusyboxImage() { - return getEnvOrDefault(JOB_POD_BUSYBOX_IMAGE, DEFAULT_JOB_POD_BUSYBOX_IMAGE); + public String getJobKubeBusyboxImage() { + return getEnvOrDefault(JOB_KUBE_BUSYBOX_IMAGE, DEFAULT_JOB_KUBE_BUSYBOX_IMAGE); } @Override - public String getJobPodCurlImage() { - return getEnvOrDefault(JOB_POD_CURL_IMAGE, DEFAULT_JOB_POD_CURL_IMAGE); + public String getJobKubeCurlImage() { + return getEnvOrDefault(JOB_KUBE_CURL_IMAGE, DEFAULT_JOB_KUBE_CURL_IMAGE); } @Override - public String getJobPodKubeNamespace() { - return getEnvOrDefault(JOB_POD_KUBE_NAMESPACE, DEFAULT_JOB_POD_KUBE_NAMESPACE); + public String getJobKubeNamespace() { + return getEnvOrDefault(JOB_KUBE_NAMESPACE, DEFAULT_JOB_KUBE_NAMESPACE); } @Override - public String getJobPodMainContainerCpuRequest() { - return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_CPU_REQUEST, DEFAULT_JOB_POD_CPU_REQUIREMENT); + public String getJobMainContainerCpuRequest() { + return getEnvOrDefault(JOB_MAIN_CONTAINER_CPU_REQUEST, DEFAULT_JOB_CPU_REQUIREMENT); } @Override - public String getJobPodMainContainerCpuLimit() { - return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_CPU_LIMIT, DEFAULT_JOB_POD_CPU_REQUIREMENT); + public String getJobMainContainerCpuLimit() { + return getEnvOrDefault(JOB_MAIN_CONTAINER_CPU_LIMIT, DEFAULT_JOB_CPU_REQUIREMENT); } @Override - public String getJobPodMainContainerMemoryRequest() { - return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST, DEFAULT_JOB_POD_MEMORY_REQUIREMENT); + public String getJobMainContainerMemoryRequest() { + return getEnvOrDefault(JOB_MAIN_CONTAINER_MEMORY_REQUEST, DEFAULT_JOB_MEMORY_REQUIREMENT); } @Override - public String getJobPodMainContainerMemoryLimit() { - return getEnvOrDefault(JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT, DEFAULT_JOB_POD_MEMORY_REQUIREMENT); + public String getJobMainContainerMemoryLimit() { + return getEnvOrDefault(JOB_MAIN_CONTAINER_MEMORY_LIMIT, DEFAULT_JOB_MEMORY_REQUIREMENT); } @Override diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java index 75ded59cf3c2..726ea236c314 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java @@ -163,45 +163,45 @@ void testTrackingStrategy() { } @Test - void testWorkerPodTolerations() { - when(function.apply(EnvConfigs.JOB_POD_TOLERATIONS)).thenReturn(null); - Assertions.assertEquals(config.getJobPodTolerations(), List.of()); + void testworkerKubeTolerations() { + when(function.apply(EnvConfigs.JOB_KUBE_TOLERATIONS)).thenReturn(null); + Assertions.assertEquals(config.getJobKubeTolerations(), List.of()); - when(function.apply(EnvConfigs.JOB_POD_TOLERATIONS)).thenReturn(";;;"); - Assertions.assertEquals(config.getJobPodTolerations(), List.of()); + when(function.apply(EnvConfigs.JOB_KUBE_TOLERATIONS)).thenReturn(";;;"); + Assertions.assertEquals(config.getJobKubeTolerations(), List.of()); - when(function.apply(EnvConfigs.JOB_POD_TOLERATIONS)).thenReturn("key=k,value=v;"); - Assertions.assertEquals(config.getJobPodTolerations(), List.of()); + when(function.apply(EnvConfigs.JOB_KUBE_TOLERATIONS)).thenReturn("key=k,value=v;"); + Assertions.assertEquals(config.getJobKubeTolerations(), List.of()); - when(function.apply(EnvConfigs.JOB_POD_TOLERATIONS)).thenReturn("key=airbyte-server,operator=Exists,effect=NoSchedule"); - Assertions.assertEquals(config.getJobPodTolerations(), List.of(new TolerationPOJO("airbyte-server", "NoSchedule", null, "Exists"))); + when(function.apply(EnvConfigs.JOB_KUBE_TOLERATIONS)).thenReturn("key=airbyte-server,operator=Exists,effect=NoSchedule"); + Assertions.assertEquals(config.getJobKubeTolerations(), List.of(new TolerationPOJO("airbyte-server", "NoSchedule", null, "Exists"))); - when(function.apply(EnvConfigs.JOB_POD_TOLERATIONS)).thenReturn("key=airbyte-server,operator=Equals,value=true,effect=NoSchedule"); - Assertions.assertEquals(config.getJobPodTolerations(), List.of(new TolerationPOJO("airbyte-server", "NoSchedule", "true", "Equals"))); + when(function.apply(EnvConfigs.JOB_KUBE_TOLERATIONS)).thenReturn("key=airbyte-server,operator=Equals,value=true,effect=NoSchedule"); + Assertions.assertEquals(config.getJobKubeTolerations(), List.of(new TolerationPOJO("airbyte-server", "NoSchedule", "true", "Equals"))); - when(function.apply(EnvConfigs.JOB_POD_TOLERATIONS)) + when(function.apply(EnvConfigs.JOB_KUBE_TOLERATIONS)) .thenReturn("key=airbyte-server,operator=Exists,effect=NoSchedule;key=airbyte-server,operator=Equals,value=true,effect=NoSchedule"); - Assertions.assertEquals(config.getJobPodTolerations(), List.of( + Assertions.assertEquals(config.getJobKubeTolerations(), List.of( new TolerationPOJO("airbyte-server", "NoSchedule", null, "Exists"), new TolerationPOJO("airbyte-server", "NoSchedule", "true", "Equals"))); } @Test - void testWorkerPodNodeSelectors() { - when(function.apply(EnvConfigs.JOB_POD_NODE_SELECTORS)).thenReturn(null); - Assertions.assertEquals(config.getJobPodNodeSelectors(), Map.of()); + void testworkerKubeNodeSelectors() { + when(function.apply(EnvConfigs.JOB_KUBE_NODE_SELECTORS)).thenReturn(null); + Assertions.assertEquals(config.getJobKubeNodeSelectors(), Map.of()); - when(function.apply(EnvConfigs.JOB_POD_NODE_SELECTORS)).thenReturn(",,,"); - Assertions.assertEquals(config.getJobPodNodeSelectors(), Map.of()); + when(function.apply(EnvConfigs.JOB_KUBE_NODE_SELECTORS)).thenReturn(",,,"); + Assertions.assertEquals(config.getJobKubeNodeSelectors(), Map.of()); - when(function.apply(EnvConfigs.JOB_POD_NODE_SELECTORS)).thenReturn("key=k,,;$%&^#"); - Assertions.assertEquals(config.getJobPodNodeSelectors(), Map.of("key", "k")); + when(function.apply(EnvConfigs.JOB_KUBE_NODE_SELECTORS)).thenReturn("key=k,,;$%&^#"); + Assertions.assertEquals(config.getJobKubeNodeSelectors(), Map.of("key", "k")); - when(function.apply(EnvConfigs.JOB_POD_NODE_SELECTORS)).thenReturn("one=two"); - Assertions.assertEquals(config.getJobPodNodeSelectors(), Map.of("one", "two")); + when(function.apply(EnvConfigs.JOB_KUBE_NODE_SELECTORS)).thenReturn("one=two"); + Assertions.assertEquals(config.getJobKubeNodeSelectors(), Map.of("one", "two")); - when(function.apply(EnvConfigs.JOB_POD_NODE_SELECTORS)).thenReturn("airbyte=server,something=nothing"); - Assertions.assertEquals(config.getJobPodNodeSelectors(), Map.of("airbyte", "server", "something", "nothing")); + when(function.apply(EnvConfigs.JOB_KUBE_NODE_SELECTORS)).thenReturn("airbyte=server,something=nothing"); + Assertions.assertEquals(config.getJobKubeNodeSelectors(), Map.of("airbyte", "server", "something", "nothing")); } } diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index 9117b4f96ecf..8710b21416dd 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -164,13 +164,13 @@ private static ProcessFactory getProcessBuilderFactory(final Configs configs, fi final KubernetesClient fabricClient = new DefaultKubernetesClient(); final String localIp = InetAddress.getLocalHost().getHostAddress(); final String kubeHeartbeatUrl = localIp + ":" + WorkerApp.KUBE_HEARTBEAT_PORT; - LOGGER.info("Using Kubernetes namespace: {}", configs.getJobPodKubeNamespace()); + LOGGER.info("Using Kubernetes namespace: {}", configs.getJobKubeNamespace()); // this needs to have two ports for the source and two ports for the destination (all four must be // exposed) KubePortManagerSingleton.init(ReplicationLauncherWorker.PORTS); - return new KubeProcessFactory(workerConfigs, configs.getJobPodKubeNamespace(), fabricClient, kubeHeartbeatUrl, false); + return new KubeProcessFactory(workerConfigs, configs.getJobKubeNamespace(), fabricClient, kubeHeartbeatUrl, false); } else { return new DockerProcessFactory( workerConfigs, diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-32-0.env b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-32-0.env index 968a1f65b695..2dd0169e2146 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-32-0.env +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/env-file-migration-test-0-32-0.env @@ -45,9 +45,9 @@ SUBMITTER_NUM_THREADS=10 # Job container images # Usually you should not need to set these, they have defaults already set -JOB_SOCAT_IMAGE= -JOB_BUSYBOX_IMAGE= -JOB_CURL_IMAGE= +JOB_KUBE_SOCAT_IMAGE= +JOB_KUBE_BUSYBOX_IMAGE= +JOB_KUBE_CURL_IMAGE= # Miscellaneous TRACKING_STRATEGY=segment diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index 16f4c1985e77..c5e8d0545f33 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -224,8 +224,8 @@ private static ProcessFactory getJobProcessFactory(final Configs configs) throws final KubernetesClient fabricClient = new DefaultKubernetesClient(); final String localIp = InetAddress.getLocalHost().getHostAddress(); final String kubeHeartbeatUrl = localIp + ":" + KUBE_HEARTBEAT_PORT; - LOGGER.info("Using Kubernetes namespace: {}", configs.getJobPodKubeNamespace()); - return new KubeProcessFactory(workerConfigs, configs.getJobPodKubeNamespace(), fabricClient, kubeHeartbeatUrl, false); + LOGGER.info("Using Kubernetes namespace: {}", configs.getJobKubeNamespace()); + return new KubeProcessFactory(workerConfigs, configs.getJobKubeNamespace(), fabricClient, kubeHeartbeatUrl, false); } else { return new DockerProcessFactory( workerConfigs, @@ -244,8 +244,8 @@ private static ProcessFactory getOrchestratorProcessFactory(final Configs config final KubernetesClient fabricClient = new DefaultKubernetesClient(); final String localIp = InetAddress.getLocalHost().getHostAddress(); final String kubeHeartbeatUrl = localIp + ":" + KUBE_HEARTBEAT_PORT; - LOGGER.info("Using Kubernetes namespace: {}", configs.getJobPodKubeNamespace()); - return new KubeProcessFactory(workerConfigs, configs.getJobPodKubeNamespace(), fabricClient, kubeHeartbeatUrl, true); + LOGGER.info("Using Kubernetes namespace: {}", configs.getJobKubeNamespace()); + return new KubeProcessFactory(workerConfigs, configs.getJobKubeNamespace(), fabricClient, kubeHeartbeatUrl, true); } else { return new DockerProcessFactory( workerConfigs, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerConfigs.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerConfigs.java index b7c97597315a..24dc126f4018 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerConfigs.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerConfigs.java @@ -14,8 +14,8 @@ public class WorkerConfigs { private final Configs.WorkerEnvironment workerEnvironment; private final ResourceRequirements resourceRequirements; - private final List workerPodTolerations; - private final Map workerPodNodeSelectors; + private final List workerKubeTolerations; + private final Map workerKubeNodeSelectors; private final String jobImagePullSecret; private final String jobImagePullPolicy; private final String jobSocatImage; @@ -25,17 +25,17 @@ public class WorkerConfigs { public WorkerConfigs(final Configs configs) { this.workerEnvironment = configs.getWorkerEnvironment(); this.resourceRequirements = new ResourceRequirements() - .withCpuRequest(configs.getJobPodMainContainerCpuRequest()) - .withCpuLimit(configs.getJobPodMainContainerCpuLimit()) - .withMemoryRequest(configs.getJobPodMainContainerMemoryRequest()) - .withMemoryLimit(configs.getJobPodMainContainerMemoryLimit()); - this.workerPodTolerations = configs.getJobPodTolerations(); - this.workerPodNodeSelectors = configs.getJobPodNodeSelectors(); - this.jobImagePullSecret = configs.getJobPodMainContainerImagePullSecret(); - this.jobImagePullPolicy = configs.getJobPodMainContainerImagePullPolicy(); - this.jobSocatImage = configs.getJobPodSocatImage(); - this.jobBusyboxImage = configs.getJobPodBusyboxImage(); - this.jobCurlImage = configs.getJobPodCurlImage(); + .withCpuRequest(configs.getJobMainContainerCpuRequest()) + .withCpuLimit(configs.getJobMainContainerCpuLimit()) + .withMemoryRequest(configs.getJobMainContainerMemoryRequest()) + .withMemoryLimit(configs.getJobMainContainerMemoryLimit()); + this.workerKubeTolerations = configs.getJobKubeTolerations(); + this.workerKubeNodeSelectors = configs.getJobKubeNodeSelectors(); + this.jobImagePullSecret = configs.getJobKubeMainContainerImagePullSecret(); + this.jobImagePullPolicy = configs.getJobKubeMainContainerImagePullPolicy(); + this.jobSocatImage = configs.getJobKubeSocatImage(); + this.jobBusyboxImage = configs.getJobKubeBusyboxImage(); + this.jobCurlImage = configs.getJobKubeCurlImage(); } public Configs.WorkerEnvironment getWorkerEnvironment() { @@ -46,12 +46,12 @@ public ResourceRequirements getResourceRequirements() { return resourceRequirements; } - public List getWorkerPodTolerations() { - return workerPodTolerations; + public List getWorkerKubeTolerations() { + return workerKubeTolerations; } - public Map getWorkerPodNodeSelectors() { - return workerPodNodeSelectors; + public Map getworkerKubeNodeSelectors() { + return workerKubeNodeSelectors; } public String getJobImagePullSecret() { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java index 9139cdaa8aba..89336507d605 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubeProcessFactory.java @@ -141,8 +141,8 @@ public Process create(final String jobId, entrypoint, resourceRequirements, workerConfigs.getJobImagePullSecret(), - workerConfigs.getWorkerPodTolerations(), - workerConfigs.getWorkerPodNodeSelectors(), + workerConfigs.getWorkerKubeTolerations(), + workerConfigs.getworkerKubeNodeSelectors(), allLabels, workerConfigs.getJobSocatImage(), workerConfigs.getJobBusyboxImage(), diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationLauncherWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationLauncherWorker.java index 0e59dda973ef..4efb5dfb825d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationLauncherWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/ReplicationLauncherWorker.java @@ -63,22 +63,22 @@ public class ReplicationLauncherWorker implements Worker ENV_VARS_TO_TRANSFER = Set.of( EnvConfigs.WORKER_ENVIRONMENT, - EnvConfigs.JOB_POD_TOLERATIONS, - EnvConfigs.JOB_POD_CURL_IMAGE, - EnvConfigs.JOB_POD_BUSYBOX_IMAGE, - EnvConfigs.JOB_POD_SOCAT_IMAGE, - EnvConfigs.JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY, - EnvConfigs.JOB_POD_MAIN_CONTAINER_IMAGE_PULL_SECRET, - EnvConfigs.JOB_POD_NODE_SELECTORS, + EnvConfigs.JOB_KUBE_TOLERATIONS, + EnvConfigs.JOB_KUBE_CURL_IMAGE, + EnvConfigs.JOB_KUBE_BUSYBOX_IMAGE, + EnvConfigs.JOB_KUBE_SOCAT_IMAGE, + EnvConfigs.JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY, + EnvConfigs.JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET, + EnvConfigs.JOB_KUBE_NODE_SELECTORS, EnvConfigs.DOCKER_NETWORK, EnvConfigs.LOCAL_DOCKER_MOUNT, EnvConfigs.WORKSPACE_DOCKER_MOUNT, EnvConfigs.WORKSPACE_ROOT, - EnvConfigs.DEFAULT_JOB_POD_KUBE_NAMESPACE, - EnvConfigs.JOB_POD_MAIN_CONTAINER_CPU_REQUEST, - EnvConfigs.JOB_POD_MAIN_CONTAINER_CPU_LIMIT, - EnvConfigs.JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST, - EnvConfigs.JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT, + EnvConfigs.DEFAULT_JOB_KUBE_NAMESPACE, + EnvConfigs.JOB_MAIN_CONTAINER_CPU_REQUEST, + EnvConfigs.JOB_MAIN_CONTAINER_CPU_LIMIT, + EnvConfigs.JOB_MAIN_CONTAINER_MEMORY_REQUEST, + EnvConfigs.JOB_MAIN_CONTAINER_MEMORY_LIMIT, EnvConfigs.LOCAL_ROOT); private final AtomicBoolean cancelled = new AtomicBoolean(false); diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 5653a0838a11..a7f3d79071f6 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -22,10 +22,10 @@ data: GOOGLE_APPLICATION_CREDENTIALS: {{ .Values.logs.gcs.credentials | quote }} INTERNAL_API_HOST: {{ include "common.names.fullname" . }}-server:{{ .Values.server.service.port }} IS_DEMO: {{ ternary "true" "false" .Values.webapp.isDemo | quote }} - JOB_POD_MAIN_CONTAINER_CPU_LIMIT: "" - JOB_POD_MAIN_CONTAINER_CPU_REQUEST: "" - JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT: "" - JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST: "" + JOB_MAIN_CONTAINER_CPU_LIMIT: "" + JOB_MAIN_CONTAINER_CPU_REQUEST: "" + JOB_MAIN_CONTAINER_MEMORY_LIMIT: "" + JOB_MAIN_CONTAINER_MEMORY_REQUEST: "" JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION: "0.29.15.001" LOCAL_ROOT: /tmp/airbyte_local RUN_DATABASE_MIGRATION_ON_STARTUP: "true" diff --git a/charts/airbyte/templates/scheduler/deployment.yaml b/charts/airbyte/templates/scheduler/deployment.yaml index 99986b00ca75..d49e3da29c9c 100644 --- a/charts/airbyte/templates/scheduler/deployment.yaml +++ b/charts/airbyte/templates/scheduler/deployment.yaml @@ -112,7 +112,7 @@ spec: key: TEMPORAL_WORKER_PORTS - name: LOG_LEVEL value: "{{ .Values.scheduler.log.level }}" - - name: JOB_POD_KUBE_NAMESPACE + - name: JOB_KUBE_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace @@ -121,26 +121,26 @@ spec: configMapKeyRef: name: airbyte-env key: SUBMITTER_NUM_THREADS - - name: JOB_POD_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_POD_MAIN_CONTAINER_CPU_LIMIT + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml index fe87f8595266..92b585dabd62 100644 --- a/charts/airbyte/templates/server/deployment.yaml +++ b/charts/airbyte/templates/server/deployment.yaml @@ -92,26 +92,26 @@ spec: key: TEMPORAL_HOST - name: LOG_LEVEL value: "{{ .Values.server.log.level }}" - - name: JOB_POD_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_POD_MAIN_CONTAINER_CPU_LIMIT + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: diff --git a/charts/airbyte/templates/worker/deployment.yaml b/charts/airbyte/templates/worker/deployment.yaml index dfa44a27ce7b..d62546dd918a 100644 --- a/charts/airbyte/templates/worker/deployment.yaml +++ b/charts/airbyte/templates/worker/deployment.yaml @@ -113,7 +113,7 @@ spec: key: TEMPORAL_WORKER_PORTS - name: LOG_LEVEL value: "{{ .Values.worker.log.level }}" - - name: JOB_POD_KUBE_NAMESPACE + - name: JOB_KUBE_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace @@ -122,26 +122,26 @@ spec: configMapKeyRef: name: airbyte-env key: SUBMITTER_NUM_THREADS - - name: JOB_POD_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_POD_MAIN_CONTAINER_CPU_LIMIT + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 4e79650b20b5..914d0d6f145c 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -433,7 +433,7 @@ worker: ## Example: ## ## extraEnv: - ## - name: JOB_POD_TOLERATIONS + ## - name: JOB_KUBE_TOLERATIONS ## value: "key=airbyte-server,operator=Equals,value=true,effect=NoSchedule" extraEnv: [] diff --git a/docker-compose.yaml b/docker-compose.yaml index cfab010b50eb..5fab17423c7f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -77,10 +77,10 @@ services: - GCS_LOG_BUCKET=${GCS_LOG_BUCKET} - LOG_LEVEL=${LOG_LEVEL} - SUBMITTER_NUM_THREADS=${SUBMITTER_NUM_THREADS} - - JOB_POD_MAIN_CONTAINER_CPU_REQUEST=${JOB_POD_MAIN_CONTAINER_CPU_REQUEST} - - JOB_POD_MAIN_CONTAINER_CPU_LIMIT=${JOB_POD_MAIN_CONTAINER_CPU_LIMIT} - - JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST} - - JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT} + - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST} + - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT} + - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST} + - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT} - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS} - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS} - INTERNAL_API_HOST=${INTERNAL_API_HOST} @@ -120,10 +120,10 @@ services: - GCS_LOG_BUCKET=${GCS_LOG_BUCKET} - LOG_LEVEL=${LOG_LEVEL} - SUBMITTER_NUM_THREADS=${SUBMITTER_NUM_THREADS} - - JOB_POD_MAIN_CONTAINER_CPU_REQUEST=${JOB_POD_MAIN_CONTAINER_CPU_REQUEST} - - JOB_POD_MAIN_CONTAINER_CPU_LIMIT=${JOB_POD_MAIN_CONTAINER_CPU_LIMIT} - - JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST} - - JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT} + - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST} + - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT} + - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST} + - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT} - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS} - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS} - INTERNAL_API_HOST=${INTERNAL_API_HOST} @@ -159,10 +159,10 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - GCS_LOG_BUCKET=${GCS_LOG_BUCKET} - LOG_LEVEL=${LOG_LEVEL} - - JOB_POD_MAIN_CONTAINER_CPU_REQUEST=${JOB_POD_MAIN_CONTAINER_CPU_REQUEST} - - JOB_POD_MAIN_CONTAINER_CPU_LIMIT=${JOB_POD_MAIN_CONTAINER_CPU_LIMIT} - - JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST} - - JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT} + - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST} + - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT} + - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST} + - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT} - SECRET_PERSISTENCE=${SECRET_PERSISTENCE} - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-} - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-} diff --git a/docs/operator-guides/scaling-airbyte.md b/docs/operator-guides/scaling-airbyte.md index 2cfcee79e6ef..9bfa0eade16d 100644 --- a/docs/operator-guides/scaling-airbyte.md +++ b/docs/operator-guides/scaling-airbyte.md @@ -27,10 +27,10 @@ As mentioned above, we are mainly concerned with scaling Sync jobs. Within a Syn This is because the Source worker reads up to 10,000 records in memory. This can present problems for database sources with tables that have large row sizes. e.g. a table with an average row size of 0.5MBs will require 0.5 \* 10000 / 1000 = 5GBs of RAM. See [this issue](https://github.com/airbytehq/airbyte/issues/3439) for more information. Our Java connectors currently follow Java's default behaviour with container memory and will only use up to 1/4 of the host's allocated memory. e.g. On a Docker agent with 8GBs of RAM configured, a Java connector limits itself to 2Gbs of RAM and will see Out-of-Memory exceptions if this goes higher. The same applies to Kubernetes pods. +You may want to customize this by setting `JOB_MAIN_CONTAINER_MEMORY_REQUEST` and `JOB_MAIN_CONTAINER_MEMORY_LIMIT` environment variables to custom values. Note that all Source database connectors are Java connectors. This means that users currently need to over-specify memory resource for Java connectors. -Improving this behaviour is on our roadmap. Please see [this issue](https://github.com/airbytehq/airbyte/issues/3440) for more information. ### Disk Space diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index e7604b9f7875..4399e074bbb7 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -52,14 +52,14 @@ STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-state-storage STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 # Docker Resource Limits -JOB_POD_MAIN_CONTAINER_CPU_REQUEST= -JOB_POD_MAIN_CONTAINER_CPU_LIMIT= -JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST= -JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT= +JOB_MAIN_CONTAINER_CPU_REQUEST= +JOB_MAIN_CONTAINER_CPU_LIMIT= +JOB_MAIN_CONTAINER_MEMORY_REQUEST= +JOB_MAIN_CONTAINER_MEMORY_LIMIT= # Worker pod tolerations and node selectors -JOB_POD_TOLERATIONS= -JOB_POD_NODE_SELECTORS= +JOB_KUBE_TOLERATIONS= +JOB_KUBE_NODE_SELECTORS= # Job image pull policy -JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY= +JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index ef0506c3f23e..b9e83e1c34f7 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -54,14 +54,14 @@ STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-state-storage STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 # Docker Resource Limits -JOB_POD_MAIN_CONTAINER_CPU_REQUEST= -JOB_POD_MAIN_CONTAINER_CPU_LIMIT= -JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST= -JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT= +JOB_MAIN_CONTAINER_CPU_REQUEST= +JOB_MAIN_CONTAINER_CPU_LIMIT= +JOB_MAIN_CONTAINER_MEMORY_REQUEST= +JOB_MAIN_CONTAINER_MEMORY_LIMIT= # Worker pod tolerations and node selectors -JOB_POD_TOLERATIONS= -JOB_POD_NODE_SELECTORS= +JOB_KUBE_TOLERATIONS= +JOB_KUBE_NODE_SELECTORS= # Job image pull policy -JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY= +JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 615998e4971c..d9448bbceace 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -54,14 +54,14 @@ STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-state-storage STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 # Docker Resource Limits -JOB_POD_MAIN_CONTAINER_CPU_REQUEST= -JOB_POD_MAIN_CONTAINER_CPU_LIMIT= -JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST= -JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT= +JOB_MAIN_CONTAINER_CPU_REQUEST= +JOB_MAIN_CONTAINER_CPU_LIMIT= +JOB_MAIN_CONTAINER_MEMORY_REQUEST= +JOB_MAIN_CONTAINER_MEMORY_LIMIT= # Worker pod tolerations and node selectors -JOB_POD_TOLERATIONS= -JOB_POD_NODE_SELECTORS= +JOB_KUBE_TOLERATIONS= +JOB_KUBE_NODE_SELECTORS= # Job image pull policy -JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY= +JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 615998e4971c..d9448bbceace 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -54,14 +54,14 @@ STATE_STORAGE_MINIO_BUCKET_NAME=airbyte-state-storage STATE_STORAGE_MINIO_ENDPOINT=http://airbyte-minio-svc:9000 # Docker Resource Limits -JOB_POD_MAIN_CONTAINER_CPU_REQUEST= -JOB_POD_MAIN_CONTAINER_CPU_LIMIT= -JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST= -JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT= +JOB_MAIN_CONTAINER_CPU_REQUEST= +JOB_MAIN_CONTAINER_CPU_LIMIT= +JOB_MAIN_CONTAINER_MEMORY_REQUEST= +JOB_MAIN_CONTAINER_MEMORY_LIMIT= # Worker pod tolerations and node selectors -JOB_POD_TOLERATIONS= -JOB_POD_NODE_SELECTORS= +JOB_KUBE_TOLERATIONS= +JOB_KUBE_NODE_SELECTORS= # Job image pull policy -JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY= +JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY= diff --git a/kube/resources/scheduler.yaml b/kube/resources/scheduler.yaml index 67cdd66298f0..82457cafd94a 100644 --- a/kube/resources/scheduler.yaml +++ b/kube/resources/scheduler.yaml @@ -93,7 +93,7 @@ spec: configMapKeyRef: name: airbyte-env key: LOG_LEVEL - - name: JOB_POD_KUBE_NAMESPACE + - name: JOB_KUBE_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace @@ -102,26 +102,26 @@ spec: configMapKeyRef: name: airbyte-env key: SUBMITTER_NUM_THREADS - - name: JOB_POD_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_POD_MAIN_CONTAINER_CPU_LIMIT + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: diff --git a/kube/resources/server.yaml b/kube/resources/server.yaml index 9c42d48f4def..02a7c3b87550 100644 --- a/kube/resources/server.yaml +++ b/kube/resources/server.yaml @@ -83,26 +83,26 @@ spec: configMapKeyRef: name: airbyte-env key: LOG_LEVEL - - name: JOB_POD_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_POD_MAIN_CONTAINER_CPU_LIMIT + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: diff --git a/kube/resources/worker.yaml b/kube/resources/worker.yaml index 80666b7b19d5..0e051e1bfaf8 100644 --- a/kube/resources/worker.yaml +++ b/kube/resources/worker.yaml @@ -95,7 +95,7 @@ spec: configMapKeyRef: name: airbyte-env key: LOG_LEVEL - - name: JOB_POD_KUBE_NAMESPACE + - name: JOB_KUBE_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace @@ -104,26 +104,26 @@ spec: configMapKeyRef: name: airbyte-env key: SUBMITTER_NUM_THREADS - - name: JOB_POD_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_REQUEST - - name: JOB_POD_MAIN_CONTAINER_CPU_LIMIT + key: JOB_MAIN_CONTAINER_CPU_REQUEST + - name: JOB_MAIN_CONTAINER_CPU_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_CPU_LIMIT - - name: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST + key: JOB_MAIN_CONTAINER_CPU_LIMIT + - name: JOB_MAIN_CONTAINER_MEMORY_REQUEST valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_REQUEST - - name: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_REQUEST + - name: JOB_MAIN_CONTAINER_MEMORY_LIMIT valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_MEMORY_LIMIT + key: JOB_MAIN_CONTAINER_MEMORY_LIMIT - name: S3_LOG_BUCKET valueFrom: configMapKeyRef: @@ -169,21 +169,21 @@ spec: configMapKeyRef: name: airbyte-env key: INTERNAL_API_HOST - - name: JOB_POD_TOLERATIONS + - name: JOB_KUBE_TOLERATIONS valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_TOLERATIONS - - name: JOB_POD_NODE_SELECTORS + key: JOB_KUBE_TOLERATIONS + - name: JOB_KUBE_NODE_SELECTORS valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_NODE_SELECTORS - - name: JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY + key: JOB_KUBE_NODE_SELECTORS + - name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY valueFrom: configMapKeyRef: name: airbyte-env - key: JOB_POD_MAIN_CONTAINER_IMAGE_PULL_POLICY + key: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY ports: - containerPort: 9000 # for heartbeat server - containerPort: 9001 # start temporal worker port pool