From a180a1040e2021c2976c83cb3ac968a9cfe3bfba Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 2 Mar 2023 17:53:38 +0100 Subject: [PATCH 01/20] feat(deployment-replica-runtime-config): look for replica node in deployment spec refs: #112 --- .../io/datacater/core/deployment/K8Deployment.java | 14 +++++++++++++- .../io/datacater/core/deployment/StaticConfig.java | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index ba0a9d48..bec30021 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -38,11 +38,13 @@ public Map create( StreamEntity streamOut, DeploymentSpec deploymentSpec, UUID deploymentId) { + final String name = StaticConfig.DEPLOYMENT_NAME_PREFIX + deploymentId; final String configmapName = StaticConfig.CONFIGMAP_NAME_PREFIX + deploymentId; final String configmapVolumeName = StaticConfig.CONFIGMAP_VOLUME_NAME_PREFIX + deploymentId; final String dataShareVolumeName = StaticConfig.DATA_SHARE_VOLUME_NAME_PREFIX + deploymentId; final String serviceName = StaticConfig.SERVICE_NAME_PREFIX + deploymentId; + final int replicaCount = getDeploymentReplicaOrDefault(deploymentSpec.deployment()); List variables = getEnvironmentVariables(streamIn, streamOut, deploymentSpec, deploymentId); @@ -55,7 +57,7 @@ public Map create( .addToLabels(getLabels(deploymentId, deploymentSpec.name(), serviceName)) .endMetadata() .withNewSpec() - .withReplicas(StaticConfig.EnvironmentVariables.REPLICAS) + .withReplicas(replicaCount) .withMinReadySeconds(StaticConfig.EnvironmentVariables.READY_SECONDS) .withNewSelector() .addToMatchLabels(getLabels(deploymentId, deploymentSpec.name(), serviceName)) @@ -456,4 +458,14 @@ private String getEnvVariableFromNode(JsonNode node, String field) { } return StaticConfig.EMPTY_STRING; } + + private int getDeploymentReplicaOrDefault(Map map){ + int replica = StaticConfig.EnvironmentVariables.REPLICAS; + try{ + replica = (int) map.get(StaticConfig.REPLICAS_TEXT); + } catch (Exception e){ + return replica; + } + return replica; + } } diff --git a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java index 73d7edf2..31012936 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java @@ -80,6 +80,7 @@ static Map getLimits() { static final String CONDITIONS = "Conditions"; static final String ERROR_TAG = "error"; static final String MESSAGE_TAG = "message"; + static final String REPLICAS_TEXT = "replicas"; static class EnvironmentVariables { private EnvironmentVariables() {} From 7b778b57ea4a3e3da76fcb70a88b9b60c64d6144 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Fri, 3 Mar 2023 10:30:59 +0100 Subject: [PATCH 02/20] feat(deployment-replica-runtime-config): add test for deployment relpicas refs: #112 --- .../core/deployment/K8Deployment.java | 6 +-- .../DatacaterDeploymentEndpointTest.java | 50 +++++++++++++++++-- .../deployment_with_custom_replicas.json | 6 +++ 3 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 platform-api/src/test/resources/deploymentTests/deployment_with_custom_replicas.json diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index bec30021..730bddda 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -459,11 +459,11 @@ private String getEnvVariableFromNode(JsonNode node, String field) { return StaticConfig.EMPTY_STRING; } - private int getDeploymentReplicaOrDefault(Map map){ + private int getDeploymentReplicaOrDefault(Map map) { int replica = StaticConfig.EnvironmentVariables.REPLICAS; - try{ + try { replica = (int) map.get(StaticConfig.REPLICAS_TEXT); - } catch (Exception e){ + } catch (Exception e) { return replica; } return replica; diff --git a/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java b/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java index c6d2c9f6..a8048bea 100644 --- a/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java +++ b/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java @@ -7,7 +7,9 @@ import com.fasterxml.jackson.databind.json.JsonMapper; import io.datacater.core.pipeline.PipelineEntity; import io.datacater.core.stream.StreamEntity; +import io.fabric8.kubernetes.client.server.mock.KubernetesServer; import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.kubernetes.client.KubernetesTestServer; import io.quarkus.test.kubernetes.client.WithKubernetesTestServer; import io.restassured.RestAssured; import io.restassured.http.ContentType; @@ -24,6 +26,9 @@ @TestMethodOrder(MethodOrderer.OrderAnnotation.class) class DatacaterDeploymentEndpointTest { private static final Logger LOGGER = Logger.getLogger(DatacaterDeploymentEndpointTest.class); + @KubernetesTestServer KubernetesServer mockServer; + + K8Deployment k8Deployment; final String baseURI = "http://localhost:8081"; final String deploymentsPath = "/deployments"; @@ -260,7 +265,7 @@ void testCreateDeploymentWithNoStreamsInPipeline() throws IOException { PipelineEntity pipeline = mapper.readValue(responsePipeline.body().asString(), PipelineEntity.class); - pipelineId = pipeline.getId(); + UUID pipelineId = pipeline.getId(); // add deployment JsonURL = ClassLoader.getSystemClassLoader().getResource(deploymentPath); @@ -295,7 +300,7 @@ void testCreateDeploymentWithNoStream() throws IOException { PipelineEntity pipeline = mapper.readValue(responsePipeline.body().asString(), PipelineEntity.class); - pipelineId = pipeline.getId(); + UUID pipelineId = pipeline.getId(); // add deployment JsonURL = ClassLoader.getSystemClassLoader().getResource(deploymentPath); @@ -345,7 +350,7 @@ void testCreateDeploymentWithNoStreamOut() throws IOException { PipelineEntity pipeline = mapper.readValue(responsePipeline.body().asString(), PipelineEntity.class); - pipelineId = pipeline.getId(); + UUID pipelineId = pipeline.getId(); // add deployment JsonURL = ClassLoader.getSystemClassLoader().getResource(deploymentPath); @@ -384,4 +389,43 @@ void testCreateDeploymentWithEmptySpec() throws IOException { Assertions.assertEquals(400, responseDeployment.getStatusCode()); } + + @Test + @Order(14) + void testCreateDeploymentWithCustomReplicas() throws IOException { + String deploymentPath = "deploymentTests/deployment_with_custom_replicas.json"; + + // add deployment + URL JsonURL = ClassLoader.getSystemClassLoader().getResource(deploymentPath); + ObjectMapper mapper = new JsonMapper(); + JsonNode json = mapper.readTree(JsonURL); + String jsonString = json.toString(); + jsonString = jsonString.replace(pipelineUUIDPlaceholder, pipelineId.toString()); + + Response responseDeployment = + given() + .contentType(ContentType.JSON) + .baseUri(baseURI) + .body(jsonString) + .post(deploymentsPath); + + DeploymentEntity deployment = + mapper.readValue(responseDeployment.body().asString(), DeploymentEntity.class); + + String k8DeploymentString = + mockServer + .getClient() + .apps() + .deployments() + .inAnyNamespace() + .withLabel("datacater.io/uuid", deployment.getId().toString()) + .list() + .toString(); + + LOGGER.info( + "testCreateDeploymentWithCustomReplicas response: " + responseDeployment.body().asString()); + + Assertions.assertEquals(200, responseDeployment.getStatusCode()); + Assertions.assertTrue(k8DeploymentString.contains("replicas=3")); + } } diff --git a/platform-api/src/test/resources/deploymentTests/deployment_with_custom_replicas.json b/platform-api/src/test/resources/deploymentTests/deployment_with_custom_replicas.json new file mode 100644 index 00000000..90538c3b --- /dev/null +++ b/platform-api/src/test/resources/deploymentTests/deployment_with_custom_replicas.json @@ -0,0 +1,6 @@ +{ + "spec": { + "pipeline": "pipelineUUIDPlaceholder", + "replicas": 3 + } +} From 5834bce89b825cff09a7838276479c9750c4916e Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 08:40:02 +0100 Subject: [PATCH 03/20] feat(deployment-replicas-config): updated health and metrics endpoints health and metrics now make their endpoint calls to the first replica refs: #112 --- .../core/deployment/DeploymentEndpoint.java | 23 +++++++++------- .../core/deployment/K8Deployment.java | 26 ++++++++++++++++++- .../core/deployment/StaticConfig.java | 4 +-- 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index 7c300a5f..6b1443db 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -92,7 +92,9 @@ public Uni getHealth(@PathParam("uuid") UUID deploymentId) { HttpClient httpClient = HttpClient.newHttpClient(); HttpRequest req = buildDeploymentServiceRequest( - deploymentId, StaticConfig.EnvironmentVariables.DEPLOYMENT_HEALTH_PATH); + deploymentId, + StaticConfig.EnvironmentVariables.DEPLOYMENT_HEALTH_PATH, + 1); HttpResponse response = httpClient.send(req, HttpResponse.BodyHandlers.ofString()); return Response.ok().entity(response.body()).build(); @@ -116,7 +118,9 @@ public Uni getMetrics(@PathParam("uuid") UUID deploymentId) { HttpClient httpClient = HttpClient.newHttpClient(); HttpRequest req = buildDeploymentServiceRequest( - deploymentId, StaticConfig.EnvironmentVariables.DEPLOYMENT_METRICS_PATH); + deploymentId, + StaticConfig.EnvironmentVariables.DEPLOYMENT_METRICS_PATH, + 1); HttpResponse response = httpClient.send(req, HttpResponse.BodyHandlers.ofString()); return Response.ok().entity(response.body()).build(); @@ -295,16 +299,17 @@ private List getDeploymentLogsAsList(UUID deploymentId) { return Arrays.asList(k8Deployment.getLogs(deploymentId).split("\n")); } - private HttpRequest buildDeploymentServiceRequest(UUID deploymentId, String path) { + private HttpRequest buildDeploymentServiceRequest(UUID deploymentId, String path, int replica) { K8Deployment k8Deployment = new K8Deployment(client); - String clusterIp = k8Deployment.getClusterIp(deploymentId); + String ip = k8Deployment.getDeploymentReplicaIp(deploymentId, replica).replace(".", "-"); + String service = StaticConfig.SERVICE_NAME_PREFIX + deploymentId; + String namespace = StaticConfig.EnvironmentVariables.NAMESPACE; + int port = StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PORT; + String protocol = StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PROTOCOL; + String uriReady = String.format( - "%s://%s:%d%s", - StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PROTOCOL, - clusterIp, - StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PORT, - path); + "%s://%s.%s.%s.svc.cluster.local:%d%s", protocol, ip, service, namespace, port, path); return HttpRequest.newBuilder() .GET() diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 730bddda..90bbee3a 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -262,6 +262,30 @@ public Map getDeployment(UUID deploymentId) { } } + public String getDeploymentReplicaIp(UUID deploymentId, int replica) { + String deploymentName = getDeploymentName(deploymentId); + final Map matchLabels = + client + .apps() + .deployments() + .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) + .withName(deploymentName) + .get() + .getSpec() + .getSelector() + .getMatchLabels(); + + final Pod first = + client + .pods() + .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) + .withLabels(matchLabels) + .list() + .getItems() + .get(replica); + return first.getStatus().getPodIP(); + } + private boolean exists(UUID deploymentId) { return !client .apps() @@ -273,7 +297,7 @@ private boolean exists(UUID deploymentId) { .isEmpty(); } - private String getDeploymentName(UUID deploymentId) { + public String getDeploymentName(UUID deploymentId) { List deployments = client .apps() diff --git a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java index 31012936..33e82055 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java @@ -93,7 +93,7 @@ private EnvironmentVariables() {} static final String FULL_IMAGE_NAME = ConfigProvider.getConfig() .getOptionalValue("datacater.deployment.image", String.class) - .orElse("datacater/pipeline:alpha-20221117"); + .orElse("datacater/pipeline:nightly"); static final Integer READY_SECONDS = ConfigProvider.getConfig() .getOptionalValue("datacater.deployment.ready-seconds", Integer.class) @@ -131,7 +131,7 @@ private EnvironmentVariables() {} static final String PYTHON_RUNNER_IMAGE_TAG = ConfigProvider.getConfig() .getOptionalValue("datacater.pythonrunner.image.version", String.class) - .orElse("alpha-20221117"); + .orElse("nightly"); static final int PYTHON_RUNNER_CONTAINER_PORT = ConfigProvider.getConfig() .getOptionalValue("datacater.pythonrunner.image.containerPort", Integer.class) From 901939601120f753148300c46b7437b5026e7695 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 10:57:52 +0100 Subject: [PATCH 04/20] feat(deployment-replica-config): added default value queries default to first replica --- .../core/deployment/DeploymentEndpoint.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index 6b1443db..b65e6178 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -76,9 +76,10 @@ public Uni> getLogs(@PathParam("uuid") UUID deploymentId) { } @GET - @Path("{uuid}/health") + @Path("{uuid}/{replica}/health") @Produces(MediaType.APPLICATION_JSON) - public Uni getHealth(@PathParam("uuid") UUID deploymentId) { + public Uni getHealth( + @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @PathParam("replica") int replica) { return dsf.withTransaction( ((session, transaction) -> session.find(DeploymentEntity.class, deploymentId))) .onItem() @@ -94,7 +95,7 @@ public Uni getHealth(@PathParam("uuid") UUID deploymentId) { buildDeploymentServiceRequest( deploymentId, StaticConfig.EnvironmentVariables.DEPLOYMENT_HEALTH_PATH, - 1); + replica); HttpResponse response = httpClient.send(req, HttpResponse.BodyHandlers.ofString()); return Response.ok().entity(response.body()).build(); @@ -102,9 +103,10 @@ public Uni getHealth(@PathParam("uuid") UUID deploymentId) { } @GET - @Path("{uuid}/metrics") + @Path("{uuid}/{replica}/metrics") @Produces(MediaType.TEXT_PLAIN) - public Uni getMetrics(@PathParam("uuid") UUID deploymentId) { + public Uni getMetrics( + @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @PathParam("replica") int replica) { return dsf.withTransaction( ((session, transaction) -> session.find(DeploymentEntity.class, deploymentId))) .onItem() @@ -120,7 +122,7 @@ public Uni getMetrics(@PathParam("uuid") UUID deploymentId) { buildDeploymentServiceRequest( deploymentId, StaticConfig.EnvironmentVariables.DEPLOYMENT_METRICS_PATH, - 1); + replica); HttpResponse response = httpClient.send(req, HttpResponse.BodyHandlers.ofString()); return Response.ok().entity(response.body()).build(); From 65fe38484010833cc4f55627d6c1f3c914b45a2b Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:17:56 +0100 Subject: [PATCH 05/20] feat(deployment-replica-config): fix problem when only 1 replica and change replica call to queryparam instead of pathparam --- .../io/datacater/core/deployment/DeploymentEndpoint.java | 8 ++++---- .../java/io/datacater/core/deployment/K8Deployment.java | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index b65e6178..198a73da 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -76,10 +76,10 @@ public Uni> getLogs(@PathParam("uuid") UUID deploymentId) { } @GET - @Path("{uuid}/{replica}/health") + @Path("{uuid}/health") @Produces(MediaType.APPLICATION_JSON) public Uni getHealth( - @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @PathParam("replica") int replica) { + @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @QueryParam("replica") int replica) { return dsf.withTransaction( ((session, transaction) -> session.find(DeploymentEntity.class, deploymentId))) .onItem() @@ -103,10 +103,10 @@ public Uni getHealth( } @GET - @Path("{uuid}/{replica}/metrics") + @Path("{uuid}/metrics") @Produces(MediaType.TEXT_PLAIN) public Uni getMetrics( - @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @PathParam("replica") int replica) { + @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @QueryParam("replica") int replica) { return dsf.withTransaction( ((session, transaction) -> session.find(DeploymentEntity.class, deploymentId))) .onItem() diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 90bbee3a..2e509b26 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -263,6 +263,10 @@ public Map getDeployment(UUID deploymentId) { } public String getDeploymentReplicaIp(UUID deploymentId, int replica) { + if (replica > 0) { + // map replica number to array position + replica--; + } String deploymentName = getDeploymentName(deploymentId); final Map matchLabels = client From db6cd9617b0c7ff26f4cfced0d642e726a8ba006 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:45:23 +0100 Subject: [PATCH 06/20] feat(deployment-replica-config): added validation check for searched replica amount refs: #112 --- .../core/deployment/DeploymentEndpoint.java | 15 +++++++++++++ .../core/deployment/K8Deployment.java | 2 +- .../DeploymentReplicaMismatchException.java | 10 +++++++++ ...loymentReplicaMismatchExceptionMapper.java | 21 +++++++++++++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchException.java create mode 100644 platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchExceptionMapper.java diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index 7e8dcff6..c44ffa31 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -8,6 +8,7 @@ import io.datacater.core.exceptions.*; import io.datacater.core.pipeline.PipelineEntity; import io.datacater.core.stream.StreamEntity; +import io.datacater.core.utilities.JsonUtilities; import io.datacater.core.utilities.StringUtilities; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.KubernetesClient; @@ -92,6 +93,7 @@ public Uni getHealth( .transform( Unchecked.function( deployment -> { + validateGivenAndActualReplica(replica, deployment.getSpec()); HttpClient httpClient = HttpClient.newHttpClient(); HttpRequest req = buildDeploymentServiceRequest( @@ -119,6 +121,7 @@ public Uni getMetrics( .transform( Unchecked.function( deployment -> { + validateGivenAndActualReplica(replica, deployment.getSpec()); HttpClient httpClient = HttpClient.newHttpClient(); HttpRequest req = buildDeploymentServiceRequest( @@ -476,4 +479,16 @@ private void watchLogsRunner(UUID deploymentId, @Context Sse sse, @Context SseEv is.close(); lw.close(); } + + private void validateGivenAndActualReplica(int given, JsonNode spec) { + Map specMap = JsonUtilities.toObjectMap(spec); + int actual = K8Deployment.getDeploymentReplicaOrDefault(specMap); + if (given > actual) { + final String errorMessage = + String.format( + "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", + given, actual); + throw new DeploymentReplicaMismatchException(errorMessage); + } + } } diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 0b174997..66813ede 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -487,7 +487,7 @@ private String getEnvVariableFromNode(JsonNode node, String field) { return StaticConfig.EMPTY_STRING; } - private int getDeploymentReplicaOrDefault(Map map) { + public static int getDeploymentReplicaOrDefault(Map map) { int replica = StaticConfig.EnvironmentVariables.REPLICAS; try { replica = (int) map.get(StaticConfig.REPLICAS_TEXT); diff --git a/platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchException.java b/platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchException.java new file mode 100644 index 00000000..e9df9f77 --- /dev/null +++ b/platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchException.java @@ -0,0 +1,10 @@ +package io.datacater.core.exceptions; + +import io.datacater.core.ExcludeFromGeneratedCoverageReport; + +@ExcludeFromGeneratedCoverageReport +public class DeploymentReplicaMismatchException extends RuntimeException { + public DeploymentReplicaMismatchException(String message) { + super(message); + } +} diff --git a/platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchExceptionMapper.java b/platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchExceptionMapper.java new file mode 100644 index 00000000..3f1e8f09 --- /dev/null +++ b/platform-api/src/main/java/io/datacater/core/exceptions/DeploymentReplicaMismatchExceptionMapper.java @@ -0,0 +1,21 @@ +package io.datacater.core.exceptions; + +import io.datacater.core.ExcludeFromGeneratedCoverageReport; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; +import javax.ws.rs.ext.Provider; + +@ExcludeFromGeneratedCoverageReport +@Provider +public class DeploymentReplicaMismatchExceptionMapper + implements ExceptionMapper { + @Override + public Response toResponse(DeploymentReplicaMismatchException exception) { + Error error = + Error.from( + Response.Status.BAD_REQUEST.getStatusCode(), + Response.Status.BAD_REQUEST, + exception.getMessage()); + return Response.status(Response.Status.BAD_REQUEST).entity(error).build(); + } +} From 1420ffb2a34c0660400e3c027550a5d433181a69 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:24:27 +0100 Subject: [PATCH 07/20] feat(deployment-replica-config): removed the validation now relying on Kubernetes client error handling and mapping to a new exception refs: #112 --- .../core/deployment/DeploymentEndpoint.java | 15 -------- .../core/deployment/K8Deployment.java | 35 ++++++++++++++----- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index c44ffa31..7e8dcff6 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -8,7 +8,6 @@ import io.datacater.core.exceptions.*; import io.datacater.core.pipeline.PipelineEntity; import io.datacater.core.stream.StreamEntity; -import io.datacater.core.utilities.JsonUtilities; import io.datacater.core.utilities.StringUtilities; import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.KubernetesClient; @@ -93,7 +92,6 @@ public Uni getHealth( .transform( Unchecked.function( deployment -> { - validateGivenAndActualReplica(replica, deployment.getSpec()); HttpClient httpClient = HttpClient.newHttpClient(); HttpRequest req = buildDeploymentServiceRequest( @@ -121,7 +119,6 @@ public Uni getMetrics( .transform( Unchecked.function( deployment -> { - validateGivenAndActualReplica(replica, deployment.getSpec()); HttpClient httpClient = HttpClient.newHttpClient(); HttpRequest req = buildDeploymentServiceRequest( @@ -479,16 +476,4 @@ private void watchLogsRunner(UUID deploymentId, @Context Sse sse, @Context SseEv is.close(); lw.close(); } - - private void validateGivenAndActualReplica(int given, JsonNode spec) { - Map specMap = JsonUtilities.toObjectMap(spec); - int actual = K8Deployment.getDeploymentReplicaOrDefault(specMap); - if (given > actual) { - final String errorMessage = - String.format( - "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", - given, actual); - throw new DeploymentReplicaMismatchException(errorMessage); - } - } } diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 66813ede..24fd2c81 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.datacater.core.exceptions.CreateDeploymentException; import io.datacater.core.exceptions.DeploymentNotFoundException; +import io.datacater.core.exceptions.DeploymentReplicaMismatchException; import io.datacater.core.pipeline.PipelineEntity; import io.datacater.core.stream.StreamEntity; import io.datacater.core.utilities.StringUtilities; @@ -14,6 +15,7 @@ import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.ResourceNotFoundException; import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import java.util.*; import javax.inject.Singleton; @@ -279,14 +281,29 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .getSelector() .getMatchLabels(); - final Pod first = - client - .pods() - .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withLabels(matchLabels) - .list() - .getItems() - .get(replica); + Pod first; + List pods = new ArrayList<>(); + try { + pods = + client + .pods() + .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) + .withLabels(matchLabels) + .list() + .getItems(); + + first = + pods.stream() + .sorted(Comparator.comparing(HasMetadata::getFullResourceName)) + .toList() + .get(replica); + } catch (ResourceNotFoundException e) { + final String errorMessage = + String.format( + "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", + replica, pods.size()); + throw new DeploymentReplicaMismatchException(errorMessage); + } return first.getStatus().getPodIP(); } @@ -487,7 +504,7 @@ private String getEnvVariableFromNode(JsonNode node, String field) { return StaticConfig.EMPTY_STRING; } - public static int getDeploymentReplicaOrDefault(Map map) { + private int getDeploymentReplicaOrDefault(Map map) { int replica = StaticConfig.EnvironmentVariables.REPLICAS; try { replica = (int) map.get(StaticConfig.REPLICAS_TEXT); From ce1a4e85d04a00ab668b219503bcc8b258f1fdc4 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:26:27 +0100 Subject: [PATCH 08/20] feat(deployment-replica-config): updated exception message refs: #112 --- .../main/java/io/datacater/core/deployment/K8Deployment.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 24fd2c81..097a0119 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -298,10 +298,7 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .toList() .get(replica); } catch (ResourceNotFoundException e) { - final String errorMessage = - String.format( - "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", - replica, pods.size()); + final String errorMessage = String.format("Replica not found: %s.", e.getCause()); throw new DeploymentReplicaMismatchException(errorMessage); } return first.getStatus().getPodIP(); From 2aecb900ebb2cd5f6fd881c854d349b00a98a3b0 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:33:34 +0100 Subject: [PATCH 09/20] feat(deployment-replica-config): change sorting criteria refs: #112 --- .../io/datacater/core/deployment/K8Deployment.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 097a0119..63c29000 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -282,19 +282,16 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .getMatchLabels(); Pod first; - List pods = new ArrayList<>(); try { - pods = + first = client .pods() .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) .withLabels(matchLabels) .list() - .getItems(); - - first = - pods.stream() - .sorted(Comparator.comparing(HasMetadata::getFullResourceName)) + .getItems() + .stream() + .sorted((Comparator.comparing(o -> o.getMetadata().getName()))) .toList() .get(replica); } catch (ResourceNotFoundException e) { From 639043755faacc27ba9a165c7c42d995231c82db Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:34:53 +0100 Subject: [PATCH 10/20] feat(deployment-replica-config): remove unused resources refs: #112 --- .../java/io/datacater/core/deployment/K8Deployment.java | 5 ----- .../java/io/datacater/core/deployment/StaticConfig.java | 7 ------- 2 files changed, 12 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 63c29000..b33cddbe 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -174,11 +174,6 @@ private static ConfigMapVolumeSource configMapVolumeSource(String deploymentName return new ConfigMapVolumeSourceBuilder().withName(deploymentName).build(); } - public String getClusterIp(UUID deploymentId) { - final String serviceName = StaticConfig.SERVICE_NAME_PREFIX + deploymentId; - return k8Service.getClusterIp(serviceName); - } - public String getLogs(UUID deploymentId) { return client .apps() diff --git a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java index 33e82055..30e9e02f 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java @@ -52,7 +52,6 @@ static Map getLimits() { static final String CONFIGMAP_VOLUME_NAME_PREFIX = "datacater-volume-"; static final String DATA_SHARE_VOLUME_NAME_PREFIX = "datacater-volume-data-"; static final String SERVICE_NAME_PREFIX = "datacater-service-"; - static final String NONE = "None"; static final String TCP_TAG = "TCP"; static final String SPEC = "spec"; static final String STREAM_OUT = "stream-out"; @@ -74,8 +73,6 @@ static Map getLimits() { static final String VALUE_SERIALIZER = "value.serializer"; static final String STREAMIN_CONFIG_TEXT = "stream-in-config"; static final String STREAMOUT_CONFIG_TEXT = "stream-out-config"; - static final String DC_STREAMIN_CONFIG_TEXT = "DATACATER_STREAMIN_CONFIG"; - static final String DC_STREAMOUT_CONFIG_TEXT = "DATACATER_STREAM_OUTCONFIG"; static final String HTTP = "http"; static final String CONDITIONS = "Conditions"; static final String ERROR_TAG = "error"; @@ -110,10 +107,6 @@ private EnvironmentVariables() {} ConfigProvider.getConfig() .getOptionalValue("datacater.deployment.metrics-path", String.class) .orElse("/q/metrics"); - public static final long DEPLOYMENT_STATS_TIMEOUT = - ConfigProvider.getConfig() - .getOptionalValue("datacater.deployment.stats.timeout", Long.class) - .orElse(10000L); static final int DEPLOYMENT_CONTAINER_PORT = ConfigProvider.getConfig() .getOptionalValue("datacater.deployment.image.container.port", Integer.class) From 6a048fb571d9a90ef0c26e0d27c6ba54039ae8a9 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:39:35 +0100 Subject: [PATCH 11/20] feat(deployment-replica-config): fix error message and replica retrieval refs: #112 --- .../core/deployment/K8Deployment.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index b33cddbe..34afac67 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -15,7 +15,6 @@ import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.ResourceNotFoundException; import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import java.util.*; import javax.inject.Singleton; @@ -260,9 +259,10 @@ public Map getDeployment(UUID deploymentId) { } public String getDeploymentReplicaIp(UUID deploymentId, int replica) { - if (replica > 0) { + int replicaPosition = replica; + if (replicaPosition > 0) { // map replica number to array position - replica--; + replicaPosition--; } String deploymentName = getDeploymentName(deploymentId); final Map matchLabels = @@ -277,20 +277,26 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .getMatchLabels(); Pod first; + List podList = new ArrayList<>(); try { - first = + podList = client .pods() .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) .withLabels(matchLabels) .list() - .getItems() - .stream() + .getItems(); + + first = + podList.stream() .sorted((Comparator.comparing(o -> o.getMetadata().getName()))) .toList() - .get(replica); - } catch (ResourceNotFoundException e) { - final String errorMessage = String.format("Replica not found: %s.", e.getCause()); + .get(replicaPosition); + } catch (ArrayIndexOutOfBoundsException e) { + final String errorMessage = + String.format( + "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", + replica, podList.size()); throw new DeploymentReplicaMismatchException(errorMessage); } return first.getStatus().getPodIP(); From 1d13cdd9b94b5ad00d3d8a236b59bc5eb4ae992f Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:39:56 +0100 Subject: [PATCH 12/20] feat(deployment-replica-config): override toString for exceptions refs: #112 --- .../main/java/io/datacater/core/exceptions/Error.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/platform-api/src/main/java/io/datacater/core/exceptions/Error.java b/platform-api/src/main/java/io/datacater/core/exceptions/Error.java index 3ed1a672..dbf1ae1e 100644 --- a/platform-api/src/main/java/io/datacater/core/exceptions/Error.java +++ b/platform-api/src/main/java/io/datacater/core/exceptions/Error.java @@ -26,4 +26,15 @@ private Error(int statusCode, Response.Status status, String message) { public static Error from(int statusCode, Response.Status status, String message) { return new Error(statusCode, status, message); } + + @Override + public String toString() { + return "{\"statusCode\":" + + this.statusCode + + ",\"status\":\"" + + this.status.toString() + + "\",\"message\":\"" + + this.message + + "\"}"; + } } From 039d38bec08e125881e832e5cd534ff798e76f1c Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Mon, 13 Mar 2023 14:09:54 +0100 Subject: [PATCH 13/20] feat(deployment-replica-config): fix bug when no configs exist refs: #112 --- .../main/java/io/datacater/core/config/ConfigUtilities.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/platform-api/src/main/java/io/datacater/core/config/ConfigUtilities.java b/platform-api/src/main/java/io/datacater/core/config/ConfigUtilities.java index b2f49f9d..5a81739d 100644 --- a/platform-api/src/main/java/io/datacater/core/config/ConfigUtilities.java +++ b/platform-api/src/main/java/io/datacater/core/config/ConfigUtilities.java @@ -20,6 +20,11 @@ private ConfigUtilities() {} public static Uni> getMappedConfigs( Map configs, DataCaterSessionFactory dsf) { + + if (configs == null) { + return Uni.createFrom().item(new ArrayList<>()); + } + return dsf.withTransaction( (session, transaction) -> session From b7f45b82712e7296e78c41a37f02fb20e314eebe Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Tue, 14 Mar 2023 16:54:02 +0100 Subject: [PATCH 14/20] feat(deployment-replica-runtime-config): updated naming refs: #112 --- .../io/datacater/core/deployment/K8Deployment.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 34afac67..0f4a3825 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -276,10 +276,10 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .getSelector() .getMatchLabels(); - Pod first; - List podList = new ArrayList<>(); + Pod searchedPod; + List allDeploymentPods = new ArrayList<>(); try { - podList = + allDeploymentPods = client .pods() .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) @@ -287,8 +287,8 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .list() .getItems(); - first = - podList.stream() + searchedPod = + allDeploymentPods.stream() .sorted((Comparator.comparing(o -> o.getMetadata().getName()))) .toList() .get(replicaPosition); @@ -296,10 +296,10 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { final String errorMessage = String.format( "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", - replica, podList.size()); + replica, allDeploymentPods.size()); throw new DeploymentReplicaMismatchException(errorMessage); } - return first.getStatus().getPodIP(); + return searchedPod.getStatus().getPodIP(); } private boolean exists(UUID deploymentId) { From 49edb14bf5ca7457ee9f1deb0755dfb2ac4e8c29 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Tue, 14 Mar 2023 17:07:53 +0100 Subject: [PATCH 15/20] feat(deployment-replica-config): add check for replica smaller than zero --- .../io/datacater/core/deployment/K8Deployment.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 0f4a3825..b5050821 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -260,10 +260,16 @@ public Map getDeployment(UUID deploymentId) { public String getDeploymentReplicaIp(UUID deploymentId, int replica) { int replicaPosition = replica; - if (replicaPosition > 0) { - // map replica number to array position - replicaPosition--; + LOGGER.info(replica); + if (replicaPosition <= 0) { + final String errorMessage = + "The deployment replica you are searching for can not be less than 1"; + throw new DeploymentReplicaMismatchException(errorMessage); } + + // map replica number to array position + replicaPosition--; + String deploymentName = getDeploymentName(deploymentId); final Map matchLabels = client @@ -288,7 +294,7 @@ public String getDeploymentReplicaIp(UUID deploymentId, int replica) { .getItems(); searchedPod = - allDeploymentPods.stream() + allDeploymentPods.stream() .sorted((Comparator.comparing(o -> o.getMetadata().getName()))) .toList() .get(replicaPosition); From 21ec22ec40ab9a9e40117a7c87169c6df129d8b9 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:08:14 +0100 Subject: [PATCH 16/20] feat(deployment-replica-config): fix log retrieval per replica refs: #178 --- .../core/deployment/DeploymentEndpoint.java | 32 ++--- .../core/deployment/K8Deployment.java | 121 ++++++++++-------- .../core/deployment/K8DeploymentTest.java | 2 +- 3 files changed, 85 insertions(+), 70 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index 7e8dcff6..5acfaca7 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -9,10 +9,9 @@ import io.datacater.core.pipeline.PipelineEntity; import io.datacater.core.stream.StreamEntity; import io.datacater.core.utilities.StringUtilities; -import io.fabric8.kubernetes.api.model.apps.Deployment; import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.dsl.ContainerResource; import io.fabric8.kubernetes.client.dsl.LogWatch; -import io.fabric8.kubernetes.client.dsl.RollableScalableResource; import io.quarkus.security.Authenticated; import io.smallrye.mutiny.Uni; import io.smallrye.mutiny.unchecked.Unchecked; @@ -66,7 +65,8 @@ public Uni getDeployment(@PathParam("uuid") UUID deploymentId) @GET @Path("{uuid}/logs") @Produces(MediaType.APPLICATION_JSON) - public Uni> getLogs(@PathParam("uuid") UUID deploymentId) { + public Uni> getLogs( + @PathParam("uuid") UUID deploymentId, @DefaultValue("1") @QueryParam("replica") int replica) { return dsf.withTransaction( ((session, transaction) -> session.find(DeploymentEntity.class, deploymentId))) .onItem() @@ -74,7 +74,8 @@ public Uni> getLogs(@PathParam("uuid") UUID deploymentId) { .failWith(new DeploymentNotFoundException(StaticConfig.LoggerMessages.DEPLOYMENT_NOT_FOUND)) .onItem() .ifNotNull() - .transform(Unchecked.function(deployment -> getDeploymentLogsAsList(deployment.getId()))); + .transform( + Unchecked.function(deployment -> getDeploymentLogsAsList(deployment.getId(), replica))); } @GET @@ -135,7 +136,10 @@ public Uni getMetrics( @Path("{uuid}/watch-logs") @Produces(MediaType.SERVER_SENT_EVENTS) public Uni watchLogs( - @PathParam("uuid") UUID deploymentId, @Context Sse sse, @Context SseEventSink eventSink) { + @PathParam("uuid") UUID deploymentId, + @DefaultValue("1") @QueryParam("replica") int replica, + @Context Sse sse, + @Context SseEventSink eventSink) { return dsf.withTransaction( ((session, transaction) -> session.find(DeploymentEntity.class, deploymentId))) .onItem() @@ -146,7 +150,7 @@ public Uni watchLogs( .transform( deployment -> { try { - watchLogsRunner(deployment.getId(), sse, eventSink); + watchLogsRunner(deployment.getId(), replica, sse, eventSink); } catch (IOException e) { throw new DatacaterException(StringUtilities.wrapString(e.getMessage())); } @@ -342,9 +346,9 @@ private Uni getStream( String.format(StaticConfig.LoggerMessages.STREAM_NOT_FOUND, key)))); } - private List getDeploymentLogsAsList(UUID deploymentId) { + private List getDeploymentLogsAsList(UUID deploymentId, int replica) { K8Deployment k8Deployment = new K8Deployment(client); - return Arrays.asList(k8Deployment.getLogs(deploymentId).split("\n")); + return Arrays.asList(k8Deployment.getLogs(deploymentId, replica).split("\n")); } private HttpRequest buildDeploymentServiceRequest(UUID deploymentId, String path, int replica) { @@ -366,9 +370,9 @@ private HttpRequest buildDeploymentServiceRequest(UUID deploymentId, String path .build(); } - private RollableScalableResource watchDeploymentLogs(UUID deploymentId) { + private ContainerResource watchDeploymentLogs(UUID deploymentId, int replica) { K8Deployment k8Deployment = new K8Deployment(client); - return k8Deployment.watchLogs(deploymentId); + return k8Deployment.watchLogs(deploymentId, replica); } private List getK8Deployments(List deployments) { @@ -459,12 +463,10 @@ private UUID getPipelineUUIDFromMap(Map map) { } } - private void watchLogsRunner(UUID deploymentId, @Context Sse sse, @Context SseEventSink eventSink) + private void watchLogsRunner( + UUID deploymentId, int replica, @Context Sse sse, @Context SseEventSink eventSink) throws IOException { - LogWatch lw = - watchDeploymentLogs(deploymentId) - .inContainer(StaticConfig.DEPLOYMENT_NAME_PREFIX + deploymentId) - .watchLog(); + LogWatch lw = watchDeploymentLogs(deploymentId, replica).watchLog(); InputStream is = lw.getOutput(); BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(is)); String line; diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index b5050821..60b38333 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -15,7 +15,7 @@ import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; -import io.fabric8.kubernetes.client.dsl.RollableScalableResource; +import io.fabric8.kubernetes.client.dsl.ContainerResource; import java.util.*; import javax.inject.Singleton; import org.jboss.logging.Logger; @@ -173,22 +173,27 @@ private static ConfigMapVolumeSource configMapVolumeSource(String deploymentName return new ConfigMapVolumeSourceBuilder().withName(deploymentName).build(); } - public String getLogs(UUID deploymentId) { + public String getLogs(UUID deploymentId, int replica) { + String deploymentName = getDeploymentName(deploymentId); + Pod pod = getDeploymentPodByReplica(deploymentName, replica); + return client - .apps() - .deployments() + .pods() .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withName(getDeploymentName(deploymentId)) + .withName(pod.getMetadata().getName()) .inContainer(StaticConfig.DEPLOYMENT_NAME_PREFIX + deploymentId) .getLog(true); } - public RollableScalableResource watchLogs(UUID deploymentId) { + public ContainerResource watchLogs(UUID deploymentId, int replica) { + String deploymentName = getDeploymentName(deploymentId); + Pod pod = getDeploymentPodByReplica(deploymentName, replica); + return client - .apps() - .deployments() + .pods() .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withName(getDeploymentName(deploymentId)); + .withName(pod.getMetadata().getName()) + .inContainer(StaticConfig.DEPLOYMENT_NAME_PREFIX + deploymentId); } public void delete(UUID deploymentId) { @@ -259,53 +264,10 @@ public Map getDeployment(UUID deploymentId) { } public String getDeploymentReplicaIp(UUID deploymentId, int replica) { - int replicaPosition = replica; - LOGGER.info(replica); - if (replicaPosition <= 0) { - final String errorMessage = - "The deployment replica you are searching for can not be less than 1"; - throw new DeploymentReplicaMismatchException(errorMessage); - } - - // map replica number to array position - replicaPosition--; - String deploymentName = getDeploymentName(deploymentId); - final Map matchLabels = - client - .apps() - .deployments() - .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withName(deploymentName) - .get() - .getSpec() - .getSelector() - .getMatchLabels(); - - Pod searchedPod; - List allDeploymentPods = new ArrayList<>(); - try { - allDeploymentPods = - client - .pods() - .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withLabels(matchLabels) - .list() - .getItems(); + Pod pod = getDeploymentPodByReplica(deploymentName, replica); - searchedPod = - allDeploymentPods.stream() - .sorted((Comparator.comparing(o -> o.getMetadata().getName()))) - .toList() - .get(replicaPosition); - } catch (ArrayIndexOutOfBoundsException e) { - final String errorMessage = - String.format( - "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", - replica, allDeploymentPods.size()); - throw new DeploymentReplicaMismatchException(errorMessage); - } - return searchedPod.getStatus().getPodIP(); + return pod.getStatus().getPodIP(); } private boolean exists(UUID deploymentId) { @@ -514,4 +476,55 @@ private int getDeploymentReplicaOrDefault(Map map) { } return replica; } + + private int replicaNumberToArrayPosition(int replica) { + int replicaPosition = replica; + if (replicaPosition <= 0) { + final String errorMessage = + "The deployment replica you are searching for can not be less than 1"; + throw new DeploymentReplicaMismatchException(errorMessage); + } + // map replica number to array position + replicaPosition--; + return replicaPosition; + } + + private Pod getDeploymentPodByReplica(String deploymentName, int replica) { + int replicaPosition = replicaNumberToArrayPosition(replica); + final Map matchLabels = + client + .apps() + .deployments() + .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) + .withName(deploymentName) + .get() + .getSpec() + .getSelector() + .getMatchLabels(); + + Pod searchedPod; + List allDeploymentPods = new ArrayList<>(); + try { + allDeploymentPods = + client + .pods() + .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) + .withLabels(matchLabels) + .list() + .getItems(); + + searchedPod = + allDeploymentPods.stream() + .sorted((Comparator.comparing(o -> o.getMetadata().getName()))) + .toList() + .get(replicaPosition); + } catch (ArrayIndexOutOfBoundsException e) { + final String errorMessage = + String.format( + "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", + replica, allDeploymentPods.size()); + throw new DeploymentReplicaMismatchException(errorMessage); + } + return searchedPod; + } } diff --git a/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java b/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java index 3c88612d..e7d7c864 100644 --- a/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java +++ b/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java @@ -110,7 +110,7 @@ void testCreateDeployment() @Test @Order(3) void testGetLogs() { - String logs = k8Deployment.getLogs(deploymentId); + String logs = k8Deployment.getLogs(deploymentId, 1); Assertions.assertNotNull(logs); } From c7d5d58dd93a413322d72f84d028dbacff1640a4 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Mon, 20 Mar 2023 17:44:06 +0100 Subject: [PATCH 17/20] feat(deployment-replica-config): remove now unneeded service refs: #112 --- .../core/deployment/DeploymentEndpoint.java | 4 +- .../core/deployment/K8Deployment.java | 4 - .../datacater/core/deployment/K8Service.java | 79 ------------------- 3 files changed, 1 insertion(+), 86 deletions(-) delete mode 100644 platform-api/src/main/java/io/datacater/core/deployment/K8Service.java diff --git a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java index 5acfaca7..bb621436 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/DeploymentEndpoint.java @@ -354,14 +354,12 @@ private List getDeploymentLogsAsList(UUID deploymentId, int replica) { private HttpRequest buildDeploymentServiceRequest(UUID deploymentId, String path, int replica) { K8Deployment k8Deployment = new K8Deployment(client); String ip = k8Deployment.getDeploymentReplicaIp(deploymentId, replica).replace(".", "-"); - String service = StaticConfig.SERVICE_NAME_PREFIX + deploymentId; String namespace = StaticConfig.EnvironmentVariables.NAMESPACE; int port = StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PORT; String protocol = StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PROTOCOL; String uriReady = - String.format( - "%s://%s.%s.%s.svc.cluster.local:%d%s", protocol, ip, service, namespace, port, path); + String.format("%s://%s.%s.pod.cluster.local:%d%s", protocol, ip, namespace, port, path); return HttpRequest.newBuilder() .GET() diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 60b38333..290a7e22 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -25,12 +25,10 @@ public class K8Deployment { private static final Logger LOGGER = Logger.getLogger(K8Deployment.class); private final KubernetesClient client; private final K8ConfigMap k8ConfigMap; - private final K8Service k8Service; public K8Deployment(KubernetesClient client) { this.client = client; this.k8ConfigMap = new K8ConfigMap(client); - this.k8Service = new K8Service(client); } public Map create( @@ -92,7 +90,6 @@ public Map create( throw new CreateDeploymentException(StaticConfig.LoggerMessages.DEPLOYMENT_NOT_CREATED); } k8ConfigMap.getOrCreate(configmapName, pe); - k8Service.create(serviceName); return getDeployment(deploymentId); } @@ -212,7 +209,6 @@ public void delete(UUID deploymentId) { // deployment to be exactly // one and continue only if that is true. if (status.size() == 1) { - k8Service.delete(serviceName); k8ConfigMap.delete(configMapName); LOGGER.info(String.format(StaticConfig.LoggerMessages.DEPLOYMENT_DELETED, name)); } else { diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Service.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Service.java deleted file mode 100644 index a4cf1da9..00000000 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Service.java +++ /dev/null @@ -1,79 +0,0 @@ -package io.datacater.core.deployment; - -import io.fabric8.kubernetes.api.model.*; -import io.fabric8.kubernetes.client.KubernetesClient; -import java.util.List; -import java.util.Map; -import javax.enterprise.context.ApplicationScoped; - -@ApplicationScoped -public class K8Service { - private final KubernetesClient client; - - K8Service(KubernetesClient client) { - this.client = client; - } - - private boolean exists(String name) { - return !getListByLabel(name).isEmpty(); - } - - public void delete(String name) { - client.services().inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE).delete(get(name)); - } - - public String getClusterIp(String serviceName) { - return client - .services() - .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withName(serviceName) - .get() - .getSpec() - .getClusterIP(); - } - - private Service get(String name) { - return getListByLabel(name).get(0); - } - - private List getListByLabel(String name) { - return client - .services() - .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .withLabel(StaticConfig.DEPLOYMENT_SERVICE_TEXT, name) - .list() - .getItems(); - } - - public void create(String name) { - if (!exists(name)) { - createService(name); - } - } - - private Service createService(String name) { - Service service = - new ServiceBuilder() - .withNewMetadata() - .withName(name) - .withLabels(Map.of(StaticConfig.DEPLOYMENT_SERVICE_TEXT, name)) - .endMetadata() - .withNewSpec() - .withSelector(Map.of(StaticConfig.DEPLOYMENT_SERVICE_TEXT, name)) - .withPorts(port()) - .endSpec() - .build(); - client - .services() - .inNamespace(StaticConfig.EnvironmentVariables.NAMESPACE) - .createOrReplace(service); - return service; - } - - private ServicePort port() { - return new ServicePortBuilder() - .withProtocol(StaticConfig.TCP_TAG) - .withPort(StaticConfig.EnvironmentVariables.DEPLOYMENT_CONTAINER_PORT) - .build(); - } -} From a714af31e7fe1bc1005cc5b989bbb362278d11ec Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Wed, 22 Mar 2023 10:39:06 +0100 Subject: [PATCH 18/20] feat(deployment-replica-config): removed unneeded references to services refs: #112 --- .../core/deployment/K8Deployment.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java index 290a7e22..32895b68 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/K8Deployment.java @@ -42,7 +42,6 @@ public Map create( final String configmapName = StaticConfig.CONFIGMAP_NAME_PREFIX + deploymentId; final String configmapVolumeName = StaticConfig.CONFIGMAP_VOLUME_NAME_PREFIX + deploymentId; final String dataShareVolumeName = StaticConfig.DATA_SHARE_VOLUME_NAME_PREFIX + deploymentId; - final String serviceName = StaticConfig.SERVICE_NAME_PREFIX + deploymentId; final int replicaCount = getDeploymentReplicaOrDefault(deploymentSpec.deployment()); List variables = @@ -53,17 +52,17 @@ public Map create( new DeploymentBuilder() .withNewMetadata() .withName(name) - .addToLabels(getLabels(deploymentId, deploymentSpec.name(), serviceName)) + .addToLabels(getLabels(deploymentId, deploymentSpec.name())) .endMetadata() .withNewSpec() .withReplicas(replicaCount) .withMinReadySeconds(StaticConfig.EnvironmentVariables.READY_SECONDS) .withNewSelector() - .addToMatchLabels(getLabels(deploymentId, deploymentSpec.name(), serviceName)) + .addToMatchLabels(getLabels(deploymentId, deploymentSpec.name())) .endSelector() .withNewTemplate() .withNewMetadata() - .addToLabels(getLabels(deploymentId, deploymentSpec.name(), serviceName)) + .addToLabels(getLabels(deploymentId, deploymentSpec.name())) .endMetadata() .withNewSpec() .addAllToContainers( @@ -93,8 +92,7 @@ public Map create( return getDeployment(deploymentId); } - private static Map getLabels( - UUID deploymentId, String prettyName, String serviceName) { + private static Map getLabels(UUID deploymentId, String prettyName) { return Map.of( StaticConfig.APP, StaticConfig.DATACATER_PIPELINE, @@ -105,9 +103,7 @@ private static Map getLabels( StaticConfig.UUID_TEXT, deploymentId.toString(), StaticConfig.DEPLOYMENT_NAME_TEXT, - prettyName, - StaticConfig.DEPLOYMENT_SERVICE_TEXT, - serviceName); + prettyName); } private Container pythonRunnerContainer(String configmapVolumeName, String dataShareVolumeName) { @@ -171,6 +167,7 @@ private static ConfigMapVolumeSource configMapVolumeSource(String deploymentName } public String getLogs(UUID deploymentId, int replica) { + String deploymentName = getDeploymentName(deploymentId); Pod pod = getDeploymentPodByReplica(deploymentName, replica); @@ -195,7 +192,6 @@ public ContainerResource watchLogs(UUID deploymentId, int replica) { public void delete(UUID deploymentId) { String name = getDeploymentName(deploymentId); - String serviceName = StaticConfig.SERVICE_NAME_PREFIX + deploymentId; String configMapName = StaticConfig.CONFIGMAP_NAME_PREFIX + deploymentId; List status = client @@ -487,6 +483,7 @@ private int replicaNumberToArrayPosition(int replica) { private Pod getDeploymentPodByReplica(String deploymentName, int replica) { int replicaPosition = replicaNumberToArrayPosition(replica); + final Map matchLabels = client .apps() @@ -515,6 +512,9 @@ private Pod getDeploymentPodByReplica(String deploymentName, int replica) { .toList() .get(replicaPosition); } catch (ArrayIndexOutOfBoundsException e) { + LOGGER.info( + String.format( + "An error occurred while trying to get replica %s: %s", replica, e.getMessage())); final String errorMessage = String.format( "The deployment replica you are searching for, %s, does not match the defined replica amount of %s.", From 10af415d4e7dd8f0ac6614a81bff518382c7bb40 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Wed, 22 Mar 2023 10:40:15 +0100 Subject: [PATCH 19/20] feat(deployment-replica-config): removed untestable tests reomved some failing tests since they can't be tested locally anymore. We need to implement integration tests for underlying kubernetes tests refs: #112 --- .../DatacaterDeploymentEndpointTest.java | 29 +++++-------------- .../core/deployment/K8DeploymentTest.java | 7 ----- 2 files changed, 7 insertions(+), 29 deletions(-) diff --git a/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java b/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java index a8048bea..7a144064 100644 --- a/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java +++ b/platform-api/src/test/java/io/datacater/core/deployment/DatacaterDeploymentEndpointTest.java @@ -28,8 +28,6 @@ class DatacaterDeploymentEndpointTest { private static final Logger LOGGER = Logger.getLogger(DatacaterDeploymentEndpointTest.class); @KubernetesTestServer KubernetesServer mockServer; - K8Deployment k8Deployment; - final String baseURI = "http://localhost:8081"; final String deploymentsPath = "/deployments"; final String streamsPath = "/streams"; @@ -130,21 +128,8 @@ void testGetDeployments() { given().baseUri(baseURI).get(deploymentsPath).then().statusCode(200); } - // only testing that logs can be fetched. - // Log validity ist harder to test, since logs will always change @Test @Order(4) - void testGetDeploymentLogs() { - given() - .pathParam("uuid", deploymentId) - .baseUri(baseURI) - .get(deploymentsPath + "/{uuid}/logs") - .then() - .statusCode(200); - } - - @Test - @Order(5) void testUpdateDeployment() throws IOException { String pipelineUUIDPlaceholder = "pipelineUUIDPlaceholder"; URL JsonURL = ClassLoader.getSystemClassLoader().getResource(deploymentPath); @@ -165,7 +150,7 @@ void testUpdateDeployment() throws IOException { } @Test - @Order(6) + @Order(5) void testDeleteDeployment() { Response response = RestAssured.given() @@ -178,7 +163,7 @@ void testDeleteDeployment() { } @Test - @Order(7) + @Order(6) void testGetDeletedDeployment() { given() .pathParam("uuid", deploymentId) @@ -189,7 +174,7 @@ void testGetDeletedDeployment() { } @Test - @Order(8) + @Order(7) void testGetUnknownDeployment() { given() .pathParam("uuid", UUID.randomUUID()) @@ -200,7 +185,7 @@ void testGetUnknownDeployment() { } @Test - @Order(9) + @Order(8) void testGetUnknownDeploymentLogs() { given() .pathParam("uuid", UUID.randomUUID()) @@ -211,7 +196,7 @@ void testGetUnknownDeploymentLogs() { } @Test - @Order(10) + @Order(9) void testWatchUnknownDeploymentLogs() { given() .pathParam("uuid", UUID.randomUUID()) @@ -370,7 +355,7 @@ void testCreateDeploymentWithNoStreamOut() throws IOException { } @Test - @Order(13) + @Order(14) void testCreateDeploymentWithEmptySpec() throws IOException { String deploymentPath = "deploymentTests/deployment_with_empty_spec.json"; @@ -391,7 +376,7 @@ void testCreateDeploymentWithEmptySpec() throws IOException { } @Test - @Order(14) + @Order(15) void testCreateDeploymentWithCustomReplicas() throws IOException { String deploymentPath = "deploymentTests/deployment_with_custom_replicas.json"; diff --git a/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java b/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java index e7d7c864..b874a7ef 100644 --- a/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java +++ b/platform-api/src/test/java/io/datacater/core/deployment/K8DeploymentTest.java @@ -107,13 +107,6 @@ void testCreateDeployment() Assertions.assertEquals(true, getDeploymentExistsMethod().invoke(k8Deployment, deploymentId)); } - @Test - @Order(3) - void testGetLogs() { - String logs = k8Deployment.getLogs(deploymentId, 1); - Assertions.assertNotNull(logs); - } - private Method getConfigMapExistsMethod() throws NoSuchMethodException { final String methodName = "exists"; Method method = K8ConfigMap.class.getDeclaredMethod(methodName, String.class); From de867fb271944f8628790e358d9f3e02c8a3b185 Mon Sep 17 00:00:00 2001 From: ChrisRousey <104754971+ChrisRousey@users.noreply.github.com> Date: Wed, 22 Mar 2023 11:43:04 +0100 Subject: [PATCH 20/20] feat(deployment-replica-config): change images to latest stable release version refs: #112 --- .../main/java/io/datacater/core/deployment/StaticConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java index 30e9e02f..5b40559d 100644 --- a/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java +++ b/platform-api/src/main/java/io/datacater/core/deployment/StaticConfig.java @@ -90,7 +90,7 @@ private EnvironmentVariables() {} static final String FULL_IMAGE_NAME = ConfigProvider.getConfig() .getOptionalValue("datacater.deployment.image", String.class) - .orElse("datacater/pipeline:nightly"); + .orElse("datacater/pipeline:2023.1"); static final Integer READY_SECONDS = ConfigProvider.getConfig() .getOptionalValue("datacater.deployment.ready-seconds", Integer.class) @@ -124,7 +124,7 @@ private EnvironmentVariables() {} static final String PYTHON_RUNNER_IMAGE_TAG = ConfigProvider.getConfig() .getOptionalValue("datacater.pythonrunner.image.version", String.class) - .orElse("nightly"); + .orElse("2023.1"); static final int PYTHON_RUNNER_CONTAINER_PORT = ConfigProvider.getConfig() .getOptionalValue("datacater.pythonrunner.image.containerPort", Integer.class)