From 7f2d861cf67b9dec63b7da1058850210e3e91262 Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 28 Mar 2023 15:59:08 +0200 Subject: [PATCH 1/7] feat: polling default ns --- .../process/KubeAPIServerProcess.java | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index 86e1e20..cc854cb 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.IOException; +import java.time.LocalTime; import java.util.ArrayList; import java.util.List; import java.util.Scanner; @@ -15,9 +16,11 @@ public class KubeAPIServerProcess { + public static final int REQUEST_WAIT_TIMEOUT = 500; private static final Logger log = LoggerFactory.getLogger(KubeAPIServerProcess.class); private static final Logger apiLog = LoggerFactory.getLogger(KubeAPIServerProcess.class .getName() + ".APIServerProcessLogs"); + public static final int POLLING_INTERVAL = 250; private final CertManager certManager; private final BinaryManager binaryManager; @@ -85,27 +88,42 @@ private List createCommand(File apiServerBinary, int apiServerPort, int public void waitUntilDefaultNamespaceCreated() { try { - AtomicBoolean started = new AtomicBoolean(false); + var startedAt = LocalTime.now(); + while (true) { + if (defaultNamespaceExists()) { + return; + } + if (LocalTime.now().isAfter(startedAt.plusSeconds(10))) { + throw new JenvtestException("API Server did not start properly"); + } + Thread.sleep(POLLING_INTERVAL); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new JenvtestException(e); + } + } + + private boolean defaultNamespaceExists() { + try { var proc = new ProcessBuilder(binaryManager.binaries().getKubectl().getPath(), "get", "ns", - "--watch").start(); - var procWaiter = new Thread(() -> { + "default").start(); + AtomicBoolean defaultFound = new AtomicBoolean(false); + var procIsReader = new Thread(() -> { log.debug("Starting proc waiter thread."); try (Scanner sc = new Scanner(proc.getInputStream())) { while (sc.hasNextLine()) { String line = sc.nextLine(); - log.debug("kubectl ns watch: {}", line); if (line.contains("default")) { - started.set(true); + defaultFound.set(true); return; } } } }); - procWaiter.start(); - procWaiter.join(KubeAPIServer.STARTUP_TIMEOUT); - if (!started.get()) { - throw new JenvtestException("API Server did not start properly"); - } + procIsReader.start(); + procIsReader.join(REQUEST_WAIT_TIMEOUT); + return defaultFound.get(); } catch (IOException e) { throw new JenvtestException(e); } catch (InterruptedException e) { From abc400d514a16be51ca6c868d2aa1303c712cab2 Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 28 Mar 2023 16:24:19 +0200 Subject: [PATCH 2/7] improvements --- .../main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java | 2 +- .../jenvtest/process/KubeAPIServerProcess.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java index e16c8c7..cf7a60a 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java @@ -12,7 +12,7 @@ public class KubeAPIServer implements UnexpectedProcessStopHandler { private static final Logger log = LoggerFactory.getLogger(KubeAPIServer.class); - public static final int STARTUP_TIMEOUT = 15_000; + public static final int STARTUP_TIMEOUT = 10_0000; private final KubeAPIServerConfig config; private final BinaryManager binaryManager; diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index cc854cb..2fff57b 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -3,6 +3,7 @@ import java.io.File; import java.io.IOException; import java.time.LocalTime; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; import java.util.Scanner; @@ -14,6 +15,8 @@ import io.javaoperatorsdk.jenvtest.*; import io.javaoperatorsdk.jenvtest.binary.BinaryManager; +import static io.javaoperatorsdk.jenvtest.KubeAPIServer.STARTUP_TIMEOUT; + public class KubeAPIServerProcess { public static final int REQUEST_WAIT_TIMEOUT = 500; @@ -93,7 +96,7 @@ public void waitUntilDefaultNamespaceCreated() { if (defaultNamespaceExists()) { return; } - if (LocalTime.now().isAfter(startedAt.plusSeconds(10))) { + if (LocalTime.now().isAfter(startedAt.plus(STARTUP_TIMEOUT, ChronoUnit.MILLIS))) { throw new JenvtestException("API Server did not start properly"); } Thread.sleep(POLLING_INTERVAL); From 20e5079530486047fc2a5867c7f280facc0b75b3 Mon Sep 17 00:00:00 2001 From: csviri Date: Tue, 28 Mar 2023 16:36:35 +0200 Subject: [PATCH 3/7] timeout fix --- .../main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java index cf7a60a..9cef811 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/KubeAPIServer.java @@ -12,7 +12,7 @@ public class KubeAPIServer implements UnexpectedProcessStopHandler { private static final Logger log = LoggerFactory.getLogger(KubeAPIServer.class); - public static final int STARTUP_TIMEOUT = 10_0000; + public static final int STARTUP_TIMEOUT = 10_000; private final KubeAPIServerConfig config; private final BinaryManager binaryManager; From 4a434daaebbb1219a5375c341c0f56a9a4f671eb Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 29 Mar 2023 10:52:38 +0200 Subject: [PATCH 4/7] feat: pollin health endpoint on startup --- .../process/KubeAPIServerProcess.java | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index 2fff57b..478144c 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -2,6 +2,11 @@ import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import java.time.LocalTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -31,6 +36,7 @@ public class KubeAPIServerProcess { private volatile Process apiServerProcess; private volatile boolean stopped = false; private final UnexpectedProcessStopHandler processStopHandler; + private int apiServerPort; public KubeAPIServerProcess(CertManager certManager, BinaryManager binaryManager, UnexpectedProcessStopHandler processStopHandler, @@ -48,7 +54,7 @@ public int startApiServer(int etcdPort) { throw new JenvtestException( "Missing binary for API Server on path: " + apiServerBinary.getAbsolutePath()); } - var apiServerPort = Utils.findFreePort(); + apiServerPort = Utils.findFreePort(); var command = createCommand(apiServerBinary, apiServerPort, etcdPort); apiServerProcess = new ProcessBuilder(command) .start(); @@ -93,7 +99,7 @@ public void waitUntilDefaultNamespaceCreated() { try { var startedAt = LocalTime.now(); while (true) { - if (defaultNamespaceExists()) { + if (healthy()) { return; } if (LocalTime.now().isAfter(startedAt.plus(STARTUP_TIMEOUT, ChronoUnit.MILLIS))) { @@ -107,27 +113,19 @@ public void waitUntilDefaultNamespaceCreated() { } } - private boolean defaultNamespaceExists() { + private boolean healthy() { try { - var proc = new ProcessBuilder(binaryManager.binaries().getKubectl().getPath(), "get", "ns", - "default").start(); - AtomicBoolean defaultFound = new AtomicBoolean(false); - var procIsReader = new Thread(() -> { - log.debug("Starting proc waiter thread."); - try (Scanner sc = new Scanner(proc.getInputStream())) { - while (sc.hasNextLine()) { - String line = sc.nextLine(); - if (line.contains("default")) { - defaultFound.set(true); - return; - } - } - } - }); - procIsReader.start(); - procIsReader.join(REQUEST_WAIT_TIMEOUT); - return defaultFound.get(); - } catch (IOException e) { + var client = HttpClient.newBuilder().build(); + HttpRequest request = HttpRequest.newBuilder() + .uri(new URI("https://localhost:"+apiServerPort+"/livez")) + .GET() + .build(); + + var response = client.send(request, HttpResponse.BodyHandlers.ofString()); + String body = response.body(); + + return true; + } catch (URISyntaxException | IOException e) { throw new JenvtestException(e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); From 80c0848164f527435115bd0292251477d6d998c5 Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 29 Mar 2023 12:37:55 +0200 Subject: [PATCH 5/7] improvements --- .../process/KubeAPIServerProcess.java | 94 +++++++++++++++---- 1 file changed, 78 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index 478144c..2dde2ad 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -2,17 +2,22 @@ import java.io.File; import java.io.IOException; +import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; import java.time.LocalTime; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.List; -import java.util.Scanner; -import java.util.concurrent.atomic.AtomicBoolean; + +import javax.net.ssl.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,11 +29,10 @@ public class KubeAPIServerProcess { - public static final int REQUEST_WAIT_TIMEOUT = 500; private static final Logger log = LoggerFactory.getLogger(KubeAPIServerProcess.class); private static final Logger apiLog = LoggerFactory.getLogger(KubeAPIServerProcess.class .getName() + ".APIServerProcessLogs"); - public static final int POLLING_INTERVAL = 250; + public static final int POLLING_INTERVAL = 150; private final CertManager certManager; private final BinaryManager binaryManager; @@ -97,9 +101,11 @@ private List createCommand(File apiServerBinary, int apiServerPort, int public void waitUntilDefaultNamespaceCreated() { try { + var client = getHttpClient(); + var request = getHttpRequest(); var startedAt = LocalTime.now(); while (true) { - if (healthy()) { + if (ready(client, request)) { return; } if (LocalTime.now().isAfter(startedAt.plus(STARTUP_TIMEOUT, ChronoUnit.MILLIS))) { @@ -113,19 +119,12 @@ public void waitUntilDefaultNamespaceCreated() { } } - private boolean healthy() { + private boolean ready(HttpClient client, HttpRequest request) { try { - var client = HttpClient.newBuilder().build(); - HttpRequest request = HttpRequest.newBuilder() - .uri(new URI("https://localhost:"+apiServerPort+"/livez")) - .GET() - .build(); - var response = client.send(request, HttpResponse.BodyHandlers.ofString()); - String body = response.body(); - - return true; - } catch (URISyntaxException | IOException e) { + log.trace("Ready Response message:{} code: {}", response.body(), response.statusCode()); + return response.statusCode() == 200; + } catch (IOException e) { throw new JenvtestException(e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -133,6 +132,69 @@ private boolean healthy() { } } + private HttpRequest getHttpRequest() { + try { + return HttpRequest.newBuilder() + .uri(new URI("https://localhost:" + apiServerPort + "/readyz")) + .GET() + .build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + private static HttpClient getHttpClient() { + try { + var sslContext = SSLContext.getInstance("TLS"); + sslContext.init( + null, + new TrustManager[] { + new X509ExtendedTrustManager() { + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, + Socket socket) throws CertificateException { + + } + + public X509Certificate[] getAcceptedIssuers() { + return null; + } + + public void checkClientTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type) {} + + public void checkServerTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type) {} + + + public void checkServerTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type, + final Socket a_socket) {} + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, + SSLEngine engine) throws CertificateException { + + } + + public void checkServerTrusted( + final X509Certificate[] a_certificates, + final String a_auth_type, + final SSLEngine a_engine) {} + } + }, + null); + return HttpClient.newBuilder() + .sslContext(sslContext) + .build(); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + throw new JenvtestException(e); + } + } + public void stopApiServer() { if (stopped) { return; From 5a76e1104c93a38a5c4d56dc8e2e9adadfd5f6ec Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 29 Mar 2023 12:54:45 +0200 Subject: [PATCH 6/7] wip --- .../javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index 2dde2ad..22c9ce4 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -135,7 +135,7 @@ private boolean ready(HttpClient client, HttpRequest request) { private HttpRequest getHttpRequest() { try { return HttpRequest.newBuilder() - .uri(new URI("https://localhost:" + apiServerPort + "/readyz")) + .uri(new URI("https://127.0.0.1:" + apiServerPort + "/readyz")) .GET() .build(); } catch (URISyntaxException e) { From 533bca10cc8b267863090fc467bc9dd7827e9e1a Mon Sep 17 00:00:00 2001 From: csviri Date: Wed, 29 Mar 2023 13:00:28 +0200 Subject: [PATCH 7/7] improvements --- .../jenvtest/process/KubeAPIServerProcess.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java index 22c9ce4..891c54c 100644 --- a/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java +++ b/core/src/main/java/io/javaoperatorsdk/jenvtest/process/KubeAPIServerProcess.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.IOException; +import java.net.ConnectException; import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; @@ -124,6 +125,10 @@ private boolean ready(HttpClient client, HttpRequest request) { var response = client.send(request, HttpResponse.BodyHandlers.ofString()); log.trace("Ready Response message:{} code: {}", response.body(), response.statusCode()); return response.statusCode() == 200; + } catch (ConnectException e) { + // still want to retry + log.warn("Cannot connect to the server", e); + return false; } catch (IOException e) { throw new JenvtestException(e); } catch (InterruptedException e) {