Add https support (#171)

* Add https support

* Use 'sudo' in update-hosts.sh

* Delete certs and generate them during the workflow run

---------

Co-authored-by: a.stamov <a.stamov@rt-solar.ru>

.docker/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3'
+services:
+  clickhouse:
+      # https://docs.docker.com/compose/compose-file/#variable-substitution
+      image: "clickhouse/clickhouse-server:${CLICKHOUSE_VERSION:-22.3}"
+      volumes:
+        - ./entrypoint.sh:/custom-entrypoint.sh
+        - ./server-config.xml:/etc/clickhouse-server/config.d/server-config.xml
+        - ./certs/server.crt:/tmp/certs/server.crt
+        - ./certs/server.key:/tmp/certs/server.key
+        - ./certs/ca.crt:/tmp/certs/ca.crt
+      expose:
+        - "8123"
+        - "8447"
+      ports:
+        - "8123:8123"
+        - "8447:8447"
+        - "9000:9000"
+      entrypoint:
+        - /custom-entrypoint.sh
+      hostname: clickhouseserver.test

.docker/entrypoint.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+CERTS_DIR=/etc/clickhouse-server/certs/
+mkdir -p $CERTS_DIR
+# Copy cert files to $CERTS_DIR and apply required rights so as not to affect the original files
+cp /tmp/certs/* $CERTS_DIR
+chown clickhouse:clickhouse $CERTS_DIR*
+chmod 644 $CERTS_DIR*
+/entrypoint.sh

.docker/generate-certs.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# Script is used to (re)generate self-signed certificates needed to run the tests
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+CERTS_DIR=$SCRIPT_DIR/certs
+rm -rf $CERTS_DIR
+mkdir -p $CERTS_DIR
+cd $CERTS_DIR
+openssl genrsa -out ca.key 2048
+openssl req -x509 -subj "/CN=clickhouseserver.test CA" -nodes -key ca.key -days 3650 -out ca.crt
+openssl req -newkey rsa:2048 -nodes -subj "/CN=clickhouseserver.test" -keyout server.key -out server.csr
+openssl x509 -req -in server.csr -out server.crt -CA ca.crt -CAkey ca.key -days 3650 -copy_extensions copy
+openssl req -newkey rsa:2048 -nodes -subj "/CN=clickhouseserver.test" -keyout client.key -out client.csr
+openssl x509 -req -in client.csr -out client.crt -CA ca.crt -CAkey ca.key -days 3650 -copy_extensions copy
+
+openssl pkcs12 -export -in client.crt -inkey client.key -out keystore.p12 -name client -CAfile ca.crt -caname 'clickhouseserver.test CA' -password pass:password
+
+keytool -importkeystore -deststorepass password -destkeypass password -destkeystore keystore.jks -deststoretype JKS -srckeystore keystore.p12 -srcstoretype PKCS12 -srcstorepass password -alias client -noprompt
+
+keytool -importcert -alias ca -file ca.crt -keystore keystore.jks -storepass password -noprompt
+rm ca.key client.key client.csr client.crt server.csr keystore.p12 

.docker/server-config.xml

@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!--
+  NOTE: User and query level settings are set up in "users.xml" file.
+  If you have accidentally specified user-level settings here, server won't start.
+  You can either move the settings to the right place inside "users.xml" file
+   or add <skip_check_for_incorrect_settings>1</skip_check_for_incorrect_settings> here.
+-->
+<yandex>
+
+    <!-- HTTP API with TLS (HTTPS).
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+    -->
+    <https_port>8447</https_port>
+
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <openSSL>
+        <server> <!-- Used for https server AND secure tcp port -->
+            <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
+            <certificateFile>/etc/clickhouse-server/certs/server.crt</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/certs/server.key</privateKeyFile>
+            <caConfig>/etc/clickhouse-server/certs/ca.crt</caConfig>
+            <!-- dhparams are optional. You can delete the <dhParamsFile> element.
+                 To generate dhparams, use the following command:
+                  openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
+                 Only file format with BEGIN DH PARAMETERS is supported.
+              -->
+            <dhParamsFile remove="remove">/etc/clickhouse-server/dhparam.pem</dhParamsFile>
+            <verificationMode>relaxed</verificationMode>
+            <cacheSessions>true</cacheSessions>
+            <preferServerCiphers>true</preferServerCiphers>
+            <loadDefaultCAFile>false</loadDefaultCAFile>
+        </server>
+    </openSSL>
+</yandex>

.docker/update-hosts.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Add 'clickhouseserver.test' to '/etc/hosts' file if it is not already there
+if ! grep -q '127.0.0.1[[:space:]]*clickhouseserver.test'  /etc/hosts; then 
+  echo '127.0.0.1   clickhouseserver.test' | sudo tee -a /etc/hosts  
+fi

.github/workflows/ci.yml

@@ -33,21 +33,27 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Docker Compose Action
-        uses: hoverkraft-tech/compose-action@v2.0.1
-        env:
-          CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
-        with:
-          compose-file: './docker-compose.yml'
-          down-flags: '--volumes'
-
       - name: Setup Java (temurin@17)
         if: matrix.java == 'temurin@17'
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: 17
 
+      - name: Update '/etc/hosts' file
+        run: ./.docker/update-hosts.sh
+
+      - name: Generate SSL certificates
+        run: ./.docker/generate-certs.sh
+
+      - name: Docker Compose Action
+        uses: hoverkraft-tech/compose-action@v2.0.1
+        env:
+          CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
+        with:
+          compose-file: './.docker/docker-compose.yml'
+          down-flags: '--volumes'
+
       - name: Cache sbt
         uses: actions/cache@v4
         with:

.gitignore

@@ -15,4 +15,11 @@ credentials.sbt
 # Scala-IDE specific
 .scala_dependencies
 .worksheet
-.idea
+.idea
+# vscode specific
+.lh
+.vscode
+.bloop
+.metals
+metals.sbt
+.docker/certs

client/src/main/scala/com/crobox/clickhouse/ClickhouseClient.scala

@@ -2,6 +2,7 @@ package com.crobox.clickhouse
 
 import org.apache.pekko.NotUsed
 import org.apache.pekko.actor.ActorSystem
+import org.apache.pekko.http.scaladsl.HttpsConnectionContext
 import org.apache.pekko.http.scaladsl.model._
 import org.apache.pekko.stream.scaladsl.{Framing, Source}
 import org.apache.pekko.util.ByteString
@@ -20,7 +21,8 @@ import scala.concurrent.{Await, ExecutionContext, Future}
  * @author Sjoerd Mulder
  * @since 31-03-17
  */
-class ClickhouseClient(configuration: Option[Config] = None)
+class ClickhouseClient(configuration: Option[Config] = None, 
+                        override val customConnectionContext: Option[HttpsConnectionContext] = None)
     extends ClickHouseExecutor
     with ClickhouseResponseParser
     with ClickhouseQueryBuilder {

client/src/main/scala/com/crobox/clickhouse/internal/ClickHouseExecutor.scala

@@ -1,7 +1,7 @@
 package com.crobox.clickhouse.internal
 
 import org.apache.pekko.actor.{ActorSystem, Terminated}
-import org.apache.pekko.http.scaladsl.Http
+import org.apache.pekko.http.scaladsl.{Http, HttpsConnectionContext}
 import org.apache.pekko.http.scaladsl.model._
 import org.apache.pekko.http.scaladsl.settings.{ClientConnectionSettings, ConnectionPoolSettings}
 import org.apache.pekko.stream._
@@ -23,6 +23,7 @@ private[clickhouse] trait ClickHouseExecutor extends LazyLogging {
   protected implicit val executionContext: ExecutionContext
   protected val hostBalancer: HostBalancer
   protected val config: Config
+  protected val customConnectionContext: Option[HttpsConnectionContext]
 
   lazy val (progressQueue, progressSource) = {
     val builtSource = QueryProgress.queryProgressStream.run()
@@ -35,7 +36,8 @@ private[clickhouse] trait ClickHouseExecutor extends LazyLogging {
       ClientConnectionSettings(system).withTransport(new StreamingProgressClickhouseTransport(progressQueue))
     )
   private lazy val http              = Http()
-  private lazy val pool              = http.superPool[Promise[HttpResponse]](settings = superPoolSettings)
+  private lazy val connectionContext = customConnectionContext.getOrElse(http.defaultClientHttpsContext)
+  private lazy val pool              = http.superPool[Promise[HttpResponse]](connectionContext = connectionContext, settings = superPoolSettings)
   private lazy val bufferSize: Int   = config.getInt("buffer-size")
   private lazy val queryRetries: Int = config.getInt("retries")
 
@@ -90,7 +92,7 @@ private[clickhouse] trait ClickHouseExecutor extends LazyLogging {
         case QueueOfferResult.Failure(e)  => Future.failed(e)
       }
     } else {
-      http.singleRequest(request)
+      http.singleRequest(request, connectionContext = connectionContext)
     }
   }
 

client/src/test/scala/com/crobox/clickhouse/ClickhouseClientTest.scala

@@ -5,6 +5,13 @@ import com.crobox.clickhouse.internal.QuerySettings
 import com.crobox.clickhouse.internal.progress.QueryProgress.{Progress, QueryAccepted, QueryFinished, QueryProgress}
 import com.typesafe.config.{ConfigFactory, ConfigValueFactory}
 import org.apache.pekko.http.scaladsl.model.headers.HttpEncodings.gzip
+import org.apache.pekko.http.scaladsl.ConnectionContext
+import java.security.KeyStore
+import javax.net.ssl.SSLContext
+import javax.net.ssl.TrustManagerFactory
+import java.io.FileInputStream
+import javax.net.ssl.KeyManagerFactory
+import java.security.SecureRandom
 
 /**
  * @author Sjoerd Mulder
@@ -29,6 +36,43 @@ class ClickhouseClientTest extends ClickhouseClientAsyncSpec {
       )
   }
 
+  it should "support SSL certs" in {
+    def createConnectionContext() = {
+      val keyStoreResource = "../.docker/certs/keystore.jks"
+      val password = "password"
+
+      val keyStore = KeyStore.getInstance("JKS")
+      val in = new FileInputStream(keyStoreResource)
+      keyStore.load(in, password.toCharArray)
+
+      val keyManagerFactory = KeyManagerFactory.getInstance("SunX509")
+      keyManagerFactory.init(keyStore, password.toCharArray)
+      val trustManagerFactory = TrustManagerFactory.getInstance("SunX509")
+      trustManagerFactory.init(keyStore)
+      val context = SSLContext.getInstance("TLS")
+      context.init(keyManagerFactory.getKeyManagers, trustManagerFactory.getTrustManagers, new SecureRandom())
+
+      val connectionContext = ConnectionContext.httpsClient(context)
+      connectionContext
+    }
+
+    new ClickhouseClient(
+      Some(config
+      .withValue("crobox.clickhouse.client.connection.port", ConfigValueFactory.fromAnyRef(8447))
+      .withValue("crobox.clickhouse.client.connection.host", ConfigValueFactory.fromAnyRef("https://clickhouseserver.test"))),
+      customConnectionContext = Some(createConnectionContext()))
+      .query("select 1 + 2")
+      .map { f =>
+        f.trim.toInt should be(3)
+      }
+      .flatMap(
+        _ =>
+          client.query("select currentDatabase()").map { f =>
+            f.trim should be("default")
+        }
+      )
+  }
+
   it should "support response compression" in {
     val client: ClickhouseClient = new ClickhouseClient(
       Some(config.resolveWith(ConfigFactory.parseString("crobox.clickhouse.client.http-compression = true")))

client/src/test/scala/com/crobox/clickhouse/internal/ClickhouseExecutorTest.scala

@@ -1,5 +1,6 @@
 package com.crobox.clickhouse.internal
 import org.apache.pekko.actor.ActorSystem
+import org.apache.pekko.http.scaladsl.HttpsConnectionContext
 import org.apache.pekko.http.scaladsl.model.{HttpResponse, Uri}
 import org.apache.pekko.stream.scaladsl.{Sink, SourceQueue}
 import org.apache.pekko.stream.{Materializer, StreamTcpException}
@@ -19,6 +20,7 @@ class ClickhouseExecutorTest extends ClickhouseClientAsyncSpec {
   private var response: Uri => Future[String] = _
   private lazy val executor = {
     new ClickHouseExecutor with ClickhouseResponseParser with ClickhouseQueryBuilder {
+      override protected val customConnectionContext: Option[HttpsConnectionContext] = None
       override protected implicit val system: ActorSystem                = self.system
       override protected implicit val executionContext: ExecutionContext = system.dispatcher
       override protected val config: Config                              = self.config.getConfig("crobox.clickhouse.client")