remove ClickHouse support

.github/workflows/build_test.yml

@@ -50,11 +50,6 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
-      clickhouse:
-        image: yandex/clickhouse-server:21.10
-        ports:
-          - 8123
-
       redis:
         image: redis:5.0
         ports:
@@ -107,8 +102,6 @@ jobs:
         POSTGRES_URI: postgres://gorse:gorse_pass@localhost:${{ job.services.postgres.ports[5432] }}/
         # MongoDB
         MONGO_URI: mongodb://root:password@localhost:${{ job.services.mongo.ports[27017] }}/
-        # ClickHouse
-        CLICKHOUSE_URI: clickhouse://localhost:${{ job.services.clickhouse.ports[8123] }}/
         # Redis
         REDIS_URI: redis://localhost:${{ job.services.redis.ports[6379] }}/
 
@@ -120,7 +113,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        database: [mysql, postgres, mongo, clickhouse]
+        database: [mysql, postgres, mongo]
     steps:
     - uses: actions/checkout@v1
 

CONTRIBUTING.md

@@ -80,7 +80,6 @@ The default database URLs are directed to these databases in `storage/docker-com
 | `MYSQL_URI`       | `mysql://root:password@tcp(127.0.0.1:3306)/` |
 | `POSTGRES_URI`    | `postgres://gorse:gorse_pass@127.0.0.1/`     |
 | `MONGO_URI`       | `mongodb://root:password@127.0.0.1:27017/`   |
-| `CLICKHOUSE_URI`  | `clickhouse://127.0.0.1:8123/`               |
 | `REDIS_URI`       | `redis://127.0.0.1:6379/`                    |
 
 For example, use TiDB as a test database by:

README.md

@@ -95,7 +95,7 @@ For more information：
 
 ## Architecture
 
-Gorse is a single-node training and distributed prediction recommender system. Gorse stores data in MySQL, MongoDB, Postgres, or ClickHouse, with intermediate results cached in Redis, MySQL, MongoDB and Postgres.
+Gorse is a single-node training and distributed prediction recommender system. Gorse stores data in MySQL, MongoDB, or Postgres, with intermediate results cached in Redis, MySQL, MongoDB and Postgres.
 
 1. The cluster consists of a master node, multiple worker nodes, and server nodes.
 1. The master node is responsible for model training, non-personalized item recommendation, configuration management, and membership management.

client/docker-compose.yml.j2

@@ -43,17 +43,6 @@ services:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: password
 
-  {% elif database == 'clickhouse' %}
-
-  clickhouse:
-    image: yandex/clickhouse-server:21.10
-    ports:
-      - 8123:8123
-    environment:
-      CLICKHOUSE_DB: gorse
-      CLICKHOUSE_USER: gorse
-      CLICKHOUSE_PASSWORD: gorse_pass
-
   {% endif %}
 
   worker:
@@ -108,8 +97,6 @@ services:
       GORSE_DATA_STORE: postgres://gorse:gorse_pass@postgres/gorse?sslmode=disable
       {% elif database == 'mongo' %}
       GORSE_DATA_STORE: mongodb://root:password@mongo:27017/gorse?authSource=admin&connect=direct
-      {% elif database == 'clickhouse' %}
-      GORSE_DATA_STORE: clickhouse://gorse:gorse_pass@clickhouse:8123/gorse?mutations_sync=2
       {% endif %}
     command: >
       -c /etc/gorse/config.toml 
@@ -127,8 +114,6 @@ services:
       - postgres
       {% elif database == 'mongo' %}
       - mongo
-      {% elif database == 'clickhouse' %}
-      - clickhouse
       {% endif %}
 
 volumes:

config/config.go

@@ -593,9 +593,6 @@ func (config *Config) Validate(oneModel bool) error {
 			storage.MySQLPrefix,
 			storage.PostgresPrefix,
 			storage.PostgreSQLPrefix,
-			storage.ClickhousePrefix,
-			storage.CHHTTPPrefix,
-			storage.CHHTTPSPrefix,
 		}
 		if oneModel {
 			prefixes = append(prefixes, storage.SQLitePrefix)

config/config.toml

@@ -9,13 +9,10 @@
 #   mongodb+srv://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
 cache_store = "redis://localhost:6379/0"
 
-# The database for persist data, support MySQL, Postgres, ClickHouse and MongoDB:
+# The database for persist data, support MySQL, Postgres and MongoDB:
 #   mysql://[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
 #   postgres://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full
 #   postgresql://bob:secret@1.2.3.4:5432/mydb?sslmode=verify-full
-#   clickhouse://user:password@host[:port]/database?param1=value1&...&paramN=valueN
-#   chhttp://user:password@host[:port]/database?param1=value1&...&paramN=valueN
-#   chhttps://user:password@host[:port]/database?param1=value1&...&paramN=valueN
 #   mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
 #   mongodb+srv://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
 data_store = "mysql://gorse:gorse_pass@tcp(localhost:3306)/gorse"

docker-compose.yml

@@ -41,17 +41,6 @@ services:
   #   volumes:
   #     - mongo_data:/data/db
 
-  # clickhouse:
-  #   image: yandex/clickhouse-server:21.10
-  #   ports:
-  #     - 8123:8123
-  #   environment:
-  #     CLICKHOUSE_DB: gorse
-  #     CLICKHOUSE_USER: gorse
-  #     CLICKHOUSE_PASSWORD: gorse_pass
-  #   volumes:
-  #     - clickhouse_data:/var/lib/clickhouse
-
   worker:
     image: zhenghaoz/gorse-worker
     restart: unless-stopped
@@ -95,7 +84,6 @@ services:
       GORSE_DATA_STORE: mysql://gorse:gorse_pass@tcp(mysql:3306)/gorse
       # GORSE_DATA_STORE: postgres://gorse:gorse_pass@postgres/gorse?sslmode=disable
       # GORSE_DATA_STORE: mongodb://root:password@mongo:27017/gorse?authSource=admin&connect=direct
-      # GORSE_DATA_STORE: clickhouse://gorse:gorse_pass@clickhouse:8123/gorse
     command: >
       -c /etc/gorse/config.toml 
       --log-path /var/log/gorse/master.log 
@@ -109,7 +97,6 @@ services:
       - mysql
       # - postgres
       # - mongo
-      # - clickhouse
 
 volumes:
   worker_data:
@@ -119,4 +106,3 @@ volumes:
   mysql_data:
   # postgres_data:
   # mongo_data:
-  # clickhouse_data:

go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/klauspost/cpuid/v2 v2.1.0
 	github.com/lafikl/consistent v0.0.0-20220512074542-bdd3606bfc3e
 	github.com/lib/pq v1.10.6
-	github.com/mailru/go-clickhouse/v2 v2.0.1-0.20221121001540-b259988ad8e5
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/orcaman/concurrent-map v1.0.0
 	github.com/prometheus/client_golang v1.13.0
@@ -61,7 +60,6 @@ require (
 	google.golang.org/grpc v1.50.1
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/yaml.v2 v2.4.0
-	gorm.io/driver/clickhouse v0.4.2
 	gorm.io/driver/mysql v1.3.4
 	gorm.io/driver/postgres v1.3.5
 	gorm.io/driver/sqlite v1.3.4
@@ -92,7 +90,6 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
-	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
@@ -165,7 +162,4 @@ require (
 	modernc.org/token v1.0.1 // indirect
 )
 
-replace (
-	gorm.io/driver/clickhouse v0.4.2 => github.com/gorse-io/clickhouse v0.3.3-0.20220715124633-688011a495bb
-	gorm.io/driver/sqlite v1.3.4 => github.com/gorse-io/sqlite v1.3.3-0.20220713123255-c322aec4e59e
-)
+replace gorm.io/driver/sqlite v1.3.4 => github.com/gorse-io/sqlite v1.3.3-0.20220713123255-c322aec4e59e

go.sum

@@ -245,26 +245,20 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
 github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
 github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorse-io/clickhouse v0.3.3-0.20220715124633-688011a495bb h1:z/oOWE+Vy0PLcwIulZmIug4FtmvE3dJ1YOGprLeHwwY=
-github.com/gorse-io/clickhouse v0.3.3-0.20220715124633-688011a495bb/go.mod h1:iILWzbul8U+gsf4kqbheF2QzBmdvVp63mloGGK8emDI=
 github.com/gorse-io/dashboard v0.0.0-20230224155550-e75876b93f90 h1:fw15FnjCLaxNTv2TM/Jq9qOw2GdZOVL98VY8NmYldPY=
 github.com/gorse-io/dashboard v0.0.0-20230224155550-e75876b93f90/go.mod h1:w74IGf70uM5ZCeXmkBhLl3Ux6D+HpBryzcc75VfZA4s=
 github.com/gorse-io/sqlite v1.3.3-0.20220713123255-c322aec4e59e h1:uPQtYQzG1QcC3Qbv+tuEe8Q2l++V4KEcqYSSwB9qobg=
 github.com/gorse-io/sqlite v1.3.3-0.20220713123255-c322aec4e59e/go.mod h1:PmIOwYnI+F1lRKd6F/PdLXGgI8GZ5H8x8z1yx0+0bmQ=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
-github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
-github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -388,8 +382,6 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mailru/go-clickhouse/v2 v2.0.1-0.20221121001540-b259988ad8e5 h1:JgQ+kJg8uKs6JjnDxnMgkKT4PPH36uU6chpYw2PQc9Q=
-github.com/mailru/go-clickhouse/v2 v2.0.1-0.20221121001540-b259988ad8e5/go.mod h1:TwxN829KnFZ7jAka9l9EoCV+U0CBFq83SFev4oLbnNU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=

server/bench_test.go

@@ -19,6 +19,16 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
 	"github.com/emicklei/go-restful/v3"
 	"github.com/go-redis/redis/v9"
 	"github.com/go-resty/resty/v2"
@@ -31,15 +41,6 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 	"go.mongodb.org/mongo-driver/mongo/options"
 	"google.golang.org/protobuf/proto"
-	"math/rand"
-	"net"
-	"net/http"
-	"os"
-	"runtime"
-	"strconv"
-	"strings"
-	"testing"
-	"time"
 )
 
 const (
@@ -60,7 +61,7 @@ func init() {
 		}
 		return defaultValue
 	}
-	benchDataStore = env("BENCH_DATA_STORE", "clickhouse://127.0.0.1:8123/")
+	benchDataStore = env("BENCH_DATA_STORE", "mysql://root:password@tcp(127.0.0.1:3306)/")
 	benchCacheStore = env("BENCH_CACHE_STORE", "redis://127.0.0.1:6379/")
 }
 
@@ -190,17 +191,6 @@ func (s *benchServer) prepareData(b *testing.B, url, benchName string) string {
 		err = db.Close()
 		require.NoError(b, err)
 		return url + strings.ToLower(dbName) + "?sslmode=disable&TimeZone=UTC"
-	} else if strings.HasPrefix(url, "clickhouse://") {
-		uri := "http://" + url[len("clickhouse://"):]
-		db, err := sql.Open("clickhouse", uri)
-		require.NoError(b, err)
-		_, err = db.Exec("DROP DATABASE IF EXISTS " + dbName)
-		require.NoError(b, err)
-		_, err = db.Exec("CREATE DATABASE " + dbName)
-		require.NoError(b, err)
-		err = db.Close()
-		require.NoError(b, err)
-		return url + dbName + "?mutations_sync=2"
 	} else if strings.HasPrefix(url, "mongodb://") {
 		ctx := context.Background()
 		cli, err := mongo.Connect(ctx, options.Client().ApplyURI(url+"?authSource=admin&connect=direct"))

server/bench_test.sh

@@ -40,9 +40,6 @@ case $CACHE_ARG in
 esac
 
 case $DATA_ARG in
-  clickhouse)
-    export BENCH_DATA_STORE='clickhouse://127.0.0.1:8123/'
-    ;;
   mysql)
     export BENCH_DATA_STORE='mysql://root:password@tcp(127.0.0.1:3306)/'
     ;;

storage/data/database.go

@@ -16,7 +16,6 @@ package data
 
 import (
 	"context"
-	"net/url"
 	"sort"
 	"strings"
 	"time"
@@ -33,7 +32,6 @@ import (
 	"go.mongodb.org/mongo-driver/x/mongo/driver/connstring"
 	"go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo"
 	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
-	"gorm.io/driver/clickhouse"
 	"gorm.io/driver/mysql"
 	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
@@ -160,7 +158,6 @@ type Database interface {
 	Init() error
 	Ping() error
 	Close() error
-	Optimize() error
 	Purge() error
 	BatchInsertItems(ctx context.Context, items []Item) error
 	BatchGetItems(ctx context.Context, itemIds []string) ([]Item, error)
@@ -232,32 +229,6 @@ func Open(path, tablePrefix string) (Database, error) {
 			return nil, errors.Trace(err)
 		}
 		return database, nil
-	} else if strings.HasPrefix(path, storage.ClickhousePrefix) || strings.HasPrefix(path, storage.CHHTTPPrefix) || strings.HasPrefix(path, storage.CHHTTPSPrefix) {
-		// replace schema
-		parsed, err := url.Parse(path)
-		if err != nil {
-			return nil, errors.Trace(err)
-		}
-		if strings.HasPrefix(path, storage.CHHTTPSPrefix) {
-			parsed.Scheme = "https"
-		} else {
-			parsed.Scheme = "http"
-		}
-		uri := parsed.String()
-		database := new(SQLDatabase)
-		database.driver = ClickHouse
-		database.TablePrefix = storage.TablePrefix(tablePrefix)
-		if database.client, err = otelsql.Open("chhttp", uri,
-			otelsql.WithAttributes(semconv.DBSystemKey.String("clickhouse")),
-			otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}),
-		); err != nil {
-			return nil, errors.Trace(err)
-		}
-		database.gormDB, err = gorm.Open(clickhouse.New(clickhouse.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix))
-		if err != nil {
-			return nil, errors.Trace(err)
-		}
-		return database, nil
 	} else if strings.HasPrefix(path, storage.MongoPrefix) || strings.HasPrefix(path, storage.MongoSrvPrefix) {
 		// connect to database
 		database := new(MongoDB)