V0.8.0 (#67)

* upgrade nebula-go version

Makefile

@@ -5,7 +5,7 @@ LDFLAGS = $(if $(DEBUGGER),,-s -w) $(shell ./hack/version.sh)
 
 DOCKER_REGISTRY ?= localhost:5000
 DOCKER_REPO ?= ${DOCKER_REGISTRY}/vesoft
-IMAGE_TAG ?= latest
+IMAGE_TAG ?= v0.8.0
 
 export GO111MODULE := on
 GOOS := $(if $(GOOS),$(GOOS),linux)

apis/apps/v1alpha1/template.go

@@ -26,9 +26,13 @@ const (
 --pid_file=pids/nebula-graphd.pid
 # Whether to enable optimizer
 --enable_optimizer=true
+# Heartbeat interval of communication between meta client and graphd service
+--heartbeat_interval_secs=10
+# Whether to use the configuration obtained from the configuration file
+--local_config=true
 
 ########## logging ##########
-# The directory to host logging files, which must already exists
+# The directory to host logging files
 --log_dir=logs
 # Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively
 --minloglevel=0
@@ -67,7 +71,7 @@ const (
 # Seconds before the idle connections are closed, 0 for never closed
 --client_idle_timeout_secs=0
 # Seconds before the idle sessions are expired, 0 for no expiration
---session_idle_timeout_secs=0
+--session_idle_timeout_secs=60000
 # The number of threads to accept incoming connections
 --num_accept_threads=1
 # The number of networking IO threads, 0 for # of CPU cores
@@ -80,19 +84,20 @@ const (
 --ws_http_port=19669
 # HTTP2 service port
 --ws_h2_port=19670
+# storage client timeout
+--storage_client_timeout_ms=60000
+# Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file
+--ws_meta_http_port=19559
 
-# The default charset when a space is created
---default_charset=utf8
-# The defaule collate when a space is created
---default_collate=utf8_bin
-
-########## authorization ##########
+########## authentication ##########
 # Enable authorization
 --enable_authorize=false
-
-########## Authentication ##########
 # User login authentication type, password for nebula authentication, ldap for ldap authentication, cloud for cloud authentication
 --auth_type=password
+
+########## memory ##########
+# System memory high watermark ratio
+--system_memory_high_watermark_ratio=0.8
 `
 	// nolint: revive
 	MetadhConfigTemplate = `
@@ -103,7 +108,7 @@ const (
 --pid_file=pids/nebula-metad.pid
 
 ########## logging ##########
-# The directory to host logging files, which must already exists
+# The directory to host logging files
 --log_dir=logs
 # Log level, 0, 1, 2, 3 for INFO, WARNING, ERROR, FATAL respectively
 --minloglevel=0
@@ -134,6 +139,8 @@ const (
 --ws_http_port=19559
 # HTTP2 service port
 --ws_h2_port=19560
+# Port to listen on Storage with HTTP protocol, it corresponds to ws_http_port in storage's configuration file
+--ws_storage_http_port=19779
 
 ########## storage ##########
 # Root data path, here should be only single path for metad
@@ -146,13 +153,20 @@ const (
 --default_replica_factor=1
 
 --heartbeat_interval_secs=10
+
+############## rocksdb Options ##############
+--rocksdb_wal_sync=true
+
 `
 	// nolint: revive
-	StoragedConfigTemplate = `########## basics ##########
+	StoragedConfigTemplate = `
+########## basics ##########
 # Whether to run as a daemon process
 --daemonize=true
 # The file to host the process id
 --pid_file=pids/nebula-storaged.pid
+# Whether to use the configuration obtained from the configuration file
+--local_config=true
 
 ########## logging ##########
 # The directory to host logging files
@@ -202,12 +216,15 @@ const (
 # One path per Rocksdb instance.
 --data_path=data/storage
 
+# Minimum reserved bytes of each data path
+--minimum_reserved_bytes=268435456
+
 # The default reserved bytes for one batch operation
 --rocksdb_batch_size=4096
 # The default block cache size used in BlockBasedTable.
 # The unit is MB.
 --rocksdb_block_cache=4
-# The type of storage engine, 'rocksdb'', 'memory', etc.
+# The type of storage engine: rocksdb, memory, etc.
 --engine_type=rocksdb
 
 # Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd
@@ -237,11 +254,6 @@ const (
 
 # Whether or not to enable rocksdb's prefix bloom filter, disabled by default.
 --enable_rocksdb_prefix_filtering=false
-# Whether or not to enable the whole key filtering.
---enable_rocksdb_whole_key_filtering=true
-# The prefix length for each key to use as the filter value.
-# can be 12 bytes(PartitionId + VertexID), or 16 bytes(PartitionId + VertexID + TagID/EdgeType).
---rocksdb_filtering_prefix_length=12
 
 ############## rocksdb Options ##############
 # rocksdb DBOptions in json, each name and value of option is a string, given as "option_name":"option_value" separated by comma

charts/index.yaml

@@ -3,9 +3,9 @@ entries:
   nebula-cluster:
   - apiVersion: v2
     appVersion: 0.1.0
-    created: "2021-07-20T16:38:43.161264382+08:00"
+    created: "2021-09-16T16:19:58.816993896+08:00"
     description: Nebula Cluster Helm chart for Kubernetes
-    digest: aa03c80a212414404e11aa1b026802b86f33063f9eaf2ea484b4195e0d301ecc
+    digest: edf2f840b19fc03b93b8a3c11bbe609a0be981062f205c23f02528ef8dd0372b
     home: https://nebula-graph.io
     keywords:
     - kubernetes
@@ -29,9 +29,9 @@ entries:
   nebula-operator:
   - apiVersion: v2
     appVersion: 0.1.0
-    created: "2021-07-20T16:38:43.162192859+08:00"
+    created: "2021-09-16T16:19:58.818392237+08:00"
     description: Nebula Operator Helm chart for Kubernetes
-    digest: 92ff3f3e6eda554ae4e3bdacdf9b5d95eaf2a2f12ab2d01a0bbc898dcb83fe21
+    digest: 6d4a55fe209d7274e8625bef9a5c22a6063c8706839db9954003c8df06417271
     home: https://nebula-graph.io
     keywords:
     - kubernetes
@@ -50,4 +50,4 @@ entries:
     urls:
     - https://vesoft-inc.github.io/nebula-operator/charts/nebula-operator-0.1.0.tgz
     version: 0.1.0
-generated: "2021-07-20T16:38:43.160050416+08:00"
+generated: "2021-09-16T16:19:58.81598799+08:00"

charts/nebula-cluster/values.yaml

@@ -1,5 +1,5 @@
 nebula:
-  version: v2.0.1
+  version: v2.5.0
   imagePullPolicy: IfNotPresent
   storageClassName: ""
   schedulerName: default-scheduler # nebula-scheduler

charts/nebula-operator/values.yaml

@@ -1,6 +1,6 @@
 image:
   nebulaOperator:
-    image: vesoft/nebula-operator:latest
+    image: vesoft/nebula-operator:v0.8.0
     imagePullPolicy: IfNotPresent
   kubeRBACProxy:
     image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0

go.mod

@@ -3,7 +3,7 @@ module github.com/vesoft-inc/nebula-operator
 go 1.15
 
 require (
-	github.com/facebook/fbthrift v0.0.0-20190922225929-2f9839604e25
+	github.com/facebook/fbthrift v0.31.1-0.20210223140454-614a73a42488
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-logr/logr v0.3.0
 	github.com/google/go-cmp v0.5.2
@@ -13,7 +13,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
-	github.com/vesoft-inc/nebula-go v0.0.0-20210322063004-e37085b865d6
+	github.com/vesoft-inc/nebula-go/v2 v2.5.0
 	go.uber.org/zap v1.15.0
 	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.19.2
@@ -22,7 +22,7 @@ require (
 	k8s.io/cli-runtime v0.19.2
 	k8s.io/client-go v0.19.2
 	k8s.io/code-generator v0.19.2
-	k8s.io/klog/v2 v2.4.0 // indirect
+	k8s.io/klog/v2 v2.4.0
 	k8s.io/kube-scheduler v0.0.0
 	k8s.io/kubectl v0.0.0
 	k8s.io/kubernetes v1.19.2

go.sum

@@ -168,8 +168,8 @@ github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQo
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
-github.com/facebook/fbthrift v0.0.0-20190922225929-2f9839604e25 h1:dezRDs9oGYxeavyvcNg/Js+dK6kIvfzERoJ7K8Xkv14=
-github.com/facebook/fbthrift v0.0.0-20190922225929-2f9839604e25/go.mod h1:2tncLx5rmw69e5kMBv/yJneERbzrr1yr5fdlnTbu8lU=
+github.com/facebook/fbthrift v0.31.1-0.20210223140454-614a73a42488 h1:A4KCT0mvTBkvb93gGN+efLPkrgTqmqMeaLDG51KVhMM=
+github.com/facebook/fbthrift v0.31.1-0.20210223140454-614a73a42488/go.mod h1:2tncLx5rmw69e5kMBv/yJneERbzrr1yr5fdlnTbu8lU=
 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -554,8 +554,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
@@ -572,8 +571,8 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
-github.com/vesoft-inc/nebula-go v0.0.0-20210322063004-e37085b865d6 h1:qdKi4VNi8DzD/gAAEHztqJr2KDpcQfkEhP1liIMRRm8=
-github.com/vesoft-inc/nebula-go v0.0.0-20210322063004-e37085b865d6/go.mod h1:wxhWNoVVP0+TLETNwaIa93H9tOECxlEv4XH+/do22Qk=
+github.com/vesoft-inc/nebula-go/v2 v2.5.0 h1:A4wZoGqR1W1mQ9y/X+tV1wCMIsDkuhFMthTQKMeT8Yc=
+github.com/vesoft-inc/nebula-go/v2 v2.5.0/go.mod h1:fehDUs97/mpmxXi9WezhznX0Dg7hmQRUoOWgDZv9zG0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
 github.com/vishvananda/netns v0.0.0-20200520041808-52d707b772fe/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
@@ -864,8 +863,9 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

pkg/controller/component/pvc_gc.go

@@ -65,7 +65,6 @@ func PvcMark(pvcClient kube.PersistentVolumeClaim, component v1alpha1.NebulaClus
 		}
 		now := time.Now().Format(time.RFC3339)
 		pvc.Annotations[annotation.AnnPVCDeferDeletingKey] = now
-		log := log.WithValues("key", annotation.AnnPVCDeferDeletingKey, now)
 		if err := pvcClient.UpdatePVC(pvc); err != nil {
 			log.Error(err, "failed to set pvc annotation", "pvcName", pvcName)
 			return err

pkg/controller/component/storaged_scaler.go

@@ -21,7 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
-	nebulago "github.com/vesoft-inc/nebula-go/nebula"
+	nebulago "github.com/vesoft-inc/nebula-go/v2/nebula"
 	"github.com/vesoft-inc/nebula-operator/apis/apps/v1alpha1"
 	"github.com/vesoft-inc/nebula-operator/pkg/kube"
 	"github.com/vesoft-inc/nebula-operator/pkg/nebula"
@@ -116,7 +116,7 @@ func (ss *storageScaler) ScaleIn(nc *v1alpha1.NebulaCluster, oldReplicas, newRep
 
 	if oldReplicas-newReplicas > 0 {
 		hosts := make([]*nebulago.HostAddr, 0, oldReplicas-newReplicas)
-		port := nebulago.Port(nc.StoragedComponent().GetPort(v1alpha1.StoragedPortNameThrift))
+		port := nc.StoragedComponent().GetPort(v1alpha1.StoragedPortNameThrift)
 		for i := oldReplicas - 1; i >= newReplicas; i-- {
 			hosts = append(hosts, &nebulago.HostAddr{
 				Host: nc.StoragedComponent().GetPodFQDN(i),

pkg/controller/nebulacluster/nebula_cluster_control_test.go

@@ -124,6 +124,7 @@ func Test_defaultNebulaClusterControl_UpdateNebulaCluster(t *testing.T) {
 	}
 }
 
+// nolint: gocritic
 func newFakeNebulaClusterControl() (
 	ControlInterface,
 	*component.FakeGraphdCluster,

pkg/nebula/client.go

@@ -17,11 +17,14 @@ limitations under the License.
 package nebula
 
 import (
+	"math"
+
 	"github.com/facebook/fbthrift/thrift/lib/go/thrift"
 )
 
 const (
 	defaultBufferSize = 128 << 10
+	frameMaxLength    = math.MaxUint32
 )
 
 func buildClientTransport(endpoint string, options ...Option) (thrift.Transport, thrift.ProtocolFactory, error) {
@@ -32,7 +35,8 @@ func buildClientTransport(endpoint string, options ...Option) (thrift.Transport,
 	if err != nil {
 		return nil, nil, err
 	}
-	transport := thrift.NewBufferedTransport(sock, defaultBufferSize)
+	bufferedTranFactory := thrift.NewBufferedTransportFactory(defaultBufferSize)
+	transport := thrift.NewFramedTransportMaxLength(bufferedTranFactory.GetTransport(sock), frameMaxLength)
 	pf := thrift.NewBinaryProtocolFactoryDefault()
 
 	return transport, pf, nil