From c03b8577550be1322c88da2018302cbfd4ced95e Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Sat, 27 Jul 2019 02:17:33 +0900 Subject: [PATCH] Change subcommands (#85) --- README.md | 74 +- ...ributor_delete.go => dispatcher_delete.go} | 2 +- .../{distributor_get.go => dispatcher_get.go} | 2 +- ...stributor_index.go => dispatcher_index.go} | 2 +- ...de_health.go => dispatcher_node_health.go} | 2 +- ...ributor_search.go => dispatcher_search.go} | 2 +- ...utor_node_start.go => dispatcher_start.go} | 4 +- ..._peers_info.go => indexer_cluster_info.go} | 2 +- ...node_leave.go => indexer_cluster_leave.go} | 4 +- ...eers_watch.go => indexer_cluster_watch.go} | 4 +- ...r_node_snapshot.go => indexer_snapshot.go} | 2 +- ...indexer_node_start.go => indexer_start.go} | 4 +- cmd/blast/main.go | 1148 ++++++++--------- ..._peers_info.go => manager_cluster_info.go} | 2 +- ...node_leave.go => manager_cluster_leave.go} | 8 +- ...eers_watch.go => manager_cluster_watch.go} | 4 +- .../{cluster_delete.go => manager_delete.go} | 2 +- cmd/blast/{cluster_get.go => manager_get.go} | 2 +- ..._node_health.go => manager_node_health.go} | 2 +- ...ster_node_info.go => manager_node_info.go} | 14 +- cmd/blast/{cluster_set.go => manager_set.go} | 2 +- ...r_node_snapshot.go => manager_snapshot.go} | 2 +- ...cluster_node_start.go => manager_start.go} | 2 +- .../{cluster_watch.go => manager_watch.go} | 2 +- ...i_bulk_index.txt => wiki_bulk_index.jsonl} | 0 25 files changed, 645 insertions(+), 649 deletions(-) rename cmd/blast/{distributor_delete.go => dispatcher_delete.go} (97%) rename cmd/blast/{distributor_get.go => dispatcher_get.go} (96%) rename cmd/blast/{distributor_index.go => dispatcher_index.go} (98%) rename cmd/blast/{distributor_node_health.go => dispatcher_node_health.go} (97%) rename cmd/blast/{distributor_search.go => dispatcher_search.go} (97%) rename cmd/blast/{distributor_node_start.go => dispatcher_start.go} (96%) rename cmd/blast/{indexer_peers_info.go => indexer_cluster_info.go} (96%) rename cmd/blast/{indexer_node_leave.go => indexer_cluster_leave.go} (92%) rename cmd/blast/{indexer_peers_watch.go => indexer_cluster_watch.go} (95%) rename cmd/blast/{indexer_node_snapshot.go => indexer_snapshot.go} (95%) rename cmd/blast/{indexer_node_start.go => indexer_start.go} (97%) rename cmd/blast/{cluster_peers_info.go => manager_cluster_info.go} (96%) rename cmd/blast/{cluster_node_leave.go => manager_cluster_leave.go} (81%) rename cmd/blast/{cluster_peers_watch.go => manager_cluster_watch.go} (95%) rename cmd/blast/{cluster_delete.go => manager_delete.go} (96%) rename cmd/blast/{cluster_get.go => manager_get.go} (96%) rename cmd/blast/{cluster_node_health.go => manager_node_health.go} (97%) rename cmd/blast/{cluster_node_info.go => manager_node_info.go} (83%) rename cmd/blast/{cluster_set.go => manager_set.go} (97%) rename cmd/blast/{cluster_node_snapshot.go => manager_snapshot.go} (95%) rename cmd/blast/{cluster_node_start.go => manager_start.go} (98%) rename cmd/blast/{cluster_watch.go => manager_watch.go} (97%) rename example/{wiki_bulk_index.txt => wiki_bulk_index.jsonl} (100%) diff --git a/README.md b/README.md index ba90827..9012660 100644 --- a/README.md +++ b/README.md @@ -171,7 +171,7 @@ You can see the binary file when build successful like so: ```bash $ ls ./bin -blast blastd +blast ``` @@ -245,7 +245,7 @@ $ make \ Running a Blast in standalone mode is easy. Start a indexer like so: ```bash -$ ./bin/blast indexer node start \ +$ ./bin/blast indexer start \ --grpc-address=:5000 \ --http-address=:8000 \ --node-id=indexer1 \ @@ -531,7 +531,7 @@ You can see the result in JSON format. The result of the above command is: Indexing documents in bulk, run the following command: ```bash -$ ./bin/blast indexer index --grpc-address=:5000 --file=./example/wiki_bulk_index.txt --bulk +$ ./bin/blast indexer index --grpc-address=:5000 --file=./example/wiki_bulk_index.jsonl --bulk ``` You can see the result in JSON format. The result of the above command is: @@ -566,7 +566,7 @@ Also you can do above commands via HTTP REST API that listened port 5002. Indexing a document via HTTP is as following: ```bash -$ curl -X PUT 'http://127.0.0.1:8000/documents/enwiki_1' --data-binary ' +$ curl -X PUT 'http://127.0.0.1:8000/documents/enwiki_1' -H 'Content-Type: application/json' --data-binary ' { "title_en": "Search engine (computing)", "text_en": "A search engine is an information retrieval system designed to help find information stored on a computer system. The search results are usually presented in a list and are commonly called hits. Search engines help to minimize the time required to find information and the amount of information which must be consulted, akin to other techniques for managing information overload. The most public, visible form of a search engine is a Web search engine which searches for information on the World Wide Web.", @@ -596,7 +596,7 @@ $ curl -X GET 'http://127.0.0.1:8000/documents/enwiki_1' Searching documents via HTTP is as following: ```bash -$ curl -X POST 'http://127.0.0.1:8000/search' --data-binary @./example/wiki_search_request.json +$ curl -X POST 'http://127.0.0.1:8000/search' -H 'Content-Type: application/json' --data-binary @./example/wiki_search_request.json ``` @@ -614,7 +614,7 @@ $ curl -X DELETE 'http://127.0.0.1:8000/documents/enwiki_1' Indexing documents in bulk via HTTP is as following: ```bash -$ curl -X PUT 'http://127.0.0.1:8000/documents?bulk=true' --data-binary @./example/wiki_bulk_index.txt +$ curl -X PUT 'http://127.0.0.1:8000/documents?bulk=true' -H 'Content-Type: application/x-ndjson' --data-binary @./example/wiki_bulk_index.jsonl ``` @@ -623,7 +623,7 @@ $ curl -X PUT 'http://127.0.0.1:8000/documents?bulk=true' --data-binary @./examp Deleting documents in bulk via HTTP is as following: ```bash -$ curl -X DELETE 'http://127.0.0.1:8000/documents' --data-binary @./example/wiki_bulk_delete.txt +$ curl -X DELETE 'http://127.0.0.1:8000/documents' -H 'Content-Type: text/plain' --data-binary @./example/wiki_bulk_delete.txt ``` @@ -636,7 +636,7 @@ Blast can easily bring up a cluster. Running a Blast in standalone is not fault First of all, start a indexer in standalone. ```bash -$ ./bin/blast indexer node start \ +$ ./bin/blast indexer start \ --grpc-address=:5000 \ --http-address=:8000 \ --node-id=indexer1 \ @@ -651,7 +651,7 @@ $ ./bin/blast indexer node start \ Then, start two more indexers. ```bash -$ ./bin/blast indexer node start \ +$ ./bin/blast indexer start \ --peer-grpc-address=:5000 \ --grpc-address=:5010 \ --http-address=:8010 \ @@ -660,7 +660,7 @@ $ ./bin/blast indexer node start \ --data-dir=/tmp/blast/indexer2 \ --raft-storage-type=boltdb -$ ./bin/blast indexer node start \ +$ ./bin/blast indexer start \ --peer-grpc-address=:5000 \ --grpc-address=:5020 \ --http-address=:8020 \ @@ -677,7 +677,7 @@ So you have a 3-node cluster. That way you can tolerate the failure of 1 node. Y ```bash -$ ./bin/blast indexer peers info --grpc-address=:5000 +$ ./bin/blast indexer cluster info --grpc-address=:5000 ``` You can see the result in JSON format. The result of the above command is: @@ -783,7 +783,7 @@ Blast provides the following type of node for federation: Manager can also bring up a cluster like an indexer. Specify a common index mapping for federation at startup. ```bash -$ ./bin/blast cluster node start \ +$ ./bin/blast manager start \ --grpc-address=:5100 \ --http-address=:8100 \ --node-id=cluster1 \ @@ -794,7 +794,7 @@ $ ./bin/blast cluster node start \ --index-type=upside_down \ --index-storage-type=boltdb -$ ./bin/blast cluster node start \ +$ ./bin/blast manager start \ --peer-grpc-address=:5100 \ --grpc-address=:5110 \ --http-address=:8110 \ @@ -803,7 +803,7 @@ $ ./bin/blast cluster node start \ --data-dir=/tmp/blast/cluster2 \ --raft-storage-type=boltdb -$ ./bin/blast cluster node start \ +$ ./bin/blast manager start \ --peer-grpc-address=:5100 \ --grpc-address=:5120 \ --http-address=:8120 \ @@ -819,8 +819,8 @@ Federated mode differs from cluster mode that it specifies the manager in start The following example starts two 3-node clusters. ```bash -$ ./bin/blast indexer node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast indexer start \ + --manager-grpc-address=:5100 \ --shard-id=shard1 \ --grpc-address=:5000 \ --http-address=:8000 \ @@ -829,8 +829,8 @@ $ ./bin/blast indexer node start \ --data-dir=/tmp/blast/indexer1 \ --raft-storage-type=boltdb -$ ./bin/blast indexer node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast indexer start \ + --manager-grpc-address=:5100 \ --shard-id=shard1 \ --grpc-address=:5010 \ --http-address=:8010 \ @@ -839,8 +839,8 @@ $ ./bin/blast indexer node start \ --data-dir=/tmp/blast/indexer2 \ --raft-storage-type=boltdb -$ ./bin/blast indexer node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast indexer start \ + --manager-grpc-address=:5100 \ --shard-id=shard1 \ --grpc-address=:5020 \ --http-address=:8020 \ @@ -849,8 +849,8 @@ $ ./bin/blast indexer node start \ --data-dir=/tmp/blast/indexer3 \ --raft-storage-type=boltdb -$ ./bin/blast indexer node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast indexer start \ + --manager-grpc-address=:5100 \ --shard-id=shard2 \ --grpc-address=:5030 \ --http-address=:8030 \ @@ -859,8 +859,8 @@ $ ./bin/blast indexer node start \ --data-dir=/tmp/blast/indexer4 \ --raft-storage-type=boltdb -$ ./bin/blast indexer node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast indexer start \ + --manager-grpc-address=:5100 \ --shard-id=shard2 \ --grpc-address=:5040 \ --http-address=:8040 \ @@ -869,8 +869,8 @@ $ ./bin/blast indexer node start \ --data-dir=/tmp/blast/indexer5 \ --raft-storage-type=boltdb -$ ./bin/blast indexer node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast indexer start \ + --manager-grpc-address=:5100 \ --shard-id=shard2 \ --grpc-address=:5050 \ --http-address=:8050 \ @@ -885,22 +885,28 @@ $ ./bin/blast indexer node start \ Finally, start the dispatcher with a manager that manages the target federation so that it can perform distributed search and indexing. ```bash -$ ./bin/blast distributor node start \ - --cluster-grpc-address=:5100 \ +$ ./bin/blast dispatcher start \ + --manager-grpc-address=:5100 \ --grpc-address=:5200 \ --http-address=:8200 ``` ```bash -$ ./bin/blast distributor index --grpc-address=:5200 --file=./example/wiki_bulk_index.txt --bulk +$ ./bin/blast manager cluster info --grpc-address=:5100 +$ ./bin/blast indexer cluster info --grpc-address=:5000 +$ ./bin/blast indexer cluster info --grpc-address=:5040 ``` ```bash -$ ./bin/blast distributor search --grpc-address=:5200 --file=./example/wiki_search_request_simple.json +$ ./bin/blast dispatcher index --grpc-address=:5200 --file=./example/wiki_bulk_index.jsonl --bulk ``` ```bash -$ ./bin/blast distributor delete --grpc-address=:5200 --file=./example/wiki_bulk_delete.txt +$ ./bin/blast dispatcher search --grpc-address=:5200 --file=./example/wiki_search_request_simple.json +``` + +```bash +$ ./bin/blast dispatcher delete --grpc-address=:5200 --file=./example/wiki_bulk_delete.txt ``` @@ -945,7 +951,7 @@ $ docker run --rm --name blast-indexer1 \ -p 5000:5000 \ -p 8000:8000 \ -v $(pwd)/example:/opt/blast/example \ - mosuka/blast:latest blast indexer node start \ + mosuka/blast:latest blast indexer start \ --grpc-address=:5000 \ --http-address=:8000 \ --node-id=blast-indexer1 \ @@ -995,7 +1001,7 @@ $ ./WikiExtractor.py -o ~/tmp/enwiki --json ~/tmp/enwiki-20190101-pages-articles ### Starting Indexer ```bash -$ ./bin/blast indexer node start \ +$ ./bin/blast indexer start \ --grpc-address=:5000 \ --http-address=:8000 \ --node-id=indexer1 \ @@ -1027,7 +1033,7 @@ This section explain how to index Spatial/Geospatial data to Blast. ### Starting Indexer with Spatial/Geospatial index mapping ```bash -$ ./bin/blast indexer node start \ +$ ./bin/blast indexer start \ --grpc-address=:5000 \ --http-address=:8000 \ --node-id=indexer1 \ diff --git a/cmd/blast/distributor_delete.go b/cmd/blast/dispatcher_delete.go similarity index 97% rename from cmd/blast/distributor_delete.go rename to cmd/blast/dispatcher_delete.go index 222af64..8466634 100644 --- a/cmd/blast/distributor_delete.go +++ b/cmd/blast/dispatcher_delete.go @@ -25,7 +25,7 @@ import ( "github.com/urfave/cli" ) -func distributorDelete(c *cli.Context) error { +func dispatcherDelete(c *cli.Context) error { grpcAddr := c.String("grpc-address") filePath := c.String("file") id := c.Args().Get(0) diff --git a/cmd/blast/distributor_get.go b/cmd/blast/dispatcher_get.go similarity index 96% rename from cmd/blast/distributor_get.go rename to cmd/blast/dispatcher_get.go index e7f0cd1..d9108fb 100644 --- a/cmd/blast/distributor_get.go +++ b/cmd/blast/dispatcher_get.go @@ -24,7 +24,7 @@ import ( "github.com/urfave/cli" ) -func distributorGet(c *cli.Context) error { +func dispatcherGet(c *cli.Context) error { grpcAddr := c.String("grpc-address") id := c.Args().Get(0) if id == "" { diff --git a/cmd/blast/distributor_index.go b/cmd/blast/dispatcher_index.go similarity index 98% rename from cmd/blast/distributor_index.go rename to cmd/blast/dispatcher_index.go index ce78c90..57f282d 100644 --- a/cmd/blast/distributor_index.go +++ b/cmd/blast/dispatcher_index.go @@ -27,7 +27,7 @@ import ( "github.com/urfave/cli" ) -func distributorIndex(c *cli.Context) error { +func dispatcherIndex(c *cli.Context) error { grpcAddr := c.String("grpc-address") filePath := c.String("file") bulk := c.Bool("bulk") diff --git a/cmd/blast/distributor_node_health.go b/cmd/blast/dispatcher_node_health.go similarity index 97% rename from cmd/blast/distributor_node_health.go rename to cmd/blast/dispatcher_node_health.go index cf0b51e..698473e 100644 --- a/cmd/blast/distributor_node_health.go +++ b/cmd/blast/dispatcher_node_health.go @@ -22,7 +22,7 @@ import ( "github.com/urfave/cli" ) -func distributorNodeHealth(c *cli.Context) error { +func dispatcherNodeHealth(c *cli.Context) error { grpcAddr := c.String("grpc-address") liveness := c.Bool("liveness") readiness := c.Bool("readiness") diff --git a/cmd/blast/distributor_search.go b/cmd/blast/dispatcher_search.go similarity index 97% rename from cmd/blast/distributor_search.go rename to cmd/blast/dispatcher_search.go index b9494cc..976e36a 100644 --- a/cmd/blast/distributor_search.go +++ b/cmd/blast/dispatcher_search.go @@ -25,7 +25,7 @@ import ( "github.com/urfave/cli" ) -func distributorSearch(c *cli.Context) error { +func dispatcherSearch(c *cli.Context) error { grpcAddr := c.String("grpc-address") searchRequestPath := c.String("file") diff --git a/cmd/blast/distributor_node_start.go b/cmd/blast/dispatcher_start.go similarity index 96% rename from cmd/blast/distributor_node_start.go rename to cmd/blast/dispatcher_start.go index bba46ee..9c9540f 100644 --- a/cmd/blast/distributor_node_start.go +++ b/cmd/blast/dispatcher_start.go @@ -25,8 +25,8 @@ import ( "github.com/urfave/cli" ) -func distributorNodeStart(c *cli.Context) error { - managerAddr := c.String("cluster-grpc-address") +func dispatcherStart(c *cli.Context) error { + managerAddr := c.String("manager-grpc-address") grpcAddr := c.String("grpc-address") httpAddr := c.String("http-address") diff --git a/cmd/blast/indexer_peers_info.go b/cmd/blast/indexer_cluster_info.go similarity index 96% rename from cmd/blast/indexer_peers_info.go rename to cmd/blast/indexer_cluster_info.go index f798ea7..3e8f1d8 100644 --- a/cmd/blast/indexer_peers_info.go +++ b/cmd/blast/indexer_cluster_info.go @@ -23,7 +23,7 @@ import ( "github.com/urfave/cli" ) -func indexerPeersInfo(c *cli.Context) error { +func indexerClusterInfo(c *cli.Context) error { grpcAddr := c.String("grpc-address") client, err := indexer.NewGRPCClient(grpcAddr) diff --git a/cmd/blast/indexer_node_leave.go b/cmd/blast/indexer_cluster_leave.go similarity index 92% rename from cmd/blast/indexer_node_leave.go rename to cmd/blast/indexer_cluster_leave.go index 5255586..b0be2d9 100644 --- a/cmd/blast/indexer_node_leave.go +++ b/cmd/blast/indexer_cluster_leave.go @@ -22,8 +22,8 @@ import ( "github.com/urfave/cli" ) -func indexerNodeLeave(c *cli.Context) error { - clusterGrpcAddr := c.String("cluster-grpc-address") +func indexerClusterLeave(c *cli.Context) error { + clusterGrpcAddr := c.String("manager-grpc-address") shardId := c.String("shard-id") peerGrpcAddr := c.String("peer-grpc-address") diff --git a/cmd/blast/indexer_peers_watch.go b/cmd/blast/indexer_cluster_watch.go similarity index 95% rename from cmd/blast/indexer_peers_watch.go rename to cmd/blast/indexer_cluster_watch.go index 1169a9d..1a5097f 100644 --- a/cmd/blast/indexer_peers_watch.go +++ b/cmd/blast/indexer_cluster_watch.go @@ -27,7 +27,7 @@ import ( "github.com/urfave/cli" ) -func indexerPeersWatch(c *cli.Context) error { +func indexerClusterWatch(c *cli.Context) error { grpcAddr := c.String("grpc-address") client, err := indexer.NewGRPCClient(grpcAddr) @@ -41,7 +41,7 @@ func indexerPeersWatch(c *cli.Context) error { } }() - err = indexerPeersInfo(c) + err = indexerClusterInfo(c) if err != nil { return err } diff --git a/cmd/blast/indexer_node_snapshot.go b/cmd/blast/indexer_snapshot.go similarity index 95% rename from cmd/blast/indexer_node_snapshot.go rename to cmd/blast/indexer_snapshot.go index d59ab92..c34459a 100644 --- a/cmd/blast/indexer_node_snapshot.go +++ b/cmd/blast/indexer_snapshot.go @@ -22,7 +22,7 @@ import ( "github.com/urfave/cli" ) -func indexerNodeSnapshot(c *cli.Context) error { +func indexerSnapshot(c *cli.Context) error { grpcAddr := c.String("grpc-address") client, err := indexer.NewGRPCClient(grpcAddr) diff --git a/cmd/blast/indexer_node_start.go b/cmd/blast/indexer_start.go similarity index 97% rename from cmd/blast/indexer_node_start.go rename to cmd/blast/indexer_start.go index 387b57e..a716efe 100644 --- a/cmd/blast/indexer_node_start.go +++ b/cmd/blast/indexer_start.go @@ -27,8 +27,8 @@ import ( "github.com/urfave/cli" ) -func indexerNodeStart(c *cli.Context) error { - clusterGRPCAddr := c.String("cluster-grpc-address") +func indexerStart(c *cli.Context) error { + clusterGRPCAddr := c.String("manager-grpc-address") shardId := c.String("shard-id") peerGRPCAddr := c.String("peer-grpc-address") diff --git a/cmd/blast/main.go b/cmd/blast/main.go index c5725e1..cdf357e 100644 --- a/cmd/blast/main.go +++ b/cmd/blast/main.go @@ -38,179 +38,179 @@ func main() { app.Commands = []cli.Command{ { - Name: "cluster", - Usage: "Command for blast cluster", + Name: "manager", + Usage: "Command for blast manager", Subcommands: []cli.Command{ + { + Name: "start", + Usage: "Start blast manager", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "peer-grpc-address", + Value: "", + EnvVar: "BLAST_MANAGER_PEER_GRPC_ADDRESS", + Usage: "The gRPC address of the peer node that exists in the cluster to be joined", + }, + cli.StringFlag{ + Name: "grpc-address", + Value: ":5100", + EnvVar: "BLAST_MANAGER_GRPC_ADDRESS", + Usage: "The gRPC listen address", + }, + cli.StringFlag{ + Name: "http-address", + Value: ":8100", + EnvVar: "BLAST_MANAGER_HTTP_ADDRESS", + Usage: "HTTP listen address", + }, + cli.StringFlag{ + Name: "node-id", + Value: "", + EnvVar: "BLAST_MANAGER_NODE_ID", + Usage: "Unique ID to identify the node", + }, + cli.StringFlag{ + Name: "node-address", + Value: ":2100", + EnvVar: "BLAST_MANAGER_NODE_ADDRESS", + Usage: "The address that should be bound to for internal cluster communications", + }, + cli.StringFlag{ + Name: "data-dir", + Value: "/tmp/blast/indexer", + EnvVar: "BLAST_MANAGER_DATA_DIR", + Usage: "A data directory for the node to store state", + }, + cli.StringFlag{ + Name: "raft-storage-type", + Value: "boltdb", + EnvVar: "BLAST_MANAGER_RAFT_STORAGE_TYPE", + Usage: "Storage type of the database that stores the state", + }, + cli.StringFlag{ + Name: "index-mapping-file", + Value: "", + EnvVar: "BLAST_MANAGER_INDEX_MAPPING_FILE", + Usage: "An index mapping file to use", + }, + cli.StringFlag{ + Name: "index-type", + Value: bleve.Config.DefaultIndexType, + EnvVar: "BLAST_MANAGER_INDEX_TYPE", + Usage: "An index type to use", + }, + cli.StringFlag{ + Name: "index-storage-type", + Value: bleve.Config.DefaultKVStore, + EnvVar: "BLAST_MANAGER_INDEX_STORAGE_TYPE", + Usage: "An index storage type to use", + }, + cli.StringFlag{ + Name: "log-level", + Value: "INFO", + EnvVar: "BLAST_MANAGER_LOG_LEVEL", + Usage: "Log level", + }, + cli.StringFlag{ + Name: "log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_MANAGER_LOG_FILE", + Usage: "Log file", + }, + cli.IntFlag{ + Name: "log-max-size", + Value: 500, + EnvVar: "BLAST_MANAGER_LOG_MAX_SIZE", + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "log-max-backups", + Value: 3, + EnvVar: "BLAST_MANAGER_LOG_MAX_BACKUPS", + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "log-max-age", + Value: 30, + EnvVar: "BLAST_MANAGER_LOG_MAX_AGE", + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "log-compress", + EnvVar: "BLAST_MANAGER_LOG_COMPRESS", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "grpc-log-level", + Value: "WARN", + EnvVar: "BLAST_MANAGER_GRPC_LOG_LEVEL", + Usage: "gRPC log level", + }, + cli.StringFlag{ + Name: "grpc-log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_MANAGER_GRPC_LOG_FILE", + Usage: "gRPC log file", + }, + cli.IntFlag{ + Name: "grpc-log-max-size", + Value: 500, + EnvVar: "BLAST_MANAGER_GRPC_LOG_MAX_SIZE", + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "grpc-log-max-backups", + Value: 3, + EnvVar: "BLAST_MANAGER_GRPC_LOG_MAX_BACKUPS", + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "grpc-log-max-age", + Value: 30, + EnvVar: "BLAST_MANAGER_GRPC_LOG_MAX_AGE", + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "grpc-log-compress", + EnvVar: "BLAST_MANAGER_GRPC_LOG_COMPRESS", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "http-log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_MANAGER_HTTP_LOG_FILE", + Usage: "HTTP access log file", + }, + cli.IntFlag{ + Name: "http-log-max-size", + Value: 500, + EnvVar: "BLAST_MANAGER_HTTP_LOG_MAX_SIZE", + Usage: "Max size of a HTTP access log file (megabytes)", + }, + cli.IntFlag{ + Name: "http-log-max-backups", + Value: 3, + EnvVar: "BLAST_MANAGER_HTTP_LOG_MAX_BACKUPS", + Usage: "Max backup count of HTTP access log files", + }, + cli.IntFlag{ + Name: "http-log-max-age", + Value: 30, + EnvVar: "BLAST_MANAGER_HTTP_LOG_MAX_AGE", + Usage: "Max age of a HTTP access log file (days)", + }, + cli.BoolFlag{ + Name: "http-log-compress", + EnvVar: "BLAST_MANAGER_HTTP_LOG_COMPRESS", + Usage: "Compress a HTTP access log", + }, + }, + Action: managerStart, + }, { Name: "node", - Usage: "Command for blast cluster node", + Usage: "Command for blast manager node", Subcommands: []cli.Command{ - { - Name: "start", - Usage: "Start blast cluster node", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "peer-grpc-address", - Value: "", - EnvVar: "BLAST_CLUSTER_PEER_GRPC_ADDRESS", - Usage: "The gRPC address of the peer node that exists in the cluster to be joined", - }, - cli.StringFlag{ - Name: "grpc-address", - Value: ":5100", - EnvVar: "BLAST_CLUSTER_GRPC_ADDRESS", - Usage: "The gRPC listen address", - }, - cli.StringFlag{ - Name: "http-address", - Value: ":8100", - EnvVar: "BLAST_CLUSTER_HTTP_ADDRESS", - Usage: "HTTP listen address", - }, - cli.StringFlag{ - Name: "node-id", - Value: "", - EnvVar: "BLAST_CLUSTER_NODE_ID", - Usage: "Unique ID to identify the node", - }, - cli.StringFlag{ - Name: "node-address", - Value: ":2100", - EnvVar: "BLAST_CLUSTER_NODE_ADDRESS", - Usage: "The address that should be bound to for internal cluster communications", - }, - cli.StringFlag{ - Name: "data-dir", - Value: "/tmp/blast/indexer", - EnvVar: "BLAST_CLUSTER_DATA_DIR", - Usage: "A data directory for the node to store state", - }, - cli.StringFlag{ - Name: "raft-storage-type", - Value: "boltdb", - EnvVar: "BLAST_CLUSTER_RAFT_STORAGE_TYPE", - Usage: "Storage type of the database that stores the state", - }, - cli.StringFlag{ - Name: "index-mapping-file", - Value: "", - EnvVar: "BLAST_CLUSTER_INDEX_MAPPING_FILE", - Usage: "An index mapping file to use", - }, - cli.StringFlag{ - Name: "index-type", - Value: bleve.Config.DefaultIndexType, - EnvVar: "BLAST_CLUSTER_INDEX_TYPE", - Usage: "An index type to use", - }, - cli.StringFlag{ - Name: "index-storage-type", - Value: bleve.Config.DefaultKVStore, - EnvVar: "BLAST_CLUSTER_INDEX_STORAGE_TYPE", - Usage: "An index storage type to use", - }, - cli.StringFlag{ - Name: "log-level", - Value: "INFO", - EnvVar: "BLAST_CLUSTER_LOG_LEVEL", - Usage: "Log level", - }, - cli.StringFlag{ - Name: "log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_CLUSTER_LOG_FILE", - Usage: "Log file", - }, - cli.IntFlag{ - Name: "log-max-size", - Value: 500, - EnvVar: "BLAST_CLUSTER_LOG_MAX_SIZE", - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "log-max-backups", - Value: 3, - EnvVar: "BLAST_CLUSTER_LOG_MAX_BACKUPS", - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "log-max-age", - Value: 30, - EnvVar: "BLAST_CLUSTER_LOG_MAX_AGE", - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "log-compress", - EnvVar: "BLAST_CLUSTER_LOG_COMPRESS", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "grpc-log-level", - Value: "WARN", - EnvVar: "BLAST_CLUSTER_GRPC_LOG_LEVEL", - Usage: "gRPC log level", - }, - cli.StringFlag{ - Name: "grpc-log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_CLUSTER_GRPC_LOG_FILE", - Usage: "gRPC log file", - }, - cli.IntFlag{ - Name: "grpc-log-max-size", - Value: 500, - EnvVar: "BLAST_CLUSTER_GRPC_LOG_MAX_SIZE", - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "grpc-log-max-backups", - Value: 3, - EnvVar: "BLAST_CLUSTER_GRPC_LOG_MAX_BACKUPS", - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "grpc-log-max-age", - Value: 30, - EnvVar: "BLAST_CLUSTER_GRPC_LOG_MAX_AGE", - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "grpc-log-compress", - EnvVar: "BLAST_CLUSTER_GRPC_LOG_COMPRESS", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "http-log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_CLUSTER_HTTP_LOG_FILE", - Usage: "HTTP access log file", - }, - cli.IntFlag{ - Name: "http-log-max-size", - Value: 500, - EnvVar: "BLAST_CLUSTER_HTTP_LOG_MAX_SIZE", - Usage: "Max size of a HTTP access log file (megabytes)", - }, - cli.IntFlag{ - Name: "http-log-max-backups", - Value: 3, - EnvVar: "BLAST_CLUSTER_HTTP_LOG_MAX_BACKUPS", - Usage: "Max backup count of HTTP access log files", - }, - cli.IntFlag{ - Name: "http-log-max-age", - Value: 30, - EnvVar: "BLAST_CLUSTER_HTTP_LOG_MAX_AGE", - Usage: "Max age of a HTTP access log file (days)", - }, - cli.BoolFlag{ - Name: "http-log-compress", - EnvVar: "BLAST_CLUSTER_HTTP_LOG_COMPRESS", - Usage: "Compress a HTTP access log", - }, - }, - Action: clusterNodeStart, - }, { Name: "info", Usage: "Get node information", @@ -231,28 +231,11 @@ func main() { Usage: "The gRPC address of the node for which to retrieve the node information", }, }, - Action: clusterNodeInfo, - }, - { - Name: "leave", - Usage: "Leave the node from the cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "peer-grpc-address", - Value: "", - Usage: "The gRPC address of the peer node that exists in the cluster to be joined", - }, - cli.StringFlag{ - Name: "node-id", - Value: "", - Usage: "The gRPC listen address", - }, - }, - Action: clusterNodeLeave, + Action: managerNodeInfo, }, { Name: "health", - Usage: "Health check", + Usage: "Health check the node", Flags: []cli.Flag{ cli.StringFlag{ Name: "grpc-address", @@ -268,29 +251,17 @@ func main() { Usage: "Readiness probe", }, }, - Action: clusterNodeHealth, - }, - { - Name: "snapshot", - Usage: "Snapshot", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-address", - Value: "", - Usage: "The gRPC listen address", - }, - }, - Action: clusterNodeSnapshot, + Action: managerNodeHealth, }, }, }, { - Name: "peers", - Usage: "Command for blast cluster peers", + Name: "cluster", + Usage: "Command for blast manager cluster", Subcommands: []cli.Command{ { Name: "info", - Usage: "Get peers", + Usage: "Get cluster information", Flags: []cli.Flag{ //cli.StringFlag{ // Name: "cluster-grpc-address", @@ -318,7 +289,7 @@ func main() { Usage: "The gRPC address of the node for which to retrieve the node information", }, }, - Action: clusterPeersInfo, + Action: managerClusterInfo, }, { Name: "watch", @@ -350,7 +321,24 @@ func main() { Usage: "The gRPC address of the node for which to retrieve the node information", }, }, - Action: clusterPeersWatch, + Action: managerClusterWatch, + }, + { + Name: "leave", + Usage: "Leave the manager from the cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "peer-grpc-address", + Value: "", + Usage: "The gRPC address of the peer node that exists in the cluster to be joined", + }, + cli.StringFlag{ + Name: "node-id", + Value: "", + Usage: "The gRPC listen address", + }, + }, + Action: managerClusterLeave, }, }, }, @@ -365,7 +353,7 @@ func main() { }, }, ArgsUsage: "[key]", - Action: clusterGet, + Action: managerGet, }, { Name: "set", @@ -383,7 +371,7 @@ func main() { }, }, ArgsUsage: "[key] [value]", - Action: clusterSet, + Action: managerSet, }, { Name: "delete", @@ -396,7 +384,7 @@ func main() { }, }, ArgsUsage: "[key]", - Action: clusterDelete, + Action: managerDelete, }, { Name: "watch", @@ -409,7 +397,19 @@ func main() { }, }, ArgsUsage: "[key]", - Action: clusterWatch, + Action: managerWatch, + }, + { + Name: "snapshot", + Usage: "Snapshot the data", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-address", + Value: "", + Usage: "The gRPC listen address", + }, + }, + Action: managerSnapshot, }, }, }, @@ -417,188 +417,188 @@ func main() { Name: "indexer", Usage: "Command for blast indexer", Subcommands: []cli.Command{ + { + Name: "start", + Usage: "Start blast indexer", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "manager-grpc-address", + Value: "", + EnvVar: "BLAST_INDEXER_MANAGER_GRPC_ADDRESS", + Usage: "The gRPC address of the existing cluster manager to be joined", + }, + cli.StringFlag{ + Name: "shard-id", + Value: "", + EnvVar: "BLAST_INDEXER_SHARD_ID", + Usage: "Shard ID registered in the existing cluster to be joined", + }, + cli.StringFlag{ + Name: "peer-grpc-address", + Value: "", + EnvVar: "BLAST_INDEXER_PEER_GRPC_ADDRESS", + Usage: "The gRPC address of the peer node that exists in the cluster to be joined", + }, + cli.StringFlag{ + Name: "grpc-address", + Value: ":5000", + EnvVar: "BLAST_INDEXER_GRPC_ADDRESS", + Usage: "The gRPC listen address", + }, + cli.StringFlag{ + Name: "http-address", + Value: ":8000", + EnvVar: "BLAST_INDEXER_HTTP_ADDRESS", + Usage: "HTTP listen address", + }, + cli.StringFlag{ + Name: "node-id", + Value: "", + EnvVar: "BLAST_INDEXER_NODE_ID", + Usage: "Unique ID to identify the node", + }, + cli.StringFlag{ + Name: "node-address", + Value: ":2000", + EnvVar: "BLAST_INDEXER_NODE_ADDRESS", + Usage: "The address that should be bound to for internal cluster communications", + }, + cli.StringFlag{ + Name: "data-dir", + Value: "/tmp/blast/indexer", + EnvVar: "BLAST_INDEXER_DATA_DIR", + Usage: "A data directory for the node to store state", + }, + cli.StringFlag{ + Name: "raft-storage-type", + Value: "boltdb", + EnvVar: "BLAST_INDEXER_RAFT_STORAGE_TYPE", + Usage: "Storage type of the database that stores the state", + }, + cli.StringFlag{ + Name: "index-mapping-file", + Value: "", + EnvVar: "BLAST_INDEXER_INDEX_MAPPING_FILE", + Usage: "An index mapping file to use", + }, + cli.StringFlag{ + Name: "index-type", + Value: bleve.Config.DefaultIndexType, + EnvVar: "BLAST_INDEXER_INDEX_TYPE", + Usage: "An index type to use", + }, + cli.StringFlag{ + Name: "index-storage-type", + Value: bleve.Config.DefaultKVStore, + EnvVar: "BLAST_INDEXER_INDEX_STORAGE_TYPE", + Usage: "An index storage type to use", + }, + cli.StringFlag{ + Name: "log-level", + Value: "INFO", + EnvVar: "BLAST_INDEXER_LOG_LEVEL", + Usage: "Log level", + }, + cli.StringFlag{ + Name: "log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_INDEXER_LOG_FILE", + Usage: "Log file", + }, + cli.IntFlag{ + Name: "log-max-size", + Value: 500, + EnvVar: "BLAST_INDEXER_LOG_MAX_SIZE", + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "log-max-backups", + Value: 3, + EnvVar: "BLAST_INDEXER_LOG_MAX_BACKUPS", + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "log-max-age", + Value: 30, + EnvVar: "BLAST_INDEXER_LOG_MAX_AGE", + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "log-compress", + EnvVar: "BLAST_INDEXER_LOG_COMPRESS", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "grpc-log-level", + Value: "WARN", + EnvVar: "BLAST_INDEXER_GRPC_LOG_LEVEL", + Usage: "gRPC log level", + }, + cli.StringFlag{ + Name: "grpc-log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_INDEXER_GRPC_LOG_FILE", + Usage: "gRPC log file", + }, + cli.IntFlag{ + Name: "grpc-log-max-size", + Value: 500, + EnvVar: "BLAST_INDEXER_GRPC_LOG_MAX_SIZE", + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "grpc-log-max-backups", + Value: 3, + EnvVar: "BLAST_INDEXER_GRPC_LOG_MAX_BACKUPS", + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "grpc-log-max-age", + Value: 30, + EnvVar: "BLAST_INDEXER_GRPC_LOG_MAX_AGE", + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "grpc-log-compress", + EnvVar: "BLAST_INDEXER_GRPC_LOG_COMPRESS", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "http-log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_INDEXER_HTTP_LOG_FILE", + Usage: "HTTP access log file", + }, + cli.IntFlag{ + Name: "http-log-max-size", + Value: 500, + EnvVar: "BLAST_INDEXER_HTTP_LOG_MAX_SIZE", + Usage: "Max size of a HTTP access log file (megabytes)", + }, + cli.IntFlag{ + Name: "http-log-max-backups", + Value: 3, + EnvVar: "BLAST_INDEXER_HTTP_LOG_MAX_BACKUPS", + Usage: "Max backup count of HTTP access log files", + }, + cli.IntFlag{ + Name: "http-log-max-age", + Value: 30, + EnvVar: "BLAST_INDEXER_HTTP_LOG_MAX_AGE", + Usage: "Max age of a HTTP access log file (days)", + }, + cli.BoolFlag{ + Name: "http-log-compress", + EnvVar: "BLAST_INDEXER_HTTP_LOG_COMPRESS", + Usage: "Compress a HTTP access log", + }, + }, + Action: indexerStart, + }, { Name: "node", Usage: "Command for blast indexer node", Subcommands: []cli.Command{ - { - Name: "start", - Usage: "Start blast indexer node", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "cluster-grpc-address", - Value: "", - EnvVar: "BLAST_INDEXER_CLUSTER_GRPC_ADDRESS", - Usage: "The gRPC address of the existing cluster node to be joined", - }, - cli.StringFlag{ - Name: "shard-id", - Value: "", - EnvVar: "BLAST_INDEXER_SHARD_ID", - Usage: "Shard ID registered in the existing cluster to be joined", - }, - cli.StringFlag{ - Name: "peer-grpc-address", - Value: "", - EnvVar: "BLAST_INDEXER_PEER_GRPC_ADDRESS", - Usage: "The gRPC address of the peer node that exists in the cluster to be joined", - }, - cli.StringFlag{ - Name: "grpc-address", - Value: ":5000", - EnvVar: "BLAST_INDEXER_GRPC_ADDRESS", - Usage: "The gRPC listen address", - }, - cli.StringFlag{ - Name: "http-address", - Value: ":8000", - EnvVar: "BLAST_INDEXER_HTTP_ADDRESS", - Usage: "HTTP listen address", - }, - cli.StringFlag{ - Name: "node-id", - Value: "", - EnvVar: "BLAST_INDEXER_NODE_ID", - Usage: "Unique ID to identify the node", - }, - cli.StringFlag{ - Name: "node-address", - Value: ":2000", - EnvVar: "BLAST_INDEXER_NODE_ADDRESS", - Usage: "The address that should be bound to for internal cluster communications", - }, - cli.StringFlag{ - Name: "data-dir", - Value: "/tmp/blast/indexer", - EnvVar: "BLAST_INDEXER_DATA_DIR", - Usage: "A data directory for the node to store state", - }, - cli.StringFlag{ - Name: "raft-storage-type", - Value: "boltdb", - EnvVar: "BLAST_INDEXER_RAFT_STORAGE_TYPE", - Usage: "Storage type of the database that stores the state", - }, - cli.StringFlag{ - Name: "index-mapping-file", - Value: "", - EnvVar: "BLAST_INDEXER_INDEX_MAPPING_FILE", - Usage: "An index mapping file to use", - }, - cli.StringFlag{ - Name: "index-type", - Value: bleve.Config.DefaultIndexType, - EnvVar: "BLAST_INDEXER_INDEX_TYPE", - Usage: "An index type to use", - }, - cli.StringFlag{ - Name: "index-storage-type", - Value: bleve.Config.DefaultKVStore, - EnvVar: "BLAST_INDEXER_INDEX_STORAGE_TYPE", - Usage: "An index storage type to use", - }, - cli.StringFlag{ - Name: "log-level", - Value: "INFO", - EnvVar: "BLAST_INDEXER_LOG_LEVEL", - Usage: "Log level", - }, - cli.StringFlag{ - Name: "log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_INDEXER_LOG_FILE", - Usage: "Log file", - }, - cli.IntFlag{ - Name: "log-max-size", - Value: 500, - EnvVar: "BLAST_INDEXER_LOG_MAX_SIZE", - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "log-max-backups", - Value: 3, - EnvVar: "BLAST_INDEXER_LOG_MAX_BACKUPS", - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "log-max-age", - Value: 30, - EnvVar: "BLAST_INDEXER_LOG_MAX_AGE", - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "log-compress", - EnvVar: "BLAST_INDEXER_LOG_COMPRESS", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "grpc-log-level", - Value: "WARN", - EnvVar: "BLAST_INDEXER_GRPC_LOG_LEVEL", - Usage: "gRPC log level", - }, - cli.StringFlag{ - Name: "grpc-log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_INDEXER_GRPC_LOG_FILE", - Usage: "gRPC log file", - }, - cli.IntFlag{ - Name: "grpc-log-max-size", - Value: 500, - EnvVar: "BLAST_INDEXER_GRPC_LOG_MAX_SIZE", - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "grpc-log-max-backups", - Value: 3, - EnvVar: "BLAST_INDEXER_GRPC_LOG_MAX_BACKUPS", - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "grpc-log-max-age", - Value: 30, - EnvVar: "BLAST_INDEXER_GRPC_LOG_MAX_AGE", - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "grpc-log-compress", - EnvVar: "BLAST_INDEXER_GRPC_LOG_COMPRESS", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "http-log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_INDEXER_HTTP_LOG_FILE", - Usage: "HTTP access log file", - }, - cli.IntFlag{ - Name: "http-log-max-size", - Value: 500, - EnvVar: "BLAST_INDEXER_HTTP_LOG_MAX_SIZE", - Usage: "Max size of a HTTP access log file (megabytes)", - }, - cli.IntFlag{ - Name: "http-log-max-backups", - Value: 3, - EnvVar: "BLAST_INDEXER_HTTP_LOG_MAX_BACKUPS", - Usage: "Max backup count of HTTP access log files", - }, - cli.IntFlag{ - Name: "http-log-max-age", - Value: 30, - EnvVar: "BLAST_INDEXER_HTTP_LOG_MAX_AGE", - Usage: "Max age of a HTTP access log file (days)", - }, - cli.BoolFlag{ - Name: "http-log-compress", - EnvVar: "BLAST_INDEXER_HTTP_LOG_COMPRESS", - Usage: "Compress a HTTP access log", - }, - }, - Action: indexerNodeStart, - }, { Name: "info", Usage: "Get node information", @@ -631,41 +631,9 @@ func main() { }, Action: indexerNodeInfo, }, - { - Name: "leave", - Usage: "Leave the node from the cluster", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "cluster-grpc-address", - Value: "", - Usage: "The gRPC address of the existing cluster node to be joined", - }, - cli.StringFlag{ - Name: "shard-id", - Value: "", - Usage: "Shard ID registered in the existing cluster to be joined", - }, - cli.StringFlag{ - Name: "peer-grpc-address", - Value: "", - Usage: "The gRPC address of the peer node that exists in the cluster to be joined", - }, - cli.StringFlag{ - Name: "grpc-address", - Value: "", - Usage: "The gRPC listen address", - }, - cli.StringFlag{ - Name: "node-id", - Value: "", - Usage: "Node ID to delete", - }, - }, - Action: indexerNodeLeave, - }, { Name: "health", - Usage: "Health check", + Usage: "Health check the node", Flags: []cli.Flag{ cli.StringFlag{ Name: "grpc-address", @@ -683,27 +651,15 @@ func main() { }, Action: indexerNodeHealth, }, - { - Name: "snapshot", - Usage: "Snapshot", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "grpc-address", - Value: "", - Usage: "The gRPC listen address", - }, - }, - Action: indexerNodeSnapshot, - }, }, }, { - Name: "peers", - Usage: "Command for blast indexer peers", + Name: "cluster", + Usage: "Command for blast indexer cluster", Subcommands: []cli.Command{ { Name: "info", - Usage: "Get peers", + Usage: "Get cluster information", Flags: []cli.Flag{ //cli.StringFlag{ // Name: "cluster-grpc-address", @@ -731,11 +687,11 @@ func main() { Usage: "The gRPC address of the node for which to retrieve the node information", }, }, - Action: indexerPeersInfo, + Action: indexerClusterInfo, }, { Name: "watch", - Usage: "Watch peers", + Usage: "Watch cluster", Flags: []cli.Flag{ //cli.StringFlag{ // Name: "cluster-grpc-address", @@ -763,7 +719,39 @@ func main() { Usage: "The gRPC address of the node for which to retrieve the node information", }, }, - Action: indexerPeersWatch, + Action: indexerClusterWatch, + }, + { + Name: "leave", + Usage: "Leave the indexer from the cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "manager-grpc-address", + Value: "", + Usage: "The gRPC address of the existing cluster node to be joined", + }, + cli.StringFlag{ + Name: "shard-id", + Value: "", + Usage: "Shard ID registered in the existing cluster to be joined", + }, + cli.StringFlag{ + Name: "peer-grpc-address", + Value: "", + Usage: "The gRPC address of the peer node that exists in the cluster to be joined", + }, + cli.StringFlag{ + Name: "grpc-address", + Value: "", + Usage: "The gRPC listen address", + }, + cli.StringFlag{ + Name: "node-id", + Value: "", + Usage: "Node ID to delete", + }, + }, + Action: indexerClusterLeave, }, }, }, @@ -782,7 +770,7 @@ func main() { Usage: "Document ID list", }, }, - ArgsUsage: "[document IDs]", + ArgsUsage: "[document ID]", Action: indexerGet, }, { @@ -822,7 +810,7 @@ func main() { Usage: "Document ID list", }, }, - ArgsUsage: "[document IDs]", + ArgsUsage: "[document ID]", Action: indexerDelete, }, { @@ -843,143 +831,155 @@ func main() { ArgsUsage: "[search request]", Action: indexerSearch, }, + { + Name: "snapshot", + Usage: "Snapshot", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "grpc-address", + Value: "", + Usage: "The gRPC listen address", + }, + }, + Action: indexerSnapshot, + }, }, }, { - Name: "distributor", - Usage: "Command for blast distributor", + Name: "dispatcher", + Usage: "Command for blast dispatcher", Subcommands: []cli.Command{ + { + Name: "start", + Usage: "Start blast dispatcher", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "manager-grpc-address", + Value: ":5100", + EnvVar: "BLAST_DISPATCHER_CLUSTER_GRPC_ADDRESS", + Usage: "The gRPC address of the existing cluster node to be joined", + }, + cli.StringFlag{ + Name: "grpc-address", + Value: ":5200", + EnvVar: "BLAST_DISPATCHER_GRPC_ADDRESS", + Usage: "The gRPC listen address", + }, + cli.StringFlag{ + Name: "http-address", + Value: ":8200", + EnvVar: "BLAST_DISPATCHER_HTTP_ADDRESS", + Usage: "HTTP listen address", + }, + cli.StringFlag{ + Name: "log-level", + Value: "INFO", + EnvVar: "BLAST_DISPATCHER_LOG_LEVEL", + Usage: "Log level", + }, + cli.StringFlag{ + Name: "log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_DISPATCHER_LOG_FILE", + Usage: "Log file", + }, + cli.IntFlag{ + Name: "log-max-size", + Value: 500, + EnvVar: "BLAST_DISPATCHER_LOG_MAX_SIZE", + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "log-max-backups", + Value: 3, + EnvVar: "BLAST_DISPATCHER_LOG_MAX_BACKUPS", + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "log-max-age", + Value: 30, + EnvVar: "BLAST_DISPATCHER_LOG_MAX_AGE", + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "log-compress", + EnvVar: "BLAST_DISPATCHER_LOG_COMPRESS", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "grpc-log-level", + Value: "WARN", + EnvVar: "BLAST_DISPATCHER_GRPC_LOG_LEVEL", + Usage: "gRPC log level", + }, + cli.StringFlag{ + Name: "grpc-log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_DISPATCHER_GRPC_LOG_FILE", + Usage: "gRPC log file", + }, + cli.IntFlag{ + Name: "grpc-log-max-size", + Value: 500, + EnvVar: "BLAST_DISPATCHER_GRPC_LOG_MAX_SIZE", + Usage: "Max size of a log file (megabytes)", + }, + cli.IntFlag{ + Name: "grpc-log-max-backups", + Value: 3, + EnvVar: "BLAST_DISPATCHER_GRPC_LOG_MAX_BACKUPS", + Usage: "Max backup count of log files", + }, + cli.IntFlag{ + Name: "grpc-log-max-age", + Value: 30, + EnvVar: "BLAST_DISPATCHER_GRPC_LOG_MAX_AGE", + Usage: "Max age of a log file (days)", + }, + cli.BoolFlag{ + Name: "grpc-log-compress", + EnvVar: "BLAST_DISPATCHER_GRPC_LOG_COMPRESS", + Usage: "Compress a log file", + }, + cli.StringFlag{ + Name: "http-log-file", + Value: os.Stderr.Name(), + EnvVar: "BLAST_DISPATCHER_HTTP_LOG_FILE", + Usage: "HTTP access log file", + }, + cli.IntFlag{ + Name: "http-log-max-size", + Value: 500, + EnvVar: "BLAST_DISPATCHER_HTTP_LOG_MAX_SIZE", + Usage: "Max size of a HTTP access log file (megabytes)", + }, + cli.IntFlag{ + Name: "http-log-max-backups", + Value: 3, + EnvVar: "BLAST_DISPATCHER_HTTP_LOG_MAX_BACKUPS", + Usage: "Max backup count of HTTP access log files", + }, + cli.IntFlag{ + Name: "http-log-max-age", + Value: 30, + EnvVar: "BLAST_DISPATCHER_HTTP_LOG_MAX_AGE", + Usage: "Max age of a HTTP access log file (days)", + }, + cli.BoolFlag{ + Name: "http-log-compress", + EnvVar: "BLAST_DISPATCHER_HTTP_LOG_COMPRESS", + Usage: "Compress a HTTP access log", + }, + }, + Action: dispatcherStart, + }, { Name: "node", - Usage: "Command for blast distributor node", + Usage: "Command for blast dispatcher node", Subcommands: []cli.Command{ - { - Name: "start", - Usage: "Start blast distributor node", - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "cluster-grpc-address", - Value: ":5100", - EnvVar: "BLAST_DISTRIBUTOR_CLUSTER_GRPC_ADDRESS", - Usage: "The gRPC address of the existing cluster node to be joined", - }, - cli.StringFlag{ - Name: "grpc-address", - Value: ":5200", - EnvVar: "BLAST_DISTRIBUTOR_GRPC_ADDRESS", - Usage: "The gRPC listen address", - }, - cli.StringFlag{ - Name: "http-address", - Value: ":8200", - EnvVar: "BLAST_DISTRIBUTOR_HTTP_ADDRESS", - Usage: "HTTP listen address", - }, - cli.StringFlag{ - Name: "log-level", - Value: "INFO", - EnvVar: "BLAST_DISTRIBUTOR_LOG_LEVEL", - Usage: "Log level", - }, - cli.StringFlag{ - Name: "log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_DISTRIBUTOR_LOG_FILE", - Usage: "Log file", - }, - cli.IntFlag{ - Name: "log-max-size", - Value: 500, - EnvVar: "BLAST_DISTRIBUTOR_LOG_MAX_SIZE", - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "log-max-backups", - Value: 3, - EnvVar: "BLAST_DISTRIBUTOR_LOG_MAX_BACKUPS", - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "log-max-age", - Value: 30, - EnvVar: "BLAST_DISTRIBUTOR_LOG_MAX_AGE", - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "log-compress", - EnvVar: "BLAST_DISTRIBUTOR_LOG_COMPRESS", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "grpc-log-level", - Value: "WARN", - EnvVar: "BLAST_DISTRIBUTOR_GRPC_LOG_LEVEL", - Usage: "gRPC log level", - }, - cli.StringFlag{ - Name: "grpc-log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_DISTRIBUTOR_GRPC_LOG_FILE", - Usage: "gRPC log file", - }, - cli.IntFlag{ - Name: "grpc-log-max-size", - Value: 500, - EnvVar: "BLAST_DISTRIBUTOR_GRPC_LOG_MAX_SIZE", - Usage: "Max size of a log file (megabytes)", - }, - cli.IntFlag{ - Name: "grpc-log-max-backups", - Value: 3, - EnvVar: "BLAST_DISTRIBUTOR_GRPC_LOG_MAX_BACKUPS", - Usage: "Max backup count of log files", - }, - cli.IntFlag{ - Name: "grpc-log-max-age", - Value: 30, - EnvVar: "BLAST_DISTRIBUTOR_GRPC_LOG_MAX_AGE", - Usage: "Max age of a log file (days)", - }, - cli.BoolFlag{ - Name: "grpc-log-compress", - EnvVar: "BLAST_DISTRIBUTOR_GRPC_LOG_COMPRESS", - Usage: "Compress a log file", - }, - cli.StringFlag{ - Name: "http-log-file", - Value: os.Stderr.Name(), - EnvVar: "BLAST_DISTRIBUTOR_HTTP_LOG_FILE", - Usage: "HTTP access log file", - }, - cli.IntFlag{ - Name: "http-log-max-size", - Value: 500, - EnvVar: "BLAST_DISTRIBUTOR_HTTP_LOG_MAX_SIZE", - Usage: "Max size of a HTTP access log file (megabytes)", - }, - cli.IntFlag{ - Name: "http-log-max-backups", - Value: 3, - EnvVar: "BLAST_DISTRIBUTOR_HTTP_LOG_MAX_BACKUPS", - Usage: "Max backup count of HTTP access log files", - }, - cli.IntFlag{ - Name: "http-log-max-age", - Value: 30, - EnvVar: "BLAST_DISTRIBUTOR_HTTP_LOG_MAX_AGE", - Usage: "Max age of a HTTP access log file (days)", - }, - cli.BoolFlag{ - Name: "http-log-compress", - EnvVar: "BLAST_DISTRIBUTOR_HTTP_LOG_COMPRESS", - Usage: "Compress a HTTP access log", - }, - }, - Action: distributorNodeStart, - }, { Name: "health", - Usage: "Health check", + Usage: "Health check the node", Flags: []cli.Flag{ cli.StringFlag{ Name: "grpc-address", @@ -995,7 +995,7 @@ func main() { Usage: "Readiness probe", }, }, - Action: distributorNodeHealth, + Action: dispatcherNodeHealth, }, }, }, @@ -1015,7 +1015,7 @@ func main() { }, }, ArgsUsage: "[document IDs]", - Action: distributorGet, + Action: dispatcherGet, }, { Name: "index", @@ -1037,7 +1037,7 @@ func main() { }, }, ArgsUsage: "[document ID] [document fields]", - Action: distributorIndex, + Action: dispatcherIndex, }, { Name: "delete", @@ -1055,7 +1055,7 @@ func main() { }, }, ArgsUsage: "[document IDs]", - Action: distributorDelete, + Action: dispatcherDelete, }, { Name: "search", @@ -1073,7 +1073,7 @@ func main() { }, }, ArgsUsage: "[search request]", - Action: distributorSearch, + Action: dispatcherSearch, }, }, }, diff --git a/cmd/blast/cluster_peers_info.go b/cmd/blast/manager_cluster_info.go similarity index 96% rename from cmd/blast/cluster_peers_info.go rename to cmd/blast/manager_cluster_info.go index 6d4ea48..8e4589c 100644 --- a/cmd/blast/cluster_peers_info.go +++ b/cmd/blast/manager_cluster_info.go @@ -23,7 +23,7 @@ import ( "github.com/urfave/cli" ) -func clusterPeersInfo(c *cli.Context) error { +func managerClusterInfo(c *cli.Context) error { grpcAddr := c.String("grpc-address") client, err := manager.NewGRPCClient(grpcAddr) diff --git a/cmd/blast/cluster_node_leave.go b/cmd/blast/manager_cluster_leave.go similarity index 81% rename from cmd/blast/cluster_node_leave.go rename to cmd/blast/manager_cluster_leave.go index 8250238..5082304 100644 --- a/cmd/blast/cluster_node_leave.go +++ b/cmd/blast/manager_cluster_leave.go @@ -22,14 +22,10 @@ import ( "github.com/urfave/cli" ) -func clusterNodeLeave(c *cli.Context) error { - clusterGrpcAddr := c.String("cluster-grpc-address") - shardId := c.String("shard-id") +func managerClusterLeave(c *cli.Context) error { peerGrpcAddr := c.String("peer-grpc-address") - if clusterGrpcAddr != "" && shardId != "" { - // get grpc address of leader node - } else if peerGrpcAddr != "" { + if peerGrpcAddr != "" { // get grpc address of leader node } diff --git a/cmd/blast/cluster_peers_watch.go b/cmd/blast/manager_cluster_watch.go similarity index 95% rename from cmd/blast/cluster_peers_watch.go rename to cmd/blast/manager_cluster_watch.go index 59d5d35..b8d4277 100644 --- a/cmd/blast/cluster_peers_watch.go +++ b/cmd/blast/manager_cluster_watch.go @@ -27,7 +27,7 @@ import ( "github.com/urfave/cli" ) -func clusterPeersWatch(c *cli.Context) error { +func managerClusterWatch(c *cli.Context) error { grpcAddr := c.String("grpc-address") client, err := manager.NewGRPCClient(grpcAddr) @@ -41,7 +41,7 @@ func clusterPeersWatch(c *cli.Context) error { } }() - err = clusterPeersInfo(c) + err = managerClusterInfo(c) if err != nil { return err } diff --git a/cmd/blast/cluster_delete.go b/cmd/blast/manager_delete.go similarity index 96% rename from cmd/blast/cluster_delete.go rename to cmd/blast/manager_delete.go index 600c70f..60c273b 100644 --- a/cmd/blast/cluster_delete.go +++ b/cmd/blast/manager_delete.go @@ -23,7 +23,7 @@ import ( "github.com/urfave/cli" ) -func clusterDelete(c *cli.Context) error { +func managerDelete(c *cli.Context) error { grpcAddr := c.String("grpc-address") key := c.Args().Get(0) diff --git a/cmd/blast/cluster_get.go b/cmd/blast/manager_get.go similarity index 96% rename from cmd/blast/cluster_get.go rename to cmd/blast/manager_get.go index fff65e3..46f0228 100644 --- a/cmd/blast/cluster_get.go +++ b/cmd/blast/manager_get.go @@ -23,7 +23,7 @@ import ( "github.com/urfave/cli" ) -func clusterGet(c *cli.Context) error { +func managerGet(c *cli.Context) error { grpcAddr := c.String("grpc-address") key := c.Args().Get(0) diff --git a/cmd/blast/cluster_node_health.go b/cmd/blast/manager_node_health.go similarity index 97% rename from cmd/blast/cluster_node_health.go rename to cmd/blast/manager_node_health.go index 39294b7..0f4d863 100644 --- a/cmd/blast/cluster_node_health.go +++ b/cmd/blast/manager_node_health.go @@ -22,7 +22,7 @@ import ( "github.com/urfave/cli" ) -func clusterNodeHealth(c *cli.Context) error { +func managerNodeHealth(c *cli.Context) error { grpcAddr := c.String("grpc-address") liveness := c.Bool("liveness") readiness := c.Bool("readiness") diff --git a/cmd/blast/cluster_node_info.go b/cmd/blast/manager_node_info.go similarity index 83% rename from cmd/blast/cluster_node_info.go rename to cmd/blast/manager_node_info.go index 517c72e..f399b30 100644 --- a/cmd/blast/cluster_node_info.go +++ b/cmd/blast/manager_node_info.go @@ -23,21 +23,15 @@ import ( "github.com/urfave/cli" ) -func clusterNodeInfo(c *cli.Context) error { - clusterGrpcAddr := c.String("cluster-grpc-address") - shardId := c.String("shard-id") +func managerNodeInfo(c *cli.Context) error { peerGrpcAddr := c.String("peer-grpc-address") + nodeId := c.String("node-id") + grpcAddr := c.String("grpc-address") - if clusterGrpcAddr != "" && shardId != "" { - - } else if peerGrpcAddr != "" { + if peerGrpcAddr != "" { } - grpcAddr := c.String("grpc-address") - - nodeId := c.Args().Get(0) - client, err := manager.NewGRPCClient(grpcAddr) if err != nil { return err diff --git a/cmd/blast/cluster_set.go b/cmd/blast/manager_set.go similarity index 97% rename from cmd/blast/cluster_set.go rename to cmd/blast/manager_set.go index 55e51f0..86542bc 100644 --- a/cmd/blast/cluster_set.go +++ b/cmd/blast/manager_set.go @@ -24,7 +24,7 @@ import ( "github.com/urfave/cli" ) -func clusterSet(c *cli.Context) error { +func managerSet(c *cli.Context) error { grpcAddr := c.String("grpc-address") key := c.Args().Get(0) diff --git a/cmd/blast/cluster_node_snapshot.go b/cmd/blast/manager_snapshot.go similarity index 95% rename from cmd/blast/cluster_node_snapshot.go rename to cmd/blast/manager_snapshot.go index ce9d62f..8dd9b71 100644 --- a/cmd/blast/cluster_node_snapshot.go +++ b/cmd/blast/manager_snapshot.go @@ -22,7 +22,7 @@ import ( "github.com/urfave/cli" ) -func clusterNodeSnapshot(c *cli.Context) error { +func managerSnapshot(c *cli.Context) error { grpcAddr := c.String("grpc-address") client, err := manager.NewGRPCClient(grpcAddr) diff --git a/cmd/blast/cluster_node_start.go b/cmd/blast/manager_start.go similarity index 98% rename from cmd/blast/cluster_node_start.go rename to cmd/blast/manager_start.go index feb56c2..e53a707 100644 --- a/cmd/blast/cluster_node_start.go +++ b/cmd/blast/manager_start.go @@ -27,7 +27,7 @@ import ( "github.com/urfave/cli" ) -func clusterNodeStart(c *cli.Context) error { +func managerStart(c *cli.Context) error { peerGrpcAddr := c.String("peer-grpc-address") grpcAddr := c.String("grpc-address") diff --git a/cmd/blast/cluster_watch.go b/cmd/blast/manager_watch.go similarity index 97% rename from cmd/blast/cluster_watch.go rename to cmd/blast/manager_watch.go index 71d2dc2..024cbbb 100644 --- a/cmd/blast/cluster_watch.go +++ b/cmd/blast/manager_watch.go @@ -27,7 +27,7 @@ import ( "github.com/urfave/cli" ) -func clusterWatch(c *cli.Context) error { +func managerWatch(c *cli.Context) error { grpcAddr := c.String("grpc-address") key := c.Args().Get(0) diff --git a/example/wiki_bulk_index.txt b/example/wiki_bulk_index.jsonl similarity index 100% rename from example/wiki_bulk_index.txt rename to example/wiki_bulk_index.jsonl