diff --git a/README.md b/README.md index 05aa84f..06373b0 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,12 @@ Note: modify the configs in basic_config.py before run the test script 2. modify server/toolchain_git_branch_commit 3. modify auth/https/serice ports if need.. ''' -python deploy_start.py all # or 'server' or 'toolchain' +python deploy_start.py all # or 'server' or 'toolchain' or 'hugegraph' + +''' +Or you can set backend with followed command +''' +python deploy_start.py hugegraph hstore # or other backend supported by hugegraph # 2. decompress dataset unzip config/dataset.zip -d config/ diff --git a/src/common/deploy_graph.py b/src/common/deploy_graph.py index 4937b03..091d5fc 100644 --- a/src/common/deploy_graph.py +++ b/src/common/deploy_graph.py @@ -5,8 +5,10 @@ create_time: 2020/4/22 5:17 下午 """ import os +import shutil import subprocess import sys +import time from config.basic_config import admin_password @@ -118,6 +120,50 @@ def set_server_properties(package_dir_path, host, server_port, gremlin_port): } ''') +def set_pd_properties(package_dir_path, host, grpc_port, rest_port, store_list, raft_port, raft_list): + """ + 修改 pd 组件配置 + :return: + """ + # 修改 application 文件 + application_conf = package_dir_path + '/conf/application.yml' + alter_properties(application_conf, + '8686', + '%d' % grpc_port) + alter_properties(application_conf, + '8620', + '%d' % rest_port) + new_store_list = ','.join([f"{host}:{port}" for port in store_list]) + alter_properties(application_conf, + 'initial-store-list: 127.0.0.1:8500', + 'initial-store-list: %s' % new_store_list) + alter_properties(application_conf, + 'address: 127.0.0.1:8610', + 'address: %s:%d' % (host, raft_port)) + new_raft_list = ','.join([f"{host}:{port}" for port in raft_list]) + alter_properties(application_conf, + 'peers-list: 127.0.0.1:8610', + 'peers-list: %s' % new_raft_list) + +def set_store_properties(package_dir_path, host, pd_list, grpc_port, raft_port, rest_port): + application_conf = package_dir_path + '/conf/application.yml' + new_pd_list = ','.join([f"{host}:{port}" for port in pd_list]) + alter_properties(application_conf, + 'address: localhost:8686', + 'address: %s' % new_pd_list) + alter_properties(application_conf, + 'port: 8500', + 'port: %d' % grpc_port) + alter_properties(application_conf, + 'address: 127.0.0.1:8510', + 'address: %s:%d' % (host, raft_port)) + alter_properties(application_conf, + 'address: 127.0.0.1:8610', + 'address: %s:%d' % (host, raft_port)) + alter_properties(application_conf, + 'port: 8520', + 'port: %d' % rest_port) + def set_hubble_properties(package_dir_path, host, port): """ @@ -128,6 +174,21 @@ def set_hubble_properties(package_dir_path, host, port): alter_properties(hubble_conf, 'server.host=localhost', 'server.host=%s' % host) alter_properties(hubble_conf, 'server.port=8088', 'server.port=%d' % port) +def ensure_start(log_file_path, target, interval = 1): + while not os.path.exists(log_file_path): + time.sleep(interval) + with open(log_file_path, 'r') as log_file: + log_file.seek(0, 2) + while True: + line = log_file.readline() + if not line: + time.sleep(interval) + continue + # 检查目标语句是否在行中 + if target in line: + return + + def start_graph(package_dir_path, graph_type): """ @@ -140,6 +201,21 @@ def start_graph(package_dir_path, graph_type): f'&& echo "{pa}" | ./bin/init-store.sh ' '&& ./bin/start-hugegraph.sh' % package_dir_path ) + elif graph_type == 'pd': + os.system( + 'cd %s ' + '&& ./bin/start-hugegraph-pd.sh' % package_dir_path + ) + elif graph_type == 'store': + os.system( + 'cd %s ' + '&& ./bin/start-hugegraph-store.sh' % package_dir_path + ) + elif graph_type == 'hugegraph_server': + os.system( + 'cd %s ' + '&& ./bin/start-hugegraph.sh' % package_dir_path + ) else: os.system( f'chmod -R 755 {package_dir_path}' @@ -151,6 +227,11 @@ def unzip_targz(file_path, file_name): cmd = f'cd {file_path} && tar -zxvf {file_name}' subprocess.check_call(cmd, shell=True) +def update_backend_properties(file_path, target_path): + if(os.path.exists(target_path)): + os.remove(target_path) + shutil.copy2(file_path, target_path) + class Deploy: """ @@ -168,6 +249,16 @@ def __init__(self, obj): self.loader_git = obj.loader_git self.tools_git = obj.tools_git self.hubble_git = obj.hubble_git + self.host = obj.host + self.pd_grpc_port = obj.pd_grpc_port + self.pd_rest_port = obj.pd_rest_port + self.store_list = obj.store_list + self.pd_raft_port = obj.pd_raft_port + self.raft_list = obj.raft_list + self.pd_list = obj.pd_list + self.store_grpc_port = obj.store_grpc_port + self.store_raft_port = obj.store_raft_port + self.store_rest_port = obj.store_rest_port @staticmethod def server(conf): @@ -189,6 +280,51 @@ def server(conf): ) start_graph(gen_dir, 'server') + @staticmethod + def pd(conf): + """ + :return: + """ + is_exists_path(conf.codebase_path) + get_code(conf.codebase_path, conf.server_git, conf.pd_local_repo) + compile_package(conf.project_path) + unzip_targz(conf.pd_path, conf.pd_tar_path.split('/')[-1]) + + gen_dir = os.path.join(conf.codebase_path, conf.pd_gen_dir) + # start graph_server + set_pd_properties( + gen_dir, + conf.host, + conf.pd_grpc_port, + conf.pd_rest_port, + conf.store_list, + conf.pd_raft_port, + conf.raft_list + ) + start_graph(gen_dir, 'pd') + + @staticmethod + def store(conf): + """ + :return: + """ + is_exists_path(conf.codebase_path) + get_code(conf.codebase_path, conf.server_git, conf.store_local_repo) + compile_package(conf.project_path) + unzip_targz(conf.store_path, conf.store_tar_path.split('/')[-1]) + + gen_dir = os.path.join(conf.codebase_path, conf.store_gen_dir) + # start graph_server + set_store_properties( + gen_dir, + conf.host, + conf.pd_list, + conf.store_grpc_port, + conf.store_raft_port, + conf.store_rest_port + ) + start_graph(gen_dir, 'store') + @staticmethod def toolchain(conf): is_exists_path(conf.codebase_path) @@ -201,6 +337,59 @@ def toolchain(conf): # set_hubble_properties(hubble_package_dir_name, conf.graph_host, conf.hubble_port) start_graph(conf.hubble_path, 'hubble') + @staticmethod + def hugegraph(conf): + is_exists_path(conf.codebase_path) + get_code(conf.codebase_path, conf.server_git, 'incubator-hugegraph') + compile_package(conf.project_path) + + unzip_targz(conf.pd_path, conf.pd_tar_path.split('/')[-1]) + unzip_targz(conf.store_path, conf.store_tar_path.split('/')[-1]) + unzip_targz(conf.server_path, conf.server_tar_path.split('/')[-1]) + + pd_gen_dir = os.path.join(conf.codebase_path, conf.pd_gen_dir) + # start graph_server + set_pd_properties( + pd_gen_dir, + conf.host, + conf.pd_grpc_port, + conf.pd_rest_port, + conf.store_list, + conf.pd_raft_port, + conf.raft_list + ) + start_graph(pd_gen_dir, 'pd') + ensure_start(pd_gen_dir + "/logs/hugegraph-pd-stdout.log", + f'Hugegraph-pd started.') + + store_gen_dir = os.path.join(conf.codebase_path, conf.store_gen_dir) + # start graph_server + set_store_properties( + store_gen_dir, + conf.host, + conf.pd_list, + conf.store_grpc_port, + conf.store_raft_port, + conf.store_rest_port + ) + start_graph(store_gen_dir, 'store') + ensure_start(store_gen_dir + "/logs/hugegraph-store.log", + f'o.a.h.s.n.StoreNodeApplication - Starting StoreNodeApplication') + + server_gen_dir = os.path.join(conf.codebase_path, conf.server_gen_dir) + # start graph_server + set_server_properties( + server_gen_dir, + conf.graph_host, + conf.server_port, + conf.gremlin_port + ) + config_file_path = os.path.dirname(os.path.realpath(__file__)) + "/../dist/" + conf.server_backend + ".properties" + target_path = os.path.join(conf.codebase_path, conf.server_gen_dir + '/conf/graphs/hugegraph.properties') + update_backend_properties(config_file_path, target_path) + start_graph(server_gen_dir, 'hugegraph_server') + + if __name__ == "__main__": pass diff --git a/src/config/basic_config.py b/src/config/basic_config.py index cb5dd61..33bc1f3 100644 --- a/src/config/basic_config.py +++ b/src/config/basic_config.py @@ -12,17 +12,25 @@ # apache release version is_incubating = 'incubating-' # TODO: consider user * instead of fixed version? -server_release_version = '1.5.0' +hugegraph_release_version = '1.5.0' toolchain_release_version = '1.3.0' +pd_local_repo = 'incubator-hugegraph/hugegraph-pd' +store_local_repo = 'incubator-hugegraph/hugegraph-store' server_local_repo = 'incubator-hugegraph/hugegraph-server' toolchain_local_repo = 'incubator-hugegraph-toolchain' -server_gen_dir = f'incubator-hugegraph/hugegraph-server/apache-hugegraph-server-{is_incubating}{server_release_version}' +pd_gen_dir = f'incubator-hugegraph/hugegraph-pd/apache-hugegraph-pd-{is_incubating}{hugegraph_release_version}' +store_gen_dir = f'incubator-hugegraph/hugegraph-store/apache-hugegraph-store-{is_incubating}{hugegraph_release_version}' +server_gen_dir = f'incubator-hugegraph/hugegraph-server/apache-hugegraph-server-{is_incubating}{hugegraph_release_version}' toolchain_gen_dir = f'incubator-hugegraph-toolchain/apache-hugegraph-toolchain-{is_incubating}{toolchain_release_version}' toolchain_obj_template = 'apache-hugegraph-{tool_name}-' + is_incubating + f'{toolchain_release_version}' project_path = os.path.join(codebase_path, 'incubator-hugegraph') +pd_path = os.path.join(codebase_path, pd_local_repo) +store_path = os.path.join(codebase_path, store_local_repo) server_path = os.path.join(codebase_path, server_local_repo) -server_tar_path = os.path.join(codebase_path, 'incubator-hugegraph/hugegraph-server', f'apache-hugegraph-server-{is_incubating}{server_release_version}' + '.tar.gz') +pd_tar_path = os.path.join(codebase_path, pd_gen_dir + '.tar.gz') +store_tar_path = os.path.join(codebase_path, store_gen_dir + '.tar.gz') +server_tar_path = os.path.join(codebase_path, server_gen_dir + '.tar.gz') toolchain_path = os.path.join(codebase_path, toolchain_local_repo) loader_path = os.path.join(codebase_path, toolchain_gen_dir, toolchain_obj_template.format(tool_name='loader')) hubble_path = os.path.join(codebase_path, toolchain_gen_dir, toolchain_obj_template.format(tool_name='hubble')) @@ -49,6 +57,18 @@ admin_password = {'admin': 'admin'} test_password = {'tester': '123456'} +host = '127.0.0.1' +pd_grpc_port = 8686 +pd_rest_port = 8620 +store_list = [8500] +pd_raft_port = 8610 +raft_list = [8610] + +pd_list = [8686] +store_grpc_port = 8500 +store_raft_port = 8510 +store_rest_port = 8520 + # toolchain (includes loader, hubble, tools) toolchain_git = { 'branch': '3b58fc6', diff --git a/src/deploy_start.py b/src/deploy_start.py index 2d278eb..f587cf2 100644 --- a/src/deploy_start.py +++ b/src/deploy_start.py @@ -24,6 +24,12 @@ def graph_deploy(param, conf_obj): Deploy.server(conf_obj) elif param == 'toolchain': Deploy.toolchain(conf_obj) + elif param == 'pd': + Deploy.pd(conf_obj) + elif param == 'store': + Deploy.store(conf_obj) + elif param == 'hugegraph': + Deploy.hugegraph(conf_obj) else: Deploy.server(conf_obj) Deploy.toolchain(conf_obj) @@ -32,8 +38,13 @@ def graph_deploy(param, conf_obj): if __name__ == "__main__": param_size = len(sys.argv) if param_size == 2 \ - and sys.argv[1] in ['all', 'server', 'toolchain']: + and sys.argv[1] in ['all', 'server', 'toolchain', 'pd', 'store', 'hugegraph']: + graph_deploy(sys.argv[1], basic_config) + elif param_size == 3 \ + and sys.argv[1] in ['all', 'server', 'toolchain', 'pd', 'store', 'hugegraph']\ + and sys.argv[2] in ['hbase', 'hstore', 'cassandra', 'mysql', 'rocksdb', 'scylladb']: + basic_config.server_backend = sys.argv[2] graph_deploy(sys.argv[1], basic_config) else: - print('failed: 执行脚本参数为[all,server,toolchain]') + print('failed: 执行脚本参数为[all, hugegraph, toolchain, server, pd, store]') exit(1) diff --git a/src/dist/cassandra.properties b/src/dist/cassandra.properties new file mode 100644 index 0000000..475636c --- /dev/null +++ b/src/dist/cassandra.properties @@ -0,0 +1,44 @@ +gremlin.graph=org.apache.hugegraph.HugeFactory + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +vertex.cache_type=l2 +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +edge.cache_type=l2 +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +store=hugegraph + +# task config +task.scheduler_type=local +task.schedule_period=10 +task.retry=0 +task.wait_timeout=10 + +# search config +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +backend=cassandra +serializer=cassandra + +# cassandra backend config +cassandra.host=localhost +cassandra.port=9042 +cassandra.username= +cassandra.password= +#cassandra.connect_timeout=5 +#cassandra.read_timeout=20 + +#cassandra.keyspace.strategy=SimpleStrategy +#cassandra.keyspace.replication=3 diff --git a/src/dist/hbase.properties b/src/dist/hbase.properties new file mode 100644 index 0000000..6101c60 --- /dev/null +++ b/src/dist/hbase.properties @@ -0,0 +1,42 @@ +gremlin.graph=org.apache.hugegraph.HugeFactory + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +vertex.cache_type=l2 +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +edge.cache_type=l2 +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +store=hugegraph + +# task config +task.scheduler_type=local +task.schedule_period=10 +task.retry=0 +task.wait_timeout=10 + +# search config +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +backend=hbase +serializer=hbase + +# hbase backend config +hbase.hosts=localhost +hbase.port=2181 +# Note: recommend to modify the HBase partition number by the actual/env data amount & RS amount before init store +# it may influence the loading speed a lot +#hbase.enable_partition=true +#hbase.vertex_partitions=10 +#hbase.edge_partitions=30 diff --git a/src/dist/hstore.properties b/src/dist/hstore.properties new file mode 100644 index 0000000..3e90c5f --- /dev/null +++ b/src/dist/hstore.properties @@ -0,0 +1,90 @@ +# gremlin entrance to create graph +# auth config: org.apache.hugegraph.auth.HugeFactoryAuthProxy +gremlin.graph=org.apache.hugegraph.HugeFactory + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +vertex.cache_type=l2 +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +edge.cache_type=l2 +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +backend=hstore +serializer=binary + +store=hugegraph + +# pd config +pd.peers=127.0.0.1:8686 + +# task config +task.scheduler_type=local +task.schedule_period=10 +task.retry=0 +task.wait_timeout=10 + +# search config +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +# rocksdb backend config +#rocksdb.data_path=/path/to/disk +#rocksdb.wal_path=/path/to/disk + + +# cassandra backend config +cassandra.host=localhost +cassandra.port=9042 +cassandra.username= +cassandra.password= +#cassandra.connect_timeout=5 +#cassandra.read_timeout=20 +#cassandra.keyspace.strategy=SimpleStrategy +#cassandra.keyspace.replication=3 + +# hbase backend config +#hbase.hosts=localhost +#hbase.port=2181 +#hbase.znode_parent=/hbase +#hbase.threads_max=64 +# IMPORTANT: recommend to modify the HBase partition number +# by the actual/env data amount & RS amount before init store +# It will influence the load speed a lot +#hbase.enable_partition=true +#hbase.vertex_partitions=10 +#hbase.edge_partitions=30 + +# mysql backend config +#jdbc.driver=com.mysql.jdbc.Driver +#jdbc.url=jdbc:mysql://127.0.0.1:3306 +#jdbc.username=root +#jdbc.password= +#jdbc.reconnect_max_times=3 +#jdbc.reconnect_interval=3 +#jdbc.ssl_mode=false + +# postgresql & cockroachdb backend config +#jdbc.driver=org.postgresql.Driver +#jdbc.url=jdbc:postgresql://localhost:5432/ +#jdbc.username=postgres +#jdbc.password= +#jdbc.postgresql.connect_database=template1 + +# palo backend config +#palo.host=127.0.0.1 +#palo.poll_interval=10 +#palo.temp_dir=./palo-data +#palo.file_limit_size=32 + +# WARNING: These raft configurations are deprecated, please use the latest version instead. +# raft.mode=false diff --git a/src/dist/mysql.properties b/src/dist/mysql.properties new file mode 100644 index 0000000..dead041 --- /dev/null +++ b/src/dist/mysql.properties @@ -0,0 +1,42 @@ +gremlin.graph=org.apache.hugegraph.HugeFactory + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +vertex.cache_type=l2 +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +edge.cache_type=l2 +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +store=hugegraph + +# task config +task.scheduler_type=local +task.schedule_period=10 +task.retry=0 +task.wait_timeout=10 + +# search config +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +backend=mysql +serializer=mysql + +# mysql backend config +jdbc.driver=com.mysql.cj.jdbc.Driver +jdbc.url=jdbc:mysql://127.0.0.1:3306 +jdbc.username= +jdbc.password= +jdbc.reconnect_max_times=3 +jdbc.reconnect_interval=3 +jdbc.ssl_mode=false diff --git a/src/dist/rocksdb.properties b/src/dist/rocksdb.properties new file mode 100644 index 0000000..52c81be --- /dev/null +++ b/src/dist/rocksdb.properties @@ -0,0 +1,90 @@ +# gremlin entrance to create graph +# auth config: org.apache.hugegraph.auth.HugeFactoryAuthProxy +gremlin.graph=org.apache.hugegraph.HugeFactory + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +vertex.cache_type=l2 +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +edge.cache_type=l2 +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +backend=rocksdb +serializer=binary + +store=hugegraph + +# pd config +#pd.peers=127.0.0.1:8686 + +# task config +task.scheduler_type=local +task.schedule_period=10 +task.retry=0 +task.wait_timeout=10 + +# search config +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +# rocksdb backend config +#rocksdb.data_path=/path/to/disk +#rocksdb.wal_path=/path/to/disk + + +# cassandra backend config +cassandra.host=localhost +cassandra.port=9042 +cassandra.username= +cassandra.password= +#cassandra.connect_timeout=5 +#cassandra.read_timeout=20 +#cassandra.keyspace.strategy=SimpleStrategy +#cassandra.keyspace.replication=3 + +# hbase backend config +#hbase.hosts=localhost +#hbase.port=2181 +#hbase.znode_parent=/hbase +#hbase.threads_max=64 +# IMPORTANT: recommend to modify the HBase partition number +# by the actual/env data amount & RS amount before init store +# It will influence the load speed a lot +#hbase.enable_partition=true +#hbase.vertex_partitions=10 +#hbase.edge_partitions=30 + +# mysql backend config +#jdbc.driver=com.mysql.jdbc.Driver +#jdbc.url=jdbc:mysql://127.0.0.1:3306 +#jdbc.username=root +#jdbc.password= +#jdbc.reconnect_max_times=3 +#jdbc.reconnect_interval=3 +#jdbc.ssl_mode=false + +# postgresql & cockroachdb backend config +#jdbc.driver=org.postgresql.Driver +#jdbc.url=jdbc:postgresql://localhost:5432/ +#jdbc.username=postgres +#jdbc.password= +#jdbc.postgresql.connect_database=template1 + +# palo backend config +#palo.host=127.0.0.1 +#palo.poll_interval=10 +#palo.temp_dir=./palo-data +#palo.file_limit_size=32 + +# WARNING: These raft configurations are deprecated, please use the latest version instead. +# raft.mode=false diff --git a/src/dist/scylladb.properties b/src/dist/scylladb.properties new file mode 100644 index 0000000..4d6f993 --- /dev/null +++ b/src/dist/scylladb.properties @@ -0,0 +1,44 @@ +gremlin.graph=org.apache.hugegraph.HugeFactory + +# cache config +#schema.cache_capacity=100000 +# vertex-cache default is 1000w, 10min expired +vertex.cache_type=l2 +#vertex.cache_capacity=10000000 +#vertex.cache_expire=600 +# edge-cache default is 100w, 10min expired +edge.cache_type=l2 +#edge.cache_capacity=1000000 +#edge.cache_expire=600 + + +# schema illegal name template +#schema.illegal_name_regex=\s+|~.* + +#vertex.default_label=vertex + +store=hugegraph + +# task config +task.scheduler_type=local +task.schedule_period=10 +task.retry=0 +task.wait_timeout=10 + +# search config +search.text_analyzer=jieba +search.text_analyzer_mode=INDEX + +backend=scylladb +serializer=scylladb + +# cassandra backend config +cassandra.host=localhost +cassandra.port=9042 +cassandra.username= +cassandra.password= +#cassandra.connect_timeout=5 +#cassandra.read_timeout=20 + +#cassandra.keyspace.strategy=SimpleStrategy +#cassandra.keyspace.replication=3