From 2fd899c224f07e68fd7a437927de2ea0ae4d0a51 Mon Sep 17 00:00:00 2001 From: liubo Date: Tue, 17 Dec 2019 10:46:50 +0800 Subject: [PATCH 01/13] add tiflash config --- conf/tiflash-learner.yml | 57 ++++++++++++++++++++ conf/tiflash-user.yml | 31 +++++++++++ conf/tiflash.yml | 38 +++++++++++++ group_vars/tiflash_servers.yml | 0 roles/local/tasks/binary_deployment.yml | 28 ++++++++++ roles/local/templates/binary_packages.yml.j2 | 5 ++ roles/tiflash/vars/tiflash-learner.yml | 57 ++++++++++++++++++++ roles/tiflash/vars/tiflash-user.yml | 31 +++++++++++ roles/tiflash/vars/tiflash.yml | 38 +++++++++++++ 9 files changed, 285 insertions(+) create mode 100644 conf/tiflash-learner.yml create mode 100644 conf/tiflash-user.yml create mode 100644 conf/tiflash.yml create mode 100644 group_vars/tiflash_servers.yml create mode 100644 roles/tiflash/vars/tiflash-learner.yml create mode 100644 roles/tiflash/vars/tiflash-user.yml create mode 100644 roles/tiflash/vars/tiflash.yml diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml new file mode 100644 index 000000000..5652a5941 --- /dev/null +++ b/conf/tiflash-learner.yml @@ -0,0 +1,57 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + labels: { "tiflash_http_port": "8123" } + engine-addr: "tiflash0:3930" + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + raftdb-path: "" + sync-log: true + max-leader-missing-duration: "22s" + abnormal-leader-missing-duration: "21s" + peer-stale-state-check-interval: "20s" + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + block-cache-size: "10GB" + + lockcf: + block-cache-size: "4GB" + + writecf: + block-cache-size: "4GB" + +raftdb: + defaultcf: + block-cache-size: "1GB" + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: \ No newline at end of file diff --git a/conf/tiflash-user.yml b/conf/tiflash-user.yml new file mode 100644 index 000000000..c6229b243 --- /dev/null +++ b/conf/tiflash-user.yml @@ -0,0 +1,31 @@ +--- + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/conf/tiflash.yml b/conf/tiflash.yml new file mode 100644 index 000000000..9107836d2 --- /dev/null +++ b/conf/tiflash.yml @@ -0,0 +1,38 @@ +--- + +global: + tmp_path: "/tmp/tiflash/data/tmp" + display_name: "TiFlash" + default_profile: "default" + users_config: "users.toml" + path: "/tmp/tiflash/data/db" + mark_cache_size: 5368709120 + listen_host: "0.0.0.0" + tcp_port: 9000 + http_port: 8123 + interserver_http_port: 9009 +flash: + tidb_status_addr: "tidb0:10080" + service_addr: "tiflash0:3930" + flash_cluster: + refresh_interval: 20 + update_rule_interval: 5 + master_ttl: 60 + cluster_manager_path: "/tics/flash_cluster_manager" + proxy: + addr: "0.0.0.0:20170" + advertise-addr: "tiflash0:20170" + data-dir: "/data" + config: "/tikv.toml" + log-file: "/log/tikv.log" +logger: + count: 10 + errorlog: "/tmp/tiflash/log/error.log" + size: "1000M" + log: "/tmp/tiflash/log/server.log" + level: "trace" +application: + runAsDaemon: true + +raft: + pd_addr: "pd0:2379" diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml new file mode 100644 index 000000000..e69de29bb diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml index e367956da..bd37a198c 100644 --- a/roles/local/tasks/binary_deployment.yml +++ b/roles/local/tasks/binary_deployment.yml @@ -48,6 +48,22 @@ - has_outbound_network - not deploy_without_tidb|default(false) +- name: download TiFlash packages + get_url: + url: "{{ item.url }}" + dest: "{{ downloads_dir }}/{{ item.name }}" + checksum: "{{ item.checksum | default(omit) }}" + force: yes + validate_certs: no + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: "{{ tiflash_packages }}" + when: + - has_outbound_network + - not deploy_without_tidb|default(false) + - name: unarchive third party binary shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf args: @@ -69,6 +85,13 @@ warn: no when: not deploy_without_tidb|default(false) +- name: unarchive tiflash + shell: tar xzf tics-latest-linux-amd64.tar.gz + args: + chdir: "{{ downloads_dir }}" + warn: no + when: not deploy_without_tidb|default(false) + - name: cp monitoring binary shell: > cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} "{{ resources_dir }}/bin/{{ item }}" @@ -88,3 +111,8 @@ shell: > cp -rfv {{ downloads_dir }}/tispark-sample-data "{{ resources_dir }}/bin/" when: not deploy_without_tidb|default(false) + +- name: cp tiflash directory + shell: > + cp -rfv {{ downloads_dir }}/tics-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" + with_items: "{{ tiflash_packages }}" diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2 index 5075ffb4e..d633a36ac 100644 --- a/roles/local/templates/binary_packages.yml.j2 +++ b/roles/local/templates/binary_packages.yml.j2 @@ -68,3 +68,8 @@ tispark_packages: url: http://download.pingcap.org/tispark-sample-data.tar.gz checksum: "sha256:bd0368a9d8663a4a8de89e39cc4cc1d91c718faf36d4bc7e1f8482c34d5bb8db" {% endif %} + +tiflash: + - name: tics-latest-linux-amd64.tar.gz + version: latest + url: http://download.pingcap.org/tics-latest-linux-amd64.tar.gz \ No newline at end of file diff --git a/roles/tiflash/vars/tiflash-learner.yml b/roles/tiflash/vars/tiflash-learner.yml new file mode 100644 index 000000000..5652a5941 --- /dev/null +++ b/roles/tiflash/vars/tiflash-learner.yml @@ -0,0 +1,57 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + labels: { "tiflash_http_port": "8123" } + engine-addr: "tiflash0:3930" + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + raftdb-path: "" + sync-log: true + max-leader-missing-duration: "22s" + abnormal-leader-missing-duration: "21s" + peer-stale-state-check-interval: "20s" + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + block-cache-size: "10GB" + + lockcf: + block-cache-size: "4GB" + + writecf: + block-cache-size: "4GB" + +raftdb: + defaultcf: + block-cache-size: "1GB" + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: \ No newline at end of file diff --git a/roles/tiflash/vars/tiflash-user.yml b/roles/tiflash/vars/tiflash-user.yml new file mode 100644 index 000000000..c6229b243 --- /dev/null +++ b/roles/tiflash/vars/tiflash-user.yml @@ -0,0 +1,31 @@ +--- + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml new file mode 100644 index 000000000..9107836d2 --- /dev/null +++ b/roles/tiflash/vars/tiflash.yml @@ -0,0 +1,38 @@ +--- + +global: + tmp_path: "/tmp/tiflash/data/tmp" + display_name: "TiFlash" + default_profile: "default" + users_config: "users.toml" + path: "/tmp/tiflash/data/db" + mark_cache_size: 5368709120 + listen_host: "0.0.0.0" + tcp_port: 9000 + http_port: 8123 + interserver_http_port: 9009 +flash: + tidb_status_addr: "tidb0:10080" + service_addr: "tiflash0:3930" + flash_cluster: + refresh_interval: 20 + update_rule_interval: 5 + master_ttl: 60 + cluster_manager_path: "/tics/flash_cluster_manager" + proxy: + addr: "0.0.0.0:20170" + advertise-addr: "tiflash0:20170" + data-dir: "/data" + config: "/tikv.toml" + log-file: "/log/tikv.log" +logger: + count: 10 + errorlog: "/tmp/tiflash/log/error.log" + size: "1000M" + log: "/tmp/tiflash/log/server.log" + level: "trace" +application: + runAsDaemon: true + +raft: + pd_addr: "pd0:2379" From abfa0373ef4c1a00de9486f9cf93f938b91a1d28 Mon Sep 17 00:00:00 2001 From: liubo Date: Wed, 18 Dec 2019 14:28:39 +0800 Subject: [PATCH 02/13] Add tiflash --- conf/tiflash.yml | 17 +--- .../tiflash_learner.yml | 3 +- .../tiflash-user.yml => conf/tiflash_user.yml | 0 group_vars/tiflash_servers.yml | 7 ++ inventory.ini | 3 + roles/local/templates/binary_packages.yml.j2 | 2 +- roles/tiflash/defaults/main.yml | 16 ++++ roles/tiflash/tasks/binary_deployment.yml | 23 ++++++ roles/tiflash/tasks/main.yml | 81 +++++++++++++++++++ roles/tiflash/tasks/supervise_deployment.yml | 8 ++ roles/tiflash/tasks/systemd_deployment.yml | 8 ++ .../templates/run_tiflash_binary.sh.j2 | 20 +++++ roles/tiflash/templates/tiflash.toml.j2 | 66 +++++++++++++++ .../tiflash/templates/tiflash_learner.toml.j2 | 81 +++++++++++++++++++ roles/tiflash/templates/tiflash_user.toml.j2 | 56 +++++++++++++ roles/tiflash/vars/tiflash.yml | 17 +--- roles/tiflash/vars/tiflash_learner.yml | 56 +++++++++++++ roles/tiflash/vars/tiflash_user.yml | 31 +++++++ 18 files changed, 460 insertions(+), 35 deletions(-) rename roles/tiflash/vars/tiflash-learner.yml => conf/tiflash_learner.yml (95%) rename roles/tiflash/vars/tiflash-user.yml => conf/tiflash_user.yml (100%) create mode 100644 roles/tiflash/defaults/main.yml create mode 100644 roles/tiflash/tasks/binary_deployment.yml create mode 100644 roles/tiflash/tasks/main.yml create mode 100644 roles/tiflash/tasks/supervise_deployment.yml create mode 100644 roles/tiflash/tasks/systemd_deployment.yml create mode 100644 roles/tiflash/templates/run_tiflash_binary.sh.j2 create mode 100644 roles/tiflash/templates/tiflash.toml.j2 create mode 100644 roles/tiflash/templates/tiflash_learner.toml.j2 create mode 100644 roles/tiflash/templates/tiflash_user.toml.j2 create mode 100644 roles/tiflash/vars/tiflash_learner.yml create mode 100644 roles/tiflash/vars/tiflash_user.yml diff --git a/conf/tiflash.yml b/conf/tiflash.yml index 9107836d2..d99ece71b 100644 --- a/conf/tiflash.yml +++ b/conf/tiflash.yml @@ -1,38 +1,23 @@ --- global: - tmp_path: "/tmp/tiflash/data/tmp" display_name: "TiFlash" default_profile: "default" - users_config: "users.toml" - path: "/tmp/tiflash/data/db" mark_cache_size: 5368709120 listen_host: "0.0.0.0" - tcp_port: 9000 - http_port: 8123 - interserver_http_port: 9009 flash: - tidb_status_addr: "tidb0:10080" - service_addr: "tiflash0:3930" flash_cluster: refresh_interval: 20 update_rule_interval: 5 master_ttl: 60 cluster_manager_path: "/tics/flash_cluster_manager" proxy: - addr: "0.0.0.0:20170" - advertise-addr: "tiflash0:20170" - data-dir: "/data" - config: "/tikv.toml" - log-file: "/log/tikv.log" + logger: count: 10 - errorlog: "/tmp/tiflash/log/error.log" size: "1000M" - log: "/tmp/tiflash/log/server.log" level: "trace" application: runAsDaemon: true raft: - pd_addr: "pd0:2379" diff --git a/roles/tiflash/vars/tiflash-learner.yml b/conf/tiflash_learner.yml similarity index 95% rename from roles/tiflash/vars/tiflash-learner.yml rename to conf/tiflash_learner.yml index 5652a5941..aee770757 100644 --- a/roles/tiflash/vars/tiflash-learner.yml +++ b/conf/tiflash_learner.yml @@ -12,7 +12,6 @@ readpool: server: labels: { "tiflash_http_port": "8123" } - engine-addr: "tiflash0:3930" storage: @@ -54,4 +53,4 @@ security: cert-path: "" key-path: "" -import: \ No newline at end of file +import: diff --git a/roles/tiflash/vars/tiflash-user.yml b/conf/tiflash_user.yml similarity index 100% rename from roles/tiflash/vars/tiflash-user.yml rename to conf/tiflash_user.yml diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml index e69de29bb..a439d94e0 100644 --- a/group_vars/tiflash_servers.yml +++ b/group_vars/tiflash_servers.yml @@ -0,0 +1,7 @@ +--- + +tcp_port: 9000 +http_port: 8123 +interserver_http_port: 9009 +flash_service_port: 3930 +flash_proxy_port: 20160 \ No newline at end of file diff --git a/inventory.ini b/inventory.ini index 0c8bc9857..6feb7aaf4 100644 --- a/inventory.ini +++ b/inventory.ini @@ -49,6 +49,9 @@ [drainer_servers] +## Tiflash Part +[tiflash_servers] + ## Group variables [pd_servers:vars] # location_labels = ["zone","rack","host"] diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2 index d633a36ac..e6aa88570 100644 --- a/roles/local/templates/binary_packages.yml.j2 +++ b/roles/local/templates/binary_packages.yml.j2 @@ -69,7 +69,7 @@ tispark_packages: checksum: "sha256:bd0368a9d8663a4a8de89e39cc4cc1d91c718faf36d4bc7e1f8482c34d5bb8db" {% endif %} -tiflash: +tiflash_packages: - name: tics-latest-linux-amd64.tar.gz version: latest url: http://download.pingcap.org/tics-latest-linux-amd64.tar.gz \ No newline at end of file diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml new file mode 100644 index 000000000..1dcb3e26e --- /dev/null +++ b/roles/tiflash/defaults/main.yml @@ -0,0 +1,16 @@ +--- + +tiflash_dir: "{{ deploy_dir }}/tiflash" +tmp_path: "{{ deploy_dir }}/tiflash/data/tmp" +path: "{{ deploy_dir }}/tiflash/data/db" +flash_cluster_manager: "{{ deploy_dir }}/bin/flash_cluster_manager" +tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log" +tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log" +tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" +user_conf: "{{ deploy_dir }}/conf/tiflash_user.toml" +flash_proxy_config: "{{ deploy_dir }}/conf/tiflash_learner.toml" +tcp_port: 9000 +http_port: 8123 +interserver_http_port: 9009 +flash_service_port: 3930 +flash_proxy_port: 20160 diff --git a/roles/tiflash/tasks/binary_deployment.yml b/roles/tiflash/tasks/binary_deployment.yml new file mode 100644 index 000000000..8305d14dd --- /dev/null +++ b/roles/tiflash/tasks/binary_deployment.yml @@ -0,0 +1,23 @@ +--- + +- name: deploy tiflash binary + copy: src="{{ resources_dir }}/bin/tiflash" dest="{{ deploy_dir }}/bin/" mode=0755 backup=yes + register: tiflash_binary + +- name: create run script + template: + src: "{{ item }}_{{ role_name }}_binary.sh.j2" + dest: "{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh" + mode: "0755" + backup: yes + with_items: + - run + vars: + role_status_dir: status/{{ role_name }} + register: tiflash_script + +- name: backup script file + command: mv "{{ tiflash_script.backup_file }}" "{{ backup_dir }}" + when: tiflash_script.changed and tiflash_script.backup_file is defined + +- include_tasks: "systemd_deployment.yml" \ No newline at end of file diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml new file mode 100644 index 000000000..917e506b9 --- /dev/null +++ b/roles/tiflash/tasks/main.yml @@ -0,0 +1,81 @@ +--- + +- name: create tiflash directories + file: path={{ item }} state=directory mode=0755 + with_items: + - "{{ tiflash_dir }}" + - "{{ tmp_path }}" + - "{{ path }}" + - "{{ data_dir }}" + +- name: "load tiflash config: tidb-ansible/conf/tiflash.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom + +- name: load tiflash default config + include_vars: file=tiflash.yml name=tiflash_conf_default + +- name: combine tiflash config + set_fact: + tiflash_conf: "{{ with_default_dicts(tiflash_conf_custom, tiflash_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_conf + +- name: create tiflash config file + template: src=tiflash.toml.j2 dest={{ deploy_dir }}/conf/tiflash.toml mode=0644 backup=yes + register: tiflash_conf_st + +- name: backup tiflash conf file + command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined + +- name: "load tiflash user config: tidb-ansible/conf/tiflash_user.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash_user.yml name=tiflash_user_conf_custom + +- name: load tiflash user default config + include_vars: file=tiflash_user.yml name=tiflash_user_conf_default + +- name: combine tiflash user config + set_fact: + tiflash_user_conf: "{{ with_default_dicts(tiflash_user_conf_custom, tiflash_user_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_user_conf + +- name: create tiflash config file + template: src=tiflash_user.toml.j2 dest={{ deploy_dir }}/conf/tiflash_user.toml mode=0644 backup=yes + register: tiflash_user_conf_st + +- name: backup tiflash user conf file + command: mv "{{ tiflash_user_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_user_conf_st.changed and tiflash_user_conf_st.backup_file is defined + +- name: "load tiflash user config: tidb-ansible/conf/tiflash_learner.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash_learner.yml name=tiflash_learner_conf_custom + +- name: load tiflash user default config + include_vars: file=tiflash_learner.yml name=tiflash_learner_conf_default + +- name: generate tiflash learner dynamic config + set_fact: + tiflash_learner_conf_generated: + server: + labels: "{{ labels }}" + +- name: generate final config + set_fact: + tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_generated, tiflash_learner_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_learner_conf + +- name: create tiflash learner config file + template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash_learner.toml mode=0644 backup=yes + register: tiflash_learner_conf_st + +- name: backup tiflash conf file + command: mv "{{ tiflash_learner_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined + +- include_tasks: "binary__deployment.yml" + +- name: prepare firewalld white list + set_fact: + firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}" \ No newline at end of file diff --git a/roles/tiflash/tasks/supervise_deployment.yml b/roles/tiflash/tasks/supervise_deployment.yml new file mode 100644 index 000000000..d2aa0d711 --- /dev/null +++ b/roles/tiflash/tasks/supervise_deployment.yml @@ -0,0 +1,8 @@ +--- + +- name: deploy supervise + include_role: + name: supervise + vars: + this_role_name: tiflash + service_name: tiflash-{{ tcp_port }} \ No newline at end of file diff --git a/roles/tiflash/tasks/systemd_deployment.yml b/roles/tiflash/tasks/systemd_deployment.yml new file mode 100644 index 000000000..2b31ffc56 --- /dev/null +++ b/roles/tiflash/tasks/systemd_deployment.yml @@ -0,0 +1,8 @@ +--- + +- name: deploy systemd + include_role: + name: systemd + vars: + this_role_name: tiflash + service_name: tiflash-{{ tcp_port }} \ No newline at end of file diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2 new file mode 100644 index 000000000..73f76e1fa --- /dev/null +++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2 @@ -0,0 +1,20 @@ +#!/bin/bash +set -e +ulimit -n 1000000 + +# WARNING: This file was auto-generated. Do not edit! +# All your edit might be overwritten! +cd "{{ deploy_dir }}" || exit 1 + +export RUST_BACKTRACE=1 + +export TZ=${TZ:-/etc/localtime} + +echo -n 'sync ... ' +stat=$(time sync) +echo ok +echo $stat + +echo $$ > "status/{{ role_name }}.pid" + +exec bin/theflash server --config-file config/tiflash.toml \ No newline at end of file diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 new file mode 100644 index 000000000..03f34eec5 --- /dev/null +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -0,0 +1,66 @@ +--- + +{% for item, value in tiflash_conf.global | dictsort -%} +{{ item }} = {{ value | to_json }} +tmp_path = "{{ tmp_path }}" +users_config = "{{ user_conf }}" +path = "{{ path }}" +tcp_port = "{{ tcp_port }}" +http_port = "{{ http_port }}" +interserver_http_port = "{{ interserver_http_port }}" +{% endfor %} + +[flash] +{% set all_tidb = [] -%} +{% for host in groups.tidb_servers -%} + {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set tidb_port = hostvars[host].tidb_status_port -%} + {% set _ = all_tidb.append("%s:%s" % (tidb_ip, tidb_port)) -%} +{% endfor %} +tidb_status_addr = "{{ all_tidb | join(',') }}" +service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" +{% for item, value in tiflash_conf.flash | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[flash.{{ item }}] +{% if item == 'flash_cluster' %} +cluster_manager_path = "{{ cluster_manager_path }}" +{% endif %} +{% if item == 'proxy' %} +addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" +advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" +data-dir = "{{ data_dir }}" +config = "{{ flash_proxy_config }}" +log-file = "{{ tiflash_tikv_log }}" +{% endif %} +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[logger] +{% for item, value in tiflash_conf.logger | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[application] +errorlog = "{{ tiflash_errlog }}" +log = "{{ tiflash_server_log }}" +{% for item, value in tiflash_conf.application | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[raft] +{% set all_pd = [] -%} +{% for host in groups.pd_servers -%} + {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set pd_port = hostvars[host].pd_client_port -%} + {% set _ = all_tidb.append("%s:%s" % (pd_ip, pd_port)) -%} +{% endfor %} +pd_addr = "{{ all_pd | join(',') }}" +{% for item, value in tiflash_conf.raft | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} \ No newline at end of file diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2 new file mode 100644 index 000000000..da86dbc9f --- /dev/null +++ b/roles/tiflash/templates/tiflash_learner.toml.j2 @@ -0,0 +1,81 @@ +--- + +[readpool] +{% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[readpool.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[server] +engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" +{% for item, value in tiflash_learner_conf.server | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[storage] +{% for item, value in tiflash_learner_conf.storage | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[pd] +{% for item, value in tiflash_learner_conf.pd | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[metric] +{% for item, value in tiflash_learner_conf.metric | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[raftstore] +{% for item, value in tiflash_learner_conf.raftstore | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[coprocessor] +{% for item, value in tiflash_learner_conf.coprocessor | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[rocksdb] +{% for item, value in tiflash_learner_conf.rocksdb | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[rocksdb.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[raftdb] +{% for item, value in tiflash_learner_conf.raftdb | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[raftdb.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[security] +{% for item, value in tiflash_learner_conf.security | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[import] +{% for item, value in tiflash_learner_conf.import | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} diff --git a/roles/tiflash/templates/tiflash_user.toml.j2 b/roles/tiflash/templates/tiflash_user.toml.j2 new file mode 100644 index 000000000..bd25d2be7 --- /dev/null +++ b/roles/tiflash/templates/tiflash_user.toml.j2 @@ -0,0 +1,56 @@ +--- + +[quotas] +{% for item, value in tiflash_user_conf.quotas | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[quotas.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[quotas.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[users] +{% for item, value in tiflash_user_conf.users | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[users.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[users.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[profiles] +{% for item, value in tiflash_user_conf.flash | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[profiles.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml index 9107836d2..d99ece71b 100644 --- a/roles/tiflash/vars/tiflash.yml +++ b/roles/tiflash/vars/tiflash.yml @@ -1,38 +1,23 @@ --- global: - tmp_path: "/tmp/tiflash/data/tmp" display_name: "TiFlash" default_profile: "default" - users_config: "users.toml" - path: "/tmp/tiflash/data/db" mark_cache_size: 5368709120 listen_host: "0.0.0.0" - tcp_port: 9000 - http_port: 8123 - interserver_http_port: 9009 flash: - tidb_status_addr: "tidb0:10080" - service_addr: "tiflash0:3930" flash_cluster: refresh_interval: 20 update_rule_interval: 5 master_ttl: 60 cluster_manager_path: "/tics/flash_cluster_manager" proxy: - addr: "0.0.0.0:20170" - advertise-addr: "tiflash0:20170" - data-dir: "/data" - config: "/tikv.toml" - log-file: "/log/tikv.log" + logger: count: 10 - errorlog: "/tmp/tiflash/log/error.log" size: "1000M" - log: "/tmp/tiflash/log/server.log" level: "trace" application: runAsDaemon: true raft: - pd_addr: "pd0:2379" diff --git a/roles/tiflash/vars/tiflash_learner.yml b/roles/tiflash/vars/tiflash_learner.yml new file mode 100644 index 000000000..aee770757 --- /dev/null +++ b/roles/tiflash/vars/tiflash_learner.yml @@ -0,0 +1,56 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + labels: { "tiflash_http_port": "8123" } + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + raftdb-path: "" + sync-log: true + max-leader-missing-duration: "22s" + abnormal-leader-missing-duration: "21s" + peer-stale-state-check-interval: "20s" + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + block-cache-size: "10GB" + + lockcf: + block-cache-size: "4GB" + + writecf: + block-cache-size: "4GB" + +raftdb: + defaultcf: + block-cache-size: "1GB" + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: diff --git a/roles/tiflash/vars/tiflash_user.yml b/roles/tiflash/vars/tiflash_user.yml new file mode 100644 index 000000000..c6229b243 --- /dev/null +++ b/roles/tiflash/vars/tiflash_user.yml @@ -0,0 +1,31 @@ +--- + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 From 97d63189f46c7b02bdd06cf0e5a155cc5ba2709d Mon Sep 17 00:00:00 2001 From: liubo Date: Wed, 18 Dec 2019 20:02:32 +0800 Subject: [PATCH 03/13] add deploy tiflash --- conf/tiflash-learner.yml | 4 +- deploy.yml | 7 +++ group_vars/tiflash_servers.yml | 2 +- roles/tiflash/defaults/main.yml | 9 +-- roles/tiflash/tasks/main.yml | 28 +++++----- roles/tiflash/templates/tiflash.toml.j2 | 8 +-- .../tiflash/templates/tiflash_learner.toml.j2 | 6 +- roles/tiflash/templates/tiflash_user.toml.j2 | 8 +-- .../tiflash/vars/tiflash-learner.yml | 2 +- .../tiflash/vars/tiflash-user.yml | 0 roles/tiflash/vars/tiflash_learner.yml | 56 ------------------- roles/tiflash/vars/tiflash_user.yml | 31 ---------- 12 files changed, 39 insertions(+), 122 deletions(-) rename conf/tiflash_learner.yml => roles/tiflash/vars/tiflash-learner.yml (95%) rename conf/tiflash_user.yml => roles/tiflash/vars/tiflash-user.yml (100%) delete mode 100644 roles/tiflash/vars/tiflash_learner.yml delete mode 100644 roles/tiflash/vars/tiflash_user.yml diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml index 5652a5941..32875ba1e 100644 --- a/conf/tiflash-learner.yml +++ b/conf/tiflash-learner.yml @@ -11,8 +11,6 @@ readpool: coprocessor: server: - labels: { "tiflash_http_port": "8123" } - engine-addr: "tiflash0:3930" storage: @@ -54,4 +52,4 @@ security: cert-path: "" key-path: "" -import: \ No newline at end of file +import: diff --git a/deploy.yml b/deploy.yml index 6c72f4eea..271420aeb 100644 --- a/deploy.yml +++ b/deploy.yml @@ -131,6 +131,13 @@ roles: - tikv +- name: deploying TiFlash cluster + hosts: tiflash_servers + tags: + - tiflash + roles: + - tiflash + - name: deploying pump cluster hosts: pump_servers tags: diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml index a439d94e0..255fd6264 100644 --- a/group_vars/tiflash_servers.yml +++ b/group_vars/tiflash_servers.yml @@ -4,4 +4,4 @@ tcp_port: 9000 http_port: 8123 interserver_http_port: 9009 flash_service_port: 3930 -flash_proxy_port: 20160 \ No newline at end of file +flash_proxy_port: 20170 diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml index 1dcb3e26e..0479f4ddc 100644 --- a/roles/tiflash/defaults/main.yml +++ b/roles/tiflash/defaults/main.yml @@ -3,14 +3,15 @@ tiflash_dir: "{{ deploy_dir }}/tiflash" tmp_path: "{{ deploy_dir }}/tiflash/data/tmp" path: "{{ deploy_dir }}/tiflash/data/db" -flash_cluster_manager: "{{ deploy_dir }}/bin/flash_cluster_manager" +cluster_manager_path: "{{ deploy_dir }}/bin/flash_cluster_manager" tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log" tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log" tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" -user_conf: "{{ deploy_dir }}/conf/tiflash_user.toml" -flash_proxy_config: "{{ deploy_dir }}/conf/tiflash_learner.toml" +user_conf: "{{ deploy_dir }}/conf/tiflash-user.toml" +flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml" +data_dir: "{{ deploy_dir }}/tiflash/data/flash" tcp_port: 9000 http_port: 8123 interserver_http_port: 9009 flash_service_port: 3930 -flash_proxy_port: 20160 +flash_proxy_port: 20170 diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml index 917e506b9..29b47bb95 100644 --- a/roles/tiflash/tasks/main.yml +++ b/roles/tiflash/tasks/main.yml @@ -16,7 +16,7 @@ - name: combine tiflash config set_fact: - tiflash_conf: "{{ with_default_dicts(tiflash_conf_custom, tiflash_conf_default) | update_default_dicts }}" + tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_default) | update_default_dicts }}" - debug: var=tiflash_conf @@ -28,31 +28,31 @@ command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}" when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined -- name: "load tiflash user config: tidb-ansible/conf/tiflash_user.yml" - include_vars: file={{ playbook_dir }}/conf/tiflash_user.yml name=tiflash_user_conf_custom +- name: "load tiflash user config: tidb-ansible/conf/tiflash-user.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash-user.yml name=tiflash_user_conf_custom - name: load tiflash user default config - include_vars: file=tiflash_user.yml name=tiflash_user_conf_default + include_vars: file=tiflash-user.yml name=tiflash_user_conf_default - name: combine tiflash user config set_fact: - tiflash_user_conf: "{{ with_default_dicts(tiflash_user_conf_custom, tiflash_user_conf_default) | update_default_dicts }}" + tiflash_user_conf: "{{ tiflash_user_conf_custom | with_default_dicts(tiflash_user_conf_default) | update_default_dicts }}" - debug: var=tiflash_user_conf -- name: create tiflash config file - template: src=tiflash_user.toml.j2 dest={{ deploy_dir }}/conf/tiflash_user.toml mode=0644 backup=yes +- name: create tiflash user config file + template: src=tiflash_user.toml.j2 dest={{ deploy_dir }}/conf/tiflash-user.toml mode=0644 backup=yes register: tiflash_user_conf_st - name: backup tiflash user conf file command: mv "{{ tiflash_user_conf_st.backup_file }}" "{{ backup_dir }}" when: tiflash_user_conf_st.changed and tiflash_user_conf_st.backup_file is defined -- name: "load tiflash user config: tidb-ansible/conf/tiflash_learner.yml" - include_vars: file={{ playbook_dir }}/conf/tiflash_learner.yml name=tiflash_learner_conf_custom +- name: "load tiflash user config: tidb-ansible/conf/tiflash-learner.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom - name: load tiflash user default config - include_vars: file=tiflash_learner.yml name=tiflash_learner_conf_default + include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default - name: generate tiflash learner dynamic config set_fact: @@ -60,22 +60,22 @@ server: labels: "{{ labels }}" -- name: generate final config +- name: generate tiflash learner config set_fact: tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_generated, tiflash_learner_conf_default) | update_default_dicts }}" - debug: var=tiflash_learner_conf - name: create tiflash learner config file - template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash_learner.toml mode=0644 backup=yes + template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash-learner.toml mode=0644 backup=yes register: tiflash_learner_conf_st - name: backup tiflash conf file command: mv "{{ tiflash_learner_conf_st.backup_file }}" "{{ backup_dir }}" when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined -- include_tasks: "binary__deployment.yml" +- include_tasks: "binary_deployment.yml" - name: prepare firewalld white list set_fact: - firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}" \ No newline at end of file + firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}" diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 index 03f34eec5..f81c5de5f 100644 --- a/roles/tiflash/templates/tiflash.toml.j2 +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -1,14 +1,12 @@ ---- - {% for item, value in tiflash_conf.global | dictsort -%} {{ item }} = {{ value | to_json }} +{% endfor %} tmp_path = "{{ tmp_path }}" users_config = "{{ user_conf }}" path = "{{ path }}" tcp_port = "{{ tcp_port }}" http_port = "{{ http_port }}" interserver_http_port = "{{ interserver_http_port }}" -{% endfor %} [flash] {% set all_tidb = [] -%} @@ -58,9 +56,9 @@ log = "{{ tiflash_server_log }}" {% for host in groups.pd_servers -%} {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} {% set pd_port = hostvars[host].pd_client_port -%} - {% set _ = all_tidb.append("%s:%s" % (pd_ip, pd_port)) -%} + {% set _ = all_pd.append("%s:%s" % (pd_ip, pd_port)) -%} {% endfor %} pd_addr = "{{ all_pd | join(',') }}" {% for item, value in tiflash_conf.raft | dictsort -%} {{ item }} = {{ value | to_json }} -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2 index da86dbc9f..e465ab57d 100644 --- a/roles/tiflash/templates/tiflash_learner.toml.j2 +++ b/roles/tiflash/templates/tiflash_learner.toml.j2 @@ -1,5 +1,3 @@ ---- - [readpool] {% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%} {% if value is not mapping -%} @@ -16,7 +14,11 @@ [server] engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" {% for item, value in tiflash_learner_conf.server | dictsort -%} +{% if item == "labels" %} +{{ item }} = {{ value | tikv_server_labels_format }} +{% else %} {{ item }} = {{ value | to_json }} +{% endif %} {% endfor %} [storage] diff --git a/roles/tiflash/templates/tiflash_user.toml.j2 b/roles/tiflash/templates/tiflash_user.toml.j2 index bd25d2be7..5d9517956 100644 --- a/roles/tiflash/templates/tiflash_user.toml.j2 +++ b/roles/tiflash/templates/tiflash_user.toml.j2 @@ -1,5 +1,3 @@ ---- - [quotas] {% for item, value in tiflash_user_conf.quotas | dictsort_by_value_type -%} {% if value is not mapping -%} @@ -8,7 +6,7 @@ [quotas.{{ item }}] {% for sub_item, sub_value in value | dictsort_by_value_type -%} -{% if value is not mapping -%} +{% if sub_value is not mapping -%} {{ sub_item }} = {{ sub_value | to_json }} {% else %} @@ -29,7 +27,7 @@ [users.{{ item }}] {% for sub_item, sub_value in value | dictsort_by_value_type -%} -{% if value is not mapping -%} +{% if sub_value is not mapping -%} {{ sub_item }} = {{ sub_value | to_json }} {% else %} @@ -43,7 +41,7 @@ {% endfor %} [profiles] -{% for item, value in tiflash_user_conf.flash | dictsort_by_value_type -%} +{% for item, value in tiflash_user_conf.profiles | dictsort_by_value_type -%} {% if value is not mapping -%} {{ item }} = {{ value | to_json }} {% else %} diff --git a/conf/tiflash_learner.yml b/roles/tiflash/vars/tiflash-learner.yml similarity index 95% rename from conf/tiflash_learner.yml rename to roles/tiflash/vars/tiflash-learner.yml index aee770757..63109d81c 100644 --- a/conf/tiflash_learner.yml +++ b/roles/tiflash/vars/tiflash-learner.yml @@ -11,7 +11,7 @@ readpool: coprocessor: server: - labels: { "tiflash_http_port": "8123" } + labels: {} storage: diff --git a/conf/tiflash_user.yml b/roles/tiflash/vars/tiflash-user.yml similarity index 100% rename from conf/tiflash_user.yml rename to roles/tiflash/vars/tiflash-user.yml diff --git a/roles/tiflash/vars/tiflash_learner.yml b/roles/tiflash/vars/tiflash_learner.yml deleted file mode 100644 index aee770757..000000000 --- a/roles/tiflash/vars/tiflash_learner.yml +++ /dev/null @@ -1,56 +0,0 @@ -# TiKV config template -# Human-readable big numbers: -# File size(based on byte): KB, MB, GB, TB, PB -# e.g.: 1_048_576: "1MB" -# Time(based on ms): ms, s, m, h -# e.g.: 78_000: "1.3m" - -readpool: - storage: - - coprocessor: - -server: - labels: { "tiflash_http_port": "8123" } - -storage: - -pd: -# This section will be overwritten by command line parameters - -metric: -#address: "172.16.30.31:9531" -#interval: "15s" -#job: "tikv" - -raftstore: - raftdb-path: "" - sync-log: true - max-leader-missing-duration: "22s" - abnormal-leader-missing-duration: "21s" - peer-stale-state-check-interval: "20s" - -coprocessor: - -rocksdb: - wal-dir: "" - - defaultcf: - block-cache-size: "10GB" - - lockcf: - block-cache-size: "4GB" - - writecf: - block-cache-size: "4GB" - -raftdb: - defaultcf: - block-cache-size: "1GB" - -security: - ca-path: "" - cert-path: "" - key-path: "" - -import: diff --git a/roles/tiflash/vars/tiflash_user.yml b/roles/tiflash/vars/tiflash_user.yml deleted file mode 100644 index c6229b243..000000000 --- a/roles/tiflash/vars/tiflash_user.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -quotas: - default: - interval: - result_rows: 0 - read_rows: 0 - execution_time: 0 - queries: 0 - errors: 0 - duration: 3600 -users: - readonly: - quota: "default" - profile: "readonly" - password: "" - networks: - ip: "::/0" - default: - quota: "default" - profile: "default" - password: "" - networks: - ip: "::/0" -profiles: - readonly: - readonly: 1 - default: - load_balancing: "random" - use_uncompressed_cache: 0 - max_memory_usage: 10000000000 From f6a1369788335665e3b9656e749ec2505fbc5e76 Mon Sep 17 00:00:00 2001 From: liubo Date: Fri, 20 Dec 2019 14:52:06 +0800 Subject: [PATCH 04/13] update --- conf/tiflash.yml | 1 - roles/tiflash/defaults/main.yml | 2 +- roles/tiflash/tasks/main.yml | 9 ++++++++- roles/tiflash/templates/run_tiflash_binary.sh.j2 | 2 +- roles/tiflash/templates/tiflash.toml.j2 | 8 ++++---- roles/tiflash/vars/tiflash.yml | 1 - 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/conf/tiflash.yml b/conf/tiflash.yml index d99ece71b..838c5713d 100644 --- a/conf/tiflash.yml +++ b/conf/tiflash.yml @@ -10,7 +10,6 @@ flash: refresh_interval: 20 update_rule_interval: 5 master_ttl: 60 - cluster_manager_path: "/tics/flash_cluster_manager" proxy: logger: diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml index 0479f4ddc..78b7697a8 100644 --- a/roles/tiflash/defaults/main.yml +++ b/roles/tiflash/defaults/main.yml @@ -3,7 +3,7 @@ tiflash_dir: "{{ deploy_dir }}/tiflash" tmp_path: "{{ deploy_dir }}/tiflash/data/tmp" path: "{{ deploy_dir }}/tiflash/data/db" -cluster_manager_path: "{{ deploy_dir }}/bin/flash_cluster_manager" +cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager" tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log" tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log" tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml index 29b47bb95..0d8f97ee0 100644 --- a/roles/tiflash/tasks/main.yml +++ b/roles/tiflash/tasks/main.yml @@ -14,9 +14,16 @@ - name: load tiflash default config include_vars: file=tiflash.yml name=tiflash_conf_default +- name: generate tiflash dynamic config + set_fact: + tiflash_conf_generated: + flash: + flash_cluster: + cluster_manager_path: "{{ cluster_manager_path }}" + - name: combine tiflash config set_fact: - tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_default) | update_default_dicts }}" + tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_generated, tiflash_conf_default) | update_default_dicts }}" - debug: var=tiflash_conf diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2 index 73f76e1fa..1e65f5522 100644 --- a/roles/tiflash/templates/run_tiflash_binary.sh.j2 +++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2 @@ -17,4 +17,4 @@ echo $stat echo $$ > "status/{{ role_name }}.pid" -exec bin/theflash server --config-file config/tiflash.toml \ No newline at end of file +exec bin/tiflash/theflash server --config-file config/tiflash.toml diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 index f81c5de5f..945f4312f 100644 --- a/roles/tiflash/templates/tiflash.toml.j2 +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -4,9 +4,9 @@ tmp_path = "{{ tmp_path }}" users_config = "{{ user_conf }}" path = "{{ path }}" -tcp_port = "{{ tcp_port }}" -http_port = "{{ http_port }}" -interserver_http_port = "{{ interserver_http_port }}" +tcp_port = {{ tcp_port }} +http_port = {{ http_port }} +interserver_http_port = {{ interserver_http_port }} [flash] {% set all_tidb = [] -%} @@ -23,7 +23,7 @@ service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_servic {% else %} [flash.{{ item }}] -{% if item == 'flash_cluster' %} +{% if item == 'cluster_manager_path' %} cluster_manager_path = "{{ cluster_manager_path }}" {% endif %} {% if item == 'proxy' %} diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml index d99ece71b..838c5713d 100644 --- a/roles/tiflash/vars/tiflash.yml +++ b/roles/tiflash/vars/tiflash.yml @@ -10,7 +10,6 @@ flash: refresh_interval: 20 update_rule_interval: 5 master_ttl: 60 - cluster_manager_path: "/tics/flash_cluster_manager" proxy: logger: From bf2dd8786442709ad07c4992c020a592734b7086 Mon Sep 17 00:00:00 2001 From: liubo Date: Fri, 27 Dec 2019 15:02:47 +0800 Subject: [PATCH 05/13] fix some error --- excessive_rolling_update.yml | 29 +++++++++++++++++++ roles/local/tasks/binary_deployment.yml | 4 +-- roles/local/templates/binary_packages.yml.j2 | 4 +-- roles/pd/vars/default.yml | 3 +- .../templates/run_tiflash_binary.sh.j2 | 3 +- roles/tiflash/templates/tiflash.toml.j2 | 4 +-- roles/tikv/vars/default.yml | 1 + rolling_update.yml | 29 +++++++++++++++++++ start.yml | 15 ++++++++++ stop.yml | 14 +++++++++ unsafe_cleanup.yml | 9 ++++++ 11 files changed, 107 insertions(+), 8 deletions(-) diff --git a/excessive_rolling_update.yml b/excessive_rolling_update.yml index b5ef331ba..3d5372256 100644 --- a/excessive_rolling_update.yml +++ b/excessive_rolling_update.yml @@ -381,6 +381,35 @@ - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml" +- name: rolling update TiFlash cluster + hosts: tiflash_servers + any_errors_fatal: true + serial: 1 + tags: + - tiflash + + pre_tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh + when: process_supervision == 'supervise' + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + roles: + - tiflash + + post_tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh + when: process_supervision == 'supervise' + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started + become: true + when: process_supervision == 'systemd' - name: rolling update pump cluster hosts: pump_servers diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml index bd37a198c..f2fd2f1a1 100644 --- a/roles/local/tasks/binary_deployment.yml +++ b/roles/local/tasks/binary_deployment.yml @@ -86,7 +86,7 @@ when: not deploy_without_tidb|default(false) - name: unarchive tiflash - shell: tar xzf tics-latest-linux-amd64.tar.gz + shell: tar xzf tiflash-latest-linux-amd64.tar.gz args: chdir: "{{ downloads_dir }}" warn: no @@ -114,5 +114,5 @@ - name: cp tiflash directory shell: > - cp -rfv {{ downloads_dir }}/tics-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" + cp -rfv {{ downloads_dir }}/tiflash-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" with_items: "{{ tiflash_packages }}" diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2 index e6aa88570..f20d1d618 100644 --- a/roles/local/templates/binary_packages.yml.j2 +++ b/roles/local/templates/binary_packages.yml.j2 @@ -70,6 +70,6 @@ tispark_packages: {% endif %} tiflash_packages: - - name: tics-latest-linux-amd64.tar.gz + - name: tiflash-latest-linux-amd64.tar.gz version: latest - url: http://download.pingcap.org/tics-latest-linux-amd64.tar.gz \ No newline at end of file + url: http://download.pingcap.org/tiflash-latest-linux-amd64.tar.gz diff --git a/roles/pd/vars/default.yml b/roles/pd/vars/default.yml index 4caa7f384..bd28b2232 100644 --- a/roles/pd/vars/default.yml +++ b/roles/pd/vars/default.yml @@ -46,7 +46,7 @@ schedule: leader-schedule-limit: 4 region-schedule-limit: 64 replica-schedule-limit: 64 - merge-schedule-limit: 8 + merge-schedule-limit: 0 enable-one-way-merge: false replication: @@ -57,3 +57,4 @@ replication: # For example, ["zone", "rack"] means that we should place replicas to # different zones first, then to different racks if we don't have enough zones. location-labels: [] + enable-placement-rules: true diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2 index 1e65f5522..168172bae 100644 --- a/roles/tiflash/templates/run_tiflash_binary.sh.j2 +++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2 @@ -9,6 +9,7 @@ cd "{{ deploy_dir }}" || exit 1 export RUST_BACKTRACE=1 export TZ=${TZ:-/etc/localtime} +export LD_LIBRARY_PATH={{ deploy_dir }}/bin/tiflash:$LD_LIBRARY_PATH echo -n 'sync ... ' stat=$(time sync) @@ -17,4 +18,4 @@ echo $stat echo $$ > "status/{{ role_name }}.pid" -exec bin/tiflash/theflash server --config-file config/tiflash.toml +exec bin/tiflash/theflash server --config-file conf/tiflash.toml diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 index 945f4312f..301d6f89a 100644 --- a/roles/tiflash/templates/tiflash.toml.j2 +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -40,13 +40,13 @@ log-file = "{{ tiflash_tikv_log }}" {% endfor %} [logger] +errorlog = "{{ tiflash_errlog }}" +log = "{{ tiflash_server_log }}" {% for item, value in tiflash_conf.logger | dictsort -%} {{ item }} = {{ value | to_json }} {% endfor %} [application] -errorlog = "{{ tiflash_errlog }}" -log = "{{ tiflash_server_log }}" {% for item, value in tiflash_conf.application | dictsort -%} {{ item }} = {{ value | to_json }} {% endfor %} diff --git a/roles/tikv/vars/default.yml b/roles/tikv/vars/default.yml index 86f8aac12..b96fa0be4 100644 --- a/roles/tikv/vars/default.yml +++ b/roles/tikv/vars/default.yml @@ -192,6 +192,7 @@ raftstore: ## Use how many threads to handle raft messages # store-pool-size: 2 + hibernate-regions: false coprocessor: diff --git a/rolling_update.yml b/rolling_update.yml index 59faa5561..b693d24de 100644 --- a/rolling_update.yml +++ b/rolling_update.yml @@ -397,6 +397,35 @@ - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml" +- name: rolling update TiFlash cluster + hosts: tiflash_servers + any_errors_fatal: true + serial: 1 + tags: + - tiflash + + pre_tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh + when: process_supervision == 'supervise' + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + roles: + - tiflash + + post_tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh + when: process_supervision == 'supervise' + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started + become: true + when: process_supervision == 'systemd' - name: rolling update pump cluster hosts: pump_servers diff --git a/start.yml b/start.yml index a9589ee7b..6c89d73a1 100644 --- a/start.yml +++ b/start.yml @@ -350,6 +350,21 @@ - enable_tls|default(false) - wait_replication|default(false) +- hosts: tiflash_servers + tags: + - tiflash + tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh + when: process_supervision == 'supervise' + with_items: + - tiflash + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started enabled=no + become: true + when: process_supervision == 'systemd' + - hosts: pump_servers tags: - pump diff --git a/stop.yml b/stop.yml index 860ae7bdc..543842d8a 100644 --- a/stop.yml +++ b/stop.yml @@ -228,6 +228,20 @@ msg: "the pump port {{ pump_port }} is not down" when: enable_binlog|default(false) +- hosts: tiflash_servers + tags: + - tiflash + tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh + when: process_supervision == 'supervise' + with_items: + - tiflash + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tco_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' - hosts: tikv_servers tags: diff --git a/unsafe_cleanup.yml b/unsafe_cleanup.yml index 1224653c1..b5c6d663c 100644 --- a/unsafe_cleanup.yml +++ b/unsafe_cleanup.yml @@ -83,6 +83,15 @@ with_items: - pump-{{ pump_port }}.service +- hosts: tiflash_servers + tasks: + - name: clean systemd config + file: path="/etc/systemd/system/{{ item }}" state=absent + become: true + when: process_supervision == 'systemd' + with_items: + - tiflash-{{ tcp_port }}.service + - hosts: tikv_servers tasks: - name: clean systemd config From 1ed62a9cf8c23276a4aa4d86458259c4470f41e0 Mon Sep 17 00:00:00 2001 From: liubo Date: Fri, 27 Dec 2019 15:10:37 +0800 Subject: [PATCH 06/13] fix typo --- stop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stop.yml b/stop.yml index 543842d8a..d503e3aef 100644 --- a/stop.yml +++ b/stop.yml @@ -239,7 +239,7 @@ - tiflash - name: stop TiFlash by systemd - systemd: name=tiflash-{{ tco_port }}.service state=stopped + systemd: name=tiflash-{{ tcp_port }}.service state=stopped become: true when: process_supervision == 'systemd' From f9817cd5ded4479cfcd577d2c5ce06878732f30f Mon Sep 17 00:00:00 2001 From: liubo Date: Thu, 2 Jan 2020 12:07:35 +0800 Subject: [PATCH 07/13] optimize labels for tiflash --- roles/tiflash/defaults/main.yml | 3 +++ roles/tiflash/tasks/main.yml | 11 ++++------- roles/tiflash/templates/tiflash_learner.toml.j2 | 1 + roles/tiflash/vars/tiflash-learner.yml | 1 - 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml index 78b7697a8..233616f77 100644 --- a/roles/tiflash/defaults/main.yml +++ b/roles/tiflash/defaults/main.yml @@ -10,6 +10,9 @@ tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" user_conf: "{{ deploy_dir }}/conf/tiflash-user.toml" flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml" data_dir: "{{ deploy_dir }}/tiflash/data/flash" +tiflash_conf_dir: "{{ deploy_dir }}/conf" +tiflash_log_dir: "{{ deploy_dir }}/log" +tiflash_scripts_dir: "{{ deploy_dir }}/scripts" tcp_port: 9000 http_port: 8123 interserver_http_port: 9009 diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml index 0d8f97ee0..47d93bd57 100644 --- a/roles/tiflash/tasks/main.yml +++ b/roles/tiflash/tasks/main.yml @@ -7,6 +7,9 @@ - "{{ tmp_path }}" - "{{ path }}" - "{{ data_dir }}" + - "{{ tiflash_conf_dir }}" + - "{{ tiflash_log_dir }}" + - "{{ tiflash_scripts_dir }}" - name: "load tiflash config: tidb-ansible/conf/tiflash.yml" include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom @@ -61,15 +64,9 @@ - name: load tiflash user default config include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default -- name: generate tiflash learner dynamic config - set_fact: - tiflash_learner_conf_generated: - server: - labels: "{{ labels }}" - - name: generate tiflash learner config set_fact: - tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_generated, tiflash_learner_conf_default) | update_default_dicts }}" + tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_default) | update_default_dicts }}" - debug: var=tiflash_learner_conf diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2 index e465ab57d..1f9d2bdfd 100644 --- a/roles/tiflash/templates/tiflash_learner.toml.j2 +++ b/roles/tiflash/templates/tiflash_learner.toml.j2 @@ -12,6 +12,7 @@ {% endfor %} [server] +labels = { "tiflash_http_port" = {{ http_port }} } engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" {% for item, value in tiflash_learner_conf.server | dictsort -%} {% if item == "labels" %} diff --git a/roles/tiflash/vars/tiflash-learner.yml b/roles/tiflash/vars/tiflash-learner.yml index 63109d81c..32875ba1e 100644 --- a/roles/tiflash/vars/tiflash-learner.yml +++ b/roles/tiflash/vars/tiflash-learner.yml @@ -11,7 +11,6 @@ readpool: coprocessor: server: - labels: {} storage: From 26551f57b2d1f574c8880d78c9193470119d54e2 Mon Sep 17 00:00:00 2001 From: liubo Date: Thu, 2 Jan 2020 13:57:07 +0800 Subject: [PATCH 08/13] optimize format --- roles/tiflash/templates/tiflash_learner.toml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2 index 1f9d2bdfd..881791bbf 100644 --- a/roles/tiflash/templates/tiflash_learner.toml.j2 +++ b/roles/tiflash/templates/tiflash_learner.toml.j2 @@ -12,7 +12,7 @@ {% endfor %} [server] -labels = { "tiflash_http_port" = {{ http_port }} } +labels = { "tiflash_http_port" = "{{ http_port }}" } engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" {% for item, value in tiflash_learner_conf.server | dictsort -%} {% if item == "labels" %} From a4dcdf6aea4c9ad423a558026004f1bfbd5de3ca Mon Sep 17 00:00:00 2001 From: Liangliang Gu Date: Wed, 8 Jan 2020 15:45:13 +0800 Subject: [PATCH 09/13] reset some config (#1096) --- roles/pd/vars/default.yml | 3 +-- roles/tikv/vars/default.yml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/roles/pd/vars/default.yml b/roles/pd/vars/default.yml index bd28b2232..4caa7f384 100644 --- a/roles/pd/vars/default.yml +++ b/roles/pd/vars/default.yml @@ -46,7 +46,7 @@ schedule: leader-schedule-limit: 4 region-schedule-limit: 64 replica-schedule-limit: 64 - merge-schedule-limit: 0 + merge-schedule-limit: 8 enable-one-way-merge: false replication: @@ -57,4 +57,3 @@ replication: # For example, ["zone", "rack"] means that we should place replicas to # different zones first, then to different racks if we don't have enough zones. location-labels: [] - enable-placement-rules: true diff --git a/roles/tikv/vars/default.yml b/roles/tikv/vars/default.yml index b96fa0be4..86f8aac12 100644 --- a/roles/tikv/vars/default.yml +++ b/roles/tikv/vars/default.yml @@ -192,7 +192,6 @@ raftstore: ## Use how many threads to handle raft messages # store-pool-size: 2 - hibernate-regions: false coprocessor: From 6bc7c030ac76ed2302a63adad3746a692ffb8f2b Mon Sep 17 00:00:00 2001 From: liubo Date: Thu, 9 Jan 2020 18:23:18 +0800 Subject: [PATCH 10/13] Merge multiple configuration files and add status check (#1102) --- conf/tiflash-user.yml | 31 ---------- conf/tiflash.yml | 30 ++++++++++ excessive_rolling_update.yml | 24 ++++++++ roles/tiflash/defaults/main.yml | 2 +- roles/tiflash/meta/main.yml | 4 ++ roles/tiflash/tasks/main.yml | 25 +-------- roles/tiflash/templates/tiflash.toml.j2 | 59 ++++++++++++++++++-- roles/tiflash/templates/tiflash_user.toml.j2 | 54 ------------------ roles/tiflash/vars/tiflash.yml | 30 ++++++++++ rolling_update.yml | 24 ++++++++ start.yml | 17 ++++++ stop.yml | 7 +++ 12 files changed, 195 insertions(+), 112 deletions(-) delete mode 100644 conf/tiflash-user.yml create mode 100644 roles/tiflash/meta/main.yml delete mode 100644 roles/tiflash/templates/tiflash_user.toml.j2 diff --git a/conf/tiflash-user.yml b/conf/tiflash-user.yml deleted file mode 100644 index c6229b243..000000000 --- a/conf/tiflash-user.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -quotas: - default: - interval: - result_rows: 0 - read_rows: 0 - execution_time: 0 - queries: 0 - errors: 0 - duration: 3600 -users: - readonly: - quota: "default" - profile: "readonly" - password: "" - networks: - ip: "::/0" - default: - quota: "default" - profile: "default" - password: "" - networks: - ip: "::/0" -profiles: - readonly: - readonly: 1 - default: - load_balancing: "random" - use_uncompressed_cache: 0 - max_memory_usage: 10000000000 diff --git a/conf/tiflash.yml b/conf/tiflash.yml index 838c5713d..3382a164e 100644 --- a/conf/tiflash.yml +++ b/conf/tiflash.yml @@ -20,3 +20,33 @@ application: runAsDaemon: true raft: + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/excessive_rolling_update.yml b/excessive_rolling_update.yml index 3d5372256..ee1ee81bd 100644 --- a/excessive_rolling_update.yml +++ b/excessive_rolling_update.yml @@ -398,6 +398,13 @@ become: true when: process_supervision == 'systemd' + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + roles: - tiflash @@ -411,6 +418,23 @@ become: true when: process_supervision == 'systemd' + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) + - name: rolling update pump cluster hosts: pump_servers any_errors_fatal: true diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml index 233616f77..a27addf5c 100644 --- a/roles/tiflash/defaults/main.yml +++ b/roles/tiflash/defaults/main.yml @@ -4,10 +4,10 @@ tiflash_dir: "{{ deploy_dir }}/tiflash" tmp_path: "{{ deploy_dir }}/tiflash/data/tmp" path: "{{ deploy_dir }}/tiflash/data/db" cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager" +cluster_manager_log: "{{ deploy_dir }}/log/tiflash_cluster_manager.log" tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log" tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log" tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" -user_conf: "{{ deploy_dir }}/conf/tiflash-user.toml" flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml" data_dir: "{{ deploy_dir }}/tiflash/data/flash" tiflash_conf_dir: "{{ deploy_dir }}/conf" diff --git a/roles/tiflash/meta/main.yml b/roles/tiflash/meta/main.yml new file mode 100644 index 000000000..beff6a6bf --- /dev/null +++ b/roles/tiflash/meta/main.yml @@ -0,0 +1,4 @@ +--- + +dependencies: + - role: common_dir diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml index 47d93bd57..93a731722 100644 --- a/roles/tiflash/tasks/main.yml +++ b/roles/tiflash/tasks/main.yml @@ -23,6 +23,7 @@ flash: flash_cluster: cluster_manager_path: "{{ cluster_manager_path }}" + cluster_manager_log: "{{ cluster_manager_log }}" - name: combine tiflash config set_fact: @@ -38,30 +39,10 @@ command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}" when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined -- name: "load tiflash user config: tidb-ansible/conf/tiflash-user.yml" - include_vars: file={{ playbook_dir }}/conf/tiflash-user.yml name=tiflash_user_conf_custom - -- name: load tiflash user default config - include_vars: file=tiflash-user.yml name=tiflash_user_conf_default - -- name: combine tiflash user config - set_fact: - tiflash_user_conf: "{{ tiflash_user_conf_custom | with_default_dicts(tiflash_user_conf_default) | update_default_dicts }}" - -- debug: var=tiflash_user_conf - -- name: create tiflash user config file - template: src=tiflash_user.toml.j2 dest={{ deploy_dir }}/conf/tiflash-user.toml mode=0644 backup=yes - register: tiflash_user_conf_st - -- name: backup tiflash user conf file - command: mv "{{ tiflash_user_conf_st.backup_file }}" "{{ backup_dir }}" - when: tiflash_user_conf_st.changed and tiflash_user_conf_st.backup_file is defined - -- name: "load tiflash user config: tidb-ansible/conf/tiflash-learner.yml" +- name: "load tiflash learner config: tidb-ansible/conf/tiflash-learner.yml" include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom -- name: load tiflash user default config +- name: load tiflash learner default config include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default - name: generate tiflash learner config diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 index 301d6f89a..0a89ca99c 100644 --- a/roles/tiflash/templates/tiflash.toml.j2 +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -2,7 +2,6 @@ {{ item }} = {{ value | to_json }} {% endfor %} tmp_path = "{{ tmp_path }}" -users_config = "{{ user_conf }}" path = "{{ path }}" tcp_port = {{ tcp_port }} http_port = {{ http_port }} @@ -23,9 +22,6 @@ service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_servic {% else %} [flash.{{ item }}] -{% if item == 'cluster_manager_path' %} -cluster_manager_path = "{{ cluster_manager_path }}" -{% endif %} {% if item == 'proxy' %} addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" @@ -62,3 +58,58 @@ pd_addr = "{{ all_pd | join(',') }}" {% for item, value in tiflash_conf.raft | dictsort -%} {{ item }} = {{ value | to_json }} {% endfor %} + +[quotas] +{% for item, value in tiflash_conf.quotas | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[quotas.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if sub_value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[quotas.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[users] +{% for item, value in tiflash_conf.users | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[users.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if sub_value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[users.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[profiles] +{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[profiles.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} diff --git a/roles/tiflash/templates/tiflash_user.toml.j2 b/roles/tiflash/templates/tiflash_user.toml.j2 deleted file mode 100644 index 5d9517956..000000000 --- a/roles/tiflash/templates/tiflash_user.toml.j2 +++ /dev/null @@ -1,54 +0,0 @@ -[quotas] -{% for item, value in tiflash_user_conf.quotas | dictsort_by_value_type -%} -{% if value is not mapping -%} -{{ item }} = {{ value | to_json }} -{% else %} - -[quotas.{{ item }}] -{% for sub_item, sub_value in value | dictsort_by_value_type -%} -{% if sub_value is not mapping -%} -{{ sub_item }} = {{ sub_value | to_json }} -{% else %} - -[quotas.{{ item }}.{{ sub_item }}] -{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} -{{ sub_sub_item }} = {{ sub_sub_value | to_json }} -{% endfor %} -{% endif %} -{% endfor %} -{% endif %} -{% endfor %} - -[users] -{% for item, value in tiflash_user_conf.users | dictsort_by_value_type -%} -{% if value is not mapping -%} -{{ item }} = {{ value | to_json }} -{% else %} - -[users.{{ item }}] -{% for sub_item, sub_value in value | dictsort_by_value_type -%} -{% if sub_value is not mapping -%} -{{ sub_item }} = {{ sub_value | to_json }} -{% else %} - -[users.{{ item }}.{{ sub_item }}] -{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} -{{ sub_sub_item }} = {{ sub_sub_value | to_json }} -{% endfor %} -{% endif %} -{% endfor %} -{% endif %} -{% endfor %} - -[profiles] -{% for item, value in tiflash_user_conf.profiles | dictsort_by_value_type -%} -{% if value is not mapping -%} -{{ item }} = {{ value | to_json }} -{% else %} - -[profiles.{{ item }}] -{% for sub_item, sub_value in value | dictsort -%} -{{ sub_item }} = {{ sub_value | to_json }} -{% endfor %} -{% endif %} -{% endfor %} diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml index 838c5713d..3382a164e 100644 --- a/roles/tiflash/vars/tiflash.yml +++ b/roles/tiflash/vars/tiflash.yml @@ -20,3 +20,33 @@ application: runAsDaemon: true raft: + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/rolling_update.yml b/rolling_update.yml index b693d24de..9110d13ec 100644 --- a/rolling_update.yml +++ b/rolling_update.yml @@ -414,6 +414,13 @@ become: true when: process_supervision == 'systemd' + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + roles: - tiflash @@ -427,6 +434,23 @@ become: true when: process_supervision == 'systemd' + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) + - name: rolling update pump cluster hosts: pump_servers any_errors_fatal: true diff --git a/start.yml b/start.yml index 6c89d73a1..7cc57d23c 100644 --- a/start.yml +++ b/start.yml @@ -365,6 +365,23 @@ become: true when: process_supervision == 'systemd' + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) + - hosts: pump_servers tags: - pump diff --git a/stop.yml b/stop.yml index d503e3aef..9e2053374 100644 --- a/stop.yml +++ b/stop.yml @@ -243,6 +243,13 @@ become: true when: process_supervision == 'systemd' + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + - hosts: tikv_servers tags: - tikv From 7b980436f3c7db281b03a748e2e47c9c34d245ee Mon Sep 17 00:00:00 2001 From: liubo Date: Tue, 14 Jan 2020 11:18:00 +0800 Subject: [PATCH 11/13] update tiflash version (#1118) --- roles/local/tasks/binary_deployment.yml | 9 +++--- roles/local/templates/binary_packages.yml.j2 | 6 ++-- roles/tiflash/vars/tiflash-user.yml | 31 -------------------- 3 files changed, 7 insertions(+), 39 deletions(-) delete mode 100644 roles/tiflash/vars/tiflash-user.yml diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml index f2fd2f1a1..064ff51e2 100644 --- a/roles/local/tasks/binary_deployment.yml +++ b/roles/local/tasks/binary_deployment.yml @@ -51,7 +51,7 @@ - name: download TiFlash packages get_url: url: "{{ item.url }}" - dest: "{{ downloads_dir }}/{{ item.name }}" + dest: "{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz" checksum: "{{ item.checksum | default(omit) }}" force: yes validate_certs: no @@ -62,7 +62,6 @@ with_items: "{{ tiflash_packages }}" when: - has_outbound_network - - not deploy_without_tidb|default(false) - name: unarchive third party binary shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf @@ -86,11 +85,11 @@ when: not deploy_without_tidb|default(false) - name: unarchive tiflash - shell: tar xzf tiflash-latest-linux-amd64.tar.gz + shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf args: chdir: "{{ downloads_dir }}" warn: no - when: not deploy_without_tidb|default(false) + with_items: "{{ tiflash_packages }}" - name: cp monitoring binary shell: > @@ -114,5 +113,5 @@ - name: cp tiflash directory shell: > - cp -rfv {{ downloads_dir }}/tiflash-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" + cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" with_items: "{{ tiflash_packages }}" diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2 index f20d1d618..16ef833b4 100644 --- a/roles/local/templates/binary_packages.yml.j2 +++ b/roles/local/templates/binary_packages.yml.j2 @@ -70,6 +70,6 @@ tispark_packages: {% endif %} tiflash_packages: - - name: tiflash-latest-linux-amd64.tar.gz - version: latest - url: http://download.pingcap.org/tiflash-latest-linux-amd64.tar.gz + - name: tiflash + version: {{ tidb_version }} + url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz diff --git a/roles/tiflash/vars/tiflash-user.yml b/roles/tiflash/vars/tiflash-user.yml deleted file mode 100644 index c6229b243..000000000 --- a/roles/tiflash/vars/tiflash-user.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -quotas: - default: - interval: - result_rows: 0 - read_rows: 0 - execution_time: 0 - queries: 0 - errors: 0 - duration: 3600 -users: - readonly: - quota: "default" - profile: "readonly" - password: "" - networks: - ip: "::/0" - default: - quota: "default" - profile: "default" - password: "" - networks: - ip: "::/0" -profiles: - readonly: - readonly: 1 - default: - load_balancing: "random" - use_uncompressed_cache: 0 - max_memory_usage: 10000000000 From ffe0ae4c51b8e4c62727724e04a72e78481dacf7 Mon Sep 17 00:00:00 2001 From: liubo Date: Tue, 14 Jan 2020 11:25:47 +0800 Subject: [PATCH 12/13] Update roles/tiflash/tasks/main.yml --- roles/tiflash/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml index 93a731722..4c660ec3e 100644 --- a/roles/tiflash/tasks/main.yml +++ b/roles/tiflash/tasks/main.yml @@ -23,7 +23,7 @@ flash: flash_cluster: cluster_manager_path: "{{ cluster_manager_path }}" - cluster_manager_log: "{{ cluster_manager_log }}" + log: "{{ cluster_manager_log }}" - name: combine tiflash config set_fact: From e03cc73ed4d12a8e18fc4d38e022bf4871421e57 Mon Sep 17 00:00:00 2001 From: liubo Date: Wed, 15 Jan 2020 16:48:22 +0800 Subject: [PATCH 13/13] create status dir for all services --- roles/common_dir/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/common_dir/tasks/main.yml b/roles/common_dir/tasks/main.yml index 3dcd01016..78ed16ccd 100644 --- a/roles/common_dir/tasks/main.yml +++ b/roles/common_dir/tasks/main.yml @@ -12,7 +12,6 @@ file: path={{ item }} state=directory mode=0755 with_items: - "{{ status_dir }}" - when: deployment_method == 'supervise' or 'tikv_servers' in group_names - name: create deploy binary directory file: path={{ item }} state=directory mode=0755