diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml new file mode 100644 index 000000000..32875ba1e --- /dev/null +++ b/conf/tiflash-learner.yml @@ -0,0 +1,55 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + raftdb-path: "" + sync-log: true + max-leader-missing-duration: "22s" + abnormal-leader-missing-duration: "21s" + peer-stale-state-check-interval: "20s" + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + block-cache-size: "10GB" + + lockcf: + block-cache-size: "4GB" + + writecf: + block-cache-size: "4GB" + +raftdb: + defaultcf: + block-cache-size: "1GB" + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: diff --git a/conf/tiflash.yml b/conf/tiflash.yml new file mode 100644 index 000000000..3382a164e --- /dev/null +++ b/conf/tiflash.yml @@ -0,0 +1,52 @@ +--- + +global: + display_name: "TiFlash" + default_profile: "default" + mark_cache_size: 5368709120 + listen_host: "0.0.0.0" +flash: + flash_cluster: + refresh_interval: 20 + update_rule_interval: 5 + master_ttl: 60 + proxy: + +logger: + count: 10 + size: "1000M" + level: "trace" +application: + runAsDaemon: true + +raft: + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/deploy.yml b/deploy.yml index 6c72f4eea..271420aeb 100644 --- a/deploy.yml +++ b/deploy.yml @@ -131,6 +131,13 @@ roles: - tikv +- name: deploying TiFlash cluster + hosts: tiflash_servers + tags: + - tiflash + roles: + - tiflash + - name: deploying pump cluster hosts: pump_servers tags: diff --git a/excessive_rolling_update.yml b/excessive_rolling_update.yml index b5ef331ba..ee1ee81bd 100644 --- a/excessive_rolling_update.yml +++ b/excessive_rolling_update.yml @@ -381,6 +381,59 @@ - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml" +- name: rolling update TiFlash cluster + hosts: tiflash_servers + any_errors_fatal: true + serial: 1 + tags: + - tiflash + + pre_tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh + when: process_supervision == 'supervise' + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + + roles: + - tiflash + + post_tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh + when: process_supervision == 'supervise' + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) - name: rolling update pump cluster hosts: pump_servers diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml new file mode 100644 index 000000000..255fd6264 --- /dev/null +++ b/group_vars/tiflash_servers.yml @@ -0,0 +1,7 @@ +--- + +tcp_port: 9000 +http_port: 8123 +interserver_http_port: 9009 +flash_service_port: 3930 +flash_proxy_port: 20170 diff --git a/inventory.ini b/inventory.ini index 0c8bc9857..6feb7aaf4 100644 --- a/inventory.ini +++ b/inventory.ini @@ -49,6 +49,9 @@ [drainer_servers] +## Tiflash Part +[tiflash_servers] + ## Group variables [pd_servers:vars] # location_labels = ["zone","rack","host"] diff --git a/roles/common_dir/tasks/main.yml b/roles/common_dir/tasks/main.yml index 3dcd01016..78ed16ccd 100644 --- a/roles/common_dir/tasks/main.yml +++ b/roles/common_dir/tasks/main.yml @@ -12,7 +12,6 @@ file: path={{ item }} state=directory mode=0755 with_items: - "{{ status_dir }}" - when: deployment_method == 'supervise' or 'tikv_servers' in group_names - name: create deploy binary directory file: path={{ item }} state=directory mode=0755 diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml index e367956da..064ff51e2 100644 --- a/roles/local/tasks/binary_deployment.yml +++ b/roles/local/tasks/binary_deployment.yml @@ -48,6 +48,21 @@ - has_outbound_network - not deploy_without_tidb|default(false) +- name: download TiFlash packages + get_url: + url: "{{ item.url }}" + dest: "{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz" + checksum: "{{ item.checksum | default(omit) }}" + force: yes + validate_certs: no + register: get_url_result + until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg" + retries: 4 + delay: "{{ retry_stagger | random + 3 }}" + with_items: "{{ tiflash_packages }}" + when: + - has_outbound_network + - name: unarchive third party binary shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf args: @@ -69,6 +84,13 @@ warn: no when: not deploy_without_tidb|default(false) +- name: unarchive tiflash + shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf + args: + chdir: "{{ downloads_dir }}" + warn: no + with_items: "{{ tiflash_packages }}" + - name: cp monitoring binary shell: > cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} "{{ resources_dir }}/bin/{{ item }}" @@ -88,3 +110,8 @@ shell: > cp -rfv {{ downloads_dir }}/tispark-sample-data "{{ resources_dir }}/bin/" when: not deploy_without_tidb|default(false) + +- name: cp tiflash directory + shell: > + cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash" + with_items: "{{ tiflash_packages }}" diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2 index 5075ffb4e..16ef833b4 100644 --- a/roles/local/templates/binary_packages.yml.j2 +++ b/roles/local/templates/binary_packages.yml.j2 @@ -68,3 +68,8 @@ tispark_packages: url: http://download.pingcap.org/tispark-sample-data.tar.gz checksum: "sha256:bd0368a9d8663a4a8de89e39cc4cc1d91c718faf36d4bc7e1f8482c34d5bb8db" {% endif %} + +tiflash_packages: + - name: tiflash + version: {{ tidb_version }} + url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml new file mode 100644 index 000000000..a27addf5c --- /dev/null +++ b/roles/tiflash/defaults/main.yml @@ -0,0 +1,20 @@ +--- + +tiflash_dir: "{{ deploy_dir }}/tiflash" +tmp_path: "{{ deploy_dir }}/tiflash/data/tmp" +path: "{{ deploy_dir }}/tiflash/data/db" +cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager" +cluster_manager_log: "{{ deploy_dir }}/log/tiflash_cluster_manager.log" +tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log" +tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log" +tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log" +flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml" +data_dir: "{{ deploy_dir }}/tiflash/data/flash" +tiflash_conf_dir: "{{ deploy_dir }}/conf" +tiflash_log_dir: "{{ deploy_dir }}/log" +tiflash_scripts_dir: "{{ deploy_dir }}/scripts" +tcp_port: 9000 +http_port: 8123 +interserver_http_port: 9009 +flash_service_port: 3930 +flash_proxy_port: 20170 diff --git a/roles/tiflash/meta/main.yml b/roles/tiflash/meta/main.yml new file mode 100644 index 000000000..beff6a6bf --- /dev/null +++ b/roles/tiflash/meta/main.yml @@ -0,0 +1,4 @@ +--- + +dependencies: + - role: common_dir diff --git a/roles/tiflash/tasks/binary_deployment.yml b/roles/tiflash/tasks/binary_deployment.yml new file mode 100644 index 000000000..8305d14dd --- /dev/null +++ b/roles/tiflash/tasks/binary_deployment.yml @@ -0,0 +1,23 @@ +--- + +- name: deploy tiflash binary + copy: src="{{ resources_dir }}/bin/tiflash" dest="{{ deploy_dir }}/bin/" mode=0755 backup=yes + register: tiflash_binary + +- name: create run script + template: + src: "{{ item }}_{{ role_name }}_binary.sh.j2" + dest: "{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh" + mode: "0755" + backup: yes + with_items: + - run + vars: + role_status_dir: status/{{ role_name }} + register: tiflash_script + +- name: backup script file + command: mv "{{ tiflash_script.backup_file }}" "{{ backup_dir }}" + when: tiflash_script.changed and tiflash_script.backup_file is defined + +- include_tasks: "systemd_deployment.yml" \ No newline at end of file diff --git a/roles/tiflash/tasks/main.yml b/roles/tiflash/tasks/main.yml new file mode 100644 index 000000000..4c660ec3e --- /dev/null +++ b/roles/tiflash/tasks/main.yml @@ -0,0 +1,66 @@ +--- + +- name: create tiflash directories + file: path={{ item }} state=directory mode=0755 + with_items: + - "{{ tiflash_dir }}" + - "{{ tmp_path }}" + - "{{ path }}" + - "{{ data_dir }}" + - "{{ tiflash_conf_dir }}" + - "{{ tiflash_log_dir }}" + - "{{ tiflash_scripts_dir }}" + +- name: "load tiflash config: tidb-ansible/conf/tiflash.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash.yml name=tiflash_conf_custom + +- name: load tiflash default config + include_vars: file=tiflash.yml name=tiflash_conf_default + +- name: generate tiflash dynamic config + set_fact: + tiflash_conf_generated: + flash: + flash_cluster: + cluster_manager_path: "{{ cluster_manager_path }}" + log: "{{ cluster_manager_log }}" + +- name: combine tiflash config + set_fact: + tiflash_conf: "{{ tiflash_conf_custom | with_default_dicts(tiflash_conf_generated, tiflash_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_conf + +- name: create tiflash config file + template: src=tiflash.toml.j2 dest={{ deploy_dir }}/conf/tiflash.toml mode=0644 backup=yes + register: tiflash_conf_st + +- name: backup tiflash conf file + command: mv "{{ tiflash_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_conf_st.changed and tiflash_conf_st.backup_file is defined + +- name: "load tiflash learner config: tidb-ansible/conf/tiflash-learner.yml" + include_vars: file={{ playbook_dir }}/conf/tiflash-learner.yml name=tiflash_learner_conf_custom + +- name: load tiflash learner default config + include_vars: file=tiflash-learner.yml name=tiflash_learner_conf_default + +- name: generate tiflash learner config + set_fact: + tiflash_learner_conf: "{{ tiflash_learner_conf_custom | with_default_dicts(tiflash_learner_conf_default) | update_default_dicts }}" + +- debug: var=tiflash_learner_conf + +- name: create tiflash learner config file + template: src=tiflash_learner.toml.j2 dest={{ deploy_dir }}/conf/tiflash-learner.toml mode=0644 backup=yes + register: tiflash_learner_conf_st + +- name: backup tiflash conf file + command: mv "{{ tiflash_learner_conf_st.backup_file }}" "{{ backup_dir }}" + when: tiflash_learner_conf_st.changed and tiflash_learner_conf_st.backup_file is defined + +- include_tasks: "binary_deployment.yml" + +- name: prepare firewalld white list + set_fact: + firewalld_ports: "{{ [tcp_port ~ '/tcp', http_port ~ '/tcp', interserver_http_port ~ '/tcp', flash_service_port ~ '/tcp', flash_proxy_port ~ '/tcp'] + firewalld_ports }}" diff --git a/roles/tiflash/tasks/supervise_deployment.yml b/roles/tiflash/tasks/supervise_deployment.yml new file mode 100644 index 000000000..d2aa0d711 --- /dev/null +++ b/roles/tiflash/tasks/supervise_deployment.yml @@ -0,0 +1,8 @@ +--- + +- name: deploy supervise + include_role: + name: supervise + vars: + this_role_name: tiflash + service_name: tiflash-{{ tcp_port }} \ No newline at end of file diff --git a/roles/tiflash/tasks/systemd_deployment.yml b/roles/tiflash/tasks/systemd_deployment.yml new file mode 100644 index 000000000..2b31ffc56 --- /dev/null +++ b/roles/tiflash/tasks/systemd_deployment.yml @@ -0,0 +1,8 @@ +--- + +- name: deploy systemd + include_role: + name: systemd + vars: + this_role_name: tiflash + service_name: tiflash-{{ tcp_port }} \ No newline at end of file diff --git a/roles/tiflash/templates/run_tiflash_binary.sh.j2 b/roles/tiflash/templates/run_tiflash_binary.sh.j2 new file mode 100644 index 000000000..168172bae --- /dev/null +++ b/roles/tiflash/templates/run_tiflash_binary.sh.j2 @@ -0,0 +1,21 @@ +#!/bin/bash +set -e +ulimit -n 1000000 + +# WARNING: This file was auto-generated. Do not edit! +# All your edit might be overwritten! +cd "{{ deploy_dir }}" || exit 1 + +export RUST_BACKTRACE=1 + +export TZ=${TZ:-/etc/localtime} +export LD_LIBRARY_PATH={{ deploy_dir }}/bin/tiflash:$LD_LIBRARY_PATH + +echo -n 'sync ... ' +stat=$(time sync) +echo ok +echo $stat + +echo $$ > "status/{{ role_name }}.pid" + +exec bin/tiflash/theflash server --config-file conf/tiflash.toml diff --git a/roles/tiflash/templates/tiflash.toml.j2 b/roles/tiflash/templates/tiflash.toml.j2 new file mode 100644 index 000000000..0a89ca99c --- /dev/null +++ b/roles/tiflash/templates/tiflash.toml.j2 @@ -0,0 +1,115 @@ +{% for item, value in tiflash_conf.global | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} +tmp_path = "{{ tmp_path }}" +path = "{{ path }}" +tcp_port = {{ tcp_port }} +http_port = {{ http_port }} +interserver_http_port = {{ interserver_http_port }} + +[flash] +{% set all_tidb = [] -%} +{% for host in groups.tidb_servers -%} + {% set tidb_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set tidb_port = hostvars[host].tidb_status_port -%} + {% set _ = all_tidb.append("%s:%s" % (tidb_ip, tidb_port)) -%} +{% endfor %} +tidb_status_addr = "{{ all_tidb | join(',') }}" +service_addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" +{% for item, value in tiflash_conf.flash | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[flash.{{ item }}] +{% if item == 'proxy' %} +addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" +advertise-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_proxy_port }}" +data-dir = "{{ data_dir }}" +config = "{{ flash_proxy_config }}" +log-file = "{{ tiflash_tikv_log }}" +{% endif %} +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[logger] +errorlog = "{{ tiflash_errlog }}" +log = "{{ tiflash_server_log }}" +{% for item, value in tiflash_conf.logger | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[application] +{% for item, value in tiflash_conf.application | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[raft] +{% set all_pd = [] -%} +{% for host in groups.pd_servers -%} + {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%} + {% set pd_port = hostvars[host].pd_client_port -%} + {% set _ = all_pd.append("%s:%s" % (pd_ip, pd_port)) -%} +{% endfor %} +pd_addr = "{{ all_pd | join(',') }}" +{% for item, value in tiflash_conf.raft | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[quotas] +{% for item, value in tiflash_conf.quotas | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[quotas.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if sub_value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[quotas.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[users] +{% for item, value in tiflash_conf.users | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[users.{{ item }}] +{% for sub_item, sub_value in value | dictsort_by_value_type -%} +{% if sub_value is not mapping -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% else %} + +[users.{{ item }}.{{ sub_item }}] +{% for sub_sub_item, sub_sub_value in sub_value | dictsort -%} +{{ sub_sub_item }} = {{ sub_sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} +{% endfor %} + +[profiles] +{% for item, value in tiflash_conf.profiles | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[profiles.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} diff --git a/roles/tiflash/templates/tiflash_learner.toml.j2 b/roles/tiflash/templates/tiflash_learner.toml.j2 new file mode 100644 index 000000000..881791bbf --- /dev/null +++ b/roles/tiflash/templates/tiflash_learner.toml.j2 @@ -0,0 +1,84 @@ +[readpool] +{% for item, value in tiflash_learner_conf.readpool | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[readpool.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[server] +labels = { "tiflash_http_port" = "{{ http_port }}" } +engine-addr = "{{ ansible_host | default(inventory_hostname) }}:{{ flash_service_port }}" +{% for item, value in tiflash_learner_conf.server | dictsort -%} +{% if item == "labels" %} +{{ item }} = {{ value | tikv_server_labels_format }} +{% else %} +{{ item }} = {{ value | to_json }} +{% endif %} +{% endfor %} + +[storage] +{% for item, value in tiflash_learner_conf.storage | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[pd] +{% for item, value in tiflash_learner_conf.pd | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[metric] +{% for item, value in tiflash_learner_conf.metric | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[raftstore] +{% for item, value in tiflash_learner_conf.raftstore | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[coprocessor] +{% for item, value in tiflash_learner_conf.coprocessor | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[rocksdb] +{% for item, value in tiflash_learner_conf.rocksdb | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[rocksdb.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[raftdb] +{% for item, value in tiflash_learner_conf.raftdb | dictsort_by_value_type -%} +{% if value is not mapping -%} +{{ item }} = {{ value | to_json }} +{% else %} + +[raftdb.{{ item }}] +{% for sub_item, sub_value in value | dictsort -%} +{{ sub_item }} = {{ sub_value | to_json }} +{% endfor %} +{% endif %} +{% endfor %} + +[security] +{% for item, value in tiflash_learner_conf.security | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} + +[import] +{% for item, value in tiflash_learner_conf.import | dictsort -%} +{{ item }} = {{ value | to_json }} +{% endfor %} diff --git a/roles/tiflash/vars/tiflash-learner.yml b/roles/tiflash/vars/tiflash-learner.yml new file mode 100644 index 000000000..32875ba1e --- /dev/null +++ b/roles/tiflash/vars/tiflash-learner.yml @@ -0,0 +1,55 @@ +# TiKV config template +# Human-readable big numbers: +# File size(based on byte): KB, MB, GB, TB, PB +# e.g.: 1_048_576: "1MB" +# Time(based on ms): ms, s, m, h +# e.g.: 78_000: "1.3m" + +readpool: + storage: + + coprocessor: + +server: + +storage: + +pd: +# This section will be overwritten by command line parameters + +metric: +#address: "172.16.30.31:9531" +#interval: "15s" +#job: "tikv" + +raftstore: + raftdb-path: "" + sync-log: true + max-leader-missing-duration: "22s" + abnormal-leader-missing-duration: "21s" + peer-stale-state-check-interval: "20s" + +coprocessor: + +rocksdb: + wal-dir: "" + + defaultcf: + block-cache-size: "10GB" + + lockcf: + block-cache-size: "4GB" + + writecf: + block-cache-size: "4GB" + +raftdb: + defaultcf: + block-cache-size: "1GB" + +security: + ca-path: "" + cert-path: "" + key-path: "" + +import: diff --git a/roles/tiflash/vars/tiflash.yml b/roles/tiflash/vars/tiflash.yml new file mode 100644 index 000000000..3382a164e --- /dev/null +++ b/roles/tiflash/vars/tiflash.yml @@ -0,0 +1,52 @@ +--- + +global: + display_name: "TiFlash" + default_profile: "default" + mark_cache_size: 5368709120 + listen_host: "0.0.0.0" +flash: + flash_cluster: + refresh_interval: 20 + update_rule_interval: 5 + master_ttl: 60 + proxy: + +logger: + count: 10 + size: "1000M" + level: "trace" +application: + runAsDaemon: true + +raft: + +quotas: + default: + interval: + result_rows: 0 + read_rows: 0 + execution_time: 0 + queries: 0 + errors: 0 + duration: 3600 +users: + readonly: + quota: "default" + profile: "readonly" + password: "" + networks: + ip: "::/0" + default: + quota: "default" + profile: "default" + password: "" + networks: + ip: "::/0" +profiles: + readonly: + readonly: 1 + default: + load_balancing: "random" + use_uncompressed_cache: 0 + max_memory_usage: 10000000000 diff --git a/rolling_update.yml b/rolling_update.yml index 59faa5561..9110d13ec 100644 --- a/rolling_update.yml +++ b/rolling_update.yml @@ -397,6 +397,59 @@ - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml" +- name: rolling update TiFlash cluster + hosts: tiflash_servers + any_errors_fatal: true + serial: 1 + tags: + - tiflash + + pre_tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh + when: process_supervision == 'supervise' + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" + + roles: + - tiflash + + post_tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh + when: process_supervision == 'supervise' + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) - name: rolling update pump cluster hosts: pump_servers diff --git a/start.yml b/start.yml index a9589ee7b..7cc57d23c 100644 --- a/start.yml +++ b/start.yml @@ -350,6 +350,38 @@ - enable_tls|default(false) - wait_replication|default(false) +- hosts: tiflash_servers + tags: + - tiflash + tasks: + - name: start TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./start_{{ item }}.sh + when: process_supervision == 'supervise' + with_items: + - tiflash + + - name: start TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=started enabled=no + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is up + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: started + msg: "the TiFlash port {{ http_port }} is not up" + + - name: wait until the TiFlash status page is available + uri: + url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()" + return_content: yes + register: tiflash_http_result + until: tiflash_http_result.status == 200 + retries: 12 + delay: 5 + when: not enable_tls|default(false) + - hosts: pump_servers tags: - pump diff --git a/stop.yml b/stop.yml index 860ae7bdc..9e2053374 100644 --- a/stop.yml +++ b/stop.yml @@ -228,6 +228,27 @@ msg: "the pump port {{ pump_port }} is not down" when: enable_binlog|default(false) +- hosts: tiflash_servers + tags: + - tiflash + tasks: + - name: stop TiFlash by supervise + shell: cd {{ deploy_dir }}/scripts && ./stop_{{ item }}.sh + when: process_supervision == 'supervise' + with_items: + - tiflash + + - name: stop TiFlash by systemd + systemd: name=tiflash-{{ tcp_port }}.service state=stopped + become: true + when: process_supervision == 'systemd' + + - name: wait until the TiFlash port is down + wait_for: + host: "{{ ansible_host }}" + port: "{{ http_port }}" + state: stopped + msg: "the TiFlash port {{ http_port }} is not down" - hosts: tikv_servers tags: diff --git a/unsafe_cleanup.yml b/unsafe_cleanup.yml index 1224653c1..b5c6d663c 100644 --- a/unsafe_cleanup.yml +++ b/unsafe_cleanup.yml @@ -83,6 +83,15 @@ with_items: - pump-{{ pump_port }}.service +- hosts: tiflash_servers + tasks: + - name: clean systemd config + file: path="/etc/systemd/system/{{ item }}" state=absent + become: true + when: process_supervision == 'systemd' + with_items: + - tiflash-{{ tcp_port }}.service + - hosts: tikv_servers tasks: - name: clean systemd config