release-3.1: support tiflash (#1148)

* Support Tiflash (#1119) * add tiflash config * Add tiflash * add deploy tiflash * update * fix some error * fix typo * optimize labels for tiflash * optimize format * reset some config (#1096) * Merge multiple configuration files and add status check (#1102) * update tiflash version (#1118) * Update roles/tiflash/tasks/main.yml * create status dir for all services Co-authored-by: Liangliang Gu <marsishandsome@gmail.com> * rename theflash (#1128) * refactor tiflash config (#1136) * add tiflash proxy status port config (#1140) * tiflash add metrics port (#1144) * add tiflash conf in prometheus for tiflash dashboard (#1146) * add tiflash conf in prometheus for tiflash dashboard Co-authored-by: Liangliang Gu <marsishandsome@gmail.com>
pingcap · Feb 18, 2020 · aa20b24 · aa20b24
1 parent 12ecbaf
commit aa20b24
Show file tree

Hide file tree

Showing 25 changed files with 808 additions and 1 deletion.
diff --git a/conf/tiflash-learner.yml b/conf/tiflash-learner.yml
@@ -0,0 +1,46 @@
+# TiKV config template
+#  Human-readable big numbers:
+#   File size(based on byte): KB, MB, GB, TB, PB
+#    e.g.: 1_048_576: "1MB"
+#   Time(based on ms): ms, s, m, h
+#    e.g.: 78_000: "1.3m"
+
+readpool:
+  storage:
+
+  coprocessor:
+
+server:
+
+storage:
+
+pd:
+# This section will be overwritten by command line parameters
+
+metric:
+#address: "172.16.30.31:9531"
+#interval: "15s"
+#job: "tikv"
+
+raftstore:
+
+coprocessor:
+
+rocksdb:
+  wal-dir: ""
+
+  defaultcf:
+
+  lockcf:
+
+  writecf:
+
+raftdb:
+  defaultcf:
+
+security:
+  ca-path: ""
+  cert-path: ""
+  key-path: ""
+
+import:
diff --git a/conf/tiflash.yml b/conf/tiflash.yml
@@ -0,0 +1,55 @@
+---
+
+global:
+  display_name: "TiFlash"
+  default_profile: "default"
+  mark_cache_size: 5368709120
+  listen_host: "0.0.0.0"
+flash:
+  flash_cluster:
+    refresh_interval: 20
+    update_rule_interval: 5
+    master_ttl: 60
+  proxy:
+
+status:
+
+logger:
+  count: 20
+  size: "1000M"
+  level: "debug"
+application:
+  runAsDaemon: true
+
+raft:
+  storage_engine: "tmt"
+
+quotas:
+  default:
+    interval:
+      result_rows: 0
+      read_rows: 0
+      execution_time: 0
+      queries: 0
+      errors: 0
+      duration: 3600
+users:
+  readonly:
+    quota: "default"
+    profile: "readonly"
+    password: ""
+    networks:
+      ip: "::/0"
+  default:
+    quota: "default"
+    profile: "default"
+    password: ""
+    networks:
+      ip: "::/0"
+profiles:
+  readonly:
+    readonly: 1
+  default:
+    load_balancing: "random"
+    use_uncompressed_cache: 0
+    max_memory_usage: 10000000000
diff --git a/deploy.yml b/deploy.yml
@@ -131,6 +131,13 @@
   roles:
     - tikv
 
+- name: deploying TiFlash cluster
+  hosts: tiflash_servers
+  tags:
+    - tiflash
+  roles:
+    - tiflash
+
 - name: deploying pump cluster
   hosts: pump_servers
   tags:

diff --git a/excessive_rolling_update.yml b/excessive_rolling_update.yml
@@ -381,6 +381,59 @@
 
     - include_tasks: "common_tasks/remove_evict_leader_scheduler.yml"
 
+- name: rolling update TiFlash cluster
+  hosts: tiflash_servers
+  any_errors_fatal: true
+  serial: 1
+  tags:
+    - tiflash
+
+  pre_tasks:
+    - name: stop TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./stop_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: stop TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=stopped
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is down
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: stopped
+        msg: "the TiFlash port {{ http_port }} is not down"
+
+  roles:
+    - tiflash
+
+  post_tasks:
+    - name: start TiFlash by supervise
+      shell: cd {{ deploy_dir }}/scripts && ./start_tiflash.sh
+      when: process_supervision == 'supervise'
+
+    - name: start TiFlash by systemd
+      systemd: name=tiflash-{{ tcp_port }}.service state=started
+      become: true
+      when: process_supervision == 'systemd'
+
+    - name: wait until the TiFlash port is up
+      wait_for:
+        host: "{{ ansible_host }}"
+        port: "{{ http_port }}"
+        state: started
+        msg: "the TiFlash port {{ http_port }} is not up"
+
+    - name: wait until the TiFlash status page is available
+      uri:
+        url: "http://{{ ansible_host }}:{{ http_port }}/?query=select%20version()"
+        return_content: yes
+      register: tiflash_http_result
+      until: tiflash_http_result.status == 200
+      retries: 12
+      delay: 5
+      when: not enable_tls|default(false)
 
 - name: rolling update pump cluster
   hosts: pump_servers

diff --git a/group_vars/tiflash_servers.yml b/group_vars/tiflash_servers.yml
@@ -0,0 +1,9 @@
+---
+
+tcp_port: 9000
+http_port: 8123
+interserver_http_port: 9009
+flash_service_port: 3930
+flash_proxy_port: 20170
+flash_proxy_status_port: 20292
+metrics_port: 8234
diff --git a/inventory.ini b/inventory.ini
@@ -49,6 +49,9 @@
 
 [drainer_servers]
 
+## Tiflash Part
+[tiflash_servers]
+
 ## Group variables
 [pd_servers:vars]
 # location_labels = ["zone","rack","host"]

diff --git a/roles/common_dir/tasks/main.yml b/roles/common_dir/tasks/main.yml
@@ -12,7 +12,6 @@
   file: path={{ item }} state=directory mode=0755
   with_items:
     - "{{ status_dir }}"
-  when: deployment_method == 'supervise' or 'tikv_servers' in group_names
 
 - name: create deploy binary directory
   file: path={{ item }} state=directory mode=0755

diff --git a/roles/local/tasks/binary_deployment.yml b/roles/local/tasks/binary_deployment.yml
@@ -48,6 +48,21 @@
     - has_outbound_network
     - not deploy_without_tidb|default(false)
 
+- name: download TiFlash packages
+  get_url:
+    url: "{{ item.url }}"
+    dest: "{{ downloads_dir }}/{{ item.name }}-{{ item.version }}.tar.gz"
+    checksum: "{{ item.checksum | default(omit) }}"
+    force: yes
+    validate_certs: no
+  register: get_url_result
+  until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items: "{{ tiflash_packages }}"
+  when:
+    - has_outbound_network
+
 - name: unarchive third party binary
   shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs -n1 tar xzf
   args:
@@ -62,6 +77,13 @@
     warn: no
   when: not deploy_without_tidb|default(false)
 
+- name: unarchive tiflash
+  shell: ls -1 {{ item.name }}-{{ item.version }}.tar.gz | xargs tar xzf
+  args:
+    chdir: "{{ downloads_dir }}"
+    warn: no
+  with_items: "{{ tiflash_packages }}"
+
 - name: cp monitoring binary
   shell: >
     cp -v {{ downloads_dir }}/{{ item }}-*/{{ item }} "{{ resources_dir }}/bin/{{ item }}"
@@ -81,3 +103,8 @@
   shell: >
     cp -rfv {{ downloads_dir }}/tispark-sample-data "{{ resources_dir }}/bin/"
   when: not deploy_without_tidb|default(false)
+
+- name: cp tiflash directory
+  shell: >
+    cp -rfv {{ downloads_dir }}/{{ item.name }}-{{ item.version }}-linux-amd64 "{{ resources_dir }}/bin/tiflash"
+  with_items: "{{ tiflash_packages }}"
diff --git a/roles/local/templates/binary_packages.yml.j2 b/roles/local/templates/binary_packages.yml.j2
@@ -55,3 +55,8 @@ tispark_packages:
     version: latest
     url: http://download.pingcap.org/tispark-sample-data.tar.gz
 {% endif %}
+
+tiflash_packages:
+  - name: tiflash
+    version: {{ tidb_version }}
+    url: http://download.pingcap.org/tiflash-{{ tidb_version }}-linux-amd64.tar.gz
diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2
@@ -109,6 +109,18 @@ alerting:
   {% set _ = tikv_status_addrs.append("%s:%s" % (tikv_ip, tikv_status_port)) -%}
 {% endfor -%}
 
+{% set tiflash_metrics_addrs = [] -%}
+{% set tiflash_http_addrs = [] -%}
+{% for host in groups.tiflash_servers -%}
+  {% set tiflash_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set tiflash_metrics_port = hostvars[host].metrics_port -%}
+  {% set tiflash_proxy_status_port = hostvars[host].flash_proxy_status_port -%}
+  {% set tiflash_http_port = hostvars[host].http_port -%}
+  {% set _ = tiflash_metrics_addrs.append("%s:%s" % (tiflash_ip, tiflash_metrics_port)) -%}
+  {% set _ = tiflash_metrics_addrs.append("%s:%s" % (tiflash_ip, tiflash_proxy_status_port)) -%}
+  {% set _ = tiflash_http_addrs.append("%s:%s" % (tiflash_ip, tiflash_http_port)) -%}
+{% endfor -%}
+
 {% set pd_addrs = [] -%}
 {% for host in groups.pd_servers -%}
   {% set pd_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
@@ -176,6 +188,16 @@ scrape_configs:
       - '{{ tikv_status_addr }}'
 {% endfor %}
 
+{% if tiflash_metrics_addrs %}
+  - job_name: 'tiflash'
+    honor_labels: true # don't overwrite job & instance labels
+    static_configs:
+    - targets:
+{% for tiflash_metrics_addr in tiflash_metrics_addrs %}
+      - '{{ tiflash_metrics_addr }}'
+{% endfor %}
+{% endif %}
+
   - job_name: "pd"
     honor_labels: true # don't overwrite job & instance labels
     static_configs:
@@ -292,6 +314,12 @@ scrape_configs:
       labels:
         group: 'tikv'
     - targets:
+{% for tiflash_http_addr in tiflash_http_addrs %}
+      - '{{ tiflash_http_addr }}'
+{% endfor %}
+      labels:
+        group: 'tiflash'
+    - targets:
 {% for pd_addr in pd_addrs %}
       - '{{ pd_addr }}'
 {% endfor %}

diff --git a/roles/tiflash/defaults/main.yml b/roles/tiflash/defaults/main.yml
@@ -0,0 +1,21 @@
+---
+
+tiflash_dir: "{{ deploy_dir }}/tiflash"
+tmp_path: "{{ deploy_dir }}/tiflash/data/tmp"
+path: "{{ deploy_dir }}/tiflash/data/db"
+cluster_manager_path: "{{ deploy_dir }}/bin/tiflash/flash_cluster_manager"
+cluster_manager_log: "{{ deploy_dir }}/log/tiflash_cluster_manager.log"
+tiflash_tikv_log: "{{ deploy_dir }}/log/tiflash_tikv.log"
+tiflash_errlog: "{{ deploy_dir }}/log/tiflash_error.log"
+tiflash_server_log: "{{ deploy_dir }}/log/tiflash.log"
+flash_proxy_config: "{{ deploy_dir }}/conf/tiflash-learner.toml"
+data_dir: "{{ deploy_dir }}/tiflash/data/flash"
+tiflash_conf_dir: "{{ deploy_dir }}/conf"
+tiflash_log_dir: "{{ deploy_dir }}/log"
+tiflash_scripts_dir: "{{ deploy_dir }}/scripts"
+tcp_port: 9000
+http_port: 8123
+flash_service_port: 3930
+flash_proxy_port: 20170
+flash_proxy_status_port: 20292
+metrics_port: 8234
diff --git a/roles/tiflash/meta/main.yml b/roles/tiflash/meta/main.yml
@@ -0,0 +1,4 @@
+---
+
+dependencies:
+  - role: common_dir
diff --git a/roles/tiflash/tasks/binary_deployment.yml b/roles/tiflash/tasks/binary_deployment.yml
@@ -0,0 +1,23 @@
+---
+
+- name: deploy tiflash binary
+  copy: src="{{ resources_dir }}/bin/tiflash" dest="{{ deploy_dir }}/bin/" mode=0755 backup=yes
+  register: tiflash_binary
+
+- name: create run script
+  template:
+    src: "{{ item }}_{{ role_name }}_binary.sh.j2"
+    dest: "{{ deploy_dir }}/scripts/{{ item }}_{{ role_name }}.sh"
+    mode: "0755"
+    backup: yes
+  with_items:
+    - run
+  vars:
+    role_status_dir: status/{{ role_name }}
+  register: tiflash_script
+
+- name: backup script file
+  command: mv "{{ tiflash_script.backup_file }}" "{{ backup_dir }}"
+  when: tiflash_script.changed and tiflash_script.backup_file is defined
+
+- include_tasks: "systemd_deployment.yml"