Merge remote-tracking branch 'remotes/origin/master' into pessimistic…

…-lock

dm/ctl/master/query_status.go

@@ -42,10 +42,11 @@ type taskInfo struct {
 // NewQueryStatusCmd creates a QueryStatus command
 func NewQueryStatusCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "query-status [-s source ...] [task-name]",
+		Use:   "query-status [-s source ...] [task-name] [--more]",
 		Short: "query task status",
 		Run:   queryStatusFunc,
 	}
+	cmd.Flags().BoolP("more", "", false, "whether to print the detailed task information")
 	return cmd
 }
 
@@ -76,7 +77,13 @@ func queryStatusFunc(cmd *cobra.Command, _ []string) {
 		return
 	}
 
-	if resp.Result && taskName == "" && len(sources) == 0 {
+	more, err := cmd.Flags().GetBool("more")
+	if err != nil {
+		common.PrintLines("%s", errors.ErrorStack(err))
+		return
+	}
+
+	if resp.Result && taskName == "" && len(sources) == 0 && !more {
 		result := wrapTaskResult(resp)
 		common.PrettyPrintInterface(result)
 	} else {

dm/dm-ansible/group_vars/dm_master_servers.yml

@@ -1,3 +1,4 @@
 ---
 
-dm_master_port: "8261"
+dm_master_port: 8261
+dm_master_peer_port: 8291

dm/dm-ansible/group_vars/dm_worker_servers.yml

@@ -1,10 +1,3 @@
 ---
 
-dm_worker_port: "8262"
-flavor: "mysql"
-enable_gtid: "false"
-relay_binlog_name: ""
-relay_binlog_gtid: ""
-
-mysql_user: "root"
-mysql_port: 3306
+dm_worker_port: 8262

dm/dm-ansible/inventory.ini

@@ -3,9 +3,9 @@
 dm_master ansible_host=172.16.10.71
 
 [dm_worker_servers]
-dm-worker1 ansible_host=172.16.10.72 source_id="mysql-replica-01" server_id=101 mysql_host=172.16.10.72 mysql_user=root mysql_password=123456 mysql_port=3306
+dm-worker1 ansible_host=172.16.10.72
 
-dm-worker2 ansible_host=172.16.10.73 source_id="mysql-replica-02" server_id=102 mysql_host=172.16.10.73 mysql_user=root mysql_password=123456 mysql_port=3306
+dm-worker2 ansible_host=172.16.10.73
 
 [dm_portal_servers]
 dm_portal ansible_host=172.16.10.71

dm/dm-ansible/roles/check_config_static/tasks/main.yml

@@ -2,7 +2,7 @@
 
 - name: Ensure dm-master host exists
   fail: msg="One dm-master host should be specified in inventory.ini file."
-  when: groups['dm_master_servers'] | length != 1
+  when: groups['dm_master_servers'] | length < 1
 
 - name: Ensure dm-worker host exists
   fail: msg="One, or more dm-worker hosts should be specified in inventory.ini file."
@@ -23,26 +23,3 @@
 - name: Check ansible_user variable
   fail: msg="ansible_user == 'root' is not supported, please ssh via normal user"
   when: ansible_user == 'root'
-
-- name: Ensure source-id valid - Get invalid source-ids of all dm-workers in cluster
-  set_fact:
-    invalid_source_ids: |-
-      [
-      {% set all_dm_workers = groups['dm_worker_servers']|unique|sort -%}
-      {% for dm_worker in all_dm_workers -%}
-        {% set source_id = hostvars[dm_worker].source_id -%}
-          {% if source_id|length > 32 -%}
-          "{{ source_id }}",
-          {% endif %}
-      {% endfor %}
-      ]
-  run_once: true
-  when: groups['dm_worker_servers'] | length >= 1
-
-- name: Ensure length of each source_id is no more than 32
-  fail:
-    msg: 'invalid source_ids: {{ invalid_source_ids | to_yaml }}'
-  run_once: true
-  when:
-    - invalid_source_ids | length > 0
-

dm/dm-ansible/roles/dm-master/defaults/main.yml

@@ -5,5 +5,8 @@
 dm_master_log_level: "info"
 dm_master_log_dir: "{{ deploy_dir }}/log"
 dm_master_log_file: "dm-master.log"
+dm_master_data_dir: "{{ deploy_dir }}/data.dm_master"
 dm_master_stdout_log_file: "dm-master-stdout.log"
 dm_master_stderr_log_file: "dm-master-stderr.log"
+dm_master_name_prefix: "dm"
+

dm/dm-ansible/roles/dm-master/tasks/main.yml

@@ -1,13 +1,25 @@
 ---
 
-- include_tasks: binary_deployment.yml
+- name: create dm-master deploy directories
+  file:
+    path: '{{ item }}'
+    state: directory
+    mode: 0755
+    owner: '{{ deploy_user }}'
+    group: '{{ deploy_user }}'
+  become: true
+  with_items:
+    - "{{ dm_master_log_dir }}"
+    - "{{ dm_master_data_dir }}"
 
-- name: create configuration file
-  template: src=dm-master.toml.j2 dest={{ deploy_dir }}/conf/dm-master.toml mode=0644 backup=yes
-  register: dm_master_conf_st
+- name: deploy dm-master binary
+  copy: src="{{ resources_dir }}/bin/dm-master" dest="{{ deploy_dir }}/bin/" mode=0755
 
-- name: backup conf file
-  command: mv "{{ dm_master_conf_st.backup_file }}" "{{ backup_dir }}"
-  when:
-    - dm_master_conf_st.changed
-    - dm_master_conf_st.backup_file is defined
+- name: create dm-master run script
+  template:
+    src: "run_{{ role_name }}.sh.j2"
+    dest: "{{ deploy_dir }}/scripts/run_{{ role_name }}.sh"
+    mode: "0755"
+    backup: yes
+
+- include_tasks: "systemd_deployment.yml"

dm/dm-ansible/roles/dm-master/templates/run_dm-master.sh.j2

@@ -5,11 +5,32 @@ ulimit -n 1000000
 DEPLOY_DIR={{ deploy_dir }}
 cd "${DEPLOY_DIR}" || exit 1
 
+{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}
+{% set my_port = hostvars[inventory_hostname].dm_master_port -%}
+{% set my_separator = "_" %}
+{% set my_hostname = hostvars[inventory_hostname]['ansible_hostname'] | default(hostvars[inventory_hostname].inventory_hostname) -%}
+{% set my_peer_id = my_separator ~ my_hostname ~ my_separator ~ my_port %}
+
+{% set all_dm_master = [] -%}
+{% for host in groups.dm_master_servers -%}
+  {% set other_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set other_port = hostvars[host]['dm_master_port'] -%}
+  {% set other_peer_port = hostvars[host]['dm_master_peer_port'] -%}
+  {% set other_dm_master_name_surfix = hostvars[host]['ansible_hostname'] | default(hostvars[host].inventory_hostname) -%}
+  {% set other_dm_master_name = dm_master_name_prefix ~ my_separator ~ other_dm_master_name_surfix ~ my_separator ~ other_port -%}
+  {% set _ = all_dm_master.append("%s=%s://%s:%s" % (other_dm_master_name, 'http', other_ip, other_peer_port)) -%}
+{% endfor -%}
+
 # WARNING: This file was auto-generated. Do not edit!
 #          All your edit might be overwritten!
 
 exec bin/dm-master \
-    --master-addr=":{{ dm_master_port }}" \
+    --name="{{ dm_master_name_prefix }}{{ my_peer_id }}" \
+    --master-addr="{{ my_ip }}:{{ dm_master_port }}" \
+    --advertise-addr="{{ my_ip }}:{{ dm_master_port }}" \
+    --peer-urls="http://{{ my_ip }}:{{ dm_master_peer_port }}" \
+    --advertise-peer-urls="http://{{ my_ip }}:{{ dm_master_peer_port }}" \
+    --initial-cluster="{{ all_dm_master | join(',') }}" \
+    --data-dir="{{ dm_master_data_dir }}" \
     -L="{{ dm_master_log_level }}" \
-    --config="conf/dm-master.toml" \
     --log-file="{{ dm_master_log_dir }}/{{ dm_master_log_file }}" >> "{{ dm_master_log_dir }}/{{ dm_master_stdout_log_file }}" 2>> "{{ dm_master_log_dir }}/{{ dm_master_stderr_log_file }}"

dm/dm-ansible/roles/dm-worker/defaults/main.yml

@@ -7,4 +7,5 @@ dm_worker_log_dir: "{{ deploy_dir }}/log"
 dm_worker_log_file: "dm-worker.log"
 dm_worker_stdout_log_file: "dm-worker-stdout.log"
 dm_worker_stderr_log_file: "dm-worker-stderr.log"
-dm_worker_relay_dir: "{{ deploy_dir }}/relay_log"
+dm_worker_name_prefix: "dm"
+

dm/dm-ansible/roles/dm-worker/tasks/main.yml

@@ -1,13 +1,27 @@
 ---
 
-- include_tasks: binary_deployment.yml
+- name: create deploy directories
+  file:
+    path: '{{ item }}'
+    state: directory
+    mode: 0755
+    owner: '{{ deploy_user }}'
+    group: '{{ deploy_user }}'
+  become: true
+  with_items:
+    - "{{ dm_worker_log_dir }}"
 
-- name: create configuration file
-  template: src=dm-worker.toml.j2 dest={{ deploy_dir }}/conf/dm-worker.toml mode=0644 backup=yes
-  register: dm_worker_conf_st
+- name: deploy dm-worker binary
+  copy: src="{{ resources_dir }}/bin/dm-worker" dest="{{ deploy_dir }}/bin/" mode=0755
 
-- name: backup conf file
-  command: mv "{{ dm_worker_conf_st.backup_file }}" "{{ backup_dir }}"
-  when:
-    - dm_worker_conf_st.changed
-    - dm_worker_conf_st.backup_file is defined
+- name: deploy mydumper binary
+  copy: src="{{ resources_dir }}/bin/mydumper" dest="{{ deploy_dir }}/bin/" mode=0755
+
+- name: create dm-worker run script
+  template:
+    src: "run_{{ role_name }}.sh.j2"
+    dest: "{{ deploy_dir }}/scripts/run_{{ role_name }}.sh"
+    mode: "0755"
+    backup: yes
+
+- include_tasks: "systemd_deployment.yml"

dm/dm-ansible/roles/dm-worker/templates/run_dm-worker.sh.j2

@@ -5,12 +5,26 @@ ulimit -n 1000000
 DEPLOY_DIR={{ deploy_dir }}
 cd "${DEPLOY_DIR}" || exit 1
 
+{% set my_ip = hostvars[inventory_hostname].ansible_host | default(hostvars[inventory_hostname].inventory_hostname) -%}
+{% set my_port = hostvars[inventory_hostname].dm_worker_port -%}
+{% set my_separator = "_" %}
+{% set my_hostname = hostvars[inventory_hostname]['ansible_hostname'] | default(hostvars[inventory_hostname].inventory_hostname) -%}
+{% set my_peer_id = my_separator ~ my_hostname ~ my_separator ~ my_port %}
+
+{% set all_dm_master = [] -%}
+{% for host in groups.dm_master_servers -%}
+  {% set other_ip = hostvars[host].ansible_host | default(hostvars[host].inventory_hostname) -%}
+  {% set other_port = hostvars[host]['dm_master_port'] -%}
+  {% set _ = all_dm_master.append("%s:%s" % (other_ip, other_port)) -%}
+{% endfor -%}
+
 # WARNING: This file was auto-generated. Do not edit!
 #          All your edit might be overwritten!
 
 exec bin/dm-worker \
-    --worker-addr=":{{ dm_worker_port }}" \
+    --name="{{ dm_worker_name_prefix }}{{ my_peer_id }}" \
+    --worker-addr="{{ my_ip }}:{{ dm_worker_port }}" \
+    --advertise-addr="{{ my_ip }}:{{ dm_worker_port }}" \
+    --join="{{ all_dm_master | join(',') }}" \
     -L="{{ dm_worker_log_level }}" \
-    --relay-dir="{{ dm_worker_relay_dir }}" \
-    --config="conf/dm-worker.toml" \
     --log-file="{{ dm_worker_log_dir }}/{{ dm_worker_log_file }}" >> "{{ dm_worker_log_dir }}/{{ dm_worker_stdout_log_file }}" 2>> "{{ dm_worker_log_dir }}/{{ dm_worker_stderr_log_file }}"

dm/dm-ansible/roles/download/templates/packages.yml.j2

@@ -10,11 +10,11 @@ dm_packages:
 
 monitoring_packages:
   - name: prometheus
-    version: 2.2.1
-    url: "http://download.pingcap.org/prometheus-2.2.1.linux-amd64.tar.gz"
+    version: 2.8.1
+    url: "https://download.pingcap.org/prometheus-2.8.1.linux-amd64.tar.gz"
   - name: alertmanager
-    version: 0.14.0
-    url: "http://download.pingcap.org/alertmanager-0.14.0.linux-amd64.tar.gz"
+    version: 0.17.0
+    url: "http://download.pingcap.org/alertmanager-0.17.0.linux-amd64.tar.gz"
   - name: grafana
-    version: 4.6.3
-    url: "http://download.pingcap.org/grafana-4.6.3.linux-x64.tar.gz"
+    version: 6.1.6
+    url: "https://download.pingcap.org/grafana-6.1.6.linux-amd64.tar.gz"

dm/dm-ansible/roles/grafana/defaults/main.yml

@@ -11,4 +11,4 @@ grafana_admin_password: "admin"
 
 grafana_exec_vars_only: false
 
-grafana_version: 4.6.3
+grafana_version: 6.1.6

dm/dm-ansible/roles/grafana/tasks/binary_deployment.yml

@@ -2,9 +2,14 @@
 
 - name: deploy grafana binary
   unarchive: >
-    creates="{{ deploy_dir }}/opt/grafana/bin/grafana-server"
     src={{ downloads_dir }}/grafana-{{ grafana_version }}.tar.gz dest={{ deploy_dir }}/opt/
 
+- name: rename grafana deploy dir
+  shell: >
+    warn=no
+    removes="{{ deploy_dir }}/opt/grafana/bin/grafana-server"
+    rm -rf {{ deploy_dir }}/opt/grafana && mv {{ deploy_dir }}/opt/grafana-{{ grafana_version }} "{{ deploy_dir }}/opt/grafana"
+
 - name: rename grafana deploy dir
   shell: >
     creates="{{ deploy_dir }}/opt/grafana/bin/grafana-server"