update lightning (#1110)

* Add create directory automatically (#1103) (#1105)

* merge from master (#1108) (#1109)

* reduce lightning config items

conf/tidb-lightning.yml

@@ -20,7 +20,6 @@ lightning:
 
   # logging
   level: "info"
-  file: "log/tidb_lightning.log"
   max-size: 128 # MB
   max-days: 28
   max-backups: 14
@@ -46,8 +45,6 @@ checkpoint:
   # keep-after-success: false
 
 tikv_importer:
-  # the listening address of tikv-importer. Change it to the actual address in tikv-importer.toml.
-  # addr: "0.0.0.0:8287"
 
 mydumper:
   # block size of file reading

conf/tikv-importer.yml

@@ -4,7 +4,6 @@
 
 global:
   # log file.
-  log-file: "log/tikv_importer.log"
   # log level: trace, debug, info, warn, error, off.
   log-level: "info"
 

group_vars/all.yml

@@ -26,9 +26,6 @@ docker_bin_dir: "/usr/bin"
 # Random shifts for retrying failed ops like downloading
 retry_stagger: 5
 
-# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address.
-tikv_importer_port: 8287
-
 # deployment methods, [binary, docker] docker deployment method is not recommended and deprecated.
 deployment_method: binary
 

group_vars/importer_server.yml

@@ -3,3 +3,6 @@ dummy:
 
 # this directory is used to store the data written by `tidb-lightning`
 import_dir: "{{ deploy_dir }}/data.import"
+
+# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address.
+tikv_importer_port: 8287

roles/tidb_lightning/defaults/main.yml

@@ -7,3 +7,6 @@ tidb_port: "{{ hostvars[groups.tidb_servers[0]].tidb_port }}"
 tidb_user: "root"
 tidb_password: ""
 tidb_status_port: "{{ hostvars[groups.tidb_servers[0]].tidb_status_port }}"
+lightning_log_dir: "{{ deploy_dir }}/log"
+lightning_conf_dir: "{{ deploy_dir }}/conf"
+lightning_log_file: "tidb_lightning.log"

roles/tidb_lightning/tasks/main.yml

@@ -1,6 +1,12 @@
 ---
 # tasks file for tidb-lightning
 
+- name: create deploy directories
+  file: path="{{ item }}" state=directory mode=0755
+  with_items:
+  - "{{ lightning_log_dir }}"
+  - "{{ lightning_conf_dir }}"
+
 - name: "load lightning customized config: tidb-ansible/conf/tidb-lightning.yml"
   include_vars: file={{ playbook_dir }}/conf/tidb-lightning.yml name=tidb_lightning_conf_custom
 
@@ -12,8 +18,7 @@
     tidb_lightning_conf_generated:
       lightning:
         pprof-port: "{{ tidb_lightning_pprof_port }}"
-      tikv_importer:
-        addr: "{{ hostvars[groups.importer_server[0]].ansible_host | default(hostvars[groups.importer_server[0]].inventory_hostname) }}:{{ tikv_importer_port }}"
+        file: "{{ lightning_log_dir }}/{{ lightning_log_file }}"
       mydumper:
         data-source-dir: "{{ data_source_dir }}"
       tidb:

roles/tidb_lightning/templates/tidb-lightning.toml.j2

@@ -17,6 +17,10 @@
 {% endfor %}
 
 [tikv-importer]
+{% set tikv_importer_node = groups.importer_server[0] -%}
+{% set tikv_importer_ip = hostvars[tikv_importer_node].ansible_host | default(hostvars[tikv_importer_node].inventory_hostname) -%}
+{% set tikv_importer_port = hostvars[tikv_importer_node].tikv_importer_port %}
+addr = "{{ tikv_importer_ip }}:{{ tikv_importer_port }}"
 {% for item, value in tidb_lightning_conf.tikv_importer | dictsort -%}
 {{ item }} = {{ value | to_json }}
 {% endfor %}

roles/tidb_lightning/vars/tidb-lightning.yml

@@ -46,8 +46,6 @@ checkpoint:
   # keep-after-success: false
 
 tikv_importer:
-  # the listening address of tikv-importer. Change it to the actual address in tikv-importer.toml.
-  # addr: "0.0.0.0:8287"
 
 mydumper:
   # block size of file reading

roles/tikv_importer/defaults/main.yml

@@ -1,3 +1,12 @@
 ---
 dummy:
 
+# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address.
+tikv_importer_port: 8287
+
+# this directory is used to store the data written by `tidb-lightning`
+import_dir: "{{ deploy_dir }}/data.import"
+
+importer_log_dir: "{{ deploy_dir }}/log"
+importer_log_file: "tikv_importer.log"
+importer_conf_dir: "{{ deploy_dir }}/conf"

roles/tikv_importer/tasks/main.yml

@@ -5,6 +5,8 @@
   file: path="{{ item }}" state=directory mode=0755
   with_items:
   - "{{ import_dir }}"
+  - "{{ importer_log_dir }}"
+  - "{{ importer_conf_dir }}"
 
 - name: "load tikv-importer customized config: tidb-ansible/conf/tikv-importer.yml"
   include_vars: file={{ playbook_dir }}/conf/tikv-importer.yml name=tikv_importer_conf_custom
@@ -15,6 +17,8 @@
 - name: generate tikv-importer dynamic config
   set_fact:
     tikv_importer_conf_generated:
+      global:
+        log-file: "{{ importer_log_dir }}/{{ importer_log_file }}"
       server:
         addr: "{{ hostvars[groups.importer_server[0]].ansible_host | default(hostvars[groups.importer_server[0]].inventory_hostname) }}:{{ tikv_importer_port }}"
       import: