From db29ad8f0b5e9c537df2bc2b5ed5a9ce284d27f7 Mon Sep 17 00:00:00 2001 From: liubo Date: Fri, 10 Jan 2020 11:09:07 +0800 Subject: [PATCH] update lightning (#1110) * Add create directory automatically (#1103) (#1105) * merge from master (#1108) (#1109) * reduce lightning config items --- conf/tidb-lightning.yml | 3 --- conf/tikv-importer.yml | 1 - group_vars/all.yml | 3 --- group_vars/importer_server.yml | 3 +++ roles/tidb_lightning/defaults/main.yml | 3 +++ roles/tidb_lightning/tasks/main.yml | 9 +++++++-- roles/tidb_lightning/templates/tidb-lightning.toml.j2 | 4 ++++ roles/tidb_lightning/vars/tidb-lightning.yml | 2 -- roles/tikv_importer/defaults/main.yml | 9 +++++++++ roles/tikv_importer/tasks/main.yml | 4 ++++ 10 files changed, 30 insertions(+), 11 deletions(-) diff --git a/conf/tidb-lightning.yml b/conf/tidb-lightning.yml index 99cdadcf5..d954416dd 100644 --- a/conf/tidb-lightning.yml +++ b/conf/tidb-lightning.yml @@ -20,7 +20,6 @@ lightning: # logging level: "info" - file: "log/tidb_lightning.log" max-size: 128 # MB max-days: 28 max-backups: 14 @@ -46,8 +45,6 @@ checkpoint: # keep-after-success: false tikv_importer: - # the listening address of tikv-importer. Change it to the actual address in tikv-importer.toml. - # addr: "0.0.0.0:8287" mydumper: # block size of file reading diff --git a/conf/tikv-importer.yml b/conf/tikv-importer.yml index e6d4e3cbc..2bb1c5821 100644 --- a/conf/tikv-importer.yml +++ b/conf/tikv-importer.yml @@ -4,7 +4,6 @@ global: # log file. - log-file: "log/tikv_importer.log" # log level: trace, debug, info, warn, error, off. log-level: "info" diff --git a/group_vars/all.yml b/group_vars/all.yml index 311bd82ee..6e5815c04 100644 --- a/group_vars/all.yml +++ b/group_vars/all.yml @@ -26,9 +26,6 @@ docker_bin_dir: "/usr/bin" # Random shifts for retrying failed ops like downloading retry_stagger: 5 -# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address. -tikv_importer_port: 8287 - # deployment methods, [binary, docker] docker deployment method is not recommended and deprecated. deployment_method: binary diff --git a/group_vars/importer_server.yml b/group_vars/importer_server.yml index e381b1709..ecf57a6d8 100644 --- a/group_vars/importer_server.yml +++ b/group_vars/importer_server.yml @@ -3,3 +3,6 @@ dummy: # this directory is used to store the data written by `tidb-lightning` import_dir: "{{ deploy_dir }}/data.import" + +# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address. +tikv_importer_port: 8287 diff --git a/roles/tidb_lightning/defaults/main.yml b/roles/tidb_lightning/defaults/main.yml index 296e15d2a..a8737fbf7 100644 --- a/roles/tidb_lightning/defaults/main.yml +++ b/roles/tidb_lightning/defaults/main.yml @@ -7,3 +7,6 @@ tidb_port: "{{ hostvars[groups.tidb_servers[0]].tidb_port }}" tidb_user: "root" tidb_password: "" tidb_status_port: "{{ hostvars[groups.tidb_servers[0]].tidb_status_port }}" +lightning_log_dir: "{{ deploy_dir }}/log" +lightning_conf_dir: "{{ deploy_dir }}/conf" +lightning_log_file: "tidb_lightning.log" diff --git a/roles/tidb_lightning/tasks/main.yml b/roles/tidb_lightning/tasks/main.yml index 4a6ad471a..9c9bd61d5 100644 --- a/roles/tidb_lightning/tasks/main.yml +++ b/roles/tidb_lightning/tasks/main.yml @@ -1,6 +1,12 @@ --- # tasks file for tidb-lightning +- name: create deploy directories + file: path="{{ item }}" state=directory mode=0755 + with_items: + - "{{ lightning_log_dir }}" + - "{{ lightning_conf_dir }}" + - name: "load lightning customized config: tidb-ansible/conf/tidb-lightning.yml" include_vars: file={{ playbook_dir }}/conf/tidb-lightning.yml name=tidb_lightning_conf_custom @@ -12,8 +18,7 @@ tidb_lightning_conf_generated: lightning: pprof-port: "{{ tidb_lightning_pprof_port }}" - tikv_importer: - addr: "{{ hostvars[groups.importer_server[0]].ansible_host | default(hostvars[groups.importer_server[0]].inventory_hostname) }}:{{ tikv_importer_port }}" + file: "{{ lightning_log_dir }}/{{ lightning_log_file }}" mydumper: data-source-dir: "{{ data_source_dir }}" tidb: diff --git a/roles/tidb_lightning/templates/tidb-lightning.toml.j2 b/roles/tidb_lightning/templates/tidb-lightning.toml.j2 index a5ee445e6..f222cc031 100644 --- a/roles/tidb_lightning/templates/tidb-lightning.toml.j2 +++ b/roles/tidb_lightning/templates/tidb-lightning.toml.j2 @@ -17,6 +17,10 @@ {% endfor %} [tikv-importer] +{% set tikv_importer_node = groups.importer_server[0] -%} +{% set tikv_importer_ip = hostvars[tikv_importer_node].ansible_host | default(hostvars[tikv_importer_node].inventory_hostname) -%} +{% set tikv_importer_port = hostvars[tikv_importer_node].tikv_importer_port %} +addr = "{{ tikv_importer_ip }}:{{ tikv_importer_port }}" {% for item, value in tidb_lightning_conf.tikv_importer | dictsort -%} {{ item }} = {{ value | to_json }} {% endfor %} diff --git a/roles/tidb_lightning/vars/tidb-lightning.yml b/roles/tidb_lightning/vars/tidb-lightning.yml index c9a133f1e..5226f7706 100644 --- a/roles/tidb_lightning/vars/tidb-lightning.yml +++ b/roles/tidb_lightning/vars/tidb-lightning.yml @@ -46,8 +46,6 @@ checkpoint: # keep-after-success: false tikv_importer: - # the listening address of tikv-importer. Change it to the actual address in tikv-importer.toml. - # addr: "0.0.0.0:8287" mydumper: # block size of file reading diff --git a/roles/tikv_importer/defaults/main.yml b/roles/tikv_importer/defaults/main.yml index 794cd28d6..c7a8958be 100644 --- a/roles/tikv_importer/defaults/main.yml +++ b/roles/tikv_importer/defaults/main.yml @@ -1,3 +1,12 @@ --- dummy: +# the listening address of tikv-importer. tidb-lightning needs to connect to this address to write data. Set it to the actual IP address. +tikv_importer_port: 8287 + +# this directory is used to store the data written by `tidb-lightning` +import_dir: "{{ deploy_dir }}/data.import" + +importer_log_dir: "{{ deploy_dir }}/log" +importer_log_file: "tikv_importer.log" +importer_conf_dir: "{{ deploy_dir }}/conf" diff --git a/roles/tikv_importer/tasks/main.yml b/roles/tikv_importer/tasks/main.yml index 96ca7d3e3..0d2888b74 100644 --- a/roles/tikv_importer/tasks/main.yml +++ b/roles/tikv_importer/tasks/main.yml @@ -5,6 +5,8 @@ file: path="{{ item }}" state=directory mode=0755 with_items: - "{{ import_dir }}" + - "{{ importer_log_dir }}" + - "{{ importer_conf_dir }}" - name: "load tikv-importer customized config: tidb-ansible/conf/tikv-importer.yml" include_vars: file={{ playbook_dir }}/conf/tikv-importer.yml name=tikv_importer_conf_custom @@ -15,6 +17,8 @@ - name: generate tikv-importer dynamic config set_fact: tikv_importer_conf_generated: + global: + log-file: "{{ importer_log_dir }}/{{ importer_log_file }}" server: addr: "{{ hostvars[groups.importer_server[0]].ansible_host | default(hostvars[groups.importer_server[0]].inventory_hostname) }}:{{ tikv_importer_port }}" import: