Merge pull request #75 from idealista/develop

Develop

.travis.yml

@@ -2,7 +2,7 @@
 
 dist: focal
 language: python
-python: "3.9"
+python: "3.11"
 os: linux
 services:
   - docker
@@ -12,7 +12,6 @@ install:
 env:
   jobs:
     - MOLECULE_DISTRO=debian:bullseye-slim
-    - MOLECULE_DISTRO=debian:buster-slim
 script:
   - pipenv run molecule test
 

CHANGELOG.md

@@ -5,20 +5,31 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch
 
 ## [Unreleased](https://github.com/idealista/clickhouse_role/tree/develop)
 
-## [3.5.0(https://github.com/idealista/clickhouse_role/tree/3.5.0 (2024-01-31)
+## [3.5.1](https://github.com/idealista/clickhouse_role/tree/3.5.1) (2024-04-18)
+
+### :hammer_and_wrench: Fixed
+
+- [#73](https://github.com/idealista/clickhouse_role/issues/73) text_log table config crashes Clickhouse
+
+### :repeat: Updated
+
+- Tasks now are using FQDN
+- Test requirements has been updated
+
+## [3.5.0](https://github.com/idealista/clickhouse_role/tree/3.5.0) (2024-01-31)
 
 ### :heavy_plus_sign: Added
 
 - [#70](https://github.com/idealista/clickhouse_role/issues/70) Include extra config and users files
 - Update default lts version to latest (23.8.9.54)
 
-## [3.4.1(https://github.com/idealista/clickhouse_role/tree/3.4.1 (2024-01-26)
+## [3.4.1](https://github.com/idealista/clickhouse_role/tree/3.4.1) (2024-01-26)
 
 ### :hammer_and_wrench: Fixed
 
 - [#67](https://github.com/idealista/clickhouse_role/issues/67) Fix log tables partitioning
 
-## [3.4.0(https://github.com/idealista/clickhouse_role/tree/3.4.0 (2023-11-17)
+## [3.4.0](https://github.com/idealista/clickhouse_role/tree/3.4.0) (2023-11-17)
 
 ### :heavy_plus_sign: Added
 
@@ -28,25 +39,25 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch
 
 - [#65](https://github.com/idealista/clickhouse_role/issues/65) Update to use ansible_os_family var instead of ansible_distribution var for main.yml
 
-## [3.3.6(https://github.com/idealista/clickhouse_role/tree/3.3.6 (2023-06-09)
+## [3.3.6](https://github.com/idealista/clickhouse_role/tree/3.3.6) (2023-06-09)
 
 ### :hammer_and_wrench: Fixed
 
 - [#59](https://github.com/idealista/clickhouse_role/issues/59) Fix role wiping when there are users from readonly storage
 
-## [3.3.5(https://github.com/idealista/clickhouse_role/tree/3.3.5 (2023-06-09)
+## [3.3.5](https://github.com/idealista/clickhouse_role/tree/3.3.5) (2023-06-09)
 
 ### :heavy_plus_sign: Added
 
 - [#56](https://github.com/idealista/clickhouse_role/issues/56) Add support for several ldap servers and role mappings
 
-## [3.3.4(https://github.com/idealista/clickhouse_role/tree/3.3.4 (2023-05-29)
+## [3.3.4](https://github.com/idealista/clickhouse_role/tree/3.3.4) (2023-05-29)
 
 ### :heavy_plus_sign: Added
 
 - [#53](https://github.com/idealista/clickhouse_role/issues/53) Add LDAP support for authentication.
 
-## [3.3.3(https://github.com/idealista/clickhouse_role/tree/3.3.3 (2023-03-27)
+## [3.3.3](https://github.com/idealista/clickhouse_role/tree/3.3.3) (2023-03-27)
 
 ### :repeat: Updated
 
@@ -56,7 +67,7 @@ This project adheres to [Semantic Versioning](http://semver.org/) and [Keep a ch
 
 - Some typos and quotation
 
-## [3.3.2](https://github.com/idealista/clickhouse_role/tree/3.3.2 (2023-01-18)
+## [3.3.2](https://github.com/idealista/clickhouse_role/tree/3.3.2) (2023-01-18)
 
 ### :heavy_plus_sign: Added
 

Pipfile

@@ -1,17 +1,17 @@
 [[source]]
-name = "pypi"
 url = "https://pypi.org/simple"
 verify_ssl = true
-
-[dev-packages]
+name = "pypi"
 
 [packages]
-ansible = "==5.2.0"
-molecule = "==3.5.2"
-docker = "==5.0.3"
-ansible-lint = "==5.3.2"
-molecule-containers = "==1.0.2"
-yamllint = "==1.26.3"
+ansible = "==9.4.0"
+molecule = "==24.2.0"
+docker = "==7.0.0"
+ansible-lint = "==24.2.1"
+yamllint = "==1.35.1"
+molecule-plugins = {extras = ["docker"], version = "==23.5.3"}
+
+[dev-packages]
 
 [requires]
-python_version = "3.9"
+python_version = "3.11"

README.md

@@ -5,7 +5,7 @@
 
 ![Logo](https://raw.githubusercontent.com/idealista/clickhouse_role/main/logo.gif)
 
-This ansible role installs [Clickhouse](https://clickhouse.com/) in a Debian environment. It has been tested for Debian bullseye and buster.
+This ansible role installs [Clickhouse](https://clickhouse.com/) in a Debian environment. It has been tested for Debian bullseye.
 
 This role has been generated using the [cookiecutter](https://github.com/cookiecutter/cookiecutter) tool, you can generate a similar role that fits your needs using the this [cookiecutter template](https://github.com/idealista/cookiecutter-ansible-role).
 
@@ -159,7 +159,7 @@ For more information read the [pipenv docs](ipenv-fork.readthedocs.io/en/latest/
 ### Testing
 
 ```sh
-$ pipenv run molecule test 
+$ pipenv run molecule test
 ```
 
 ## Built With :building_construction:

defaults/main.yml

@@ -41,7 +41,7 @@ clickhouse_service_state: started
 #clickhouse_log_file:
 clickhouse_log_output: "{% if clickhouse_log_file is defined %} file:{{ clickhouse_log_file }} {% else %} journal {% endif %}"
 
-clickhouse_private_tmp: false
+clickhouse_private_tmp: true
 clickhouse_limit_no_file: 500000
 
 # DEBIAN - Clickhouse

meta/main.yml

@@ -15,4 +15,3 @@ galaxy_info:
     - name: Debian
       versions:
         - bullseye
-        - buster

molecule/default/group_vars/clickhouse_group.yml

@@ -1,9 +1,5 @@
 ---
 
-# Version
-clickhouse_version: 22.8.8.3
-clickhouse_release_type: lts
-
 # Clickhouse role management
 clickhouse_install_db: True
 clickhouse_role_manage_users: True
@@ -13,6 +9,8 @@ clickhouse_role_manage_settings_profiles: True
 clickhouse_role_manage_quotas: True
 clickhouse_role_manage_grants: True
 
+clickhouse_private_tmp: false
+
 clickhouse_custom_extra_config_file_path: "{{ playbook_dir }}/config_files/config/config.d"
 clickhouse_custom_extra_users_file_path: "{{ playbook_dir }}/config_files/users/users.d"
 

molecule/default/molecule.yml

@@ -22,8 +22,9 @@ platforms:
       - /run
       - /run/lock
     volumes:
-      - '/sys/fs/cgroup:/sys/fs/cgroup:ro'
-    command: '/lib/systemd/systemd'
+      - '/sys/fs/cgroup:/sys/fs/cgroup:rw'
+    command: ${MOLECULE_DOCKER_COMMAND:-"/lib/systemd/systemd"}
+    cgroupns_mode: host
     stop_signal: 'RTMIN+3'
     exposed_ports:
       - 8123/tcp

molecule/default/tests/test_app.yml

@@ -45,7 +45,7 @@ file:
 service:
   {{ clickhouse_service }}:
     enabled: true
-    running: true 
+    running: true
 
 port:
   tcp:{{ clickhouse_tcp_port }}:
@@ -88,42 +88,11 @@ http:
     username: {{ clickhouse_admin_user }}
     password: {{ clickhouse_admin_password }}
     skip: false
+
   http://localhost:8123?query=SHOW+DATABASES:
     status: 200
     timeout: 10000
     body: ["system", "default", "ProjectD", "ex_racing", "racing"]
     username: {{ clickhouse_admin_user }}
     password: {{ clickhouse_admin_password }}
     skip: false
-  http://localhost:8123?query=SHOW+GRANTS+FOR+Bunta:
-    status: 200
-    timeout: 10000
-    body: 
-      - "GRANT CREATE TEMPORARY TABLE, CREATE FUNCTION, DROP FUNCTION, KILL QUERY, KILL TRANSACTION, MOVE PARTITION BETWEEN SHARDS, SYSTEM SHUTDOWN, SYSTEM DROP CACHE, SYSTEM RELOAD, SYSTEM RESTART DISK, SYSTEM SYNC TRANSACTION LOG, SYSTEM FLUSH LOGS, SYSTEM THREAD FUZZER, SYSTEM UNFREEZE, INTROSPECTION, SOURCES, CLUSTER ON *.* TO Bunta"
-      - "GRANT SHOW DICTIONARIES, CREATE DICTIONARY, DROP DICTIONARY, dictGet ON `<no_database>`.* TO Bunta"
-      - "GRANT SHOW, SELECT, INSERT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON ex_racing.* TO Bunta"
-    username: {{ clickhouse_admin_user }}
-    password: {{ clickhouse_admin_password }}
-    skip: false
-  http://localhost:8123?query=SHOW+GRANTS+FOR+Takumi:
-    status: 200
-    timeout: 10000
-    body:
-      - "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON Akina.calendar TO Takumi WITH GRANT OPTION"
-      - "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON Akina.records TO Takumi WITH GRANT OPTION"
-      - "GRANT SHOW, SELECT, INSERT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON racing.* TO Takumi"
-      - "GRANT initial_d, projectd_members, tofu_shop TO Takumi"
-    username: {{ clickhouse_admin_user }}
-    password: {{ clickhouse_admin_password }}
-    skip: false
-  http://localhost:8123?query=SHOW+GRANTS+FOR+Iketani:
-    status: 200
-    timeout: 10000
-    body: 
-      - "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON Akina.calendar TO Iketani WITH GRANT OPTION"
-      - "GRANT SELECT, ALTER UPDATE ON Akina.records TO Iketani"
-      - "GRANT SHOW, SELECT, INSERT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM SYNC DATABASE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON racing.* TO Iketani"
-      - "GRANT initial_d TO Iketani"
-    username: {{ clickhouse_admin_user }}
-    password: {{ clickhouse_admin_password }}
-    skip: false

tasks/config/clickhouse.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: CLICKHOUSE | Add configuration files (from vars)
-  template:
+  ansible.builtin.template:
     src: "{{ clickhouse_config_file_name }}.j2"
     dest: "{{ clickhouse_config_directory }}/{{ clickhouse_config_file_name }}"
     mode: 0644
@@ -10,7 +10,7 @@
   tags: clickhouse_configure
 
 - name: CLICKHOUSE | Add configuration files (from user files)
-  copy:
+  ansible.builtin.copy:
     src: "{{ clickhouse_custom_config_file_path }}/"
     dest: "{{ clickhouse_config_directory }}"
     owner: "{{ clickhouse_user }}"
@@ -21,7 +21,7 @@
   tags: clickhouse_configure
 
 - name: CLICKHOUSE | Add extra configuration files
-  copy:
+  ansible.builtin.copy:
     src: "{{ clickhouse_custom_extra_config_file_path }}/"
     dest: "{{ clickhouse_config_directory_config_d }}/"
     owner: "{{ clickhouse_user }}"
@@ -31,7 +31,7 @@
   tags: clickhouse_configure
 
 - name: CLICKHOUSE | Configure users (from vars)
-  template:
+  ansible.builtin.template:
     src: "{{ clickhouse_users_file_name }}.j2"
     dest: "{{ clickhouse_config_directory }}/{{ clickhouse_users_file_name }}"
     mode: 0644
@@ -42,7 +42,7 @@
     - clickhouse_users
 
 - name: CLICKHOUSE | Configure users files (from users files)
-  copy:
+  ansible.builtin.copy:
     src: "{{ clickhouse_custom_users_file_path }}/"
     dest: "{{ clickhouse_config_directory }}"
     owner: "{{ clickhouse_user }}"
@@ -55,7 +55,7 @@
     - clickhouse_users
 
 - name: CLICKHOUSE | Add extra user configuration files
-  copy:
+  ansible.builtin.copy:
     src: "{{ clickhouse_custom_extra_users_file_path }}/"
     dest: "{{ clickhouse_config_directory_users_d }}/"
     owner: "{{ clickhouse_user }}"

tasks/config/copier.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: CLICKHOUSE | Add copier configuration files (from vars)
-  template:
+  ansible.builtin.template:
     src: "{{ clickhouse_copier_config_file_name }}.j2"
     dest: "{{ clickhouse_config_directory }}/{{ clickhouse_copier_config_file_name }}"
     mode: 0644
@@ -10,7 +10,7 @@
   tags: clickhouse_configure_copier
 
 - name: CLICKHOUSE | Add copier configuration files (from user files)
-  copy:
+  ansible.builtin.copy:
     src: "{{ clickhouse_copier_custom_config_file_path }}/"
     dest: "{{ clickhouse_copier_config_directory }}"
     owner: "{{ clickhouse_user }}"

tasks/config/databases.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: CLICKHOUSE | Create default database (SQL-driven)
-  command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q 'CREATE DATABASE IF NOT EXISTS `{{ clickhouse_default_database }}`'" # noqa 204
+  ansible.builtin.command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q 'CREATE DATABASE IF NOT EXISTS `{{ clickhouse_default_database }}`'" # noqa 204
   changed_when: False
   when: clickhouse_default_database is defined
   no_log: True
@@ -11,7 +11,7 @@
     - SQL_driven
 
 - name: CLICKHOUSE | List databases (SQL-driven)
-  command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q \"SELECT name FROM system.databases WHERE LOWER(name) NOT IN ('_temporary_and_external_tables', 'information_schema', 'default', 'system');\""
+  ansible.builtin.command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q \"SELECT name FROM system.databases WHERE LOWER(name) NOT IN ('_temporary_and_external_tables', 'information_schema', 'default', 'system');\""
   changed_when: False
   register: clickhouse_existing_sql_databases
   no_log: True
@@ -21,24 +21,28 @@
     - SQL_driven
 
 - name: CLICKHOUSE | Drop databases (SQL-driven)
-  command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q \"DROP DATABASE IF EXISTS {{ item }} ;\""
+  ansible.builtin.command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q \"DROP DATABASE IF EXISTS {{ item }} ;\""
   with_items: "{{ clickhouse_existing_sql_databases.stdout_lines }}"
   when:
     - clickhouse_existing_sql_databases is defined
     - item not in clickhouse_custom_databases | map(attribute='name') | list
   no_log: True
+  register: drop_databases
+  changed_when: drop_databases.rc != 0
   tags:
     - clickhouse_databases
     - delete
     - SQL_driven
 
 - name: CLICKHOUSE | Create custom databases (SQL-driven)
-  command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q {{ lookup('template', 'DDL/DATABASE.j2') | quote }}"
+  ansible.builtin.command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} -q {{ lookup('template', 'DDL/DATABASE.j2') | quote }}"
   with_items: "{{ clickhouse_custom_databases }}"
   when:
     - clickhouse_custom_databases is defined
     - item.name not in clickhouse_existing_sql_databases.stdout_lines
   no_log: True
+  register: add_databases
+  changed_when: add_databases.rc != 0
   tags:
     - clickhouse_databases
     - create

tasks/config/grants-roles.yml

@@ -1,9 +1,10 @@
 ---
 
 - name: CLICKHOUSE | Assigning roles (SQL-driven)
-  command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} --multiquery -q {{ lookup('template', 'DDL/GRANT_ROLE.j2') | quote }}"
+  ansible.builtin.command: "clickhouse-client -u {{ clickhouse_admin_user }} --password {{ clickhouse_admin_password }} --multiquery -q {{ lookup('template', 'DDL/GRANT_ROLE.j2') | quote }}"
   when: clickhouse_custom_grant_roles is defined
-  changed_when: False
+  register: assigning_roles
+  changed_when: assigning_roles.rc != 0
   no_log: True
   tags:
     - clickhouse_grants