added clickhouse state

states/clickhouse/config.sls

@@ -0,0 +1,46 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context -%}
+---
+clickhouse-config-file-settings:
+  file.managed:
+    - name: /etc/clickhouse-server/config.d/settings.xml
+    - source: salt://clickhouse/templates/settings.xml.j2
+    - user: clickhouse
+    - group: clickhouse
+    - mode: 600
+    - template: jinja
+
+clickhouse-config-file-macros:
+  file.managed:
+    - name: /etc/clickhouse-server/config.d/macros.xml
+    - source: salt://clickhouse/templates/macros.xml.j2
+    - user: clickhouse
+    - group: clickhouse
+    - mode: 600
+    - template: jinja
+
+clickhouse-config-file-replica:
+  file.managed:
+    - name: /etc/clickhouse-server/config.d/replica.xml
+    - source: salt://clickhouse/templates/replica.xml.j2
+    - user: clickhouse
+    - group: clickhouse
+    - mode: 600
+    - template: jinja
+
+clickhouse-config-file-keeper:
+  file.managed:
+    - name: /etc/clickhouse-server/config.d/keeper.xml
+    - source: salt://clickhouse/templates/keeper.xml.j2
+    - user: clickhouse
+    - group: clickhouse
+    - mode: 600
+    - template: jinja
+
+clickhouse-config-file-tls:
+  file.managed:
+    - name: /etc/clickhouse-server/config.d/tls.xml
+    - source: salt://clickhouse/templates/tls.xml.j2
+    - user: clickhouse
+    - group: clickhouse
+    - mode: 600
+    - template: jinja

states/clickhouse/defaults.yaml

@@ -0,0 +1,12 @@
+---
+clickhouse:
+  enabled: true
+  pkgs:
+    - clickhouse-server
+    - clickhouse-client
+  databases: {}
+  users: {}
+  cluster: paulbsd
+  clusters:
+    paulbsd:
+      nodes: {}

states/clickhouse/init.sls

@@ -0,0 +1,6 @@
+---
+include:
+  - .install
+  - .config
+  - .service
+  - .users

states/clickhouse/install.sls

@@ -0,0 +1,6 @@
+---
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+clickhouse-install:
+  pkg.installed:
+    - name: clickhouse-install
+    - pkgs: {{ clickhouse.pkgs }}

states/clickhouse/map.jinja

@@ -0,0 +1,5 @@
+{%- import_yaml "clickhouse/defaults.yaml" as default_settings -%}
+
+{%- set defaults = salt['grains.filter_by'](default_settings, default='clickhouse') -%}
+
+{%- set clickhouse = salt['pillar.get']('clickhouse', default=defaults, merge=True) -%}

states/clickhouse/service.sls

@@ -0,0 +1,6 @@
+---
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+clickhouse-service:
+  service.running:
+    - name: clickhouse-server
+    - enable: true

states/clickhouse/templates/keeper.xml.j2

@@ -0,0 +1,35 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context -%}
+<clickhouse>
+  <zookeeper>
+{%- for nodename, settings in clickhouse.clusters[clickhouse.cluster].nodes.items() %}
+    <node>
+      <host>{{ nodename }}</host>
+      <port>{{ settings.port|default(9281) }}</port>
+      <secure>1</secure>
+    </node>
+{%- endfor %}
+  </zookeeper>
+  <keeper_server>
+    <tcp_port_secure>{{ settings.port|default(9281) }}</tcp_port_secure>
+    <server_id>{{ clickhouse.clusters[clickhouse.cluster].nodes[salt['grains.get']('fqdn')].id }}</server_id>
+    <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+    <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+
+    <coordination_settings>
+      <operation_timeout_ms>10000</operation_timeout_ms>
+      <session_timeout_ms>30000</session_timeout_ms>
+      <raft_logs_level>fatal</raft_logs_level>
+    </coordination_settings>
+
+    <raft_configuration>
+      <secure>true</secure>
+{%- for nodename, settings in clickhouse.clusters[clickhouse.cluster].nodes.items() %}
+      <server>
+        <id>{{ settings.id }}</id>
+        <hostname>{{ nodename }}</hostname>
+        <port>{{ settings.port|default(9444) }}</port>
+      </server>
+{%- endfor %}
+    </raft_configuration>
+  </keeper_server>
+</clickhouse>

states/clickhouse/templates/macros.xml.j2

@@ -0,0 +1,8 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+<clickhouse>
+  <macros>
+    <cluster>{{ clickhouse.cluster }}</cluster>
+    <shard>{{ clickhouse.clusters[clickhouse.cluster].nodes[salt['grains.get']('fqdn')].shard }}</shard>
+    <replica>{{ salt['grains.get']('fqdn') }}</replica>
+  </macros>
+</clickhouse>

states/clickhouse/templates/replica.xml.j2

@@ -0,0 +1,19 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+<clickhouse>
+  <remote_servers>
+    <{{ clickhouse.cluster }}>
+      <shard>
+        <internal_replication>false</internal_replication>
+{% for nodename, settings in clickhouse.clusters[clickhouse.cluster].nodes.items() %}
+        <replica>
+          <host>{{ nodename }}</host>
+          <port>9440</port>
+          <user>default</user>
+          <password></password>
+          <secure>1</secure>
+        </replica>
+{% endfor %}
+      </shard>
+      </{{ clickhouse.cluster }}>
+  </remote_servers>
+</clickhouse>

states/clickhouse/templates/settings.xml.j2

@@ -0,0 +1,14 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+<clickhouse>
+  <listen_host>::</listen_host>
+  <default_replica_name>{replica}</default_replica_name>
+  <default_replica_path>/clickhouse/{cluster}/tables/{uuid}/{database}/{table}-{shard}</default_replica_path>
+  <logger>
+    <level>fatal</level>
+  </logger>
+  <max_server_memory_usage_to_ram_ratio>0.5</max_server_memory_usage_to_ram_ratio>
+  <tcp_port_secure>9440</tcp_port_secure>
+  <https_port>8443</https_port>
+  <interserver_http_port remove="true"></interserver_http_port>
+  <interserver_https_port>9010</interserver_https_port>
+</clickhouse>

states/clickhouse/templates/tls.xml.j2

@@ -0,0 +1,25 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+<clickhouse>
+  <openSSL>
+    <server>
+        <certificateFile>/etc/clickhouse-server/certs/{{ salt['grains.get']('fqdn') }}.crt</certificateFile>
+        <privateKeyFile>/etc/clickhouse-server/certs/{{ salt['grains.get']('fqdn') }}.key</privateKeyFile>
+        <verificationMode>relaxed</verificationMode>
+        <caConfig>/etc/clickhouse-server/certs/{{ clickhouse.cluster }}_ca.crt</caConfig>
+        <cacheSessions>true</cacheSessions>
+        <disableProtocols>sslv2,sslv3</disableProtocols>
+        <preferServerCiphers>true</preferServerCiphers>
+    </server>
+    <client>
+        <loadDefaultCAFile>false</loadDefaultCAFile>
+        <caConfig>/etc/clickhouse-server/certs/{{ clickhouse.cluster }}_ca.crt</caConfig>
+        <cacheSessions>true</cacheSessions>
+        <disableProtocols>sslv2,sslv3</disableProtocols>
+        <preferServerCiphers>true</preferServerCiphers>
+        <verificationMode>relaxed</verificationMode>
+        <invalidCertificateHandler>
+            <name>RejectCertificateHandler</name>
+        </invalidCertificateHandler>
+    </client>
+  </openSSL>
+</clickhouse>

states/clickhouse/templates/user.xml.j2

@@ -0,0 +1,24 @@
+{%- from "clickhouse/map.jinja" import clickhouse with context -%}
+<clickhouse>
+  <users>
+    <{{ name }}>
+      <networks>
+      {%- for network in params.networks|default(["127.0.0.1/8"]) %}
+        <ip>{{ network }}</ip>
+      {%- endfor %}
+      </networks>
+      <profile>{{ params.profile|default("default") }}</profile>
+      <quota>{{ params.quota|default("default") }}</quota>
+      <default_database>{{ params.default_database|default("default") }}</default_database>
+      <access_management>{{ params.access_management|default(0)}}</access_management>
+      {%- if params.databases|default([])|length > 0 %}
+      <allow_databases>
+      {%- for database in params.databases|default([]) %}
+        <database>{{ database }}</database>
+      {%- endfor %}
+      </allow_databases>
+      {%- endif %}
+      <password>{{ params.password|default("") }}</password>
+    </{{ name }}>
+  </users>
+</clickhouse>

states/clickhouse/users.sls

@@ -0,0 +1,15 @@
+---
+{%- from "clickhouse/map.jinja" import clickhouse with context %}
+{% for user, params in clickhouse.users.items() %}
+clickhouse-user-{{ user }}:
+  file.managed:
+    - name: /etc/clickhouse-server/users.d/{{ user }}.xml
+    - source: salt://clickhouse/templates/user.xml.j2
+    - user: clickhouse
+    - group: clickhouse
+    - mode: 600
+    - template: jinja
+    - context:
+        name: {{ user }}
+        params: {{ params }}
+{% endfor %}