initial commit

README.md

@@ -0,0 +1,48 @@
+# clickhouse_ddl_export
+
+## Usage
+
+```
+usage: clickhouse_ddl_export [-h] [--export | --print-statements] [dumpfile]
+
+Exports clickhouse DDL, useful when adding replicas in a cluster
+
+positional arguments:
+  dumpfile
+
+options:
+  -h, --help          show this help message and exit
+  --export
+  --print-statements
+```
+
+## License
+
+```text
+Copyright (c) 2024 PaulBSD
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies,
+either expressed or implied, of this project.
+```

clickhouse_ddl_export.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+# install:
+# #pip3 install clickhouse-connect
+
+import argparse
+import re
+import json
+import clickhouse_connect
+
+DUMPFILE = "/tmp/ch_export"
+createre = re.compile("(?P<begin>CREATE (TABLE|(MATERIALIZED )?VIEW)) (?P<database>\w+)\.(?P<table>\w+) (?P<query>.*)")
+
+database_exclude = {
+    "default": True,
+    "INFORMATION_SCHEMA": True,
+    "information_schema": True,
+    "system": True,
+}
+
+def get_databases_tables(client):
+    databases = {}
+    q_db = client.query("show databases;")
+    for d in q_db.result_rows:
+        dbname = d[0]
+        if database_exclude.get(dbname):
+            continue
+        databases[dbname] = {"schema":"", "tables":{}}
+        q_tables = client.query(f"show tables from {dbname};")
+        for t in q_tables.result_rows:
+            tablename = t[0]
+            q_table_schema = client.query(f"SELECT create_table_query, uuid from system.tables where database='{dbname}' and table='{tablename}';")
+            for schema in q_table_schema.result_rows:
+                create_table_database = schema[0]
+                uuid = schema[1]
+                resre = createre.match(create_table_database)
+                begin = resre.group("begin")
+                database = resre.group("database")
+                table = resre.group("table")
+                query = resre.group("query")
+                fullquery = f"{begin} {database}.{table} UUID '{uuid}' {query}"
+                databases[dbname]["tables"][tablename] = fullquery
+    return databases
+
+def export(dumpfile=DUMPFILE):
+    client = clickhouse_connect.get_client(host='localhost', username='default', password='')
+    databases = get_databases_tables(client)
+    with open(dumpfile, "w") as f:
+        json.dump(databases, f, indent=4)
+    return
+
+def dump(dumpfile=DUMPFILE):
+    with open(dumpfile, "r") as f:
+        a = json.load(f)
+        for database,values in a.items():
+            for table,c in values["tables"].items():
+                print(f"# {database}.{table}")
+                print(f"{c}\n")
+
+def main():
+    parser = argparse.ArgumentParser(
+                    prog="clickhouse_ddl_export",
+                    description="Exports clickhouse DDL, useful when adding replicas in a cluster")
+    parser.add_argument("dumpfile", default=DUMPFILE, nargs="?")
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument("--export", action="store_true")
+    group.add_argument("--print-statements", action="store_true")
+    args = parser.parse_args()
+    if args.export:
+        export(args.dumpfile)
+    if args.print_statements:
+        dump(args.dumpfile)
+
+if __name__ == "__main__":
+    main()