clickhouse_ddl_export/clickhouse_ddl_export.go

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"

	"github.com/ClickHouse/clickhouse-go/v2"
	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
)

var dbexclude = []string{
	"'default'",
	"'system'",
	"'INFORMATION_SCHEMA'",
	"'information_schema'",
}

const dbquery = "SELECT database,toString(uuid) FROM system.databases WHERE database NOT IN (%s) AND database NOT LIKE '.%%';"

var phoneRENamedCaps = `(?P<begin>CREATE (DICTIONARY|TABLE|(MATERIALIZED )?VIEW)) (?P<database>\w+)\.(?P<table>\w+) (?P<query>.*)`
var re = regexp.MustCompile(phoneRENamedCaps)

var Host = "127.0.0.1"
var Port = 9000
var Filename = "/tmp/ch_export"
var Mode = "export"

func main() {
	flag.StringVar(&Mode, "mode", "export", "Action")
	flag.StringVar(&Filename, "filename", Filename, "Export file")
	flag.StringVar(&Host, "host", Host, "Hostname")
	flag.IntVar(&Port, "port", Port, "Hostname")
	flag.Parse()

	switch Mode {
	case "export":
		Export()
	case "dump":
		Dump()
	default:
		fmt.Println("choose mode")
	}
}

func Connect() (conn driver.Conn, err error) {
	conn, err = clickhouse.Open(&clickhouse.Options{
		Addr: []string{fmt.Sprintf("%s:%d", Host, Port)},
		Auth: clickhouse.Auth{
			Database: "default",
			Username: "default",
			Password: "",
		},
	})
	return
}

func GetDatabasesTables() (result Data, err error) {
	result = make(Data)
	c, err := Connect()
	if err != nil {
		fmt.Println(err)
	}
	rows, err := c.Query(context.Background(), fmt.Sprintf(dbquery, strings.Join(dbexclude, ",")))
	if err != nil {
		fmt.Println(err)
	}
	for rows.Next() {
		var dbname string
		var dbuuid string
		err = rows.Scan(&dbname, &dbuuid)
		var dbstatement = fmt.Sprintf("CREATE DATABASE %s;", dbname)
		if dbuuid == "00000000-0000-0000-0000-000000000000" {
			dbstatement = fmt.Sprintf("CREATE DATABASE %s;", dbname)
		}
		var db = Database{
			DDL:    dbstatement,
			Tables: make(map[string]string),
		}
		rowstable, err := c.Query(context.Background(), fmt.Sprintf("SHOW TABLES FROM %s;", dbname))
		if err != nil {
			fmt.Println(err)
		}
		for rowstable.Next() {
			var tablename string
			err = rowstable.Scan(&tablename)
			r, err := c.Query(context.Background(), fmt.Sprintf("SELECT create_table_query, uuid from system.tables where database='%s' and table='%s';", dbname, tablename))
			if err != nil {
				fmt.Println(err)
			}
			for r.Next() {
				var tablecreate string
				var tableuuid string
				err = r.Scan(&tablecreate, &tableuuid)
				matches := re.FindStringSubmatch(tablecreate)
				if len(matches) < 7 {
					fmt.Printf("error with table %s\n", tablename)
				}
				tablestatement := fmt.Sprintf("%s %s.%s UUID '%s' %s;", matches[1], matches[4], matches[5], tableuuid, matches[6])
				if tableuuid == "00000000-0000-0000-0000-000000000000" {
					tablestatement = fmt.Sprintf("%s %s.%s %s;", matches[1], matches[4], matches[5], matches[6])
				}
				db.Tables[tablename] = tablestatement
			}
		}
		result[dbname] = db
	}

	return
}

func Export() {
	res, err := GetDatabasesTables()

	f, err := os.Create(Filename)
	defer f.Close()
	if err != nil {
		fmt.Println(err)
	}
	data, err := json.MarshalIndent(res, " ", " ")
	if err != nil {
		fmt.Println(err)
	}
	f.Write(data)
}

func Dump() {
	f, err := os.Open(Filename)
	defer f.Close()
	if err != nil {
		fmt.Println(err)
	}
	dataraw, err := io.ReadAll(f)
	if err != nil {
		fmt.Println(err)
	}
	var data Data
	err = json.Unmarshal(dataraw, &data)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("-- Databases --")
	for d, dv := range data {
		fmt.Printf("%s %s\n\n", d, dv.DDL)
	}
	fmt.Println("")
	fmt.Println("-- Tables --")
	for _, dv := range data {
		for _, tv := range dv.Tables {
			fmt.Printf("%s\n\n", tv)
		}
	}
}

type Database struct {
	DDL    string            `json:"ddl"`
	Tables map[string]string `json:"tables"`
}
type Data map[string]Database