From f02e2563ca0f1d097723b80dfb1a33312fe483ef Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Wed, 7 Feb 2024 22:00:25 +0100 Subject: [PATCH] updated kopia state --- states/kopia/templates/kopia_backup.py.j2 | 138 +++++++++++----------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/states/kopia/templates/kopia_backup.py.j2 b/states/kopia/templates/kopia_backup.py.j2 index 4f0d0ee..94d8ba8 100644 --- a/states/kopia/templates/kopia_backup.py.j2 +++ b/states/kopia/templates/kopia_backup.py.j2 @@ -21,88 +21,92 @@ parser = argparse.ArgumentParser(prog='{{ name }} backup') def init(mode="filesystem", path=None, bucket=None, prefix=None, gateway=None, region=None, ak=None, sak=None): - if mode == "filesystem" and path: - repo_connect = Popen(f"kopia repository connect filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}", shell=True, stdout=PIPE, stderr=PIPE) - repo_create = Popen(f"kopia repository create filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}" , shell=True, stdout=PIPE, stderr=PIPE) - elif mode == "s3" and prefix and gateway and region and ak and sak: - cmd_create = f"kopia repository create s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'" - cmd_connect = f"kopia repository connect s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'" - run_create = Popen(cmd_create, shell=True, stdout=PIPE, stderr=PIPE) - run_create.wait() - if run_create.returncode == 0: - print("successfully created repository") - run_connect = Popen(cmd_connect, shell=True, stdout=PIPE, stderr=PIPE) - run_connect.wait() - if run_connect.returncode == 0: - print("successfully connected to repository") - else: - print("no valid mode or missing informations") - return None - return True + if mode == "filesystem" and path: + repo_connect = Popen(f"kopia repository connect filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}", shell=True, stdout=PIPE, stderr=PIPE) + repo_create = Popen(f"kopia repository create filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}" , shell=True, stdout=PIPE, stderr=PIPE) + elif mode == "s3" and prefix and gateway and region and ak and sak: + cmd_create = f"kopia repository create s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'" + cmd_connect = f"kopia repository connect s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'" + run_create = Popen(cmd_create, shell=True, stdout=PIPE, stderr=PIPE) + run_create.wait() + if run_create.returncode == 0: + print("successfully created repository") + run_connect = Popen(cmd_connect, shell=True, stdout=PIPE, stderr=PIPE) + run_connect.wait() + if run_connect.returncode == 0: + print("successfully connected to repository") + else: + print("no valid mode or missing informations") + return None + return True def set_policy(compression="zstd"): - cmd_policy = f"kopia policy set --global --config-file={KOPIA_CONFIG_PATH} --compression={compression} --keep-latest={{ params.keep_latest|default(7) }} --keep-hourly 0 --keep-daily {{ params.keep_daily|default(7) }} --keep-weekly {{ params.keep_weekly|default(4) }} --keep-monthly {{ params.keep_monthly|default(6) }} --keep-annual 0 --one-file-system=true" - run_policy = Popen(cmd_policy, shell=True, stdout=PIPE, stderr=PIPE) - run_policy.wait() - if run_policy.returncode == 0: - print("successfully set repository policy") - return run_policy.returncode == 0 + cmd_policy = f"kopia policy set --global --config-file={KOPIA_CONFIG_PATH} --compression={compression} --keep-latest={{ params.keep_latest|default(7) }} --keep-hourly 0 --keep-daily {{ params.keep_daily|default(7) }} --keep-weekly {{ params.keep_weekly|default(4) }} --keep-monthly {{ params.keep_monthly|default(6) }} --keep-annual 0 --one-file-system=true" + run_policy = Popen(cmd_policy, shell=True, stdout=PIPE, stderr=PIPE) + run_policy.wait() + if run_policy.returncode == 0: + print("successfully set repository policy") + return run_policy.returncode == 0 def set_cache(): - rcs = [] - cmds_cache = [f"kopia cache set --content-cache-size-mb 1000 --config-file={KOPIA_CONFIG_PATH}", - f"kopia cache set --content-cache-size-limit-mb 2000 --config-file={KOPIA_CONFIG_PATH}"] - for cmd_cache in cmds_cache: - run_cache = Popen(cmd_cache, shell=True, stdout=PIPE, stderr=PIPE) - run_cache.wait() - rcs.append(run_cache.returncode == 0) - if all(rcs): - print("successfully set cache policy") - return all(rcs) + rcs = [] + cmds_cache = [f"kopia cache set --content-cache-size-mb 1000 --config-file={KOPIA_CONFIG_PATH}", + f"kopia cache set --content-cache-size-limit-mb 2000 --config-file={KOPIA_CONFIG_PATH}"] + for cmd_cache in cmds_cache: + run_cache = Popen(cmd_cache, shell=True, stdout=PIPE, stderr=PIPE) + run_cache.wait() + rcs.append(run_cache.returncode == 0) + if all(rcs): + print("successfully set cache policy") + return all(rcs) def run(before_tasks=[]): - rcs = [] - for cmd_task in before_tasks: - run_task = Popen(cmd_task, shell=True) - run_task.wait() - rcs.append(run_task.returncode == 0) - if not all(rcs): - return False + rcs = [] + for cmd_task in before_tasks: + run_task = Popen(cmd_task, shell=True) + run_task.wait() + rcs.append(run_task.returncode == 0) + if not all(rcs): + return False - cmd_launch = f"kopia snapshot create {{ params.dirs|map(attribute='path')|join(' ') }} --config-file={KOPIA_CONFIG_PATH}" - run_launch = Popen(cmd_launch, shell=True) - run_launch.wait() - return run_launch.returncode == 0 + cmd_launch = f"kopia snapshot create {{ params.dirs|map(attribute='path')|join(' ') }} --config-file={KOPIA_CONFIG_PATH}" + run_launch = Popen(cmd_launch, shell=True) + run_launch.wait() + return run_launch.returncode == 0 def list_snapshots(): - cmd_list = f"kopia snapshot list --config-file={KOPIA_CONFIG_PATH}" - run_list = Popen(cmd_list, shell=True) - run_list.wait() - return run_list.returncode == 0 + cmd_list = f"kopia snapshot list --config-file={KOPIA_CONFIG_PATH}" + run_list = Popen(cmd_list, shell=True) + run_list.wait() + return run_list.returncode == 0 if __name__ == "__main__": - parser.add_argument('action', nargs="?") - args = parser.parse_args() + parser.add_argument('action', nargs="?") + args = parser.parse_args() - res_init = init(mode="s3", bucket="kopia", prefix="{{ name }}", gateway="{{ kopia.repos[params.repo].gateway|default('gateway.storjshare.io') }}", region="{{ kopia.repos[params.repo].region|default('EU1') }}", ak="{{ kopia.repos[params.repo].ak }}", sak="{{ kopia.repos[params.repo].sak }}") - if not res_init: - print("init error") - sys.exit(1) + res_init = init(mode="s3", bucket="kopia", prefix="{{ name }}", gateway="{{ kopia.repos[params.repo].gateway|default('gateway.storjshare.io') }}", region="{{ kopia.repos[params.repo].region|default('EU1') }}", ak="{{ kopia.repos[params.repo].ak }}", sak="{{ kopia.repos[params.repo].sak }}") + if not res_init: + print("init error") + sys.exit(1) - if args.action == "run": - if not set_policy(): - print("set policy error") - sys.exit(1) - if not set_cache(): - print("set cache error") - sys.exit(1) - if not run(before_tasks={{ params.before_tasks|default([]) }}): - print("run error") - sys.exit(1) - else: - list_snapshots() + if args.action == "run": + if not set_policy(): + print("set policy error") + sys.exit(1) + if not set_cache(): + print("set cache error") + sys.exit(1) + if not run(before_tasks={{ params.before_tasks|default([]) }}): + print("run error") + sys.exit(1) + elif args.action == "env": + print();print("# source this to use kopia with this repo") + for k,v in ENV.items(): + print(f"export {k}=\"{v}\"") + else: + list_snapshots()