paulbsd/paulbsd-salt
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

updated kopia state

Browse Source
This commit is contained in:
Paul 2024-02-07 22:00:25 +01:00
parent 7f5b3f51a3
commit f02e2563ca
1 changed files with 71 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

138
states/kopia/templates/kopia_backup.py.j2
View File

@ -21,88 +21,92 @@ parser = argparse.ArgumentParser(prog='{{ name }} backup')
def init(mode="filesystem", path=None, bucket=None, prefix=None, gateway=None, region=None, ak=None, sak=None): def init(mode="filesystem", path=None, bucket=None, prefix=None, gateway=None, region=None, ak=None, sak=None):
if mode == "filesystem" and path: if mode == "filesystem" and path:
repo_connect = Popen(f"kopia repository connect filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}", shell=True, stdout=PIPE, stderr=PIPE) repo_connect = Popen(f"kopia repository connect filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}", shell=True, stdout=PIPE, stderr=PIPE)
repo_create = Popen(f"kopia repository create filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}" , shell=True, stdout=PIPE, stderr=PIPE) repo_create = Popen(f"kopia repository create filesystem --config-file={KOPIA_CONFIG_PATH} --path={path} -p {KOPIA_PASSWORD}" , shell=True, stdout=PIPE, stderr=PIPE)
elif mode == "s3" and prefix and gateway and region and ak and sak: elif mode == "s3" and prefix and gateway and region and ak and sak:
cmd_create = f"kopia repository create s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'" cmd_create = f"kopia repository create s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'"
cmd_connect = f"kopia repository connect s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'" cmd_connect = f"kopia repository connect s3 --config-file={KOPIA_CONFIG_PATH} --bucket={bucket} --prefix={prefix}/ --endpoint={gateway} --region={region} --access-key={ak} --secret-access-key={sak} -p {KOPIA_PASSWORD} --no-check-for-updates --description='{{ name }} repository'"
run_create = Popen(cmd_create, shell=True, stdout=PIPE, stderr=PIPE) run_create = Popen(cmd_create, shell=True, stdout=PIPE, stderr=PIPE)
run_create.wait() run_create.wait()
if run_create.returncode == 0: if run_create.returncode == 0:
print("successfully created repository") print("successfully created repository")
run_connect = Popen(cmd_connect, shell=True, stdout=PIPE, stderr=PIPE) run_connect = Popen(cmd_connect, shell=True, stdout=PIPE, stderr=PIPE)
run_connect.wait() run_connect.wait()
if run_connect.returncode == 0: if run_connect.returncode == 0:
print("successfully connected to repository") print("successfully connected to repository")
else: else:
print("no valid mode or missing informations") print("no valid mode or missing informations")
return None return None
return True return True
def set_policy(compression="zstd"): def set_policy(compression="zstd"):
cmd_policy = f"kopia policy set --global --config-file={KOPIA_CONFIG_PATH} --compression={compression} --keep-latest={{ params.keep_latest|default(7) }} --keep-hourly 0 --keep-daily {{ params.keep_daily|default(7) }} --keep-weekly {{ params.keep_weekly|default(4) }} --keep-monthly {{ params.keep_monthly|default(6) }} --keep-annual 0 --one-file-system=true" cmd_policy = f"kopia policy set --global --config-file={KOPIA_CONFIG_PATH} --compression={compression} --keep-latest={{ params.keep_latest|default(7) }} --keep-hourly 0 --keep-daily {{ params.keep_daily|default(7) }} --keep-weekly {{ params.keep_weekly|default(4) }} --keep-monthly {{ params.keep_monthly|default(6) }} --keep-annual 0 --one-file-system=true"
run_policy = Popen(cmd_policy, shell=True, stdout=PIPE, stderr=PIPE) run_policy = Popen(cmd_policy, shell=True, stdout=PIPE, stderr=PIPE)
run_policy.wait() run_policy.wait()
if run_policy.returncode == 0: if run_policy.returncode == 0:
print("successfully set repository policy") print("successfully set repository policy")
return run_policy.returncode == 0 return run_policy.returncode == 0
def set_cache(): def set_cache():
rcs = [] rcs = []
cmds_cache = [f"kopia cache set --content-cache-size-mb 1000 --config-file={KOPIA_CONFIG_PATH}", cmds_cache = [f"kopia cache set --content-cache-size-mb 1000 --config-file={KOPIA_CONFIG_PATH}",
f"kopia cache set --content-cache-size-limit-mb 2000 --config-file={KOPIA_CONFIG_PATH}"] f"kopia cache set --content-cache-size-limit-mb 2000 --config-file={KOPIA_CONFIG_PATH}"]
for cmd_cache in cmds_cache: for cmd_cache in cmds_cache:
run_cache = Popen(cmd_cache, shell=True, stdout=PIPE, stderr=PIPE) run_cache = Popen(cmd_cache, shell=True, stdout=PIPE, stderr=PIPE)
run_cache.wait() run_cache.wait()
rcs.append(run_cache.returncode == 0) rcs.append(run_cache.returncode == 0)
if all(rcs): if all(rcs):
print("successfully set cache policy") print("successfully set cache policy")
return all(rcs) return all(rcs)
def run(before_tasks=[]): def run(before_tasks=[]):
rcs = [] rcs = []
for cmd_task in before_tasks: for cmd_task in before_tasks:
run_task = Popen(cmd_task, shell=True) run_task = Popen(cmd_task, shell=True)
run_task.wait() run_task.wait()
rcs.append(run_task.returncode == 0) rcs.append(run_task.returncode == 0)
if not all(rcs): if not all(rcs):
return False return False
cmd_launch = f"kopia snapshot create {{ params.dirs|map(attribute='path')|join(' ') }} --config-file={KOPIA_CONFIG_PATH}" cmd_launch = f"kopia snapshot create {{ params.dirs|map(attribute='path')|join(' ') }} --config-file={KOPIA_CONFIG_PATH}"
run_launch = Popen(cmd_launch, shell=True) run_launch = Popen(cmd_launch, shell=True)
run_launch.wait() run_launch.wait()
return run_launch.returncode == 0 return run_launch.returncode == 0
def list_snapshots(): def list_snapshots():
cmd_list = f"kopia snapshot list --config-file={KOPIA_CONFIG_PATH}" cmd_list = f"kopia snapshot list --config-file={KOPIA_CONFIG_PATH}"
run_list = Popen(cmd_list, shell=True) run_list = Popen(cmd_list, shell=True)
run_list.wait() run_list.wait()
return run_list.returncode == 0 return run_list.returncode == 0
if __name__ == "__main__": if __name__ == "__main__":
parser.add_argument('action', nargs="?") parser.add_argument('action', nargs="?")
args = parser.parse_args() args = parser.parse_args()
res_init = init(mode="s3", bucket="kopia", prefix="{{ name }}", gateway="{{ kopia.repos[params.repo].gateway|default('gateway.storjshare.io') }}", region="{{ kopia.repos[params.repo].region|default('EU1') }}", ak="{{ kopia.repos[params.repo].ak }}", sak="{{ kopia.repos[params.repo].sak }}") res_init = init(mode="s3", bucket="kopia", prefix="{{ name }}", gateway="{{ kopia.repos[params.repo].gateway|default('gateway.storjshare.io') }}", region="{{ kopia.repos[params.repo].region|default('EU1') }}", ak="{{ kopia.repos[params.repo].ak }}", sak="{{ kopia.repos[params.repo].sak }}")
if not res_init: if not res_init:
print("init error") print("init error")
sys.exit(1) sys.exit(1)
if args.action == "run": if args.action == "run":
if not set_policy(): if not set_policy():
print("set policy error") print("set policy error")
sys.exit(1) sys.exit(1)
if not set_cache(): if not set_cache():
print("set cache error") print("set cache error")
sys.exit(1) sys.exit(1)
if not run(before_tasks={{ params.before_tasks|default([]) }}): if not run(before_tasks={{ params.before_tasks|default([]) }}):
print("run error") print("run error")
sys.exit(1) sys.exit(1)
else: elif args.action == "env":
list_snapshots() print();print("# source this to use kopia with this repo")
for k,v in ENV.items():
print(f"export {k}=\"{v}\"")
else:
list_snapshots()