From 1b59cdacdd0cee29a477624e4bf5d2777302aadd Mon Sep 17 00:00:00 2001 From: Paul Lecuq Date: Mon, 21 Feb 2022 13:52:46 +0100 Subject: [PATCH] * Update salt states - bl module updated - borg state updated - config state updated - netbox state updated - nginx state updated - misc updates --- states/_modules/bl.py | 2 +- states/borg/config.sls | 1 + states/borg/defaults.yaml | 1 + states/borg/install.sls | 5 ++++ states/borg/templates/borg_job.sh.j2 | 4 ++-- states/config/config.toml.j2 | 2 ++ states/config/config.yml.j2 | 2 ++ states/config/defaults.yaml | 1 + states/config/init.sls | 8 +++++++ states/dip/dip.service.j2 | 2 +- states/dkron/dkron.service.j2 | 2 +- states/g2g/install.sls | 6 +++++ states/gitea/defaults.yaml | 1 + states/go-aptproxy/go-aptproxy.service.j2 | 2 +- states/influxdb/influxdb.service.j2 | 1 + states/ipbl/ipbl.service.j2 | 2 +- states/ipfs/ipfs.service.j2 | 1 + states/misc/init.sls | 7 ------ states/netbox/defaults.yaml | 1 - states/netbox/install.sls | 10 ++++++++ states/netbox/templates/gunicorn.py.j2 | 2 +- states/netbox/templates/netbox.service.j2 | 3 +-- states/nginx/auth.sls | 17 +++++++------- states/nginx/map.jinja | 7 +++--- states/nginx/templates/nginx.conf.j2 | 28 +++++++++++++---------- states/pki/pki.service.j2 | 2 +- states/postgresql/install.sls | 1 + states/provision/init.sls | 8 ------- states/qrz/qrz.service.j2 | 2 +- states/syncthing/syncthing.service.j2 | 1 + states/telegraf/telegraf.conf.j2 | 2 +- states/telegraf/telegraf.service.j2 | 2 +- states/zsh/zprofile.j2 | 2 +- 33 files changed, 84 insertions(+), 54 deletions(-) create mode 100644 states/config/config.toml.j2 create mode 100644 states/config/config.yml.j2 diff --git a/states/_modules/bl.py b/states/_modules/bl.py index f247a00..2868d5f 100644 --- a/states/_modules/bl.py +++ b/states/_modules/bl.py @@ -6,7 +6,7 @@ from urllib.request import urlopen, Request def get_ips(url="https://ipbl.paulbsd.com"): """get_ips fetch ips blacklists from ipbl""" - fullurl = f"{url}/ips" + fullurl = f"{url}/ips/last" req = Request(method="GET", url=fullurl) res = urlopen(req) results = json.loads(res.read()) diff --git a/states/borg/config.sls b/states/borg/config.sls index 20da6fe..1299312 100644 --- a/states/borg/config.sls +++ b/states/borg/config.sls @@ -13,6 +13,7 @@ borg-config-script-{{ key }}: key: {{ key }} job: {{ job }} working_dir: {{ borg.working_dir }} + log_dir: {{ borg.log_dir }} {% endfor %} borg-config-sudo: diff --git a/states/borg/defaults.yaml b/states/borg/defaults.yaml index 1e4f5c9..0072e8e 100644 --- a/states/borg/defaults.yaml +++ b/states/borg/defaults.yaml @@ -3,6 +3,7 @@ borg: enabled: true run_user: dkron working_dir: /opt/borg + log_dir: /var/log/borg default_dirs: - logs - scripts diff --git a/states/borg/install.sls b/states/borg/install.sls index e54be34..4c9eb59 100644 --- a/states/borg/install.sls +++ b/states/borg/install.sls @@ -11,3 +11,8 @@ borg-working-dir-{{ dir }}: - name: {{ borg.working_dir }}/{{ dir }} - mode: 0700 {% endfor %} + +borg-log-dir: + file.directory: + - name: {{ borg.log_dir }} + - mode: 0700 diff --git a/states/borg/templates/borg_job.sh.j2 b/states/borg/templates/borg_job.sh.j2 index fd83a9c..386f949 100644 --- a/states/borg/templates/borg_job.sh.j2 +++ b/states/borg/templates/borg_job.sh.j2 @@ -50,7 +50,7 @@ borg create \ {%- for include in job.included_dirs %} {{ include }} \ {%- endfor %} - 2>> {{ working_dir }}/logs/${backup_name}-$(date +%Y-%m-%d).log + 2>> {{ log_dir }}/${backup_name}-$(date +%Y-%m-%d).log backup_exit=$? @@ -64,7 +64,7 @@ borg prune \ --keep-monthly {{ job.keep_monthly|default(6) }} \ info "Cleaning up logs" -find {{ working_dir }}/logs/${backup_name}-*.log -mtime +{{ job.keep_logs_days|default(7) }} -delete +find {{ log_dir }}/${backup_name}-*.log -mtime +{{ job.keep_logs_days|default(7) }} -delete info "Starting after tasks" {%- for task in job.after_tasks|default([]) %} diff --git a/states/config/config.toml.j2 b/states/config/config.toml.j2 new file mode 100644 index 0000000..05a28cc --- /dev/null +++ b/states/config/config.toml.j2 @@ -0,0 +1,2 @@ +{%- from "config/map.jinja" import config with context -%} +{{ config.config|toml }} diff --git a/states/config/config.yml.j2 b/states/config/config.yml.j2 new file mode 100644 index 0000000..4252bff --- /dev/null +++ b/states/config/config.yml.j2 @@ -0,0 +1,2 @@ +{%- from "config/map.jinja" import config with context -%} +{{ config.config|yaml(False) }} diff --git a/states/config/defaults.yaml b/states/config/defaults.yaml index c7f1d1e..d551135 100644 --- a/states/config/defaults.yaml +++ b/states/config/defaults.yaml @@ -4,4 +4,5 @@ config: install_dir: "/usr/local/apps/config" ini_filename: "common.ini" json_filename: "common.json" + yaml_filename: "common.yml" config: {} diff --git a/states/config/init.sls b/states/config/init.sls index 36760ea..6bf2fc3 100644 --- a/states/config/init.sls +++ b/states/config/init.sls @@ -25,3 +25,11 @@ config-json-content: - user: root - mode: "0755" - template: jinja + +config-yaml-content: + file.managed: + - name: {{ config.install_dir }}/{{ config.yaml_filename }} + - source: salt://config/config.yml.j2 + - user: root + - mode: "0755" + - template: jinja diff --git a/states/dip/dip.service.j2 b/states/dip/dip.service.j2 index 1a5f1b2..f5f6c67 100644 --- a/states/dip/dip.service.j2 +++ b/states/dip/dip.service.j2 @@ -1,5 +1,5 @@ +{%- from "dip/map.jinja" import dip with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "dip/map.jinja" import dip with context %} [Unit] Description=DIP After=network.target diff --git a/states/dkron/dkron.service.j2 b/states/dkron/dkron.service.j2 index b7d9181..f8ae126 100644 --- a/states/dkron/dkron.service.j2 +++ b/states/dkron/dkron.service.j2 @@ -1,5 +1,5 @@ +{%- from "dkron/map.jinja" import dkron with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "dkron/map.jinja" import dkron with context %} [Unit] Description=dkron - Open Source task scheduler Documentation=https://dkron.io/ diff --git a/states/g2g/install.sls b/states/g2g/install.sls index 3b1db504..dd4e52f 100644 --- a/states/g2g/install.sls +++ b/states/g2g/install.sls @@ -14,3 +14,9 @@ g2g-binary-symlink: - target: {{ g2g.release_dir }}/g2g-{{ g2g.version }} - require: - archive: g2g-archive-extract + +g2g-cleanup: + software.cleanup: + - name: g2g + - path: {{ g2g.release_dir }} + - version: "{{ g2g.version }}" diff --git a/states/gitea/defaults.yaml b/states/gitea/defaults.yaml index e25c80f..c96698f 100644 --- a/states/gitea/defaults.yaml +++ b/states/gitea/defaults.yaml @@ -77,6 +77,7 @@ gitea: cookie_secure: "true" cookie_name: i_like_gitea log: + root_path: /var/log/gitea mode: file level: Info security: diff --git a/states/go-aptproxy/go-aptproxy.service.j2 b/states/go-aptproxy/go-aptproxy.service.j2 index b37cbed..812921c 100644 --- a/states/go-aptproxy/go-aptproxy.service.j2 +++ b/states/go-aptproxy/go-aptproxy.service.j2 @@ -1,5 +1,5 @@ +{%- from "go-aptproxy/map.jinja" import goaptproxy with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "go-aptproxy/map.jinja" import goaptproxy with context %} [Unit] Description=go-aptproxy Documentation=https://git.paulbsd.com/paulbsd/go-aptproxy diff --git a/states/influxdb/influxdb.service.j2 b/states/influxdb/influxdb.service.j2 index f022a21..0258cd0 100644 --- a/states/influxdb/influxdb.service.j2 +++ b/states/influxdb/influxdb.service.j2 @@ -1,4 +1,5 @@ {%- from "influxdb/map.jinja" import influxdb with context -%} +## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} [Unit] Description=InfluxDB is an open-source, distributed, time series database Documentation=https://docs.influxdata.com/influxdb/ diff --git a/states/ipbl/ipbl.service.j2 b/states/ipbl/ipbl.service.j2 index 799510f..8d7ae51 100644 --- a/states/ipbl/ipbl.service.j2 +++ b/states/ipbl/ipbl.service.j2 @@ -1,5 +1,5 @@ +{%- from "ipbl/map.jinja" import ipbl with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "ipbl/map.jinja" import ipbl with context %} [Unit] Description=ipbl After=network.target postgresql.service diff --git a/states/ipfs/ipfs.service.j2 b/states/ipfs/ipfs.service.j2 index afa5ba3..7495b37 100644 --- a/states/ipfs/ipfs.service.j2 +++ b/states/ipfs/ipfs.service.j2 @@ -1,3 +1,4 @@ +## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} [Unit] Description=ipfs p2p daemon After=network.target diff --git a/states/misc/init.sls b/states/misc/init.sls index 3a6deb6..fd4b994 100644 --- a/states/misc/init.sls +++ b/states/misc/init.sls @@ -16,10 +16,3 @@ misc-public-dir: - group: root - mode: 644 {%- endfor %} - -misc-files-delete: - file.absent: - - names: -{%- for file in salt['pillar.get']('misc_files_delete') %} - - {{ file }} -{%- endfor %} diff --git a/states/netbox/defaults.yaml b/states/netbox/defaults.yaml index eaa3569..0810766 100644 --- a/states/netbox/defaults.yaml +++ b/states/netbox/defaults.yaml @@ -58,7 +58,6 @@ netbox: banner_bottom: "" banner_login: "NetBox" base_path: "" - cache_timeout: 900 changelog_retention: 90 cors_origin_allow_all: false cors_origin_whitelist: [] diff --git a/states/netbox/install.sls b/states/netbox/install.sls index ec9df55..b8a8f86 100644 --- a/states/netbox/install.sls +++ b/states/netbox/install.sls @@ -43,6 +43,8 @@ netbox-gunicorn-execfile: - group: root - mode: 0644 - template: jinja + - watch_in: + - service: netbox-service netbox-config-file: file.managed: @@ -52,6 +54,8 @@ netbox-config-file: - group: root - mode: 0644 - template: jinja + - watch_in: + - service: netbox-service netbox-migration: module.run: @@ -70,3 +74,9 @@ netbox-install-static-files: - bin_env: {{ netbox.install_dir }}/netbox - require: - virtualenv: netbox-virtualenv + +netbox-cleanup: + software.cleanup: + - name: netbox + - path: {{ netbox.release_dir }} + - version: "{{ netbox.version }}" diff --git a/states/netbox/templates/gunicorn.py.j2 b/states/netbox/templates/gunicorn.py.j2 index b93cd7f..f4f1fe4 100644 --- a/states/netbox/templates/gunicorn.py.j2 +++ b/states/netbox/templates/gunicorn.py.j2 @@ -1,6 +1,6 @@ +{%- from "netbox/map.jinja" import netbox with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "netbox/map.jinja" import netbox with context -%} command = '{{ netbox.install_dir }}/netbox/bin/gunicorn' pythonpath = '{{ netbox.install_dir }}/netbox/netbox' bind = '{{ netbox.gunicorn.bind_addr }}:{{ netbox.gunicorn.bind_port }}' diff --git a/states/netbox/templates/netbox.service.j2 b/states/netbox/templates/netbox.service.j2 index aa44253..46e4225 100644 --- a/states/netbox/templates/netbox.service.j2 +++ b/states/netbox/templates/netbox.service.j2 @@ -1,6 +1,5 @@ -## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} - {%- from "netbox/map.jinja" import netbox with context -%} +## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} [Unit] Description=Netbox After=network.target postgresql.service redis.service diff --git a/states/nginx/auth.sls b/states/nginx/auth.sls index 271ee16..6968771 100644 --- a/states/nginx/auth.sls +++ b/states/nginx/auth.sls @@ -1,23 +1,24 @@ --- {%- from "nginx/map.jinja" import nginx with context %} +{%- from "nginx/map.jinja" import users with context %} -{%- for user in salt['pillar.get']('htpasswds') %} -{%- if user.state == 'present' %} +{%- for key, value in users.items() %} +{%- if value.state == 'present' %} -nginx-auth-present-{{ user.name }}: +nginx-auth-present-{{ key }}: webutil.user_exists: - - name: {{ user.name }} -{%- if user.password is defined %} - - password: {{ user.password }} + - name: {{ key }} +{%- if value.password is defined %} + - password: {{ value.password }} {%- endif %} - htpasswd_file: {{ nginx.config.dir }}/auth/htpasswd - require: - file: nginx-config-dir-auth {%- else %} -nginx-auth-absent-{{ user.name }}: +nginx-auth-absent-{{ key }}: webutil.user_absent: - - name: {{ user.name }} + - name: {{ key }} - htpasswd_file: {{ nginx.config.dir }}/auth/htpasswd - require: - file: nginx-config-dir-auth diff --git a/states/nginx/map.jinja b/states/nginx/map.jinja index 85bd16d..f5b1159 100644 --- a/states/nginx/map.jinja +++ b/states/nginx/map.jinja @@ -1,5 +1,6 @@ -{%- import_yaml "nginx/defaults.yaml" as defaults %} +{%- import_yaml "nginx/defaults.yaml" as defaults -%} -{%- set nginx = salt['pillar.get']('nginx', default=defaults.nginx, merge=True) %} +{%- set nginx = salt['pillar.get']('nginx', default=defaults.nginx, merge=True) -%} -{%- set net = salt['pillar.get']('net') %} \ No newline at end of file +{%- set users = salt['pillar.get']('htpasswds', merge=True) -%} +{%- set net = salt['pillar.get']('net') -%} \ No newline at end of file diff --git a/states/nginx/templates/nginx.conf.j2 b/states/nginx/templates/nginx.conf.j2 index c9634bf..39c0dd4 100644 --- a/states/nginx/templates/nginx.conf.j2 +++ b/states/nginx/templates/nginx.conf.j2 @@ -12,21 +12,25 @@ events { } http { - include fastcgi_params; - include proxy_params; - include mime.types; - include ssl_params; - charset utf-8; + include fastcgi_params; + include proxy_params; + include mime.types; + include ssl_params; + charset utf-8; - include sites-enabled/*; + include sites-enabled/*; - access_log /var/log/nginx/$host.access.log; - access_log syslog:server=localhost:514 combined; + log_format main '$remote_addr - $remote_user [$time_iso8601] ' + '"$request" $status $body_bytes_sent ' + '"$http_referer" "$http_user_agent"'; - default_type application/octet-stream; - sendfile on; - keepalive_timeout 60; - server_tokens off; + access_log /var/log/nginx/$host.access.log main; + access_log syslog:server=localhost:514 main; + + default_type application/octet-stream; + sendfile on; + keepalive_timeout 60; + server_tokens off; proxy_intercept_errors on; fastcgi_intercept_errors on; diff --git a/states/pki/pki.service.j2 b/states/pki/pki.service.j2 index 5a398d6..2896deb 100644 --- a/states/pki/pki.service.j2 +++ b/states/pki/pki.service.j2 @@ -1,5 +1,5 @@ +{%- from "pki/map.jinja" import pki with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "pki/map.jinja" import pki with context %} [Unit] Description=PKI Let's encrypt database After=network.target diff --git a/states/postgresql/install.sls b/states/postgresql/install.sls index a993c89..3b3de99 100644 --- a/states/postgresql/install.sls +++ b/states/postgresql/install.sls @@ -4,5 +4,6 @@ postgresql-install: - pkgs: - postgresql - postgresql-client + - postgresql-13-cron - postgresql-13-repmgr - repmgr diff --git a/states/provision/init.sls b/states/provision/init.sls index 514002f..4477440 100644 --- a/states/provision/init.sls +++ b/states/provision/init.sls @@ -14,14 +14,6 @@ schedule_saltutil_clear_cache: - days: 7 - run_on_start: false -schedule_pkg_refresh: - schedule.absent: - - name: schedule_pkg_refresh - -schedule_pkg_upgrade: - schedule.absent: - - name: schedule_pkg_upgrade - {%- if specs.default.items()|length > 0 and salt['grains.get']('id') in specs.keys() %} {%- for key, value in specs.default.items() %} {{ key }}: diff --git a/states/qrz/qrz.service.j2 b/states/qrz/qrz.service.j2 index b3534c2..dcc484f 100644 --- a/states/qrz/qrz.service.j2 +++ b/states/qrz/qrz.service.j2 @@ -1,5 +1,5 @@ +{%- from "qrz/map.jinja" import qrz with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "qrz/map.jinja" import qrz with context %} [Unit] Description=QRZ After=network.target postgresql.service diff --git a/states/syncthing/syncthing.service.j2 b/states/syncthing/syncthing.service.j2 index aeb8be7..f4f37b6 100644 --- a/states/syncthing/syncthing.service.j2 +++ b/states/syncthing/syncthing.service.j2 @@ -1,4 +1,5 @@ {%- from "syncthing/map.jinja" import syncthing with context -%} +## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} [Unit] Description=Syncthing - Open Source Continuous File Synchronization for %I Documentation=man:syncthing(1) diff --git a/states/telegraf/telegraf.conf.j2 b/states/telegraf/telegraf.conf.j2 index 3a777f4..1dc1f6a 100644 --- a/states/telegraf/telegraf.conf.j2 +++ b/states/telegraf/telegraf.conf.j2 @@ -7,7 +7,7 @@ metric_batch_size = 1000 metric_buffer_limit = 10000 collection_jitter = "0s" - flush_interval = "10s" + flush_interval = "30s" flush_jitter = "0s" precision = "" hostname = "{{ salt['grains.get']('fqdn') }}" diff --git a/states/telegraf/telegraf.service.j2 b/states/telegraf/telegraf.service.j2 index 8405500..431cebe 100644 --- a/states/telegraf/telegraf.service.j2 +++ b/states/telegraf/telegraf.service.j2 @@ -1,5 +1,5 @@ +{%- from "telegraf/map.jinja" import telegraf with context -%} ## {{ salt['pillar.get']('salt_managed', default='Salt Managed') }} -{%- from "telegraf/map.jinja" import telegraf with context %} [Unit] Description=The plugin-driven server agent for reporting metrics into InfluxDB Documentation=https://github.com/influxdata/telegraf diff --git a/states/zsh/zprofile.j2 b/states/zsh/zprofile.j2 index d02cd1f..2153280 100644 --- a/states/zsh/zprofile.j2 +++ b/states/zsh/zprofile.j2 @@ -7,7 +7,7 @@ export JAVA_OPTS='-XX:+IgnoreUnrecognizedVMOptions --add-modules java.se.ee' {% if zsh.config.golang -%} # Golang settings -if [[ -d /usr/local/apps/go ]] +if [[ -d /usr/local/apps/golang ]] then export GOPATH=~/go export GOROOT=/usr/local/apps/golang