Merge pull request #5 from car-da/scheduling
aded scheduling options for backup client in rsync engine
diff --git a/.kitchen.yml b/.kitchen.yml
index 3fb19aa..050f800 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -36,7 +36,7 @@
platforms:
- name: <%=ENV['PLATFORM'] || 'saltstack-ubuntu-xenial-salt-stable' %>
driver_config:
- image: <%=ENV['PLATFORM'] || 'epcim/salt-formulas:saltstack-ubuntu-xenial-salt-stable'%>
+ image: <%=ENV['PLATFORM'] || 'epcim/salt:saltstack-ubuntu-xenial-salt-stable'%>
platform: ubuntu
suites:
@@ -51,11 +51,21 @@
pillars-from-files:
backupninja.sls: tests/pillar/client_rsync.sls
+ - name: client_rsync_backup_times
+ provisioner:
+ pillars-from-files:
+ backupninja.sls: tests/pillar/client_rsync_backup_times.sls
+
- name: client_s3
provisioner:
pillars-from-files:
backupninja.sls: tests/pillar/client_s3.sls
+ - name: client_s3_disabled_auto
+ provisioner:
+ pillars-from-files:
+ backupninja.sls: tests/pillar/client_s3_disabled_auto.sls
+
- name: client_webdav
provisioner:
pillars-from-files:
diff --git a/.travis.yml b/.travis.yml
index 0629a98..3456794 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -17,18 +17,32 @@
- bundle install
env:
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=server-rdiff
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=client-rsync
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=client-s3
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2016.3 SUITE=client-webdav
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=server-rdiff
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-rsync
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-s3
- - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-webdav
-# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=server-rdiff
-# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=client-rsync
-# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=client-s3
-# - PLATFORM=epcim/salt-formulas:saltstack-ubuntu-bionic-salt-2017.7 SUITE=client-webdav
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2016.3 SUITE=server-rdiff
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2016.3 SUITE=client-rsync
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2016.3 SUITE=client-s3
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2016.3 SUITE=client-webdav
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 SUITE=server-rdiff
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-rsync
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-rsync-backup-times
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-s3
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-s3-disabled-auto
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2017.7 SUITE=client-webdav
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 SUITE=server-rdiff
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 SUITE=client-rsync
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 SUITE=client-rsync-backup-times
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 SUITE=client-s3
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 SUITE=client-s3-disabled-auto
+ - PLATFORM=epcim/salt:saltstack-ubuntu-xenial-salt-2018.3 SUITE=client-webdav
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 SUITE=server-rdiff
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 SUITE=client-rsync
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 SUITE=client-s3
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2017.7 SUITE=client-webdav
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2018.3 SUITE=server-rdiff
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2018.3 SUITE=client-rsync
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2018.3 SUITE=client-rsync-backup-times
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2018.3 SUITE=client-s3
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2018.3 SUITE=client-s3-disabled-auto
+# - PLATFORM=epcim/salt:saltstack-ubuntu-bionic-salt-2018.3 SUITE=client-webdav
before_script:
- set -o pipefail
diff --git a/README.rst b/README.rst
index d60e1bd..987e654 100644
--- a/README.rst
+++ b/README.rst
@@ -30,6 +30,20 @@
host: 10.10.10.208
user: backupninja
+Backup client with ssh/rsync remote target with specific rsync options
+
+.. code-block:: yaml
+
+ backupninja:
+ client:
+ enabled: true
+ target:
+ engine: rsync
+ engine_opts: "-av --delete --recursive --safe-links"
+ home_dir: /srv/volumes/backup/backupninja
+ host: 10.10.10.208
+ user: backupninja
+
Backup client with s3 remote target
.. code-block:: yaml
@@ -59,6 +73,43 @@
principal: host/${linux:network:fqdn}
keytab: /etc/krb5.keytab
+Backup client with exact backup times
+
+.. code-block:: yaml
+
+ backupninja:
+ client:
+ enabled: true
+ auto_backup_disabled: false
+ backup_times:
+ day_of_week: 1
+ hour: 2
+ minute: 32
+
+.. code-block:: yaml
+
+ backupninja:
+ client:
+ enabled: true
+ auto_backup_disabled: false
+ backup_times:
+ day_of_month: 24
+ hour: 14
+ minute: 12
+
+Backup client with defined number of backups to keep
+
+..code-block:: yaml
+
+ backupninja:
+ client:
+ enabled: true
+ target:
+ engine: rsync
+ host: 10.10.10.208
+ user: backupninja
+ keep: 4
+
Backup server rsync/rdiff
.. code-block:: yaml
@@ -72,6 +123,13 @@
enabled: true
key: ssh-key
+Backup server without strict client policy restriction
+
+.. code-block:: yaml
+
+ backupninja:
+ server:
+ restrict_clients: false
Backup client with local storage
diff --git a/backupninja/client.sls b/backupninja/client.sls
index fd0f0bd..8b61b8f 100644
--- a/backupninja/client.sls
+++ b/backupninja/client.sls
@@ -20,6 +20,36 @@
- user: root
- group: root
+{%- if client.backup_times is defined %}
+
+delete_cron_file:
+ file.absent:
+ - name: /etc/cron.d/backupninja
+ - require:
+ - pkg: backupninja_packages
+
+create_cron_job:
+ cron.present:
+ - name: if [ -x /usr/sbin/backupninja ]; then /usr/sbin/backupninja --run; fi
+ - user: root
+ {%- if client.backup_times.day_of_week is defined %}
+ - dayweek: {{ client.backup_times.day_of_week }}
+ {%- endif %}
+ {%- if client.backup_times.day_of_month is defined %}
+ - daymonth: {{ client.backup_times.day_of_month }}
+ {%- endif %}
+ {%- if client.backup_times.hour is defined %}
+ - hour: {{ client.backup_times.hour }}
+ {%- endif %}
+ {%- if client.backup_times.minute is defined %}
+ - minute: {{ client.backup_times.minute }}
+ {%- endif %}
+ {%- if client.get('auto_backup_disable', False) %}
+ - commented: True
+ {%- endif %}
+
+{%- endif %}
+
{%- if pillar.postgresql is defined or pillar.maas is defined %}
backupninja_postgresql_handler:
file.managed:
diff --git a/backupninja/files/authorized_keys b/backupninja/files/authorized_keys
new file mode 100644
index 0000000..6f54c9b
--- /dev/null
+++ b/backupninja/files/authorized_keys
@@ -0,0 +1,19 @@
+{%- from "backupninja/map.jinja" import server with context %}
+{%- for key_name, key in server.key.iteritems() %}
+{%- if key.get('enabled', False) %}
+{%- set clients = [] %}
+{%- if server.restrict_clients %}
+ {%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
+ {%- if node_grains.get('backupninja', {}).get('client') %}
+ {%- set client = node_grains.backupninja.get("client") %}
+ {%- if client.get('addresses') and client.get('addresses', []) is iterable %}
+ {%- for address in client.addresses %}
+ {%- do clients.append(address|string) %}
+ {%- endfor %}
+ {%- endif %}
+ {%- endif %}
+ {%- endfor %}
+{%- endif %}
+no-pty{%- if clients %},from="{{ clients|join(',') }}"{%- endif %} {{ key.key }}
+{%- endif %}
+{%- endfor %}
\ No newline at end of file
diff --git a/backupninja/files/rdiff.conf b/backupninja/files/rdiff.conf
index 91d3c7c..a2f49dd 100644
--- a/backupninja/files/rdiff.conf
+++ b/backupninja/files/rdiff.conf
@@ -29,6 +29,9 @@
##
## Default:
# options =
+{% if client.target.engine_opts is defined %}
+options = {{ client.target.engine_opts }}
+{%- endif %}
## default is 0, but set to 19 if you want to lower the priority.
## an example setting would be:
diff --git a/backupninja/files/rsync.conf b/backupninja/files/rsync.conf
index 91c02e4..e045ab3 100644
--- a/backupninja/files/rsync.conf
+++ b/backupninja/files/rsync.conf
@@ -35,7 +35,7 @@
# backup partition mountpoint or backup main folder
# this doesn't need to be a real partition, but should be at least the
# main folder where the backup is being stored
-mountpoint = /srv/backupninja
+mountpoint = {{ client.target.get('home_dir', '/srv/backupninja') }}
# folder relative do mountpoint where the backup should be stored
backupdir = {{ system.name }}.{{ system.domain }}
@@ -71,7 +71,7 @@
# that with the days config your backup gets a .0 suffix at the destination
# folder, making it easier to turn it later to an incremental backup.
#
-days = 7
+days = {{ client.target.keep|default(7) }}
# for long storage format, specify the number of daily backup increments
#keepdaily = 7
@@ -139,7 +139,11 @@
#rsync = $RSYNC
# rsync command options
+{% if client.target.engine_opts is defined %}
+rsync_options = {{ client.target.engine_opts }}
+{%- else %}
#rsync_options = "-av --delete --recursive"
+{%- endif %}
# when set to 1, use numeric ids instead of user/group mappings on rsync
#numericids =
diff --git a/backupninja/files/rsync.sh b/backupninja/files/rsync.sh
index 354e451..c3d8241 100644
--- a/backupninja/files/rsync.sh
+++ b/backupninja/files/rsync.sh
@@ -259,7 +259,7 @@
fi
ssh_cmd_base="ssh -T -o PasswordAuthentication=no -p $port -i $id_file"
- ssh_cmd="$ssh_cmd_base $user@$host"
+ ssh_cmd="$ssh_cmd_base $user@$host bash"
if [ "$from" == "remote" ] || [ "$dest" == "remote" ]; then
if [ "$testconnect" == "yes" ] && [ "$protocol" == "ssh" ]; then
diff --git a/backupninja/map.jinja b/backupninja/map.jinja
index 2a5db08..048d43c 100644
--- a/backupninja/map.jinja
+++ b/backupninja/map.jinja
@@ -20,11 +20,13 @@
'Debian': {
'pkgs': ['rsync'],
'home_dir': '/srv/backupninja',
+ 'restrict_clients': True,
'keys': [],
},
'RedHat': {
'pkgs': ['rsync'],
'home_dir': '/srv/backupninja',
+ 'restrict_clients': True,
'keys': [],
},
}, merge=salt['pillar.get']('backupninja:server')) %}
diff --git a/backupninja/meta/salt.yml b/backupninja/meta/salt.yml
index 41080b9..d7f60d4 100644
--- a/backupninja/meta/salt.yml
+++ b/backupninja/meta/salt.yml
@@ -3,7 +3,22 @@
{%- if pillar.get('backupninja', {}).get('client') %}
{%- from "backupninja/map.jinja" import service_grains with context %}
{{ service_grains|yaml(False)|indent(4) }}
+ {#- client addresses #}
+ {%- set addresses = [] %}
+ {%- set ips = salt['grains.get']("fqdn_ip4")|list %}
+ {%- if ips %}
+ {%- for ip in ips %}
+ {%- if not (ip|string).startswith('127.') %}
+ {%- do addresses.append(ip) %}
+ {%- endif %}
+ {%- endfor %}
+ {%- endif %}
+ {%- if addresses %}
+ client:
+ addresses: {{ addresses|yaml }}
+ {%- endif %}
{%- else %}
backupninja:
backup: {}
{%- endif %}
+
diff --git a/backupninja/server.sls b/backupninja/server.sls
index 4be4fc4..58bf986 100644
--- a/backupninja/server.sls
+++ b/backupninja/server.sls
@@ -9,9 +9,9 @@
user.present:
- name: backupninja
- system: true
- - home: /srv/backupninja
+ - home: {{ server.home_dir }}
-/srv/backupninja:
+{{ server.home_dir }}:
file.directory:
- mode: 700
- user: backupninja
@@ -21,27 +21,30 @@
- user: backupninja_user
- pkg: backupninja_server_packages
-{%- for key_name, key in server.key.iteritems() %}
-
-{%- if key.get('enabled', False) %}
-
-backupninja_key_{{ key.key }}:
- ssh_auth.present:
+{{ server.home_dir }}/.ssh:
+ file.directory:
+ - mode: 700
- user: backupninja
- - name: {{ key.key }}
+ - group: backupninja
- require:
- - file: /srv/backupninja
+ - file: {{ server.home_dir }}
-{%- endif %}
-
-{%- endfor %}
+{{ server.home_dir }}/.ssh/authorized_keys:
+ file.managed:
+ - user: backupninja
+ - group: backupninja
+ - template: jinja
+ - source: salt://backupninja/files/authorized_keys
+ - require:
+ - file: {{ server.home_dir }}
+ - file: {{ server.home_dir }}/.ssh
{%- for node_name, node_grains in salt['mine.get']('*', 'grains.items').iteritems() %}
{%- for backup_name, backup in node_grains.get('backupninja', {}).get('backup', {}).iteritems() %}
{%- for fs_include in backup.fs_includes %}
-/srv/backupninja/{{ node_name }}{{ fs_include }}:
+{{ server.home_dir }}/{{ node_name }}{{ fs_include }}:
file.directory:
- mode: 700
- user: backupninja
@@ -56,4 +59,4 @@
{%- endfor %}
-{%- endif %}
\ No newline at end of file
+{%- endif %}
diff --git a/metadata.yml b/metadata.yml
index 39bc145..d527801 100644
--- a/metadata.yml
+++ b/metadata.yml
@@ -1,3 +1,3 @@
name: "backupninja"
version: "0.2"
-source: "https://github.com/tcpcloud/salt-formula-backupninja"
+source: "https://github.com/salt-formulas/salt-formula-backupninja"
diff --git a/tests/pillar/client_rsync_backup_times.sls b/tests/pillar/client_rsync_backup_times.sls
new file mode 100644
index 0000000..adb5a78
--- /dev/null
+++ b/tests/pillar/client_rsync_backup_times.sls
@@ -0,0 +1,15 @@
+backupninja:
+ client:
+ enabled: true
+ backup_times:
+ day_of_week: 1
+ hour: 4
+ minute: 52
+ target:
+ engine: rsync
+ host: 10.10.10.208
+ user: backupninja
+linux:
+ system:
+ name: hostname
+ domain: domain
\ No newline at end of file
diff --git a/tests/pillar/client_s3_disabled_auto.sls b/tests/pillar/client_s3_disabled_auto.sls
new file mode 100644
index 0000000..8ee38bf
--- /dev/null
+++ b/tests/pillar/client_s3_disabled_auto.sls
@@ -0,0 +1,10 @@
+backupninja:
+ client:
+ enabled: true
+ auto_backup_disabled: true
+ target:
+ engine: dup
+ url: s3+http://bucket-name/folder-name
+ auth:
+ awsaccesskeyid: awsaccesskeyid
+ awssecretaccesskey: awssecretaccesskey
\ No newline at end of file