Integrate backup and monitoring operations
This commit is contained in:
@@ -28,4 +28,29 @@ ansible-playbook guestops.yml \
|
||||
|
||||
The controller verifies the source package before transfer. The host independently checks the transferred archive checksum, extracts into a commit-specific directory, verifies the package again, selects the commit-tagged images and disabled send/FAQ defaults in the private environment, builds the API and worker images, records immutable image IDs, runs the offline preflight, validates Compose without printing expanded secrets, starts with `--no-build`, and waits for loopback readiness. It then enables Docker and Nginx at boot, validates and installs the reviewed proxy site, rejects public API or MongoDB listeners, and checks the public redirect and certificate-backed HTTPS readiness before changing the `current` symlink. The source archive remains in the restricted controller store; the temporary host copy is removed after success.
|
||||
|
||||
This playbook does not provision DNS, TLS, Nginx, firewall rules, backup keys, monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities.
|
||||
The release playbook installs the reviewed Nginx site but does not provision DNS, certificates, firewall rules, durable logging, backup keys, central monitoring, or the private environment. Those remain explicit Milestone 10 and 11 acceptance activities.
|
||||
|
||||
## Backup and monitoring operations
|
||||
|
||||
After the exact release is deployed, run `guestops-operations.yml`. It installs the repository systemd units, verifies their definitions, imports only the approved public recovery key, enables transfer retry and monitoring, connects the aggregate status file to Zabbix, and retains non-sensitive installation evidence. It never creates a recovery private key or writes transfer credentials.
|
||||
|
||||
Before running it, provision these private host files through Ansible Vault or the established secret process:
|
||||
|
||||
- `/etc/guestops/backup.env` mode `0600`;
|
||||
- `/etc/guestops/backup-transfer.env` mode `0600`;
|
||||
- `/etc/guestops/backup-transfer.key` mode `0600`;
|
||||
- `/etc/guestops/backup-known-hosts` mode `0644`.
|
||||
|
||||
The recovery public key must be available on the controller. Its private key and protected recovery copy must remain off the Debian host. Confirm durable restricted logging separately and pass `guestops_durable_logs_configured=true` only after that review. The daily backup timer defaults to disabled; set `guestops_enable_backup_schedule=true` only after the manual backup and service-recovery exercise passes.
|
||||
|
||||
```sh
|
||||
ansible-playbook guestops-operations.yml \
|
||||
-l guestops_sandbox \
|
||||
-e guestops_release_commit=FULL_40_CHARACTER_SHA \
|
||||
-e guestops_backup_recipient=FULL_40_CHARACTER_GPG_FINGERPRINT \
|
||||
-e guestops_recovery_public_key=/secure/recovery/guestops-public.asc \
|
||||
-e guestops_evidence_directory=/secure/evidence/guestops/0.2.1 \
|
||||
-e guestops_durable_logs_configured=true
|
||||
```
|
||||
|
||||
Run the manual encrypted backup, transfer, isolated restore and rollback exercises separately under the approved maintenance procedure. After they pass, rerun with `guestops_enable_backup_schedule=true`. Configure Zabbix trigger thresholds and escalation recipients in the central Zabbix environment; the playbook exposes only the non-sensitive `guestops.status` item.
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
---
|
||||
- name: Install GuestOps backup and monitoring operations
|
||||
hosts: guestops
|
||||
become: true
|
||||
gather_facts: true
|
||||
|
||||
vars:
|
||||
guestops_root: /srv/guestops
|
||||
guestops_current: "{{ guestops_root }}/current"
|
||||
guestops_config_root: /etc/guestops
|
||||
guestops_backup_root: /var/backups/guestops
|
||||
guestops_gnupg_root: /var/lib/guestops-backup/gnupg
|
||||
guestops_zabbix_service: zabbix-agent2
|
||||
guestops_zabbix_include_directory: /etc/zabbix/zabbix_agent2.d
|
||||
guestops_enable_backup_schedule: false
|
||||
guestops_durable_logs_configured: false
|
||||
|
||||
pre_tasks:
|
||||
- name: Validate required operational variables
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- guestops_release_commit is match('^[0-9a-f]{40}$')
|
||||
- guestops_backup_recipient is match('^[A-Fa-f0-9]{40}$')
|
||||
- guestops_recovery_public_key | length > 0
|
||||
- guestops_evidence_directory | length > 0
|
||||
- guestops_durable_logs_configured | bool
|
||||
fail_msg: Release commit, recovery public key/fingerprint, evidence directory and reviewed durable logging are required.
|
||||
|
||||
- name: Confirm selected release link
|
||||
ansible.builtin.stat:
|
||||
path: "{{ guestops_current }}"
|
||||
follow: false
|
||||
register: guestops_selected_release
|
||||
|
||||
- name: Require deployed release before operations installation
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- guestops_selected_release.stat.exists
|
||||
- guestops_selected_release.stat.islnk
|
||||
- guestops_selected_release.stat.lnk_source == guestops_root + '/releases/' + guestops_release_commit
|
||||
fail_msg: /srv/guestops/current must select the exact approved release commit.
|
||||
|
||||
- name: Confirm selected release target exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ guestops_root }}/releases/{{ guestops_release_commit }}"
|
||||
follow: true
|
||||
register: guestops_release_target
|
||||
|
||||
- name: Require selected release directory
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- guestops_release_target.stat.exists
|
||||
- guestops_release_target.stat.isdir
|
||||
fail_msg: The selected release target must be an installed directory.
|
||||
|
||||
- name: Inspect pre-provisioned private operational files
|
||||
ansible.builtin.stat:
|
||||
path: "{{ item.path }}"
|
||||
follow: false
|
||||
loop:
|
||||
- { path: /etc/guestops/backup.env, mode: "0600" }
|
||||
- { path: /etc/guestops/backup-transfer.env, mode: "0600" }
|
||||
- { path: /etc/guestops/backup-transfer.key, mode: "0600" }
|
||||
- { path: /etc/guestops/backup-known-hosts, mode: "0644" }
|
||||
register: guestops_operational_files
|
||||
|
||||
- name: Require private backup and transfer configuration
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.stat.exists
|
||||
- item.stat.isreg
|
||||
- not item.stat.islnk
|
||||
- item.stat.mode == item.item.mode
|
||||
fail_msg: "{{ item.item.path }} must be a regular non-symlink file with mode {{ item.item.mode }}."
|
||||
loop: "{{ guestops_operational_files.results }}"
|
||||
no_log: true
|
||||
|
||||
tasks:
|
||||
- name: Create private backup directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0700"
|
||||
loop:
|
||||
- "{{ guestops_backup_root }}"
|
||||
- "{{ guestops_gnupg_root }}"
|
||||
|
||||
- name: Stage recovery public key
|
||||
ansible.builtin.copy:
|
||||
src: "{{ guestops_recovery_public_key }}"
|
||||
dest: /etc/guestops/recovery-public.asc
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0600"
|
||||
|
||||
- name: Import recovery public key only
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- gpg
|
||||
- --batch
|
||||
- --homedir
|
||||
- "{{ guestops_gnupg_root }}"
|
||||
- --import
|
||||
- /etc/guestops/recovery-public.asc
|
||||
no_log: true
|
||||
changed_when: true
|
||||
|
||||
- name: Verify configured recovery fingerprint
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- gpg
|
||||
- --batch
|
||||
- --homedir
|
||||
- "{{ guestops_gnupg_root }}"
|
||||
- --list-keys
|
||||
- "{{ guestops_backup_recipient }}"
|
||||
changed_when: false
|
||||
no_log: true
|
||||
|
||||
- name: Remove staged recovery public key
|
||||
ansible.builtin.file:
|
||||
path: /etc/guestops/recovery-public.asc
|
||||
state: absent
|
||||
|
||||
- name: Install GuestOps systemd units
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../systemd/{{ item }}"
|
||||
dest: "/etc/systemd/system/{{ item }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
loop:
|
||||
- guestops-backup.service
|
||||
- guestops-backup.timer
|
||||
- guestops-backup-transfer.service
|
||||
- guestops-backup-transfer.timer
|
||||
- guestops-monitor-status.service
|
||||
- guestops-monitor-status.timer
|
||||
register: guestops_systemd_units
|
||||
|
||||
- name: Verify installed systemd units
|
||||
ansible.builtin.command:
|
||||
argv:
|
||||
- systemd-analyze
|
||||
- verify
|
||||
- /etc/systemd/system/guestops-backup.service
|
||||
- /etc/systemd/system/guestops-backup.timer
|
||||
- /etc/systemd/system/guestops-backup-transfer.service
|
||||
- /etc/systemd/system/guestops-backup-transfer.timer
|
||||
- /etc/systemd/system/guestops-monitor-status.service
|
||||
- /etc/systemd/system/guestops-monitor-status.timer
|
||||
changed_when: false
|
||||
|
||||
- name: Reload systemd unit definitions
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
when: guestops_systemd_units.changed
|
||||
|
||||
- name: Enable transfer retry and monitoring timers
|
||||
ansible.builtin.systemd_service:
|
||||
name: "{{ item }}"
|
||||
enabled: true
|
||||
state: started
|
||||
loop:
|
||||
- guestops-backup-transfer.timer
|
||||
- guestops-monitor-status.timer
|
||||
|
||||
- name: Select backup schedule state
|
||||
ansible.builtin.systemd_service:
|
||||
name: guestops-backup.timer
|
||||
enabled: "{{ guestops_enable_backup_schedule | bool }}"
|
||||
state: "{{ 'started' if guestops_enable_backup_schedule | bool else 'stopped' }}"
|
||||
|
||||
- name: Require Zabbix include directory
|
||||
ansible.builtin.stat:
|
||||
path: "{{ guestops_zabbix_include_directory }}"
|
||||
register: guestops_zabbix_directory
|
||||
|
||||
- name: Confirm Zabbix agent configuration path
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- guestops_zabbix_directory.stat.exists
|
||||
- guestops_zabbix_directory.stat.isdir
|
||||
fail_msg: Override guestops_zabbix_include_directory for the installed Zabbix agent.
|
||||
|
||||
- name: Install restricted Zabbix status item
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../systemd/zabbix-agent-guestops.conf.example"
|
||||
dest: "{{ guestops_zabbix_include_directory }}/guestops.conf"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: guestops_zabbix_configuration
|
||||
|
||||
- name: Restart Zabbix agent after configuration change
|
||||
ansible.builtin.service:
|
||||
name: "{{ guestops_zabbix_service }}"
|
||||
enabled: true
|
||||
state: restarted
|
||||
when: guestops_zabbix_configuration.changed
|
||||
|
||||
- name: Generate initial non-sensitive monitoring status
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, start, guestops-monitor-status.service]
|
||||
changed_when: true
|
||||
|
||||
- name: Read installed timer schedule
|
||||
ansible.builtin.command:
|
||||
argv: [systemctl, list-timers, --all, --no-pager, guestops-backup.timer, guestops-backup-transfer.timer, guestops-monitor-status.timer]
|
||||
register: guestops_timer_status
|
||||
changed_when: false
|
||||
|
||||
- name: Retain monitoring status on the Ansible controller
|
||||
ansible.builtin.fetch:
|
||||
src: /run/guestops-monitor/status.json
|
||||
dest: "{{ guestops_evidence_directory }}/monitor-status.json"
|
||||
flat: true
|
||||
|
||||
- name: Retain timer status on the Ansible controller
|
||||
ansible.builtin.copy:
|
||||
content: "{{ guestops_timer_status.stdout }}\n"
|
||||
dest: "{{ guestops_evidence_directory }}/systemd-timers.txt"
|
||||
mode: "0600"
|
||||
delegate_to: localhost
|
||||
become: false
|
||||
@@ -5,7 +5,7 @@
|
||||
gather_facts: true
|
||||
|
||||
vars:
|
||||
guestops_root: /opt/guestops
|
||||
guestops_root: /srv/guestops
|
||||
guestops_config_root: /etc/guestops
|
||||
guestops_public_hostname: sandbox-guestops.futuresens.co.uk
|
||||
guestops_public_origin: "https://{{ guestops_public_hostname }}"
|
||||
|
||||
@@ -2,15 +2,15 @@
|
||||
Description=Transfer GuestOps encrypted backups to restricted storage
|
||||
Wants=network-online.target
|
||||
After=network-online.target guestops-backup.service
|
||||
ConditionPathIsDirectory=/srv/guestops
|
||||
ConditionPathIsDirectory=/srv/guestops/current
|
||||
ConditionPathIsDirectory=/var/backups/guestops
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/guestops
|
||||
WorkingDirectory=/srv/guestops/current
|
||||
EnvironmentFile=/etc/guestops/backup-transfer.env
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/deploy/backup_transfer.py --prune-verified --retention-days 7
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/backup_transfer.py --prune-verified --retention-days 7
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
|
||||
@@ -2,17 +2,17 @@
|
||||
Description=GuestOps encrypted maintenance backup
|
||||
Requires=docker.service
|
||||
After=docker.service
|
||||
ConditionPathIsDirectory=/srv/guestops
|
||||
ConditionPathIsDirectory=/srv/guestops/current
|
||||
ConditionPathIsDirectory=/var/backups/guestops
|
||||
ConditionPathIsDirectory=/var/lib/guestops-backup/gnupg
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/guestops
|
||||
WorkingDirectory=/srv/guestops/current
|
||||
EnvironmentFile=/etc/guestops/backup.env
|
||||
Environment=GNUPGHOME=/var/lib/guestops-backup/gnupg
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/deploy/ops.py scheduled-backup --confirm-maintenance
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/ops.py scheduled-backup --confirm-maintenance
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
Description=Write non-sensitive GuestOps monitoring status
|
||||
Requires=docker.service
|
||||
After=docker.service network-online.target
|
||||
ConditionPathIsDirectory=/srv/guestops
|
||||
ConditionPathIsDirectory=/srv/guestops/current
|
||||
ConditionPathIsDirectory=/var/backups/guestops
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/guestops
|
||||
WorkingDirectory=/srv/guestops/current
|
||||
UMask=0022
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/deploy/monitor_status.py
|
||||
ExecStart=/usr/bin/python3 /srv/guestops/current/deploy/monitor_status.py
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
ProtectHome=true
|
||||
|
||||
Reference in New Issue
Block a user