diff --git a/README.md b/README.md index cd62fed..fd1fc91 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,21 @@ If you want to add a service host, place it in the dedicated service group in `h - `services_map` for the map stack (`batman`, `batctl`, `grafana_public`, `meshviewer`) - `services_web` for the web stack (`batman`, `batctl`, `website`, `firmware_release`, `firmware_selector`) +### Website deployment + +The website role runs on `web-ffsh`. The main domain is served as a static site +from `/var/www/web-static`; the beta domain remains at `/var/www/web-beta`. +The previous WordPress files at `/var/www/web` and the PHP runtime are retained +for rollback, but the main-domain vhost no longer serves them. + +The Hugo repository's CI should build the site and deploy pushes to `main` only +to beta, with production deployments restricted to version tags. Use separate +SSH keys for the two destinations. Add their public keys to the Ansible Vault +variables `vault_website_beta_deploy_public_key` and +`vault_website_prod_deploy_public_key` in `group_vars/secrets.yml`; keep the +matching private keys in the Hugo repository's CI secrets. The CI workflow is +separate from this Ansible repository. + The standalone DNS host belongs to the `dns` group. The DNS play only installs node_exporter and bind_exporter; it does not manage the existing BIND service, zones, or listening addresses. diff --git a/group_vars/secrets.yml b/group_vars/secrets.yml index 24a66af..f6af88c 100644 --- a/group_vars/secrets.yml +++ b/group_vars/secrets.yml @@ -1,22 +1,27 @@ $ANSIBLE_VAULT;1.2;AES256;fastd_key -64336137376336663336323131373233636138353362646638393732386563326136323465306639 -6561383436386463343731363635663764323266343930350a353438303533313836363836656430 -65653432366132396138343530336662316261656263346564353938623835653731356631366439 -3235383332646361610a616330666139653637356239643936376137623462313864616532363738 -65346532343666333638623338613137333961333762353737323636316361393236666330343061 -66383163653666383262666165633139383436643063653335666230393866653935383436646261 -66616235633839383539333531623834396163623232646532316634383137623064323534323562 -65636130653837646663393964623931316239616533306366363730353162366635376434343332 -36386438613532363865633531663564633663623263326362333332653666653132643033316339 -65633361393831663032643566613432613430376337366636666662333730656232333461396666 -37343731313163653531373333363634303230343138633965393538643566623266333030656534 -64643836623066653132313062313463383633363638353266666430343239373262386264333637 -35373238333835363564373062656666313662363639333934616337613364363338663435613230 -38336462613064336633373730303636373536356536303839663434643530656434323465623766 -37353136666162363735303664633462396663306234613161646166646537303937653433613034 -33616365663439626162656665316637356634643663343631626262393930653635396637366631 -37386233653733346261643238303230643236343761303537363130313333633435353834666234 -35666137353662666361643033616534656262633464346638303161323630363663346339326131 -66306637373963383261363865336638393132613436336566663036326538373834663139366235 -32653633393939386536626634356435303533646333396366396539316538653836646230323463 -6638 +66623338626636623361306163353237636232363038613432346536663430656234323463646432 +3866396231303831386566633339373336333663383230370a393239646266343361653532326663 +37643231306661663331336639383563626231326337323432383638393764636664616366663238 +3632656439646138310a386130626663616339646462623963656264336634386336386432396237 +64636661613965343439653734373436303335343765636636313466323165646130363135663764 +61373334633638663264376639323939646230363330346563383435646465303865646565636333 +61666533316130313262383732383135373531333961633730623730376237333131383534366664 +39656234333838373839623064366639306439333861366637396261356538613063633934643931 +31316330363566336235306638386661643434386534333836393133646165666131613930623832 +31306261323661323731393033613330646536363233383335613133613836376230613738623033 +31306162613162626664383666386330356137353238633430336338633633343665666530623533 +33363931346337376439366564626661393266373065303763333530313762366131373035376232 +62353566636465323433396665313063366463333063633362326330366239306231363964316437 +30323432316461616265306366313662373933343030646662393064343666373739323930633738 +65366365633234313365626436353134303563336333633834393238333964636566366630306366 +62333361623133383666326561333332363362393730623035663763633432336539663233643235 +62613333636139643935363936353337663266313537626665303937333930333265356135393436 +62643939376162663636303261343939643231616438643930316638306331623538633138353964 +32353133663037656365303862383937666134626132383864376564336262343734336437366234 +63366537626236643333663663663335323462646436616432663339343630633031336261633834 +35633236353263633666356132393761303366636166333537663539633964666130323938393139 +32373332383865616531333762333436356265356263646664623136376262326630333664646336 +62303832613938353633333562613633313530363037376638316666393331656135326466393965 +34383435636566623139333931626437393563643566363865613665316538626134333261333163 +37383531663333633030343436356166383339663866623833663334363833616463326132333835 +3531353863366538623664386562306266323339376139643036 diff --git a/group_vars/services_web.yml b/group_vars/services_web.yml index 99ff66e..8ccada7 100644 --- a/group_vars/services_web.yml +++ b/group_vars/services_web.yml @@ -2,19 +2,21 @@ website_vhosts: - name: freifunk-suedholstein.de aliases: - www.freifunk-suedholstein.de - root: /var/www/web - index: index.php - type: php + root: /var/www/web-static + owner: website-deploy + group: www-data + mode: "0755" + index: index.html + type: static tls: true redirect_http_to_https: true - autoindex: true + autoindex: false stats_proxy: http://127.0.0.1:9100/ cors_enabled: true cors_allowed_origins: - localhost - www.freifunk-suedholstein.de - freifunk-suedholstein.de - php_fpm_socket: /var/run/php/php8.3-fpm.sock - name: firmware.freifunk-suedholstein.de aliases: @@ -33,7 +35,7 @@ website_vhosts: root: /var/www/web-beta owner: beta-deploy group: www-data - mode: "0775" + mode: "0755" index: index.html type: static tls: true @@ -42,17 +44,20 @@ website_vhosts: cors_enabled: false cors_allow_all: false -website_deploy_user: - username: beta-deploy - group: www-data - groups: - - www-data - shell: /bin/bash - web_root: /var/www/web-beta - web_root_mode: "0775" - ssh_key: "{{ vault_secrets.vault_website_beta_deploy_public_key | default('') }}" +website_deploy_users: + - username: beta-deploy + group: www-data + shell: /bin/bash + web_root: /var/www/web-beta + web_root_mode: "0755" + ssh_key: "{{ vault_secrets.vault_website_beta_deploy_public_key | default('') }}" + - username: website-deploy + group: www-data + shell: /bin/bash + web_root: /var/www/web-static + web_root_mode: "0755" + ssh_key: "{{ vault_secrets.vault_website_prod_deploy_public_key | default('') }}" website_nginx_sites_available_dir: /etc/nginx/sites-available website_nginx_sites_enabled_dir: /etc/nginx/sites-enabled -website_php_fpm_socket: /var/run/php/php8.3-fpm.sock website_default_email: noc@freifunk-suedholstein.de diff --git a/plan.md b/plan.md deleted file mode 100644 index 92bab66..0000000 --- a/plan.md +++ /dev/null @@ -1,68 +0,0 @@ -# Plan: Add Alertmanager with Blackbox reachability alerts → Telegram - -## TL;DR -Add a new `alertmanager` role (installed via apt, same pattern as `prometheus`/`blackbox_exporter`), wire it into the existing (currently commented-out) `alerting:`/`rule_files:` sections of `roles/prometheus/templates/prometheus.yml.j2`, add a Prometheus alert rule that fires when a `blackbox-http` probe fails for 5m, and route Alertmanager notifications to a Telegram channel via a bot (native `telegram_configs` receiver, no extra webhook service needed). Telegram bot token + chat ID go into vault-encrypted `group_vars/secrets.yml` as `vault_secrets.alertmanager.telegram_bot_token` / `vault_secrets.alertmanager.telegram_chat_id` (user will add these manually via `ansible-vault edit`). - -## Steps - -### Phase 1: New `alertmanager` role -1. Create `roles/alertmanager/tasks/main.yml` — mirrors `roles/blackbox_exporter/tasks/main.yml` pattern: - - `ansible.builtin.apt` install `prometheus-alertmanager`, `update_cache: true` - - `ansible.builtin.template` deploy `alertmanager.yml.j2` → `/etc/prometheus/alertmanager.yml`, owner/group root, mode 0644, `notify: Restart alertmanager` - - `ansible.builtin.systemd` enable+start `prometheus-alertmanager` -2. Create `roles/alertmanager/handlers/main.yml` — handler `Restart alertmanager` (systemd restart `prometheus-alertmanager`), same shape as `roles/blackbox_exporter/handlers/main.yml`. -3. Create `roles/alertmanager/templates/alertmanager.yml.j2`: - - `route:` default receiver = `telegram`, `group_by: ['alertname', 'instance']`, `group_wait`/`group_interval`/`repeat_interval` reasonable defaults (e.g. 30s/5m/3h) - - `receivers:` one receiver `telegram` using native `telegram_configs` - - Listen address bound to localhost only - -### Phase 2: Alert rule for blackbox reachability -4. Create `roles/prometheus/templates/alert_rules.yml.j2` with a rule group: - - `groups: - name: blackbox` containing rule `alert: ProbeFailed`, `expr: probe_success == 0`, `for: 5m`, `labels: severity: critical`, `annotations: summary/description` referencing `$labels.instance` and `$labels.job` -5. Update `roles/prometheus/tasks/main.yml` — add a task templating `alert_rules.yml.j2` → `/etc/prometheus/alert_rules.yml` (owner/group root, mode 0644) before the restart task. - -### Phase 3: Wire Prometheus → Alertmanager -6. Edit `roles/prometheus/templates/prometheus.yml.j2`: - - Uncomment `alerting.alertmanagers[0].static_configs[0].targets` and set to `["127.0.0.1:9093"]` - - Uncomment `rule_files:` and set to `["/etc/prometheus/alert_rules.yml"]` - -### Phase 4: Playbook wiring -7. Edit `setup.yml` monitoring play — add `{role: alertmanager, tags: "alertmanager"}` to the `roles:` list. - -### Phase 5: Secrets (manual, user-performed) -8. User runs `ansible-vault edit group_vars/secrets.yml` and adds: - ```yaml - vault_secrets: - alertmanager: - telegram_bot_token: "" - telegram_chat_id: - ``` - -## Relevant files -- `roles/alertmanager/tasks/main.yml` — new, install + configure + start -- `roles/alertmanager/handlers/main.yml` — new, restart handler -- `roles/alertmanager/templates/alertmanager.yml.j2` — new, route + telegram receiver -- `roles/prometheus/templates/alert_rules.yml.j2` — new, `ProbeFailed` rule on `probe_success == 0` -- `roles/prometheus/tasks/main.yml` — add template task for alert_rules.yml -- `roles/prometheus/templates/prometheus.yml.j2` — uncomment/fill `alerting:` and `rule_files:` sections -- `setup.yml` — add `alertmanager` role to monitoring play -- `group_vars/secrets.yml` — user adds `vault_secrets.alertmanager.telegram_bot_token`/`telegram_chat_id` manually via `ansible-vault edit` - -## Verification -1. `ansible-playbook setup.yml --syntax-check` after edits -2. `ansible-playbook setup.yml --limit monitoring --tags alertmanager,prometheus --check --diff` dry run -3. Deploy for real: `ansible-playbook setup.yml --limit monitoring --tags alertmanager,prometheus` -4. On monitoring host: `systemctl status prometheus-alertmanager`, `curl -s http://127.0.0.1:9093/-/healthy` -5. Prometheus UI → Status → Rules: confirm `ProbeFailed` rule loaded; Status → Alertmanagers: confirm target `127.0.0.1:9093` shown as up -6. Trigger a real test: temporarily break reachability for one target and confirm a Telegram message arrives in the channel after ~5m -7. `promtool check config` / `promtool check rules` on rendered files if promtool available on monitoring host - -## Decisions -- Alertmanager installed via apt (`prometheus-alertmanager` Debian package) for consistency with existing prometheus/blackbox_exporter roles — no version pinning in `group_vars/versions.yml`. -- Alert firing delay: `for: 5m` on `probe_success == 0`. -- Telegram integration uses Alertmanager's native `telegram_configs` receiver (built into Alertmanager since v0.25) rather than a separate telegram-webhook bridge service. -- Scope: only blackbox probe reachability alerts for now. Not adding alerts for node_exporter/gateway/ffshmon/disk/CPU etc. in this pass. -- Alertmanager bound to localhost (127.0.0.1:9093), reached by Prometheus on the same monitoring host — not exposed externally. - -## Further Considerations -1. Telegram bot/channel setup guidance (creating bot via @BotFather, obtaining numeric chat ID for a channel) provided as part of implementation. diff --git a/roles/website/defaults/main.yml b/roles/website/defaults/main.yml index 22020bb..2e5fe89 100644 --- a/roles/website/defaults/main.yml +++ b/roles/website/defaults/main.yml @@ -8,6 +8,6 @@ website_default_tls: true website_default_ssl_dhparam: /etc/nginx/dhparam.pem website_default_email: noc@freifunk-suedholstein.de website_auto_issue_tls: false -website_deploy_user: {} +website_deploy_users: [] website_vhosts: [] diff --git a/roles/website/tasks/main.yml b/roles/website/tasks/main.yml index d6bcdae..a01a679 100644 --- a/roles/website/tasks/main.yml +++ b/roles/website/tasks/main.yml @@ -8,57 +8,57 @@ - python3-certbot-nginx state: present -- name: Ensure deployment user exists when configured +- name: Ensure website deployment users exist ansible.builtin.user: - name: "{{ website_deploy_user.username }}" + name: "{{ deploy_user.username }}" state: present create_home: true - shell: "{{ website_deploy_user.shell | default('/bin/bash') }}" - group: "{{ website_deploy_user.group | default('www-data') }}" - groups: "{{ website_deploy_user.groups | default(['www-data']) }}" - append: true - when: - - website_deploy_user is defined - - website_deploy_user.username is defined - - website_deploy_user.username | length > 0 + shell: "{{ deploy_user.shell | default('/bin/bash') }}" + group: "{{ deploy_user.group | default('www-data') }}" + loop: "{{ website_deploy_users }}" + loop_control: + loop_var: deploy_user + label: "{{ deploy_user.username }}" -- name: Ensure beta deployment directory exists and is writable +- name: Ensure website deployment directories exist ansible.builtin.file: - path: "{{ website_deploy_user.web_root }}" + path: "{{ deploy_user.web_root }}" state: directory - owner: "{{ website_deploy_user.username }}" - group: "{{ website_deploy_user.group | default('www-data') }}" - mode: "{{ website_deploy_user.web_root_mode | default('0775') }}" - when: - - website_deploy_user is defined - - website_deploy_user.web_root is defined - - website_deploy_user.web_root | length > 0 + owner: "{{ deploy_user.username }}" + group: "{{ deploy_user.group | default('www-data') }}" + mode: "{{ deploy_user.web_root_mode | default('0755') }}" + loop: "{{ website_deploy_users }}" + loop_control: + loop_var: deploy_user + label: "{{ deploy_user.web_root }}" -- name: Ensure beta deployment directory contents are writable for deploy user and web group +- name: Ensure website deployment directory contents are owned by each deploy user ansible.builtin.file: - path: "{{ website_deploy_user.web_root }}" + path: "{{ deploy_user.web_root }}" state: directory recurse: true - owner: "{{ website_deploy_user.username }}" - group: "{{ website_deploy_user.group | default('www-data') }}" - mode: "u=rwX,g=rwX,o=rX" - when: - - website_deploy_user is defined - - website_deploy_user.web_root is defined - - website_deploy_user.web_root | length > 0 + owner: "{{ deploy_user.username }}" + group: "{{ deploy_user.group | default('www-data') }}" + mode: "u=rwX,g=rX,o=rX" + loop: "{{ website_deploy_users }}" + loop_control: + loop_var: deploy_user + label: "{{ deploy_user.web_root }}" -- name: Install deploy SSH key for the deployment user +- name: Install website deployment SSH keys ansible.posix.authorized_key: - user: "{{ website_deploy_user.username }}" + user: "{{ deploy_user.username }}" state: present - key: "{{ website_deploy_user.ssh_key }}" + key: "{{ deploy_user.ssh_key }}" exclusive: false manage_dir: true + loop: "{{ website_deploy_users }}" + loop_control: + loop_var: deploy_user + label: "{{ deploy_user.username }}" when: - - website_deploy_user is defined - - website_deploy_user.username is defined - - website_deploy_user.ssh_key is defined - - website_deploy_user.ssh_key | length > 0 + - deploy_user.ssh_key is defined + - deploy_user.ssh_key | length > 0 - name: Ensure all website document roots exist ansible.builtin.file: