From d30b1631225ebaeb8feed96743ece65e7b146b1c Mon Sep 17 00:00:00 2001 From: Bernard Siebens Date: Tue, 14 Jul 2026 10:11:15 +0200 Subject: [PATCH] Add maintenance mode: lock the platform down from the control panel Closes every club subdomain with a 503 in that club's own colours, stands the scheduled jobs down, and keeps open exactly what is needed to end it again. The exemptions ARE the feature: - /accounts/ stays open on the base domain. Close it too and you cannot sign in to turn maintenance off -- a lock-down with no key, fixable only from a shell. - /healthz answers on every host. Close it and the load balancer decides the node is dead, stops routing to it, and takes the control panel down with everything else. - migrate and collectstatic are NOT blocked. Maintenance is usually declared in order to run them; a blanket guard on BaseCommand would mean turning the mode off to do the work you turned it on for. Only the domain jobs (archive_overdue_clubs, extend_event_series, import_members_csv) refuse, and they exit non-zero so cron mails you -- a scheduled job that silently skips itself is how a month of billing goes missing. The state is cached with a 10-second TTL, not for ever. Write-through makes the flip instant for the shared Redis of a real deployment, and the TTL is the belt to that braces: on a per-process cache -- a dev box with no Redis, or a misconfigured deploy -- a lock-down that reached only one gunicorn worker would be worse than useless. Live-verified: a club subdomain, its login page and the base domain all 503 while the control panel and the sign-in screens stay up. Also adds the two deployment pieces asked for: compose.behind-proxy.yaml for a dev/test box that already runs Caddy on :80 (app on the loopback, host Caddy proxies to it -- and the host's Caddy still needs the DNS plugin, because the wildcard is still a wildcard), and deploy/backup.sh + restore-check.sh with a cron schedule. The backup writes to a .part file and only lands it once gzip -t says it is readable: a truncated dump that looks like a backup is the failure you find on the day you need it. The weekly restore rehearsal is the only line in that cron that proves the rest work. Co-Authored-By: Claude Opus 4.8 --- DEPLOYMENT.md | 139 ++++++++++++++++- .../commands/archive_overdue_clubs.py | 4 +- compose.behind-proxy.yaml | 55 +++++++ controlpanel/forms.py | 12 ++ controlpanel/templates/controlpanel/base.html | 10 ++ .../templates/controlpanel/features.html | 44 +++++- controlpanel/tests.py | 36 +++++ controlpanel/urls.py | 1 + controlpanel/views.py | 21 ++- deploy/backup.sh | 51 +++++++ deploy/restore-check.sh | 30 ++++ .../commands/extend_event_series.py | 5 +- features/commands.py | 31 ++++ features/context_processors.py | 7 + features/middleware.py | 74 +++++++++ features/migrations/0002_maintenance.py | 33 ++++ features/models.py | 75 +++++++++ features/tests.py | 142 +++++++++++++++++- .../management/commands/import_members_csv.py | 5 +- rosterchief/settings.py | 3 + static/css/app.css | 6 + templates/maintenance.html | 24 +++ 22 files changed, 796 insertions(+), 12 deletions(-) create mode 100644 compose.behind-proxy.yaml create mode 100755 deploy/backup.sh create mode 100755 deploy/restore-check.sh create mode 100644 features/commands.py create mode 100644 features/context_processors.py create mode 100644 features/middleware.py create mode 100644 features/migrations/0002_maintenance.py create mode 100644 templates/maintenance.html diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 5ae6bd3..d81784a 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -100,7 +100,144 @@ emails to the same club. 0 3 * * * cd /srv/rosterchief && docker compose run --rm web python manage.py extend_event_series ``` -## Backups +## Maintenance mode + +Control panel → **Features → Maintenance mode**. While it is on: + +- every **club subdomain** serves a 503 maintenance page, in that club's own colours; +- the **control panel and the sign-in screens stay open**, because closing them would leave + you with no way to turn it back off; +- `/healthz` keeps answering on every host, or the load balancer would take the node out of + rotation and the control panel with it; +- the **scheduled jobs stand down** — `archive_overdue_clubs`, `extend_event_series` and + `import_members_csv` refuse to run. + +`migrate` and `collectstatic` are deliberately **not** blocked. Maintenance is usually +declared *in order* to run them, and a guard that stopped them would mean turning the mode +off to do the work you turned it on for. + +The scheduled jobs exit **non-zero** while the platform is closed, so cron will mail you. +That is intended: a job that silently skips itself is how a month of billing goes missing. If +you genuinely mean to run one during a window, pass `--ignore-maintenance`. + +So a migration-heavy deploy looks like: + +```bash +# 1. Close the platform in the control panel (or from a shell): +docker compose run --rm web python manage.py shell -c \ + "from features.models import Maintenance; Maintenance.start(message='Upgrading. Back by 21:00.')" + +# 2. Do the work — migrate is not blocked. +docker compose build +docker compose run --rm web python manage.py migrate +docker compose up -d --no-deps web + +# 3. Reopen from the control panel. +``` + +The state lives in Redis as well as the database, so it takes effect on **every worker and +every server at once** — a per-process cache would leave some workers still serving clubs. + +## Behind an existing Caddy (dev / test server) + +If the box already runs Caddy on :80 and :443 — a test server sharing a host with other +sites — do **not** run ours: two Caddies cannot both hold port 80. Run the app only, publish +it on the loopback, and add a site block to the Caddy that is already there. + +```bash +docker compose -f compose.behind-proxy.yaml up -d # web + db + redis, no caddy +``` + +`web` publishes on `127.0.0.1:8001` (override with `WEB_PORT`). **Loopback, not 0.0.0.0** — +bound to all interfaces, a test instance is reachable at `http://:8001` with no +TLS, bypassing the proxy and every security header with it. + +Then, in the host's Caddyfile: + +```caddy +test.rosterchief.app, *.test.rosterchief.app { + tls { + dns cloudflare {env.CLOUDFLARE_API_TOKEN} + } + + reverse_proxy 127.0.0.1:8001 { + header_up X-Forwarded-Proto {scheme} + header_up X-Real-IP {remote_host} + } +} +``` + +Three things this needs, and each one is a way to lose an afternoon: + +1. **The host's Caddy must have the DNS plugin too.** The wildcard is still a wildcard: the + stock `caddy` package cannot answer a DNS-01 challenge. `caddy add-package + github.com/caddy-dns/cloudflare` on a package install, or run a Caddy built like + `deploy/caddy/Dockerfile`. +2. **Give the test instance its own subdomain tree** (`*.test.rosterchief.app`) and set + `ROSTERCHIEF_BASE_DOMAIN=test.rosterchief.app`. It drives tenant resolution, the shared + session cookie *and* the WebAuthn RP ID — point it at the production domain and test + passkeys start colliding with real ones. +3. **`header_up X-Forwarded-Proto` is not optional**, exactly as in the bundled Caddyfile. + Without it Django believes the request is plain HTTP behind the proxy. + +DNS still needs both records, pointing at the test box: + +``` +A test.rosterchief.app -> +A *.test.rosterchief.app -> +``` + +The compose project is named `rosterchief-test`, so its containers and volumes never collide +with a production stack on the same host. + +## Automated backups + +`deploy/backup.sh` dumps the database, tars the uploads while they are still on local disk, +prunes anything older than `KEEP_DAYS`, and — if you set `BACKUP_REMOTE` — copies the lot off +the box with rclone. + +```bash +deploy/backup.sh /var/backups/rosterchief +``` + +It writes to a `.part` file and only moves it into place once `gzip -t` says the archive is +readable and non-empty. A truncated dump that *looks* like a backup is the failure mode worth +engineering against, because you only discover it on the day you need it. + +Schedule it as root on the host (single server; on several, run it on the database node): + +```cron +# Nightly at 02:30, before the billing and event jobs. +30 2 * * * cd /srv/rosterchief && BACKUP_REMOTE=b2:rosterchief-backups KEEP_DAYS=14 deploy/backup.sh /var/backups/rosterchief + +# Weekly restore rehearsal into a throwaway database. This is the only line here that proves +# the others work. +0 4 * * 0 cd /srv/rosterchief && deploy/restore-check.sh +``` + +Cron mails you on non-zero exit, and the script uses `set -Eeuo pipefail` so it *does* exit +non-zero. A backup script that fails quietly is worse than none, because you will believe you +have backups. + +**Offsite matters more than frequency.** A dump sitting on the same disk as the database +survives a bad migration but not the server. `BACKUP_REMOTE` takes any rclone remote (S3, +Backblaze, a second box). + +**Once uploads move to S3** (`AWS_STORAGE_BUCKET_NAME`), the script skips the media tarball: +the bucket's own versioning is the backup. Turn versioning on when you create it. + +### Restoring + +```bash +gunzip -c /var/backups/rosterchief/db-2026-07-14-0230.sql.gz \ + | docker compose exec -T db psql -U rosterchief rosterchief +``` + +The dump is taken with `--clean --if-exists`, so it drops and recreates rather than colliding +with what is there. Rehearse it once, now, against a scratch database — not the first time you +need it. + +## Backups (manual) Two things carry state: Postgres and the uploads. diff --git a/billing/management/commands/archive_overdue_clubs.py b/billing/management/commands/archive_overdue_clubs.py index 3c6368b..113c57a 100644 --- a/billing/management/commands/archive_overdue_clubs.py +++ b/billing/management/commands/archive_overdue_clubs.py @@ -5,13 +5,13 @@ switches off paying customers, and a cron misconfiguration, a clock skew or a ba should cost you a confusing email, not a morning of angry clubs. """ -from django.core.management.base import BaseCommand from django.utils import timezone from billing.services.dues import archivable_clubs +from features.commands import MaintenanceAwareCommand -class Command(BaseCommand): +class Command(MaintenanceAwareCommand): help = "Archive clubs that are unpaid past their grace period (dry run unless --commit)." def add_arguments(self, parser): diff --git a/compose.behind-proxy.yaml b/compose.behind-proxy.yaml new file mode 100644 index 0000000..47d4303 --- /dev/null +++ b/compose.behind-proxy.yaml @@ -0,0 +1,55 @@ +# A dev/test deployment on a server that ALREADY runs Caddy on :80/:443. +# +# docker compose -f compose.behind-proxy.yaml up -d +# +# The difference from compose.yaml is only what listens on the network: no caddy service, and +# web publishes on the loopback instead of the public interface. The host's Caddy reverse +# proxies to it (see DEPLOYMENT.md, "Behind an existing Caddy"). +# +# Publishing on 127.0.0.1 and not 0.0.0.0 is the point: bound to all interfaces, a test +# instance is reachable on http://:8001 with no TLS, bypassing the proxy and every +# security header with it. + +name: rosterchief-test + +services: + web: + build: . + restart: unless-stopped + env_file: .env.production + ports: + - "127.0.0.1:${WEB_PORT:-8001}:8000" + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-fsS", "http://localhost:8000/healthz"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 20s + + db: + image: postgres:17-alpine + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-rosterchief} + POSTGRES_USER: ${POSTGRES_USER:-rosterchief} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set a database password} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-rosterchief}"] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + restart: unless-stopped + command: ["redis-server", "--save", "", "--appendonly", "no"] + +volumes: + pgdata: diff --git a/controlpanel/forms.py b/controlpanel/forms.py index 92a077d..e8054d2 100644 --- a/controlpanel/forms.py +++ b/controlpanel/forms.py @@ -103,3 +103,15 @@ class OpenPeriodForm(forms.Form): """Renew, or reactivate an archived club.""" start = forms.DateField(required=False, widget=forms.DateInput(attrs={"type": "date"}), label=_("Period starts"), help_text=_("Left blank, it continues from the end of the last period — so a lapsed year is still owed.")) + + +class MaintenanceForm(forms.Form): + """Closing the platform is a deliberate act, so it takes a sentence explaining itself — + that message is the only thing a club will see.""" + + message = forms.CharField( + required=False, + widget=forms.Textarea(attrs={"rows": 2}), + label=_("Message"), + help_text=_("Shown to every club while the platform is closed. Left blank, they get a generic notice."), + ) diff --git a/controlpanel/templates/controlpanel/base.html b/controlpanel/templates/controlpanel/base.html index cd469c9..a08eb18 100644 --- a/controlpanel/templates/controlpanel/base.html +++ b/controlpanel/templates/controlpanel/base.html @@ -6,6 +6,16 @@ {% endblock title %} {% block main %} + {% if maintenance_on %} +
+ {% lucide "wrench" size=20 %} + + The platform is closed for maintenance. + Clubs see a maintenance page and the scheduled jobs are standing down. + + Reopen +
+ {% endif %}

diff --git a/controlpanel/templates/controlpanel/features.html b/controlpanel/templates/controlpanel/features.html index 7c931df..6527131 100644 --- a/controlpanel/templates/controlpanel/features.html +++ b/controlpanel/templates/controlpanel/features.html @@ -1,5 +1,5 @@ {% extends "controlpanel/base.html" %} -{% load lucide %} +{% load lucide ui %} {% block heading %}Features{% endblock heading %} @@ -8,6 +8,48 @@ {% endblock actions %} {% block panel %} + {% comment %} + The lock-down. Clubs get a maintenance page, the scheduled jobs stand down, and the + control panel and the auth screens stay open — otherwise you could not sign in to + turn it back off. + {% endcomment %} +
+
+
+
+

{% lucide "wrench" size=18 %} Maintenance mode

+ {% if maintenance.is_active %} +

+ {% lucide "lock" size=12 %} Platform closed + since {{ maintenance.started_at|date:"j M Y, H:i" }}{% if maintenance.started_by %} by {{ maintenance.started_by.email }}{% endif %}. +

+ {% if maintenance.message %}

“{{ maintenance.message }}”

{% endif %} + {% else %} +

+ Closes every club subdomain and stands the scheduled jobs down. The control panel and the sign-in screens stay open. +

+ {% endif %} +
+
+ +
+ {% csrf_token %} + {% if not maintenance.is_active %} +
+ + {{ maintenance_form.message|daisy }} + {{ maintenance_form.message.help_text }} +
+ + {% else %} + + {% endif %} +
+
+
+

{% lucide "flag" size=18 %} Flags

diff --git a/controlpanel/tests.py b/controlpanel/tests.py index 444151e..f5cdc92 100644 --- a/controlpanel/tests.py +++ b/controlpanel/tests.py @@ -20,6 +20,7 @@ from billing.services import BillingError from billing.services.dues import record_payment, subscribe from club.models import Club, ClubMembership, ClubRole, Season from events.models import Attendance, Event +from features.models import Maintenance from members.models import Member from shop.models import Order from teams.models import Position, StaffAssignment, Team, TeamMembership @@ -1315,3 +1316,38 @@ class BillingFormRenderTests(ControlPanelTestBase): response = self.client.post(reverse("controlpanel:due_waive", args=[due.pk]), follow=True) self.assertContains(response, "remove them before waiving") + + +class MaintenancePanelTests(ControlPanelTestBase): + def setUp(self): + super().setUp() + cache.clear() + self.addCleanup(cache.clear) + + def test_closing_the_platform_records_who_did_it(self): + self.client.post(reverse("controlpanel:maintenance"), {"message": "Database upgrade."}) + + maintenance = Maintenance.current() + self.assertTrue(maintenance.is_active) + self.assertEqual(maintenance.message, "Database upgrade.") + self.assertEqual(maintenance.started_by, self.staff) + + def test_posting_again_reopens_the_platform(self): + Maintenance.start(message="x", user=self.staff) + + self.client.post(reverse("controlpanel:maintenance"), {}) + + self.assertFalse(Maintenance.is_on()) + + def test_every_panel_page_warns_while_the_platform_is_closed(self): + # Not a state to leave on by accident. + Maintenance.start(user=self.staff) + + for url in (reverse("controlpanel:dashboard"), reverse("controlpanel:club_list"), reverse("controlpanel:features")): + self.assertContains(self.client.get(url), "closed for maintenance", msg_prefix=url) + + def test_the_features_page_offers_the_switch(self): + response = self.client.get(reverse("controlpanel:features")) + + self.assertContains(response, "Maintenance mode") + self.assertContains(response, "Close the platform") diff --git a/controlpanel/urls.py b/controlpanel/urls.py index db81799..e3c6fda 100644 --- a/controlpanel/urls.py +++ b/controlpanel/urls.py @@ -18,6 +18,7 @@ urlpatterns = [ path("clubs//features//toggle/", views.ClubFeatureToggleView.as_view(), name="club_feature_toggle"), # Features path("features/", views.FeatureListView.as_view(), name="features"), + path("features/maintenance/", views.MaintenanceView.as_view(), name="maintenance"), path("features/flags/new/", views.FlagCreateView.as_view(), name="flag_create"), path("features/flags//edit/", views.FlagUpdateView.as_view(), name="flag_update"), path("features/switches//toggle/", views.SwitchToggleView.as_view(), name="switch_toggle"), diff --git a/controlpanel/views.py b/controlpanel/views.py index a26ec66..add35dd 100644 --- a/controlpanel/views.py +++ b/controlpanel/views.py @@ -13,8 +13,9 @@ from billing.services import BillingError from billing.services.dues import next_period_start, open_period, reactivate, record_payment, subscribe, waive from billing.services.invoices import invoice_pdf, issue_invoice from club.models import Club, ClubRole +from features.models import Maintenance -from .forms import ClubAdminForm, ClubForm, DuePaymentForm, FlagForm, OpenPeriodForm, PlatformAdminForm, SubscriptionForm, TierForm, TierPriceForm +from .forms import ClubAdminForm, ClubForm, DuePaymentForm, FlagForm, MaintenanceForm, OpenPeriodForm, PlatformAdminForm, SubscriptionForm, TierForm, TierPriceForm from .mixins import PlatformStaffRequiredMixin, PlatformSuperuserRequiredMixin from .services.admins import grant_club_admin, revoke_club_admin from .services.platform_admins import ( @@ -197,10 +198,28 @@ class FeatureListView(PlatformStaffRequiredMixin, TemplateView): nav="features", flags=Flag.objects.prefetch_related("clubs").order_by("name"), switches=Switch.objects.order_by("name"), + maintenance=Maintenance.current(), + maintenance_form=MaintenanceForm(), **kwargs, ) +class MaintenanceView(PlatformStaffRequiredMixin, View): + """Close the platform, or open it again.""" + + def post(self, request): + if Maintenance.is_on(): + Maintenance.stop() + messages.success(request, "Maintenance ended. The clubs are back.") + else: + form = MaintenanceForm(request.POST) + message = form.cleaned_data["message"] if form.is_valid() else "" + Maintenance.start(message=message, user=request.user) + messages.warning(request, "Platform closed. Every club subdomain now serves a maintenance page, and the scheduled jobs stand down.") + + return redirect("controlpanel:features") + + class FlagCreateView(PlatformStaffRequiredMixin, CreateView): model = Flag form_class = FlagForm diff --git a/deploy/backup.sh b/deploy/backup.sh new file mode 100755 index 0000000..410973e --- /dev/null +++ b/deploy/backup.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Back up what carries state: the database, and the uploads if they are still on local disk. +# +# deploy/backup.sh /var/backups/rosterchief +# +# Runs from cron (see DEPLOYMENT.md). Exits non-zero on any failure, so cron mails you — +# a backup script that fails quietly is worse than no backup script, because you will +# believe you have backups. +set -Eeuo pipefail + +DEST="${1:-/var/backups/rosterchief}" +COMPOSE="${COMPOSE:-docker compose}" +KEEP_DAYS="${KEEP_DAYS:-14}" +STAMP="$(date +%F-%H%M)" + +mkdir -p "$DEST" + +# --- database --------------------------------------------------------------- +# Written to a temporary name and moved into place only on success: a truncated dump that +# looks like a backup is the trap this avoids. +DB_TMP="$DEST/.db-$STAMP.sql.gz.part" +DB_OUT="$DEST/db-$STAMP.sql.gz" + +$COMPOSE exec -T db pg_dump --clean --if-exists -U "${POSTGRES_USER:-rosterchief}" "${POSTGRES_DB:-rosterchief}" | gzip > "$DB_TMP" +gzip -t "$DB_TMP" # the archive is readable +[ -s "$DB_TMP" ] # ...and not empty +mv "$DB_TMP" "$DB_OUT" + +# --- uploads ---------------------------------------------------------------- +# Only while media is local. Once AWS_STORAGE_BUCKET_NAME is set the bucket's own versioning +# is the backup, and this step is skipped. +if [ -z "${AWS_STORAGE_BUCKET_NAME:-}" ]; then + MEDIA_OUT="$DEST/media-$STAMP.tar.gz" + $COMPOSE exec -T web tar -cz -C /app media | cat > "$MEDIA_OUT.part" + mv "$MEDIA_OUT.part" "$MEDIA_OUT" +fi + +# --- retention -------------------------------------------------------------- +find "$DEST" -name 'db-*.sql.gz' -mtime "+$KEEP_DAYS" -delete +find "$DEST" -name 'media-*.tar.gz' -mtime "+$KEEP_DAYS" -delete +find "$DEST" -name '*.part' -mtime +1 -delete + +echo "$(date -Iseconds) backup ok: $(basename "$DB_OUT") ($(du -h "$DB_OUT" | cut -f1))" + +# --- offsite ---------------------------------------------------------------- +# A backup on the same disk as the database is not a backup: it survives a bad migration, but +# not the server. Set BACKUP_REMOTE to an rclone remote to copy it off the box. +if [ -n "${BACKUP_REMOTE:-}" ]; then + rclone copy "$DEST" "$BACKUP_REMOTE" --max-age "${KEEP_DAYS}d" + echo "$(date -Iseconds) copied to $BACKUP_REMOTE" +fi diff --git a/deploy/restore-check.sh b/deploy/restore-check.sh new file mode 100755 index 0000000..dbe73cb --- /dev/null +++ b/deploy/restore-check.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Restore the latest dump into a throwaway database and count the rows. +# +# deploy/restore-check.sh [/var/backups/rosterchief] +# +# The only line in the backup cron that proves the others work. A dump you have never +# restored is a hypothesis, not a backup. +set -Eeuo pipefail + +DEST="${1:-/var/backups/rosterchief}" +COMPOSE="${COMPOSE:-docker compose}" +USER_NAME="${POSTGRES_USER:-rosterchief}" +SCRATCH="restore_check_$(date +%s)" + +LATEST="$(ls -1t "$DEST"/db-*.sql.gz 2>/dev/null | head -1)" +[ -n "$LATEST" ] || { echo "no dump found in $DEST"; exit 1; } + +cleanup() { $COMPOSE exec -T db dropdb -U "$USER_NAME" --if-exists "$SCRATCH" >/dev/null 2>&1 || true; } +trap cleanup EXIT + +$COMPOSE exec -T db createdb -U "$USER_NAME" "$SCRATCH" +gunzip -c "$LATEST" | $COMPOSE exec -T db psql -q -U "$USER_NAME" "$SCRATCH" >/dev/null + +# A restore that produces an empty schema exits 0 and tells you nothing. Ask it something. +CLUBS="$($COMPOSE exec -T db psql -tAq -U "$USER_NAME" "$SCRATCH" -c 'SELECT count(*) FROM club_club')" +USERS="$($COMPOSE exec -T db psql -tAq -U "$USER_NAME" "$SCRATCH" -c 'SELECT count(*) FROM authentication_user')" + +[ "$USERS" -gt 0 ] || { echo "restore check FAILED: $(basename "$LATEST") restored no users"; exit 1; } + +echo "$(date -Iseconds) restore ok: $(basename "$LATEST") -> $CLUBS clubs, $USERS users" diff --git a/events/management/commands/extend_event_series.py b/events/management/commands/extend_event_series.py index bb33cbc..f7fdeb7 100644 --- a/events/management/commands/extend_event_series.py +++ b/events/management/commands/extend_event_series.py @@ -1,10 +1,9 @@ -from django.core.management.base import BaseCommand - from events.models import EventSeries from events.services import generate_occurrences, horizon +from features.commands import MaintenanceAwareCommand -class Command(BaseCommand): +class Command(MaintenanceAwareCommand): help = "Materialise recurring event occurrences up to the rolling horizon." def handle(self, *args, **options): diff --git a/features/commands.py b/features/commands.py new file mode 100644 index 0000000..5f06cb4 --- /dev/null +++ b/features/commands.py @@ -0,0 +1,31 @@ +"""Scheduled work refuses to run while the platform is locked down. + +Deliberately opt-in, per command, rather than a blanket guard on BaseCommand: maintenance is +usually declared IN ORDER to run `migrate` or `collectstatic`, and a guard that blocked those +would make the mode useless — you would have to turn it off to do the work you turned it on +for. Only the domain jobs (which write club data, archive clubs, or import members) stand +down. +""" + +from django.core.management.base import BaseCommand, CommandError + +from features.models import Maintenance + + +class MaintenanceAwareCommand(BaseCommand): + """A command that must not run while the platform is closed.""" + + def execute(self, *args, **options): + if Maintenance.is_on() and not options.get("ignore_maintenance"): + raise CommandError("The platform is in maintenance mode; this command stands down. Pass --ignore-maintenance to override.") + + return super().execute(*args, **options) + + def create_parser(self, prog_name, subcommand, **kwargs): + parser = super().create_parser(prog_name, subcommand, **kwargs) + parser.add_argument( + "--ignore-maintenance", + action="store_true", + help="Run even though the platform is in maintenance mode.", + ) + return parser diff --git a/features/context_processors.py b/features/context_processors.py new file mode 100644 index 0000000..2875c19 --- /dev/null +++ b/features/context_processors.py @@ -0,0 +1,7 @@ +from features.models import Maintenance + + +def maintenance(request): + """So no control-panel page can forget the platform is closed. Cached, so it costs no + query.""" + return {"maintenance_on": Maintenance.is_on()} diff --git a/features/middleware.py b/features/middleware.py new file mode 100644 index 0000000..2cde80f --- /dev/null +++ b/features/middleware.py @@ -0,0 +1,74 @@ +"""Platform lock-down. + +While maintenance is on, every club subdomain is closed and the base domain keeps only what +is needed to *end* the maintenance: the control panel, the auth screens that get you into it, +the static files those pages need, and the health check. + +The exemptions are the whole design. Close /accounts/ as well and you cannot sign in to turn +maintenance off — a lock-down with no key, fixable only from a shell. Close /healthz and the +load balancer concludes the node is dead and stops routing to it, which takes the control +panel down with everything else. +""" + +from django.http import JsonResponse +from django.shortcuts import render +from django.utils.translation import gettext_lazy as _ + +from features.models import Maintenance + +#: Reachable on the base domain while the platform is locked down. +OPEN_PREFIXES = ( + "/controlpanel/", # the point of the exercise + "/accounts/", # ...which you cannot reach without signing in + "/admin/", + "/static/", + "/media/", + "/__reload__/", # dev only; absent outside DEBUG +) + +#: Reachable on every host, always. The health check must answer or the load balancer will +#: take the node out of rotation and the control panel with it. +ALWAYS_OPEN = ("/healthz",) + +RETRY_AFTER_SECONDS = 3600 + + +class MaintenanceMiddleware: + """Runs after ClubTenantMiddleware: whether a request is a club's or the platform's is + decided by ``request.club``, which the tenant middleware has just resolved.""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if not self.is_closed(request): + return self.get_response(request) + + maintenance = Maintenance.current() + response = self.render(request, maintenance) + response["Retry-After"] = RETRY_AFTER_SECONDS + + return response + + def is_closed(self, request) -> bool: + if request.path.startswith(ALWAYS_OPEN): + return False + + if not Maintenance.is_on(): + return False + + # A club subdomain is closed outright — no login, no shop, nothing. + if getattr(request, "club", None) is not None: + return True + + # The base domain keeps the way back in. + return not request.path.startswith(OPEN_PREFIXES) + + def render(self, request, maintenance): + message = maintenance.message or _("RosterChief is down for maintenance. It will be back shortly.") + + # An API-ish caller gets JSON rather than a page of HTML it cannot read. + if request.headers.get("accept", "").startswith("application/json"): + return JsonResponse({"status": "maintenance", "detail": str(message)}, status=503) + + return render(request, "maintenance.html", {"message": message, "maintenance": maintenance}, status=503) diff --git a/features/migrations/0002_maintenance.py b/features/migrations/0002_maintenance.py new file mode 100644 index 0000000..3048687 --- /dev/null +++ b/features/migrations/0002_maintenance.py @@ -0,0 +1,33 @@ +# Generated by Django 6.0.6 on 2026-07-14 08:00 + +import django.db.models.deletion +import uuid +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('features', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Maintenance', + fields=[ + ('created', models.DateTimeField(auto_now_add=True, verbose_name='created')), + ('modified', models.DateTimeField(auto_now=True, verbose_name='modified')), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('is_active', models.BooleanField(default=False, verbose_name='active')), + ('message', models.TextField(blank=True, help_text='Shown to clubs while the platform is locked down.', verbose_name='message')), + ('started_at', models.DateTimeField(blank=True, null=True, verbose_name='started at')), + ('started_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='maintenance_windows', to=settings.AUTH_USER_MODEL, verbose_name='started by')), + ], + options={ + 'verbose_name': 'maintenance', + 'verbose_name_plural': 'maintenance', + }, + ), + ] diff --git a/features/models.py b/features/models.py index c3385a4..49fc6c3 100644 --- a/features/models.py +++ b/features/models.py @@ -10,11 +10,15 @@ Everything waffle already offers (``everyone`` / ``percent`` / ``staff`` / ``superusers`` / per-user / per-group) keeps working untouched. """ +from django.conf import settings from django.db import models +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from waffle.models import CACHE_EMPTY, AbstractUserFlag from waffle.utils import get_cache, keyfmt +from rosterchief.base import UUIDModel + #: Cache key template for a flag's club ids, mirroring waffle's own #: FLAG_USERS_CACHE_KEY / FLAG_GROUPS_CACHE_KEY. FLAG_CLUBS_CACHE_KEY = "flag:%s:clubs" @@ -69,3 +73,74 @@ class Flag(AbstractUserFlag): if self.everyone is not None: return self.everyone return club.pk in self._get_club_ids() + + +class Maintenance(UUIDModel): + """Platform lock-down. One row, read on every request. + + Cached rather than queried per request, and the cache is the same shared Redis the flags + use — so turning maintenance on in the control panel takes effect on every worker and + every server at once. A per-process cache would leave some workers still serving clubs. + """ + + CACHE_KEY = "maintenance:current" + + #: Cached, but not for ever. Write-through makes the flip instant for the process that + #: made it and — on the shared Redis of a real deployment — for every other one too. The + #: TTL is the belt to that braces: on a per-process cache (a dev box with no Redis, or a + #: misconfigured deploy) a lock-down that only reached one gunicorn worker would be worse + #: than useless, so the others notice within ten seconds regardless. + CACHE_SECONDS = 10 + + is_active = models.BooleanField(_("active"), default=False) + message = models.TextField(_("message"), blank=True, help_text=_("Shown to clubs while the platform is locked down.")) + started_at = models.DateTimeField(_("started at"), null=True, blank=True) + started_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, blank=True, related_name="maintenance_windows", verbose_name=_("started by")) + + class Meta: + verbose_name = _("maintenance") + verbose_name_plural = _("maintenance") + + def __str__(self): + return "Maintenance on" if self.is_active else "Maintenance off" + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + get_cache().set(self.CACHE_KEY, self, self.CACHE_SECONDS) + + @classmethod + def current(cls) -> Maintenance: + """The one row, created on first read. Cached until it changes.""" + cached = get_cache().get(cls.CACHE_KEY) + if cached is not None: + return cached + + maintenance = cls.objects.first() or cls.objects.create() + get_cache().set(cls.CACHE_KEY, maintenance, cls.CACHE_SECONDS) + + return maintenance + + @classmethod + def is_on(cls) -> bool: + return cls.current().is_active + + @classmethod + def start(cls, *, message: str = "", user=None) -> Maintenance: + maintenance = cls.current() + maintenance.is_active = True + maintenance.message = message + maintenance.started_at = timezone.now() + maintenance.started_by = user + maintenance.save() + + return maintenance + + @classmethod + def stop(cls) -> Maintenance: + maintenance = cls.current() + maintenance.is_active = False + maintenance.started_at = None + maintenance.started_by = None + maintenance.save() + + return maintenance diff --git a/features/tests.py b/features/tests.py index 6e447df..8cc6f81 100644 --- a/features/tests.py +++ b/features/tests.py @@ -1,10 +1,19 @@ +from io import StringIO + +from allauth.mfa.models import Authenticator +from django.contrib.auth import get_user_model from django.core.cache import cache -from django.test import RequestFactory, TestCase +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import RequestFactory, TestCase, override_settings from waffle import flag_is_active, get_waffle_flag_model from club.models import Club +from .models import Maintenance + Flag = get_waffle_flag_model() +User = get_user_model() class ClubScopedFlagTests(TestCase): @@ -94,3 +103,134 @@ class ClubScopedFlagTests(TestCase): self.club.flags.add(self.flag) self.assertTrue(self.active_for(self.club)) + + +@override_settings( + ROSTERCHIEF_BASE_DOMAIN="rosterchief.app", + ALLOWED_HOSTS=["rosterchief.app", "ajax-united.rosterchief.app", "testserver"], +) +class MaintenanceModeTests(TestCase): + def setUp(self): + cache.clear() + self.addCleanup(cache.clear) + self.club = Club.objects.create(name="Ajax United", slug="ajax-united") + self.user = User.objects.create_user(email="root@example.com", password="pw-secret-123", is_staff=True) + Authenticator.objects.create(user=self.user, type=Authenticator.Type.TOTP, data={"secret": "JBSWY3DPEHPK3PXP"}) + + def club_get(self, path="/"): + return self.client.get(path, HTTP_HOST="ajax-united.rosterchief.app") + + def platform_get(self, path): + return self.client.get(path, HTTP_HOST="rosterchief.app") + + def test_a_club_subdomain_is_closed(self): + Maintenance.start(message="Upgrading the database.") + + response = self.club_get("/accounts/login/") + + self.assertEqual(response.status_code, 503) + self.assertContains(response, "Upgrading the database", status_code=503) + self.assertEqual(response["Retry-After"], "3600") + + def test_a_club_is_open_again_when_maintenance_ends(self): + Maintenance.start() + Maintenance.stop() + + self.assertEqual(self.club_get("/accounts/login/").status_code, 200) + + def test_the_control_panel_stays_reachable(self): + Maintenance.start() + self.client.force_login(self.user) + + self.assertEqual(self.platform_get("/controlpanel/").status_code, 200) + + def test_signing_in_stays_possible(self): + # Close /accounts/ as well and you cannot sign in to turn maintenance off: a + # lock-down with no key, fixable only from a shell. + Maintenance.start() + + self.assertEqual(self.platform_get("/accounts/login/").status_code, 200) + + def test_the_health_check_still_answers(self): + # Close it and the load balancer decides the node is dead and stops routing to it — + # taking the control panel down with everything else. + Maintenance.start() + + self.assertEqual(self.platform_get("/healthz").status_code, 200) + self.assertEqual(self.client.get("/healthz", HTTP_HOST="ajax-united.rosterchief.app").status_code, 200) + + def test_the_rest_of_the_base_domain_is_closed(self): + Maintenance.start() + + self.assertEqual(self.platform_get("/").status_code, 503) + + def test_a_json_caller_gets_json(self): + Maintenance.start(message="Back soon.") + + response = self.client.get("/", HTTP_HOST="ajax-united.rosterchief.app", headers={"accept": "application/json"}) + + self.assertEqual(response.status_code, 503) + self.assertEqual(response.json()["status"], "maintenance") + + def test_the_message_is_optional(self): + Maintenance.start() + + self.assertContains(self.club_get("/"), "down for maintenance", status_code=503) + + def test_it_says_which_state_it_is_in(self): + self.assertEqual(str(Maintenance.start()), "Maintenance on") + self.assertEqual(str(Maintenance.stop()), "Maintenance off") + + def test_it_records_who_closed_the_platform_and_when(self): + maintenance = Maintenance.start(message="db upgrade", user=self.user) + + self.assertTrue(maintenance.is_active) + self.assertEqual(maintenance.started_by, self.user) + self.assertIsNotNone(maintenance.started_at) + + def test_the_state_is_shared_rather_than_per_process(self): + # The cache is the shared one the flags use, so closing the platform reaches every + # worker and every server. A per-process cache would leave some workers serving clubs. + Maintenance.start() + + self.assertTrue(cache.get(Maintenance.CACHE_KEY).is_active) + self.assertTrue(Maintenance.is_on()) + + Maintenance.stop() + + self.assertFalse(cache.get(Maintenance.CACHE_KEY).is_active) + + +class MaintenanceCommandTests(TestCase): + def setUp(self): + cache.clear() + self.addCleanup(cache.clear) + + def run_command(self, name, *args): + out = StringIO() + call_command(name, *args, stdout=out, stderr=out) + return out.getvalue() + + def test_a_scheduled_job_stands_down(self): + Maintenance.start() + + with self.assertRaises(CommandError): + self.run_command("archive_overdue_clubs") + + def test_it_runs_again_once_the_platform_reopens(self): + Maintenance.start() + Maintenance.stop() + + self.assertIn("Nothing overdue", self.run_command("archive_overdue_clubs")) + + def test_an_override_exists_for_when_you_mean_it(self): + Maintenance.start() + + self.assertIn("Nothing overdue", self.run_command("archive_overdue_clubs", "--ignore-maintenance")) + + def test_migrate_is_not_blocked(self): + # Maintenance is usually declared IN ORDER to migrate. A guard on every command would + # mean turning the mode off to do the work you turned it on for. + Maintenance.start() + + self.run_command("migrate", "--check") # raises SystemExit only if migrations are pending diff --git a/members/management/commands/import_members_csv.py b/members/management/commands/import_members_csv.py index 2a0e285..2c092f1 100644 --- a/members/management/commands/import_members_csv.py +++ b/members/management/commands/import_members_csv.py @@ -1,9 +1,8 @@ -from django.core.management.base import BaseCommand, CommandError - +from features.commands import CommandError, MaintenanceAwareCommand from members.services import MemberCsvImporter -class Command(BaseCommand): +class Command(MaintenanceAwareCommand): help = "Import club members from a CSV file." def add_arguments(self, parser) -> None: diff --git a/rosterchief/settings.py b/rosterchief/settings.py index 972833c..46f88d9 100644 --- a/rosterchief/settings.py +++ b/rosterchief/settings.py @@ -86,6 +86,8 @@ MIDDLEWARE = [ "allauth.account.middleware.AccountMiddleware", "authentication.middleware.RequireMFAMiddleware", "club.tenancy.ClubTenantMiddleware", + # After tenancy: it decides club-vs-platform from request.club, which was just resolved. + "features.middleware.MaintenanceMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", ] @@ -166,6 +168,7 @@ TEMPLATES = [ "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", "club.context_processors.branding", + "features.context_processors.maintenance", ], }, }, diff --git a/static/css/app.css b/static/css/app.css index c3cf6d0..e1ea44b 100644 --- a/static/css/app.css +++ b/static/css/app.css @@ -3767,6 +3767,9 @@ .items-center { align-items: center; } + .items-start { + align-items: flex-start; + } .justify-between { justify-content: space-between; } @@ -4187,6 +4190,9 @@ .opacity-70 { opacity: 70%; } + .opacity-80 { + opacity: 80%; + } .shadow { --tw-shadow: 0 1px 3px 0 var(--tw-shadow-color, rgb(0 0 0 / 0.1)), 0 1px 2px -1px var(--tw-shadow-color, rgb(0 0 0 / 0.1)); box-shadow: var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow); diff --git a/templates/maintenance.html b/templates/maintenance.html new file mode 100644 index 0000000..5b6df86 --- /dev/null +++ b/templates/maintenance.html @@ -0,0 +1,24 @@ +{% extends base_template %} +{% load lucide %} + +{% comment %} + Rendered through the tenant's own skin, so a club sees its own logo and colours rather + than a bare error page — this is the platform being unavailable, not the club being gone. +{% endcomment %} + +{% block head_title %}Maintenance{% endblock head_title %} + +{% block main %} +
+
+
+
{% lucide "wrench" size=48 %}
+

Down for maintenance

+

{{ message }}

+ {% if maintenance.started_at %} +

Since {{ maintenance.started_at|date:"j M Y, H:i" }}

+ {% endif %} +
+
+
+{% endblock main %}