Add maintenance mode: lock the platform down from the control panel

Closes every club subdomain with a 503 in that club's own colours, stands the
scheduled jobs down, and keeps open exactly what is needed to end it again.

The exemptions ARE the feature:

- /accounts/ stays open on the base domain. Close it too and you cannot sign in to
  turn maintenance off -- a lock-down with no key, fixable only from a shell.
- /healthz answers on every host. Close it and the load balancer decides the node
  is dead, stops routing to it, and takes the control panel down with everything
  else.
- migrate and collectstatic are NOT blocked. Maintenance is usually declared in
  order to run them; a blanket guard on BaseCommand would mean turning the mode off
  to do the work you turned it on for. Only the domain jobs (archive_overdue_clubs,
  extend_event_series, import_members_csv) refuse, and they exit non-zero so cron
  mails you -- a scheduled job that silently skips itself is how a month of billing
  goes missing.

The state is cached with a 10-second TTL, not for ever. Write-through makes the
flip instant for the shared Redis of a real deployment, and the TTL is the belt to
that braces: on a per-process cache -- a dev box with no Redis, or a misconfigured
deploy -- a lock-down that reached only one gunicorn worker would be worse than
useless. Live-verified: a club subdomain, its login page and the base domain all
503 while the control panel and the sign-in screens stay up.

Also adds the two deployment pieces asked for: compose.behind-proxy.yaml for a
dev/test box that already runs Caddy on :80 (app on the loopback, host Caddy proxies
to it -- and the host's Caddy still needs the DNS plugin, because the wildcard is
still a wildcard), and deploy/backup.sh + restore-check.sh with a cron schedule. The
backup writes to a .part file and only lands it once gzip -t says it is readable: a
truncated dump that looks like a backup is the failure you find on the day you need
it. The weekly restore rehearsal is the only line in that cron that proves the rest
work.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

features/tests.py

@@ -1,10 +1,19 @@
+from io import StringIO
+
+from allauth.mfa.models import Authenticator
+from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.test import RequestFactory, TestCase
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.test import RequestFactory, TestCase, override_settings
 from waffle import flag_is_active, get_waffle_flag_model
 
 from club.models import Club
 
+from .models import Maintenance
+
 Flag = get_waffle_flag_model()
+User = get_user_model()
 
 
 class ClubScopedFlagTests(TestCase):
@@ -94,3 +103,134 @@ class ClubScopedFlagTests(TestCase):
         self.club.flags.add(self.flag)
 
         self.assertTrue(self.active_for(self.club))
+
+
+@override_settings(
+    ROSTERCHIEF_BASE_DOMAIN="rosterchief.app",
+    ALLOWED_HOSTS=["rosterchief.app", "ajax-united.rosterchief.app", "testserver"],
+)
+class MaintenanceModeTests(TestCase):
+    def setUp(self):
+        cache.clear()
+        self.addCleanup(cache.clear)
+        self.club = Club.objects.create(name="Ajax United", slug="ajax-united")
+        self.user = User.objects.create_user(email="root@example.com", password="pw-secret-123", is_staff=True)
+        Authenticator.objects.create(user=self.user, type=Authenticator.Type.TOTP, data={"secret": "JBSWY3DPEHPK3PXP"})
+
+    def club_get(self, path="/"):
+        return self.client.get(path, HTTP_HOST="ajax-united.rosterchief.app")
+
+    def platform_get(self, path):
+        return self.client.get(path, HTTP_HOST="rosterchief.app")
+
+    def test_a_club_subdomain_is_closed(self):
+        Maintenance.start(message="Upgrading the database.")
+
+        response = self.club_get("/accounts/login/")
+
+        self.assertEqual(response.status_code, 503)
+        self.assertContains(response, "Upgrading the database", status_code=503)
+        self.assertEqual(response["Retry-After"], "3600")
+
+    def test_a_club_is_open_again_when_maintenance_ends(self):
+        Maintenance.start()
+        Maintenance.stop()
+
+        self.assertEqual(self.club_get("/accounts/login/").status_code, 200)
+
+    def test_the_control_panel_stays_reachable(self):
+        Maintenance.start()
+        self.client.force_login(self.user)
+
+        self.assertEqual(self.platform_get("/controlpanel/").status_code, 200)
+
+    def test_signing_in_stays_possible(self):
+        # Close /accounts/ as well and you cannot sign in to turn maintenance off: a
+        # lock-down with no key, fixable only from a shell.
+        Maintenance.start()
+
+        self.assertEqual(self.platform_get("/accounts/login/").status_code, 200)
+
+    def test_the_health_check_still_answers(self):
+        # Close it and the load balancer decides the node is dead and stops routing to it —
+        # taking the control panel down with everything else.
+        Maintenance.start()
+
+        self.assertEqual(self.platform_get("/healthz").status_code, 200)
+        self.assertEqual(self.client.get("/healthz", HTTP_HOST="ajax-united.rosterchief.app").status_code, 200)
+
+    def test_the_rest_of_the_base_domain_is_closed(self):
+        Maintenance.start()
+
+        self.assertEqual(self.platform_get("/").status_code, 503)
+
+    def test_a_json_caller_gets_json(self):
+        Maintenance.start(message="Back soon.")
+
+        response = self.client.get("/", HTTP_HOST="ajax-united.rosterchief.app", headers={"accept": "application/json"})
+
+        self.assertEqual(response.status_code, 503)
+        self.assertEqual(response.json()["status"], "maintenance")
+
+    def test_the_message_is_optional(self):
+        Maintenance.start()
+
+        self.assertContains(self.club_get("/"), "down for maintenance", status_code=503)
+
+    def test_it_says_which_state_it_is_in(self):
+        self.assertEqual(str(Maintenance.start()), "Maintenance on")
+        self.assertEqual(str(Maintenance.stop()), "Maintenance off")
+
+    def test_it_records_who_closed_the_platform_and_when(self):
+        maintenance = Maintenance.start(message="db upgrade", user=self.user)
+
+        self.assertTrue(maintenance.is_active)
+        self.assertEqual(maintenance.started_by, self.user)
+        self.assertIsNotNone(maintenance.started_at)
+
+    def test_the_state_is_shared_rather_than_per_process(self):
+        # The cache is the shared one the flags use, so closing the platform reaches every
+        # worker and every server. A per-process cache would leave some workers serving clubs.
+        Maintenance.start()
+
+        self.assertTrue(cache.get(Maintenance.CACHE_KEY).is_active)
+        self.assertTrue(Maintenance.is_on())
+
+        Maintenance.stop()
+
+        self.assertFalse(cache.get(Maintenance.CACHE_KEY).is_active)
+
+
+class MaintenanceCommandTests(TestCase):
+    def setUp(self):
+        cache.clear()
+        self.addCleanup(cache.clear)
+
+    def run_command(self, name, *args):
+        out = StringIO()
+        call_command(name, *args, stdout=out, stderr=out)
+        return out.getvalue()
+
+    def test_a_scheduled_job_stands_down(self):
+        Maintenance.start()
+
+        with self.assertRaises(CommandError):
+            self.run_command("archive_overdue_clubs")
+
+    def test_it_runs_again_once_the_platform_reopens(self):
+        Maintenance.start()
+        Maintenance.stop()
+
+        self.assertIn("Nothing overdue", self.run_command("archive_overdue_clubs"))
+
+    def test_an_override_exists_for_when_you_mean_it(self):
+        Maintenance.start()
+
+        self.assertIn("Nothing overdue", self.run_command("archive_overdue_clubs", "--ignore-maintenance"))
+
+    def test_migrate_is_not_blocked(self):
+        # Maintenance is usually declared IN ORDER to migrate. A guard on every command would
+        # mean turning the mode off to do the work you turned it on for.
+        Maintenance.start()
+
+        self.run_command("migrate", "--check")  # raises SystemExit only if migrations are pending