fix(ops): harden deploy/restore/verify scripts
Some checks failed
CI / validate (push) Has been cancelled
CI / dependency-scanning (push) Has been cancelled
CI / docs (push) Has been cancelled
CI / deploy (push) Has been cancelled
CI / ruff (push) Successful in 9s
CI / pytest (push) Has been cancelled

- deploy.sh: add DB health wait before migrations, prune old Docker images
- restore.sh: add redis-exporter to stop list, replace sleep with DB health wait
- verify-server.sh: add redis-exporter to expected containers, add Sentry + Redis exporter checks

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-28 00:23:14 +01:00
parent 35d1559162
commit 93a2d9baff
3 changed files with 85 additions and 15 deletions

View File

@@ -226,7 +226,7 @@ if [ "$MODE" = "prod" ]; then
section "3. Docker Containers"
# -----------------------------------------------------------------------
EXPECTED_CONTAINERS="db redis api celery-worker celery-beat flower prometheus grafana node-exporter cadvisor alertmanager"
EXPECTED_CONTAINERS="db redis api celery-worker celery-beat flower prometheus grafana node-exporter cadvisor alertmanager redis-exporter"
for name in $EXPECTED_CONTAINERS; do
container=$(docker compose --profile full -f "$ORION_DIR/docker-compose.yml" ps --format '{{.Name}}' 2>/dev/null | grep "$name" || true)
if [ -n "$container" ]; then
@@ -338,7 +338,44 @@ if [ "$MODE" = "prod" ]; then
fi
# -----------------------------------------------------------------------
section "9. DNS Resolution"
section "9. Sentry"
# -----------------------------------------------------------------------
if [ -f "$ORION_DIR/.env" ]; then
SENTRY_DSN=$(grep -E '^SENTRY_DSN=' "$ORION_DIR/.env" 2>/dev/null | cut -d= -f2- || echo "")
if [ -n "$SENTRY_DSN" ] && [ "$SENTRY_DSN" != "None" ]; then
pass "SENTRY_DSN is configured"
else
warn "SENTRY_DSN not set — error tracking disabled"
fi
SENTRY_ENV=$(grep -E '^SENTRY_ENVIRONMENT=' "$ORION_DIR/.env" 2>/dev/null | cut -d= -f2- || echo "")
if [ "$SENTRY_ENV" = "production" ]; then
pass "SENTRY_ENVIRONMENT is 'production'"
elif [ -n "$SENTRY_ENV" ]; then
warn "SENTRY_ENVIRONMENT is '$SENTRY_ENV' (expected 'production')"
fi
fi
# -----------------------------------------------------------------------
section "10. Redis Exporter"
# -----------------------------------------------------------------------
redis_exporter_status=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:9121/health 2>/dev/null || echo "000")
if [ "$redis_exporter_status" = "200" ]; then
pass "Redis exporter: accessible (HTTP 200)"
redis_up=$(curl -s http://localhost:9121/metrics 2>/dev/null | grep '^redis_up ' | awk '{print $2}' || echo "0")
if [ "$redis_up" = "1" ]; then
pass "Redis exporter: redis_up = 1"
else
fail "Redis exporter: redis_up = $redis_up (Redis unreachable)"
fi
else
fail "Redis exporter: HTTP $redis_exporter_status (expected 200)"
fi
# -----------------------------------------------------------------------
section "11. DNS Resolution"
# -----------------------------------------------------------------------
EXPECTED_DOMAINS="wizard.lu api.wizard.lu git.wizard.lu grafana.wizard.lu flower.wizard.lu omsflow.lu rewardflow.lu"
@@ -352,7 +389,7 @@ if [ "$MODE" = "prod" ]; then
done
# -----------------------------------------------------------------------
section "10. Health Endpoints"
section "12. Health Endpoints"
# -----------------------------------------------------------------------
HEALTH_URL="http://localhost:8001/health"
@@ -383,7 +420,7 @@ if [ "$MODE" = "prod" ]; then
fi
# -----------------------------------------------------------------------
section "11. Prometheus Targets"
section "13. Prometheus Targets"
# -----------------------------------------------------------------------
targets=$(curl -s http://localhost:9090/api/v1/targets 2>/dev/null || echo "")
@@ -402,7 +439,7 @@ if [ "$MODE" = "prod" ]; then
fi
# -----------------------------------------------------------------------
section "12. Grafana"
section "14. Grafana"
# -----------------------------------------------------------------------
grafana_status=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:3001/api/health 2>/dev/null || echo "000")