Samir Boulahtit
2792414395
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
108 lines
2.9 KiB
YAML
108 lines
2.9 KiB
YAML
# docker-compose.yml
|
|
services:
|
|
db:
|
|
image: postgres:15
|
|
restart: always
|
|
environment:
|
|
POSTGRES_DB: wizamart_db
|
|
POSTGRES_USER: wizamart_user
|
|
POSTGRES_PASSWORD: secure_password
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
|
|
ports:
|
|
- "5432:5432"
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U wizamart_user -d wizamart_db"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
|
|
redis:
|
|
image: redis:7-alpine
|
|
restart: always
|
|
ports:
|
|
- "6380:6379" # Use 6380 to avoid conflict with host Redis
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
|
|
api:
|
|
build: .
|
|
restart: always
|
|
profiles:
|
|
- full # Only start with: docker compose --profile full up -d
|
|
ports:
|
|
- "8001:8000" # Use 8001 to avoid conflict with local dev server
|
|
environment:
|
|
DATABASE_URL: postgresql://wizamart_user:secure_password@db:5432/wizamart_db
|
|
JWT_SECRET_KEY: ${JWT_SECRET_KEY:-your-super-secret-key}
|
|
REDIS_URL: redis://redis:6379/0
|
|
USE_CELERY: "true"
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
volumes:
|
|
- ./logs:/app/logs
|
|
- ./uploads:/app/uploads
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 3
|
|
|
|
# Celery worker for processing background tasks
|
|
celery-worker:
|
|
build: .
|
|
restart: always
|
|
profiles:
|
|
- full # Only start with: docker compose --profile full up -d
|
|
command: celery -A app.core.celery_config worker --loglevel=info -Q default,long_running,scheduled
|
|
environment:
|
|
DATABASE_URL: postgresql://wizamart_user:secure_password@db:5432/wizamart_db
|
|
REDIS_URL: redis://redis:6379/0
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
redis:
|
|
condition: service_healthy
|
|
volumes:
|
|
- ./logs:/app/logs
|
|
- ./exports:/app/exports
|
|
|
|
# Celery beat for scheduled tasks
|
|
celery-beat:
|
|
build: .
|
|
restart: always
|
|
profiles:
|
|
- full # Only start with: docker compose --profile full up -d
|
|
command: celery -A app.core.celery_config beat --loglevel=info
|
|
environment:
|
|
REDIS_URL: redis://redis:6379/0
|
|
depends_on:
|
|
redis:
|
|
condition: service_healthy
|
|
|
|
# Flower monitoring dashboard
|
|
flower:
|
|
build: .
|
|
restart: always
|
|
profiles:
|
|
- full # Only start with: docker compose --profile full up -d
|
|
command: celery -A app.core.celery_config flower --port=5555
|
|
ports:
|
|
- "5555:5555"
|
|
environment:
|
|
REDIS_URL: redis://redis:6379/0
|
|
FLOWER_BASIC_AUTH: ${FLOWER_USER:-admin}:${FLOWER_PASSWORD:-changeme}
|
|
depends_on:
|
|
redis:
|
|
condition: service_healthy
|
|
|
|
volumes:
|
|
postgres_data:
|