feat: add Celery/Redis task queue with feature flag support

Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis
for persistent task queuing, retries, and scheduled jobs.

Key changes:
- Add Celery configuration with Redis broker/backend
- Create task dispatcher with USE_CELERY feature flag for gradual rollout
- Add Celery task wrappers for all background operations:
  - Marketplace imports
  - Letzshop historical imports
  - Product exports
  - Code quality scans
  - Test runs
  - Subscription scheduled tasks (via Celery Beat)
- Add celery_task_id column to job tables for Flower integration
- Add Flower dashboard link to admin background tasks page
- Update docker-compose.yml with worker, beat, and flower services
- Add Makefile targets: celery-worker, celery-beat, celery-dev, flower

When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks
for development without Redis dependency.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-11 17:35:16 +01:00
parent 879ac0caea
commit 2792414395
30 changed files with 2218 additions and 79 deletions

View File

@@ -45,9 +45,9 @@ class Settings(BaseSettings):
"""
# =============================================================================
# DATABASE
# DATABASE (PostgreSQL only)
# =============================================================================
database_url: str = "sqlite:///./wizamart.db"
database_url: str = "postgresql://wizamart_user:secure_password@localhost:5432/wizamart_db"
# =============================================================================
# ADMIN INITIALIZATION (for init_production.py)
@@ -174,6 +174,19 @@ class Settings(BaseSettings):
seed_products_per_vendor: int = 20 # Products per vendor
seed_orders_per_vendor: int = 10 # Orders per vendor
# =============================================================================
# CELERY / REDIS TASK QUEUE
# =============================================================================
# Redis URL for Celery broker and result backend
redis_url: str = "redis://localhost:6379/0"
# Feature flag: enable Celery for background tasks (False = use FastAPI BackgroundTasks)
use_celery: bool = False
# Flower monitoring dashboard
flower_url: str = "http://localhost:5555"
flower_password: str = "changeme" # CHANGE IN PRODUCTION!
model_config = {"env_file": ".env"}
@@ -238,6 +251,27 @@ def is_staging_environment() -> bool:
# =============================================================================
def validate_database_url() -> None:
"""
Validate that database URL is PostgreSQL.
Raises:
ValueError: If database URL is not PostgreSQL
"""
if settings.database_url.startswith("sqlite"):
raise ValueError(
"SQLite is not supported. Please use PostgreSQL.\n"
"Set DATABASE_URL environment variable to a PostgreSQL connection string.\n"
"Example: postgresql://user:password@localhost:5432/dbname\n"
"For local development, run: docker-compose up -d db"
)
if not settings.database_url.startswith("postgresql"):
raise ValueError(
f"Unsupported database: {settings.database_url.split(':')[0]}\n"
"Only PostgreSQL is supported."
)
def validate_production_settings() -> list[str]:
"""
Validate settings for production environment.
@@ -304,6 +338,7 @@ __all__ = [
"is_development_environment",
"is_staging_environment",
# Validation
"validate_database_url",
"validate_production_settings",
"print_environment_info",
]