feat: add Celery/Redis task queue with feature flag support
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -45,9 +45,9 @@ class Settings(BaseSettings):
|
||||
"""
|
||||
|
||||
# =============================================================================
|
||||
# DATABASE
|
||||
# DATABASE (PostgreSQL only)
|
||||
# =============================================================================
|
||||
database_url: str = "sqlite:///./wizamart.db"
|
||||
database_url: str = "postgresql://wizamart_user:secure_password@localhost:5432/wizamart_db"
|
||||
|
||||
# =============================================================================
|
||||
# ADMIN INITIALIZATION (for init_production.py)
|
||||
@@ -174,6 +174,19 @@ class Settings(BaseSettings):
|
||||
seed_products_per_vendor: int = 20 # Products per vendor
|
||||
seed_orders_per_vendor: int = 10 # Orders per vendor
|
||||
|
||||
# =============================================================================
|
||||
# CELERY / REDIS TASK QUEUE
|
||||
# =============================================================================
|
||||
# Redis URL for Celery broker and result backend
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
|
||||
# Feature flag: enable Celery for background tasks (False = use FastAPI BackgroundTasks)
|
||||
use_celery: bool = False
|
||||
|
||||
# Flower monitoring dashboard
|
||||
flower_url: str = "http://localhost:5555"
|
||||
flower_password: str = "changeme" # CHANGE IN PRODUCTION!
|
||||
|
||||
model_config = {"env_file": ".env"}
|
||||
|
||||
|
||||
@@ -238,6 +251,27 @@ def is_staging_environment() -> bool:
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def validate_database_url() -> None:
|
||||
"""
|
||||
Validate that database URL is PostgreSQL.
|
||||
|
||||
Raises:
|
||||
ValueError: If database URL is not PostgreSQL
|
||||
"""
|
||||
if settings.database_url.startswith("sqlite"):
|
||||
raise ValueError(
|
||||
"SQLite is not supported. Please use PostgreSQL.\n"
|
||||
"Set DATABASE_URL environment variable to a PostgreSQL connection string.\n"
|
||||
"Example: postgresql://user:password@localhost:5432/dbname\n"
|
||||
"For local development, run: docker-compose up -d db"
|
||||
)
|
||||
if not settings.database_url.startswith("postgresql"):
|
||||
raise ValueError(
|
||||
f"Unsupported database: {settings.database_url.split(':')[0]}\n"
|
||||
"Only PostgreSQL is supported."
|
||||
)
|
||||
|
||||
|
||||
def validate_production_settings() -> list[str]:
|
||||
"""
|
||||
Validate settings for production environment.
|
||||
@@ -304,6 +338,7 @@ __all__ = [
|
||||
"is_development_environment",
|
||||
"is_staging_environment",
|
||||
# Validation
|
||||
"validate_database_url",
|
||||
"validate_production_settings",
|
||||
"print_environment_info",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user