feat: add Celery/Redis task queue with feature flag support
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
24
app/api/v1/vendor/onboarding.py
vendored
24
app/api/v1/vendor/onboarding.py
vendored
@@ -227,11 +227,27 @@ def trigger_order_sync(
|
||||
|
||||
# Queue background task to process the import
|
||||
if result.get("success") and result.get("job_id"):
|
||||
background_tasks.add_task(
|
||||
process_historical_import,
|
||||
result["job_id"],
|
||||
current_user.token_vendor_id,
|
||||
from app.tasks.dispatcher import task_dispatcher
|
||||
|
||||
celery_task_id = task_dispatcher.dispatch_historical_import(
|
||||
background_tasks=background_tasks,
|
||||
job_id=result["job_id"],
|
||||
vendor_id=current_user.token_vendor_id,
|
||||
)
|
||||
|
||||
# Store Celery task ID if using Celery
|
||||
if celery_task_id:
|
||||
from models.database.letzshop import LetzshopHistoricalImportJob
|
||||
|
||||
job = (
|
||||
db.query(LetzshopHistoricalImportJob)
|
||||
.filter(LetzshopHistoricalImportJob.id == result["job_id"])
|
||||
.first()
|
||||
)
|
||||
if job:
|
||||
job.celery_task_id = celery_task_id
|
||||
db.commit()
|
||||
|
||||
logger.info(f"Queued historical import task for job {result['job_id']}")
|
||||
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user