feat: add Celery/Redis task queue with feature flag support
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -382,24 +382,46 @@ def export_vendor_products_letzshop_to_folder(
|
||||
automated product sync.
|
||||
|
||||
**Behavior:**
|
||||
- When Celery is enabled: Queues export as background task, returns immediately
|
||||
- When Celery is disabled: Runs synchronously and returns file paths
|
||||
- Creates CSV files for each language (fr, de, en)
|
||||
- Places files in: exports/letzshop/{vendor_code}/
|
||||
- Filename format: {vendor_code}_products_{language}.csv
|
||||
|
||||
Returns:
|
||||
JSON with export status and file paths
|
||||
JSON with export status and file paths (or task_id if async)
|
||||
"""
|
||||
import os
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path as FilePath
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.letzshop_export_service import letzshop_export_service
|
||||
|
||||
vendor = vendor_service.get_vendor_by_identifier(db, vendor_identifier)
|
||||
|
||||
include_inactive = request.include_inactive if request else False
|
||||
|
||||
# Track export start time
|
||||
# If Celery is enabled, dispatch as async task
|
||||
if settings.use_celery:
|
||||
from app.tasks.dispatcher import task_dispatcher
|
||||
|
||||
celery_task_id = task_dispatcher.dispatch_product_export(
|
||||
vendor_id=vendor.id,
|
||||
triggered_by=f"admin:{current_admin.id}",
|
||||
include_inactive=include_inactive,
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"Export task queued for vendor {vendor.vendor_code}. Check Flower for status.",
|
||||
"vendor_code": vendor.vendor_code,
|
||||
"export_directory": f"exports/letzshop/{vendor.vendor_code.lower()}",
|
||||
"files": [],
|
||||
"celery_task_id": celery_task_id,
|
||||
"is_async": True,
|
||||
}
|
||||
|
||||
# Synchronous export (when Celery is disabled)
|
||||
started_at = datetime.now(UTC)
|
||||
|
||||
# Create export directory
|
||||
@@ -445,7 +467,6 @@ def export_vendor_products_letzshop_to_folder(
|
||||
})
|
||||
|
||||
# Log the export operation via service
|
||||
# For exports, track files (languages) as the unit of work
|
||||
completed_at = datetime.now(UTC)
|
||||
letzshop_export_service.log_export(
|
||||
db=db,
|
||||
@@ -467,4 +488,5 @@ def export_vendor_products_letzshop_to_folder(
|
||||
"vendor_code": vendor.vendor_code,
|
||||
"export_directory": str(export_dir),
|
||||
"files": exported_files,
|
||||
"is_async": False,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user