sboulahtit/orion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add Celery/Redis task queue with feature flag support

Browse Source 
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis
for persistent task queuing, retries, and scheduled jobs.

Key changes:
- Add Celery configuration with Redis broker/backend
- Create task dispatcher with USE_CELERY feature flag for gradual rollout
- Add Celery task wrappers for all background operations:
  - Marketplace imports
  - Letzshop historical imports
  - Product exports
  - Code quality scans
  - Test runs
  - Subscription scheduled tasks (via Celery Beat)
- Add celery_task_id column to job tables for Flower integration
- Add Flower dashboard link to admin background tasks page
- Update docker-compose.yml with worker, beat, and flower services
- Add Makefile targets: celery-worker, celery-beat, celery-dev, flower

When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks
for development without Redis dependency.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Samir Boulahtit
2026-01-11 17:35:16 +01:00
parent 879ac0caea
commit 2792414395
30 changed files with 2218 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/api/v1/admin/background_tasks.py
View File

@@ -31,6 +31,7 @@ class BackgroundTaskResponse(BaseModel):
triggered_by: str | None
error_message: str | None
details: dict | None
celery_task_id: str | None = None # Celery task ID for Flower linking
class BackgroundTasksStatsResponse(BaseModel):
@@ -77,6 +78,7 @@ def _convert_import_to_response(job) -> BackgroundTaskResponse:
"errors": job.error_count,
"total_processed": job.total_processed,
},
celery_task_id=getattr(job, "celery_task_id", None),
)
@@ -105,6 +107,7 @@ def _convert_test_run_to_response(run) -> BackgroundTaskResponse:
"pass_rate": run.pass_rate,
"git_branch": run.git_branch,
},
celery_task_id=getattr(run, "celery_task_id", None),
)
@@ -141,6 +144,7 @@ def _convert_scan_to_response(scan) -> BackgroundTaskResponse:
"git_commit_hash": scan.git_commit_hash,
"progress_message": scan.progress_message,
},
celery_task_id=getattr(scan, "celery_task_id", None),
)

14
app/api/v1/admin/code_quality.py
View File

@@ -213,14 +213,24 @@ async def trigger_scan(
scan_jobs = []
triggered_by = f"manual:{current_user.username}"
# Import dispatcher for Celery support
from app.tasks.dispatcher import task_dispatcher
for vtype in request.validator_types:
# Create scan record with pending status via service
scan = code_quality_service.create_pending_scan(
db, validator_type=vtype.value, triggered_by=triggered_by
)
# Queue background task
background_tasks.add_task(execute_code_quality_scan, scan.id)
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
celery_task_id = task_dispatcher.dispatch_code_quality_scan(
background_tasks=background_tasks,
scan_id=scan.id,
)
# Store Celery task ID if using Celery
if celery_task_id:
scan.celery_task_id = celery_task_id
scan_jobs.append(
ScanJobResponse(

17
app/api/v1/admin/letzshop.py
View File

@@ -823,13 +823,20 @@ def start_historical_import(
logger.info(f"Created historical import job {job.id} for vendor {vendor_id}")
# Queue background task
background_tasks.add_task(
process_historical_import,
job.id,
vendor_id,
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
from app.tasks.dispatcher import task_dispatcher
celery_task_id = task_dispatcher.dispatch_historical_import(
background_tasks=background_tasks,
job_id=job.id,
vendor_id=vendor_id,
)
# Store Celery task ID if using Celery
if celery_task_id:
job.celery_task_id = celery_task_id
db.commit()
return LetzshopHistoricalImportStartResponse(
job_id=job.id,
status="pending",

24
app/api/v1/admin/marketplace.py
View File

@@ -97,16 +97,24 @@ async def create_marketplace_import_job(
f"for vendor {vendor.vendor_code} (language={request.language})"
)
background_tasks.add_task(
process_marketplace_import,
job.id,
request.source_url,
request.marketplace,
vendor.id,
request.batch_size or 1000,
request.language, # Pass language to background task
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
from app.tasks.dispatcher import task_dispatcher
celery_task_id = task_dispatcher.dispatch_marketplace_import(
background_tasks=background_tasks,
job_id=job.id,
url=request.source_url,
marketplace=request.marketplace,
vendor_id=vendor.id,
batch_size=request.batch_size or 1000,
language=request.language,
)
# Store Celery task ID if using Celery
if celery_task_id:
job.celery_task_id = celery_task_id
db.commit()
return marketplace_import_job_service.convert_to_response_model(job)

19
app/api/v1/admin/tests.py
View File

@@ -128,14 +128,21 @@ async def run_tests(
)
db.commit()
# Start background execution
background_tasks.add_task(
execute_test_run,
run.id,
test_path,
extra_args,
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
from app.tasks.dispatcher import task_dispatcher
celery_task_id = task_dispatcher.dispatch_test_run(
background_tasks=background_tasks,
run_id=run.id,
test_path=test_path,
extra_args=extra_args,
)
# Store Celery task ID if using Celery
if celery_task_id:
run.celery_task_id = celery_task_id
db.commit()
return TestRunResponse(
id=run.id,
timestamp=run.timestamp.isoformat(),

30
app/api/v1/admin/vendors.py
View File

@@ -382,24 +382,46 @@ def export_vendor_products_letzshop_to_folder(
automated product sync.
**Behavior:**
- When Celery is enabled: Queues export as background task, returns immediately
- When Celery is disabled: Runs synchronously and returns file paths
- Creates CSV files for each language (fr, de, en)
- Places files in: exports/letzshop/{vendor_code}/
- Filename format: {vendor_code}_products_{language}.csv
Returns:
JSON with export status and file paths
JSON with export status and file paths (or task_id if async)
"""
import os
from datetime import UTC, datetime
from pathlib import Path as FilePath
from app.core.config import settings
from app.services.letzshop_export_service import letzshop_export_service
vendor = vendor_service.get_vendor_by_identifier(db, vendor_identifier)
include_inactive = request.include_inactive if request else False
# Track export start time
# If Celery is enabled, dispatch as async task
if settings.use_celery:
from app.tasks.dispatcher import task_dispatcher
celery_task_id = task_dispatcher.dispatch_product_export(
vendor_id=vendor.id,
triggered_by=f"admin:{current_admin.id}",
include_inactive=include_inactive,
)
return {
"success": True,
"message": f"Export task queued for vendor {vendor.vendor_code}. Check Flower for status.",
"vendor_code": vendor.vendor_code,
"export_directory": f"exports/letzshop/{vendor.vendor_code.lower()}",
"files": [],
"celery_task_id": celery_task_id,
"is_async": True,
}
# Synchronous export (when Celery is disabled)
started_at = datetime.now(UTC)
# Create export directory
@@ -445,7 +467,6 @@ def export_vendor_products_letzshop_to_folder(
})
# Log the export operation via service
# For exports, track files (languages) as the unit of work
completed_at = datetime.now(UTC)
letzshop_export_service.log_export(
db=db,
@@ -467,4 +488,5 @@ def export_vendor_products_letzshop_to_folder(
"vendor_code": vendor.vendor_code,
"export_directory": str(export_dir),
"files": exported_files,
"is_async": False,
}