Files
orion/app/api/v1/admin/marketplace.py
Samir Boulahtit 2792414395 feat: add Celery/Redis task queue with feature flag support
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis
for persistent task queuing, retries, and scheduled jobs.

Key changes:
- Add Celery configuration with Redis broker/backend
- Create task dispatcher with USE_CELERY feature flag for gradual rollout
- Add Celery task wrappers for all background operations:
  - Marketplace imports
  - Letzshop historical imports
  - Product exports
  - Code quality scans
  - Test runs
  - Subscription scheduled tasks (via Celery Beat)
- Add celery_task_id column to job tables for Flower integration
- Add Flower dashboard link to admin background tasks page
- Update docker-compose.yml with worker, beat, and flower services
- Add Makefile targets: celery-worker, celery-beat, celery-dev, flower

When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks
for development without Redis dependency.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-11 17:35:16 +01:00

174 lines
5.6 KiB
Python

# app/api/v1/admin/marketplace.py
"""
Marketplace import job monitoring endpoints for admin.
"""
import logging
from fastapi import APIRouter, BackgroundTasks, Depends, Query
from sqlalchemy.orm import Session
from app.api.deps import get_current_admin_api
from app.core.database import get_db
from app.services.marketplace_import_job_service import marketplace_import_job_service
from app.services.stats_service import stats_service
from app.services.vendor_service import vendor_service
from app.tasks.background_tasks import process_marketplace_import
from models.database.user import User
from models.schema.marketplace_import_job import (
AdminMarketplaceImportJobListResponse,
AdminMarketplaceImportJobRequest,
AdminMarketplaceImportJobResponse,
MarketplaceImportErrorListResponse,
MarketplaceImportErrorResponse,
MarketplaceImportJobRequest,
MarketplaceImportJobResponse,
)
from models.schema.stats import ImportStatsResponse
router = APIRouter(prefix="/marketplace-import-jobs")
logger = logging.getLogger(__name__)
@router.get("", response_model=AdminMarketplaceImportJobListResponse)
def get_all_marketplace_import_jobs(
marketplace: str | None = Query(None),
status: str | None = Query(None),
page: int = Query(1, ge=1),
limit: int = Query(100, ge=1, le=100),
db: Session = Depends(get_db),
current_admin: User = Depends(get_current_admin_api),
):
"""Get all marketplace import jobs with pagination (Admin only)."""
jobs, total = marketplace_import_job_service.get_all_import_jobs_paginated(
db=db,
marketplace=marketplace,
status=status,
page=page,
limit=limit,
)
return AdminMarketplaceImportJobListResponse(
items=[
marketplace_import_job_service.convert_to_admin_response_model(job)
for job in jobs
],
total=total,
page=page,
limit=limit,
)
@router.post("", response_model=MarketplaceImportJobResponse)
async def create_marketplace_import_job(
request: AdminMarketplaceImportJobRequest,
background_tasks: BackgroundTasks,
db: Session = Depends(get_db),
current_admin: User = Depends(get_current_admin_api),
):
"""
Create a new marketplace import job (Admin only).
Admins can trigger imports for any vendor by specifying vendor_id.
The import is processed asynchronously in the background.
The `language` parameter specifies the language code for product
translations (e.g., 'en', 'fr', 'de'). Default is 'en'.
"""
vendor = vendor_service.get_vendor_by_id(db, request.vendor_id)
job_request = MarketplaceImportJobRequest(
source_url=request.source_url,
marketplace=request.marketplace,
batch_size=request.batch_size,
language=request.language,
)
job = marketplace_import_job_service.create_import_job(
db=db,
request=job_request,
vendor=vendor,
user=current_admin,
)
db.commit()
logger.info(
f"Admin {current_admin.username} created import job {job.id} "
f"for vendor {vendor.vendor_code} (language={request.language})"
)
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
from app.tasks.dispatcher import task_dispatcher
celery_task_id = task_dispatcher.dispatch_marketplace_import(
background_tasks=background_tasks,
job_id=job.id,
url=request.source_url,
marketplace=request.marketplace,
vendor_id=vendor.id,
batch_size=request.batch_size or 1000,
language=request.language,
)
# Store Celery task ID if using Celery
if celery_task_id:
job.celery_task_id = celery_task_id
db.commit()
return marketplace_import_job_service.convert_to_response_model(job)
# NOTE: /stats must be defined BEFORE /{job_id} to avoid route conflicts
@router.get("/stats", response_model=ImportStatsResponse)
def get_import_statistics(
db: Session = Depends(get_db),
current_admin: User = Depends(get_current_admin_api),
):
"""Get marketplace import statistics (Admin only)."""
stats = stats_service.get_import_statistics(db)
return ImportStatsResponse(**stats)
@router.get("/{job_id}", response_model=AdminMarketplaceImportJobResponse)
def get_marketplace_import_job(
job_id: int,
db: Session = Depends(get_db),
current_admin: User = Depends(get_current_admin_api),
):
"""Get a single marketplace import job by ID (Admin only)."""
job = marketplace_import_job_service.get_import_job_by_id_admin(db, job_id)
return marketplace_import_job_service.convert_to_admin_response_model(job)
@router.get("/{job_id}/errors", response_model=MarketplaceImportErrorListResponse)
def get_import_job_errors(
job_id: int,
page: int = Query(1, ge=1),
limit: int = Query(50, ge=1, le=100),
error_type: str | None = Query(None, description="Filter by error type"),
db: Session = Depends(get_db),
current_admin: User = Depends(get_current_admin_api),
):
"""Get import errors for a specific job (Admin only).
Returns detailed error information including row number, identifier,
error type, error message, and raw row data for review.
"""
# Verify job exists
marketplace_import_job_service.get_import_job_by_id_admin(db, job_id)
# Get errors from service
errors, total = marketplace_import_job_service.get_import_job_errors(
db=db,
job_id=job_id,
error_type=error_type,
page=page,
limit=limit,
)
return MarketplaceImportErrorListResponse(
errors=[MarketplaceImportErrorResponse.model_validate(e) for e in errors],
total=total,
import_job_id=job_id,
)