Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
174 lines
5.6 KiB
Python
174 lines
5.6 KiB
Python
# app/api/v1/admin/marketplace.py
|
|
"""
|
|
Marketplace import job monitoring endpoints for admin.
|
|
"""
|
|
|
|
import logging
|
|
|
|
from fastapi import APIRouter, BackgroundTasks, Depends, Query
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.api.deps import get_current_admin_api
|
|
from app.core.database import get_db
|
|
from app.services.marketplace_import_job_service import marketplace_import_job_service
|
|
from app.services.stats_service import stats_service
|
|
from app.services.vendor_service import vendor_service
|
|
from app.tasks.background_tasks import process_marketplace_import
|
|
from models.database.user import User
|
|
from models.schema.marketplace_import_job import (
|
|
AdminMarketplaceImportJobListResponse,
|
|
AdminMarketplaceImportJobRequest,
|
|
AdminMarketplaceImportJobResponse,
|
|
MarketplaceImportErrorListResponse,
|
|
MarketplaceImportErrorResponse,
|
|
MarketplaceImportJobRequest,
|
|
MarketplaceImportJobResponse,
|
|
)
|
|
from models.schema.stats import ImportStatsResponse
|
|
|
|
router = APIRouter(prefix="/marketplace-import-jobs")
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@router.get("", response_model=AdminMarketplaceImportJobListResponse)
|
|
def get_all_marketplace_import_jobs(
|
|
marketplace: str | None = Query(None),
|
|
status: str | None = Query(None),
|
|
page: int = Query(1, ge=1),
|
|
limit: int = Query(100, ge=1, le=100),
|
|
db: Session = Depends(get_db),
|
|
current_admin: User = Depends(get_current_admin_api),
|
|
):
|
|
"""Get all marketplace import jobs with pagination (Admin only)."""
|
|
jobs, total = marketplace_import_job_service.get_all_import_jobs_paginated(
|
|
db=db,
|
|
marketplace=marketplace,
|
|
status=status,
|
|
page=page,
|
|
limit=limit,
|
|
)
|
|
|
|
return AdminMarketplaceImportJobListResponse(
|
|
items=[
|
|
marketplace_import_job_service.convert_to_admin_response_model(job)
|
|
for job in jobs
|
|
],
|
|
total=total,
|
|
page=page,
|
|
limit=limit,
|
|
)
|
|
|
|
|
|
@router.post("", response_model=MarketplaceImportJobResponse)
|
|
async def create_marketplace_import_job(
|
|
request: AdminMarketplaceImportJobRequest,
|
|
background_tasks: BackgroundTasks,
|
|
db: Session = Depends(get_db),
|
|
current_admin: User = Depends(get_current_admin_api),
|
|
):
|
|
"""
|
|
Create a new marketplace import job (Admin only).
|
|
|
|
Admins can trigger imports for any vendor by specifying vendor_id.
|
|
The import is processed asynchronously in the background.
|
|
|
|
The `language` parameter specifies the language code for product
|
|
translations (e.g., 'en', 'fr', 'de'). Default is 'en'.
|
|
"""
|
|
vendor = vendor_service.get_vendor_by_id(db, request.vendor_id)
|
|
|
|
job_request = MarketplaceImportJobRequest(
|
|
source_url=request.source_url,
|
|
marketplace=request.marketplace,
|
|
batch_size=request.batch_size,
|
|
language=request.language,
|
|
)
|
|
|
|
job = marketplace_import_job_service.create_import_job(
|
|
db=db,
|
|
request=job_request,
|
|
vendor=vendor,
|
|
user=current_admin,
|
|
)
|
|
db.commit()
|
|
|
|
logger.info(
|
|
f"Admin {current_admin.username} created import job {job.id} "
|
|
f"for vendor {vendor.vendor_code} (language={request.language})"
|
|
)
|
|
|
|
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
|
|
from app.tasks.dispatcher import task_dispatcher
|
|
|
|
celery_task_id = task_dispatcher.dispatch_marketplace_import(
|
|
background_tasks=background_tasks,
|
|
job_id=job.id,
|
|
url=request.source_url,
|
|
marketplace=request.marketplace,
|
|
vendor_id=vendor.id,
|
|
batch_size=request.batch_size or 1000,
|
|
language=request.language,
|
|
)
|
|
|
|
# Store Celery task ID if using Celery
|
|
if celery_task_id:
|
|
job.celery_task_id = celery_task_id
|
|
db.commit()
|
|
|
|
return marketplace_import_job_service.convert_to_response_model(job)
|
|
|
|
|
|
# NOTE: /stats must be defined BEFORE /{job_id} to avoid route conflicts
|
|
@router.get("/stats", response_model=ImportStatsResponse)
|
|
def get_import_statistics(
|
|
db: Session = Depends(get_db),
|
|
current_admin: User = Depends(get_current_admin_api),
|
|
):
|
|
"""Get marketplace import statistics (Admin only)."""
|
|
stats = stats_service.get_import_statistics(db)
|
|
return ImportStatsResponse(**stats)
|
|
|
|
|
|
@router.get("/{job_id}", response_model=AdminMarketplaceImportJobResponse)
|
|
def get_marketplace_import_job(
|
|
job_id: int,
|
|
db: Session = Depends(get_db),
|
|
current_admin: User = Depends(get_current_admin_api),
|
|
):
|
|
"""Get a single marketplace import job by ID (Admin only)."""
|
|
job = marketplace_import_job_service.get_import_job_by_id_admin(db, job_id)
|
|
return marketplace_import_job_service.convert_to_admin_response_model(job)
|
|
|
|
|
|
@router.get("/{job_id}/errors", response_model=MarketplaceImportErrorListResponse)
|
|
def get_import_job_errors(
|
|
job_id: int,
|
|
page: int = Query(1, ge=1),
|
|
limit: int = Query(50, ge=1, le=100),
|
|
error_type: str | None = Query(None, description="Filter by error type"),
|
|
db: Session = Depends(get_db),
|
|
current_admin: User = Depends(get_current_admin_api),
|
|
):
|
|
"""Get import errors for a specific job (Admin only).
|
|
|
|
Returns detailed error information including row number, identifier,
|
|
error type, error message, and raw row data for review.
|
|
"""
|
|
# Verify job exists
|
|
marketplace_import_job_service.get_import_job_by_id_admin(db, job_id)
|
|
|
|
# Get errors from service
|
|
errors, total = marketplace_import_job_service.get_import_job_errors(
|
|
db=db,
|
|
job_id=job_id,
|
|
error_type=error_type,
|
|
page=page,
|
|
limit=limit,
|
|
)
|
|
|
|
return MarketplaceImportErrorListResponse(
|
|
errors=[MarketplaceImportErrorResponse.model_validate(e) for e in errors],
|
|
total=total,
|
|
import_job_id=job_id,
|
|
)
|