Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
134 lines
4.6 KiB
Python
134 lines
4.6 KiB
Python
# app/api/v1/vendor/marketplace.py
|
|
"""
|
|
Marketplace import endpoints for vendors.
|
|
|
|
Vendor Context: Uses token_vendor_id from JWT token (authenticated vendor API pattern).
|
|
The get_current_vendor_api dependency guarantees token_vendor_id is present.
|
|
"""
|
|
|
|
import logging
|
|
|
|
from fastapi import APIRouter, BackgroundTasks, Depends, Query
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.api.deps import get_current_vendor_api
|
|
from app.core.database import get_db
|
|
from app.services.marketplace_import_job_service import marketplace_import_job_service
|
|
from app.services.vendor_service import vendor_service
|
|
from app.tasks.background_tasks import process_marketplace_import
|
|
from middleware.decorators import rate_limit
|
|
from models.database.user import User
|
|
from models.schema.marketplace_import_job import (
|
|
MarketplaceImportJobRequest,
|
|
MarketplaceImportJobResponse,
|
|
)
|
|
|
|
router = APIRouter(prefix="/marketplace")
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@router.post("/import", response_model=MarketplaceImportJobResponse)
|
|
@rate_limit(max_requests=10, window_seconds=3600)
|
|
async def import_products_from_marketplace(
|
|
request: MarketplaceImportJobRequest,
|
|
background_tasks: BackgroundTasks,
|
|
current_user: User = Depends(get_current_vendor_api),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Import products from marketplace CSV with background processing (Protected).
|
|
|
|
The `language` parameter specifies the language code for product
|
|
translations (e.g., 'en', 'fr', 'de'). Default is 'en'.
|
|
|
|
For multi-language imports, call this endpoint multiple times with
|
|
different language codes and CSV files containing translations.
|
|
"""
|
|
vendor = vendor_service.get_vendor_by_id(db, current_user.token_vendor_id)
|
|
|
|
logger.info(
|
|
f"Starting marketplace import: {request.marketplace} for vendor {vendor.vendor_code} "
|
|
f"by user {current_user.username} (language={request.language})"
|
|
)
|
|
|
|
# Create import job (vendor comes from token)
|
|
import_job = marketplace_import_job_service.create_import_job(
|
|
db, request, vendor, current_user
|
|
)
|
|
db.commit()
|
|
|
|
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
|
|
from app.tasks.dispatcher import task_dispatcher
|
|
|
|
celery_task_id = task_dispatcher.dispatch_marketplace_import(
|
|
background_tasks=background_tasks,
|
|
job_id=import_job.id,
|
|
url=request.source_url,
|
|
marketplace=request.marketplace,
|
|
vendor_id=vendor.id,
|
|
batch_size=request.batch_size or 1000,
|
|
language=request.language,
|
|
)
|
|
|
|
# Store Celery task ID if using Celery
|
|
if celery_task_id:
|
|
import_job.celery_task_id = celery_task_id
|
|
db.commit()
|
|
|
|
return MarketplaceImportJobResponse(
|
|
job_id=import_job.id,
|
|
status="pending",
|
|
marketplace=request.marketplace,
|
|
vendor_id=import_job.vendor_id,
|
|
vendor_code=vendor.vendor_code,
|
|
vendor_name=vendor.name,
|
|
source_url=request.source_url,
|
|
language=request.language,
|
|
message=f"Marketplace import started from {request.marketplace}. "
|
|
f"Check status with /import-status/{import_job.id}",
|
|
imported=0,
|
|
updated=0,
|
|
total_processed=0,
|
|
error_count=0,
|
|
created_at=import_job.created_at,
|
|
)
|
|
|
|
|
|
@router.get("/imports/{job_id}", response_model=MarketplaceImportJobResponse)
|
|
def get_marketplace_import_status(
|
|
job_id: int,
|
|
current_user: User = Depends(get_current_vendor_api),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Get status of marketplace import job (Protected)."""
|
|
# Service validates that job belongs to vendor and raises UnauthorizedVendorAccessException if not
|
|
job = marketplace_import_job_service.get_import_job_for_vendor(
|
|
db, job_id, current_user.token_vendor_id
|
|
)
|
|
|
|
return marketplace_import_job_service.convert_to_response_model(job)
|
|
|
|
|
|
@router.get("/imports", response_model=list[MarketplaceImportJobResponse])
|
|
def get_marketplace_import_jobs(
|
|
marketplace: str | None = Query(None, description="Filter by marketplace"),
|
|
skip: int = Query(0, ge=0),
|
|
limit: int = Query(50, ge=1, le=100),
|
|
current_user: User = Depends(get_current_vendor_api),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""Get marketplace import jobs for current vendor (Protected)."""
|
|
vendor = vendor_service.get_vendor_by_id(db, current_user.token_vendor_id)
|
|
|
|
jobs = marketplace_import_job_service.get_import_jobs(
|
|
db=db,
|
|
vendor=vendor,
|
|
user=current_user,
|
|
marketplace=marketplace,
|
|
skip=skip,
|
|
limit=limit,
|
|
)
|
|
|
|
return [
|
|
marketplace_import_job_service.convert_to_response_model(job) for job in jobs
|
|
]
|