Files
orion/app/api/v1/vendor/marketplace.py
Samir Boulahtit 2792414395 feat: add Celery/Redis task queue with feature flag support
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis
for persistent task queuing, retries, and scheduled jobs.

Key changes:
- Add Celery configuration with Redis broker/backend
- Create task dispatcher with USE_CELERY feature flag for gradual rollout
- Add Celery task wrappers for all background operations:
  - Marketplace imports
  - Letzshop historical imports
  - Product exports
  - Code quality scans
  - Test runs
  - Subscription scheduled tasks (via Celery Beat)
- Add celery_task_id column to job tables for Flower integration
- Add Flower dashboard link to admin background tasks page
- Update docker-compose.yml with worker, beat, and flower services
- Add Makefile targets: celery-worker, celery-beat, celery-dev, flower

When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks
for development without Redis dependency.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-11 17:35:16 +01:00

134 lines
4.6 KiB
Python

# app/api/v1/vendor/marketplace.py
"""
Marketplace import endpoints for vendors.
Vendor Context: Uses token_vendor_id from JWT token (authenticated vendor API pattern).
The get_current_vendor_api dependency guarantees token_vendor_id is present.
"""
import logging
from fastapi import APIRouter, BackgroundTasks, Depends, Query
from sqlalchemy.orm import Session
from app.api.deps import get_current_vendor_api
from app.core.database import get_db
from app.services.marketplace_import_job_service import marketplace_import_job_service
from app.services.vendor_service import vendor_service
from app.tasks.background_tasks import process_marketplace_import
from middleware.decorators import rate_limit
from models.database.user import User
from models.schema.marketplace_import_job import (
MarketplaceImportJobRequest,
MarketplaceImportJobResponse,
)
router = APIRouter(prefix="/marketplace")
logger = logging.getLogger(__name__)
@router.post("/import", response_model=MarketplaceImportJobResponse)
@rate_limit(max_requests=10, window_seconds=3600)
async def import_products_from_marketplace(
request: MarketplaceImportJobRequest,
background_tasks: BackgroundTasks,
current_user: User = Depends(get_current_vendor_api),
db: Session = Depends(get_db),
):
"""Import products from marketplace CSV with background processing (Protected).
The `language` parameter specifies the language code for product
translations (e.g., 'en', 'fr', 'de'). Default is 'en'.
For multi-language imports, call this endpoint multiple times with
different language codes and CSV files containing translations.
"""
vendor = vendor_service.get_vendor_by_id(db, current_user.token_vendor_id)
logger.info(
f"Starting marketplace import: {request.marketplace} for vendor {vendor.vendor_code} "
f"by user {current_user.username} (language={request.language})"
)
# Create import job (vendor comes from token)
import_job = marketplace_import_job_service.create_import_job(
db, request, vendor, current_user
)
db.commit()
# Dispatch via task dispatcher (supports Celery or BackgroundTasks)
from app.tasks.dispatcher import task_dispatcher
celery_task_id = task_dispatcher.dispatch_marketplace_import(
background_tasks=background_tasks,
job_id=import_job.id,
url=request.source_url,
marketplace=request.marketplace,
vendor_id=vendor.id,
batch_size=request.batch_size or 1000,
language=request.language,
)
# Store Celery task ID if using Celery
if celery_task_id:
import_job.celery_task_id = celery_task_id
db.commit()
return MarketplaceImportJobResponse(
job_id=import_job.id,
status="pending",
marketplace=request.marketplace,
vendor_id=import_job.vendor_id,
vendor_code=vendor.vendor_code,
vendor_name=vendor.name,
source_url=request.source_url,
language=request.language,
message=f"Marketplace import started from {request.marketplace}. "
f"Check status with /import-status/{import_job.id}",
imported=0,
updated=0,
total_processed=0,
error_count=0,
created_at=import_job.created_at,
)
@router.get("/imports/{job_id}", response_model=MarketplaceImportJobResponse)
def get_marketplace_import_status(
job_id: int,
current_user: User = Depends(get_current_vendor_api),
db: Session = Depends(get_db),
):
"""Get status of marketplace import job (Protected)."""
# Service validates that job belongs to vendor and raises UnauthorizedVendorAccessException if not
job = marketplace_import_job_service.get_import_job_for_vendor(
db, job_id, current_user.token_vendor_id
)
return marketplace_import_job_service.convert_to_response_model(job)
@router.get("/imports", response_model=list[MarketplaceImportJobResponse])
def get_marketplace_import_jobs(
marketplace: str | None = Query(None, description="Filter by marketplace"),
skip: int = Query(0, ge=0),
limit: int = Query(50, ge=1, le=100),
current_user: User = Depends(get_current_vendor_api),
db: Session = Depends(get_db),
):
"""Get marketplace import jobs for current vendor (Protected)."""
vendor = vendor_service.get_vendor_by_id(db, current_user.token_vendor_id)
jobs = marketplace_import_job_service.get_import_jobs(
db=db,
vendor=vendor,
user=current_user,
marketplace=marketplace,
skip=skip,
limit=limit,
)
return [
marketplace_import_job_service.convert_to_response_model(job) for job in jobs
]