Samir Boulahtit
2792414395
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
273 lines
10 KiB
Python
273 lines
10 KiB
Python
# app/tasks/celery_tasks/letzshop.py
|
|
"""
|
|
Celery tasks for Letzshop historical order imports.
|
|
|
|
Wraps the existing process_historical_import function for Celery execution.
|
|
"""
|
|
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from typing import Callable
|
|
|
|
from app.core.celery_config import celery_app
|
|
from app.services.admin_notification_service import admin_notification_service
|
|
from app.services.letzshop import LetzshopClientError
|
|
from app.services.letzshop.credentials_service import LetzshopCredentialsService
|
|
from app.services.letzshop.order_service import LetzshopOrderService
|
|
from app.tasks.celery_tasks.base import DatabaseTask
|
|
from models.database.letzshop import LetzshopHistoricalImportJob
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _get_credentials_service(db) -> LetzshopCredentialsService:
|
|
"""Create a credentials service instance."""
|
|
return LetzshopCredentialsService(db)
|
|
|
|
|
|
def _get_order_service(db) -> LetzshopOrderService:
|
|
"""Create an order service instance."""
|
|
return LetzshopOrderService(db)
|
|
|
|
|
|
@celery_app.task(
|
|
bind=True,
|
|
base=DatabaseTask,
|
|
name="app.tasks.celery_tasks.letzshop.process_historical_import",
|
|
max_retries=2,
|
|
default_retry_delay=120,
|
|
autoretry_for=(Exception,),
|
|
retry_backoff=True,
|
|
)
|
|
def process_historical_import(self, job_id: int, vendor_id: int):
|
|
"""
|
|
Celery task for historical order import with progress tracking.
|
|
|
|
Imports both confirmed and unconfirmed orders from Letzshop API,
|
|
updating job progress in the database for frontend polling.
|
|
|
|
Args:
|
|
job_id: ID of the LetzshopHistoricalImportJob record
|
|
vendor_id: ID of the vendor to import orders for
|
|
|
|
Returns:
|
|
dict: Import statistics
|
|
"""
|
|
with self.get_db() as db:
|
|
# Get the import job
|
|
job = (
|
|
db.query(LetzshopHistoricalImportJob)
|
|
.filter(LetzshopHistoricalImportJob.id == job_id)
|
|
.first()
|
|
)
|
|
if not job:
|
|
logger.error(f"Historical import job {job_id} not found")
|
|
return {"error": f"Job {job_id} not found"}
|
|
|
|
# Store Celery task ID
|
|
job.celery_task_id = self.request.id
|
|
|
|
try:
|
|
# Mark as started
|
|
job.status = "fetching"
|
|
job.started_at = datetime.now(UTC)
|
|
db.commit()
|
|
|
|
creds_service = _get_credentials_service(db)
|
|
order_service = _get_order_service(db)
|
|
|
|
# Create progress callback for fetching
|
|
def fetch_progress_callback(page: int, total_fetched: int):
|
|
"""Update fetch progress in database."""
|
|
job.current_page = page
|
|
job.shipments_fetched = total_fetched
|
|
db.commit()
|
|
|
|
# Create progress callback for processing
|
|
def create_processing_callback(
|
|
phase: str,
|
|
) -> Callable[[int, int, int, int], None]:
|
|
"""Create a processing progress callback for a phase."""
|
|
|
|
def callback(processed: int, imported: int, updated: int, skipped: int):
|
|
job.orders_processed = processed
|
|
job.orders_imported = imported
|
|
job.orders_updated = updated
|
|
job.orders_skipped = skipped
|
|
db.commit()
|
|
|
|
return callback
|
|
|
|
with creds_service.create_client(vendor_id) as client:
|
|
# ================================================================
|
|
# Phase 1: Import confirmed orders
|
|
# ================================================================
|
|
job.current_phase = "confirmed"
|
|
job.current_page = 0
|
|
job.shipments_fetched = 0
|
|
db.commit()
|
|
|
|
logger.info(f"Job {job_id}: Fetching confirmed shipments for vendor {vendor_id}")
|
|
|
|
confirmed_shipments = client.get_all_shipments_paginated(
|
|
state="confirmed",
|
|
page_size=50,
|
|
progress_callback=fetch_progress_callback,
|
|
)
|
|
|
|
logger.info(f"Job {job_id}: Fetched {len(confirmed_shipments)} confirmed shipments")
|
|
|
|
# Process confirmed shipments
|
|
job.status = "processing"
|
|
job.orders_processed = 0
|
|
job.orders_imported = 0
|
|
job.orders_updated = 0
|
|
job.orders_skipped = 0
|
|
db.commit()
|
|
|
|
confirmed_stats = order_service.import_historical_shipments(
|
|
vendor_id=vendor_id,
|
|
shipments=confirmed_shipments,
|
|
match_products=True,
|
|
progress_callback=create_processing_callback("confirmed"),
|
|
)
|
|
|
|
# Store confirmed stats
|
|
job.confirmed_stats = {
|
|
"total": confirmed_stats["total"],
|
|
"imported": confirmed_stats["imported"],
|
|
"updated": confirmed_stats["updated"],
|
|
"skipped": confirmed_stats["skipped"],
|
|
"products_matched": confirmed_stats["products_matched"],
|
|
"products_not_found": confirmed_stats["products_not_found"],
|
|
}
|
|
job.products_matched = confirmed_stats["products_matched"]
|
|
job.products_not_found = confirmed_stats["products_not_found"]
|
|
db.commit()
|
|
|
|
logger.info(
|
|
f"Job {job_id}: Confirmed phase complete - "
|
|
f"imported={confirmed_stats['imported']}, "
|
|
f"updated={confirmed_stats['updated']}, "
|
|
f"skipped={confirmed_stats['skipped']}"
|
|
)
|
|
|
|
# ================================================================
|
|
# Phase 2: Import unconfirmed (pending) orders
|
|
# ================================================================
|
|
job.current_phase = "unconfirmed"
|
|
job.status = "fetching"
|
|
job.current_page = 0
|
|
job.shipments_fetched = 0
|
|
db.commit()
|
|
|
|
logger.info(f"Job {job_id}: Fetching unconfirmed shipments for vendor {vendor_id}")
|
|
|
|
unconfirmed_shipments = client.get_all_shipments_paginated(
|
|
state="unconfirmed",
|
|
page_size=50,
|
|
progress_callback=fetch_progress_callback,
|
|
)
|
|
|
|
logger.info(
|
|
f"Job {job_id}: Fetched {len(unconfirmed_shipments)} unconfirmed shipments"
|
|
)
|
|
|
|
# Process unconfirmed shipments
|
|
job.status = "processing"
|
|
job.orders_processed = 0
|
|
db.commit()
|
|
|
|
unconfirmed_stats = order_service.import_historical_shipments(
|
|
vendor_id=vendor_id,
|
|
shipments=unconfirmed_shipments,
|
|
match_products=True,
|
|
progress_callback=create_processing_callback("unconfirmed"),
|
|
)
|
|
|
|
# Store unconfirmed stats
|
|
job.declined_stats = {
|
|
"total": unconfirmed_stats["total"],
|
|
"imported": unconfirmed_stats["imported"],
|
|
"updated": unconfirmed_stats["updated"],
|
|
"skipped": unconfirmed_stats["skipped"],
|
|
"products_matched": unconfirmed_stats["products_matched"],
|
|
"products_not_found": unconfirmed_stats["products_not_found"],
|
|
}
|
|
|
|
# Add to cumulative product matching stats
|
|
job.products_matched += unconfirmed_stats["products_matched"]
|
|
job.products_not_found += unconfirmed_stats["products_not_found"]
|
|
|
|
logger.info(
|
|
f"Job {job_id}: Unconfirmed phase complete - "
|
|
f"imported={unconfirmed_stats['imported']}, "
|
|
f"updated={unconfirmed_stats['updated']}, "
|
|
f"skipped={unconfirmed_stats['skipped']}"
|
|
)
|
|
|
|
# ================================================================
|
|
# Complete
|
|
# ================================================================
|
|
job.status = "completed"
|
|
job.completed_at = datetime.now(UTC)
|
|
db.commit()
|
|
|
|
# Update credentials sync status
|
|
creds_service.update_sync_status(vendor_id, "success", None)
|
|
|
|
logger.info(f"Job {job_id}: Historical import completed successfully")
|
|
|
|
return {
|
|
"job_id": job_id,
|
|
"confirmed": confirmed_stats,
|
|
"unconfirmed": unconfirmed_stats,
|
|
}
|
|
|
|
except LetzshopClientError as e:
|
|
logger.error(f"Job {job_id}: Letzshop API error: {e}")
|
|
job.status = "failed"
|
|
job.error_message = f"Letzshop API error: {e}"
|
|
job.completed_at = datetime.now(UTC)
|
|
|
|
# Get vendor name for notification
|
|
order_service = _get_order_service(db)
|
|
vendor = order_service.get_vendor(vendor_id)
|
|
vendor_name = vendor.name if vendor else f"Vendor {vendor_id}"
|
|
|
|
# Create admin notification
|
|
admin_notification_service.notify_order_sync_failure(
|
|
db=db,
|
|
vendor_name=vendor_name,
|
|
error_message=f"Historical import failed: {str(e)[:150]}",
|
|
vendor_id=vendor_id,
|
|
)
|
|
|
|
db.commit()
|
|
|
|
creds_service = _get_credentials_service(db)
|
|
creds_service.update_sync_status(vendor_id, "failed", str(e))
|
|
raise # Re-raise for Celery retry
|
|
|
|
except Exception as e:
|
|
logger.error(f"Job {job_id}: Unexpected error: {e}", exc_info=True)
|
|
job.status = "failed"
|
|
job.error_message = str(e)[:500]
|
|
job.completed_at = datetime.now(UTC)
|
|
|
|
# Get vendor name for notification
|
|
order_service = _get_order_service(db)
|
|
vendor = order_service.get_vendor(vendor_id)
|
|
vendor_name = vendor.name if vendor else f"Vendor {vendor_id}"
|
|
|
|
# Create admin notification
|
|
admin_notification_service.notify_critical_error(
|
|
db=db,
|
|
error_type="Historical Import",
|
|
error_message=f"Import job {job_id} failed for {vendor_name}: {str(e)[:150]}",
|
|
details={"job_id": job_id, "vendor_id": vendor_id, "vendor_name": vendor_name},
|
|
)
|
|
|
|
db.commit()
|
|
raise # Re-raise for Celery retry
|