Samir Boulahtit
2792414395
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
117 lines
3.7 KiB
Python
117 lines
3.7 KiB
Python
from sqlalchemy import JSON, Column, DateTime, ForeignKey, Index, Integer, String, Text
|
|
from sqlalchemy.orm import relationship
|
|
|
|
from app.core.database import Base
|
|
from models.database.base import TimestampMixin
|
|
|
|
|
|
class MarketplaceImportError(Base, TimestampMixin):
|
|
"""
|
|
Stores detailed information about individual import errors.
|
|
|
|
Each row that fails during import creates an error record with:
|
|
- Row number from the source file
|
|
- Identifier (marketplace_product_id if available)
|
|
- Error type and message
|
|
- Raw row data for review
|
|
"""
|
|
|
|
__tablename__ = "marketplace_import_errors"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
import_job_id = Column(
|
|
Integer,
|
|
ForeignKey("marketplace_import_jobs.id", ondelete="CASCADE"),
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
|
|
# Error location
|
|
row_number = Column(Integer, nullable=False)
|
|
|
|
# Identifier from the row (if available)
|
|
identifier = Column(String) # marketplace_product_id, gtin, mpn, etc.
|
|
|
|
# Error details
|
|
error_type = Column(
|
|
String(50), nullable=False
|
|
) # missing_title, missing_id, parse_error, etc.
|
|
error_message = Column(Text, nullable=False)
|
|
|
|
# Raw row data for review (JSON)
|
|
row_data = Column(JSON)
|
|
|
|
# Relationship
|
|
import_job = relationship("MarketplaceImportJob", back_populates="errors")
|
|
|
|
__table_args__ = (
|
|
Index("idx_import_error_job_id", "import_job_id"),
|
|
Index("idx_import_error_type", "error_type"),
|
|
)
|
|
|
|
def __repr__(self):
|
|
return (
|
|
f"<MarketplaceImportError(id={self.id}, job_id={self.import_job_id}, "
|
|
f"row={self.row_number}, type='{self.error_type}')>"
|
|
)
|
|
|
|
|
|
class MarketplaceImportJob(Base, TimestampMixin):
|
|
__tablename__ = "marketplace_import_jobs"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
vendor_id = Column(Integer, ForeignKey("vendors.id"), nullable=False, index=True)
|
|
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
|
|
|
# Import configuration
|
|
marketplace = Column(String, nullable=False, index=True, default="Letzshop")
|
|
source_url = Column(String, nullable=False)
|
|
language = Column(
|
|
String(5), nullable=False, default="en"
|
|
) # Language for translations
|
|
|
|
# Status tracking
|
|
status = Column(
|
|
String, nullable=False, default="pending"
|
|
) # pending, processing, completed, failed, completed_with_errors
|
|
|
|
# Results
|
|
imported_count = Column(Integer, default=0)
|
|
updated_count = Column(Integer, default=0)
|
|
error_count = Column(Integer, default=0)
|
|
total_processed = Column(Integer, default=0)
|
|
|
|
# Error handling
|
|
error_message = Column(Text)
|
|
|
|
# Celery task tracking (optional - for USE_CELERY=true)
|
|
celery_task_id = Column(String(255), nullable=True, index=True)
|
|
|
|
# Timestamps
|
|
started_at = Column(DateTime(timezone=True))
|
|
completed_at = Column(DateTime(timezone=True))
|
|
|
|
# Relationships
|
|
vendor = relationship("Vendor", back_populates="marketplace_import_jobs")
|
|
user = relationship("User", foreign_keys=[user_id])
|
|
errors = relationship(
|
|
"MarketplaceImportError",
|
|
back_populates="import_job",
|
|
cascade="all, delete-orphan",
|
|
order_by="MarketplaceImportError.row_number",
|
|
)
|
|
|
|
# Indexes for performance
|
|
__table_args__ = (
|
|
Index("idx_import_vendor_status", "vendor_id", "status"),
|
|
Index("idx_import_vendor_created", "vendor_id", "created_at"),
|
|
Index("idx_import_user_marketplace", "user_id", "marketplace"),
|
|
)
|
|
|
|
def __repr__(self):
|
|
return (
|
|
f"<MarketplaceImportJob(id={self.id}, vendor_id={self.vendor_id}, "
|
|
f"marketplace='{self.marketplace}', status='{self.status}', "
|
|
f"imported={self.imported_count})>"
|
|
)
|