Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
203 lines
7.3 KiB
Python
203 lines
7.3 KiB
Python
"""
|
|
Architecture Scan Models
|
|
Database models for tracking code quality scans and violations
|
|
"""
|
|
|
|
from sqlalchemy import (
|
|
JSON,
|
|
Boolean,
|
|
Column,
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Integer,
|
|
String,
|
|
Text,
|
|
)
|
|
from sqlalchemy.orm import relationship
|
|
from sqlalchemy.sql import func
|
|
|
|
from app.core.database import Base
|
|
|
|
|
|
class ArchitectureScan(Base):
|
|
"""Represents a single run of a code quality validator"""
|
|
|
|
__tablename__ = "architecture_scans"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
timestamp = Column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False, index=True
|
|
)
|
|
validator_type = Column(
|
|
String(20), nullable=False, index=True, default="architecture"
|
|
) # 'architecture', 'security', 'performance'
|
|
|
|
# Background task status fields (harmonized architecture)
|
|
status = Column(
|
|
String(30), nullable=False, default="pending", index=True
|
|
) # 'pending', 'running', 'completed', 'failed', 'completed_with_warnings'
|
|
started_at = Column(DateTime(timezone=True), nullable=True)
|
|
completed_at = Column(DateTime(timezone=True), nullable=True)
|
|
error_message = Column(Text, nullable=True)
|
|
progress_message = Column(String(255), nullable=True) # Current step description
|
|
|
|
# Scan results
|
|
total_files = Column(Integer, default=0)
|
|
total_violations = Column(Integer, default=0)
|
|
errors = Column(Integer, default=0)
|
|
warnings = Column(Integer, default=0)
|
|
duration_seconds = Column(Float, default=0.0)
|
|
triggered_by = Column(String(100)) # 'manual:username', 'scheduled', 'ci/cd'
|
|
git_commit_hash = Column(String(40))
|
|
|
|
# Celery task tracking (optional - for USE_CELERY=true)
|
|
celery_task_id = Column(String(255), nullable=True, index=True)
|
|
|
|
# Relationship to violations
|
|
violations = relationship(
|
|
"ArchitectureViolation", back_populates="scan", cascade="all, delete-orphan"
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<ArchitectureScan(id={self.id}, violations={self.total_violations}, errors={self.errors})>"
|
|
|
|
|
|
class ArchitectureViolation(Base):
|
|
"""Represents a single code quality violation found during a scan"""
|
|
|
|
__tablename__ = "architecture_violations"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
scan_id = Column(
|
|
Integer, ForeignKey("architecture_scans.id"), nullable=False, index=True
|
|
)
|
|
validator_type = Column(
|
|
String(20), nullable=False, index=True, default="architecture"
|
|
) # 'architecture', 'security', 'performance'
|
|
rule_id = Column(String(20), nullable=False, index=True) # e.g., 'API-001', 'SEC-001', 'PERF-001'
|
|
rule_name = Column(String(200), nullable=False)
|
|
severity = Column(
|
|
String(10), nullable=False, index=True
|
|
) # 'error', 'warning', 'info'
|
|
file_path = Column(String(500), nullable=False, index=True)
|
|
line_number = Column(Integer, nullable=False)
|
|
message = Column(Text, nullable=False)
|
|
context = Column(Text) # Code snippet
|
|
suggestion = Column(Text)
|
|
status = Column(
|
|
String(20), default="open", index=True
|
|
) # 'open', 'assigned', 'resolved', 'ignored', 'technical_debt'
|
|
assigned_to = Column(Integer, ForeignKey("users.id"))
|
|
resolved_at = Column(DateTime(timezone=True))
|
|
resolved_by = Column(Integer, ForeignKey("users.id"))
|
|
resolution_note = Column(Text)
|
|
created_at = Column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
|
)
|
|
|
|
# Relationships
|
|
scan = relationship("ArchitectureScan", back_populates="violations")
|
|
assigned_user = relationship(
|
|
"User", foreign_keys=[assigned_to], backref="assigned_violations"
|
|
)
|
|
resolver = relationship(
|
|
"User", foreign_keys=[resolved_by], backref="resolved_violations"
|
|
)
|
|
assignments = relationship(
|
|
"ViolationAssignment", back_populates="violation", cascade="all, delete-orphan"
|
|
)
|
|
comments = relationship(
|
|
"ViolationComment", back_populates="violation", cascade="all, delete-orphan"
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<ArchitectureViolation(id={self.id}, rule={self.rule_id}, file={self.file_path}:{self.line_number})>"
|
|
|
|
|
|
class ArchitectureRule(Base):
|
|
"""Code quality rules configuration (from YAML with database overrides)"""
|
|
|
|
__tablename__ = "architecture_rules"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
rule_id = Column(
|
|
String(20), unique=True, nullable=False, index=True
|
|
) # e.g., 'API-001', 'SEC-001', 'PERF-001'
|
|
validator_type = Column(
|
|
String(20), nullable=False, index=True, default="architecture"
|
|
) # 'architecture', 'security', 'performance'
|
|
category = Column(
|
|
String(50), nullable=False
|
|
) # 'api_endpoint', 'service_layer', 'authentication', 'database', etc.
|
|
name = Column(String(200), nullable=False)
|
|
description = Column(Text)
|
|
severity = Column(String(10), nullable=False) # Can override default from YAML
|
|
enabled = Column(Boolean, default=True, nullable=False)
|
|
custom_config = Column(JSON) # For rule-specific settings
|
|
created_at = Column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
|
)
|
|
updated_at = Column(
|
|
DateTime(timezone=True),
|
|
server_default=func.now(),
|
|
onupdate=func.now(),
|
|
nullable=False,
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<ArchitectureRule(id={self.rule_id}, name={self.name}, enabled={self.enabled})>"
|
|
|
|
|
|
class ViolationAssignment(Base):
|
|
"""Tracks assignment of violations to developers"""
|
|
|
|
__tablename__ = "violation_assignments"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
violation_id = Column(
|
|
Integer, ForeignKey("architecture_violations.id"), nullable=False, index=True
|
|
)
|
|
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
|
assigned_at = Column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
|
)
|
|
assigned_by = Column(Integer, ForeignKey("users.id"))
|
|
due_date = Column(DateTime(timezone=True))
|
|
priority = Column(
|
|
String(10), default="medium"
|
|
) # 'low', 'medium', 'high', 'critical'
|
|
|
|
# Relationships
|
|
violation = relationship("ArchitectureViolation", back_populates="assignments")
|
|
user = relationship("User", foreign_keys=[user_id], backref="violation_assignments")
|
|
assigner = relationship(
|
|
"User", foreign_keys=[assigned_by], backref="assigned_by_me"
|
|
)
|
|
|
|
def __repr__(self):
|
|
return f"<ViolationAssignment(id={self.id}, violation_id={self.violation_id}, user_id={self.user_id})>"
|
|
|
|
|
|
class ViolationComment(Base):
|
|
"""Comments on violations for collaboration"""
|
|
|
|
__tablename__ = "violation_comments"
|
|
|
|
id = Column(Integer, primary_key=True, index=True)
|
|
violation_id = Column(
|
|
Integer, ForeignKey("architecture_violations.id"), nullable=False, index=True
|
|
)
|
|
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
|
|
comment = Column(Text, nullable=False)
|
|
created_at = Column(
|
|
DateTime(timezone=True), server_default=func.now(), nullable=False
|
|
)
|
|
|
|
# Relationships
|
|
violation = relationship("ArchitectureViolation", back_populates="comments")
|
|
user = relationship("User", backref="violation_comments")
|
|
|
|
def __repr__(self):
|
|
return f"<ViolationComment(id={self.id}, violation_id={self.violation_id}, user_id={self.user_id})>"
|