sboulahtit/orion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: PostgreSQL migration compatibility and infrastructure improvements

Browse Source 
Database & Migrations:
- Update all Alembic migrations for PostgreSQL compatibility
- Remove SQLite-specific syntax (AUTOINCREMENT, etc.)
- Add database utility helpers for PostgreSQL operations
- Fix services to use PostgreSQL-compatible queries

Documentation:
- Add comprehensive Docker deployment guide
- Add production deployment documentation
- Add infrastructure architecture documentation
- Update database setup guide for PostgreSQL-only
- Expand troubleshooting guide

Architecture & Validation:
- Add migration.yaml rules for SQL compatibility checking
- Enhance validate_architecture.py with migration validation
- Update architecture rules to validate Alembic migrations

Development:
- Fix duplicate install-all target in Makefile
- Add Celery/Redis validation to install.py script
- Add docker-compose.test.yml for CI testing
- Add squash_migrations.py utility script
- Update tests for PostgreSQL compatibility
- Improve test fixtures in conftest.py

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Samir Boulahtit
2026-01-11 17:52:28 +01:00
parent 2792414395
commit 3614d448e4
45 changed files with 3179 additions and 507 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
app/core/database.py
View File

@@ -3,45 +3,33 @@
Database configuration and session management.
This module provides classes and functions for:
- Database engine creation and configuration
- PostgreSQL database engine creation and configuration
- Session management with connection pooling
- Database dependency for FastAPI routes
Note: This project uses PostgreSQL only. SQLite is not supported.
"""
import logging
from sqlalchemy import create_engine, event
from sqlalchemy import create_engine
from sqlalchemy.orm import declarative_base, sessionmaker
from sqlalchemy.pool import QueuePool
from .config import settings
from .config import settings, validate_database_url
# Validate database URL on import
validate_database_url()
def _configure_sqlite_connection(dbapi_connection, connection_record):
"""Configure SQLite connection for better concurrency.
- WAL mode: Allows concurrent reads during writes
- busy_timeout: Wait up to 30 seconds if database is locked
- synchronous=NORMAL: Balance between safety and performance
"""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=30000")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.close()
# Create engine with SQLite-specific configuration
engine_kwargs = {}
# Add SQLite-specific settings for better concurrent access
if settings.database_url.startswith("sqlite"):
engine_kwargs["connect_args"] = {"check_same_thread": False}
engine = create_engine(settings.database_url, **engine_kwargs)
# Configure SQLite pragmas on connection
if settings.database_url.startswith("sqlite"):
event.listen(engine, "connect", _configure_sqlite_connection)
# Create PostgreSQL engine with connection pooling
engine = create_engine(
settings.database_url,
poolclass=QueuePool,
pool_size=10,
max_overflow=20,
pool_pre_ping=True,
echo=False,
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

12
app/services/email_service.py
View File

@@ -1012,10 +1012,12 @@ class EmailService:
def _has_feature(self, vendor_id: int, feature_code: str) -> bool:
"""Check if vendor has a specific feature enabled."""
if vendor_id not in self._feature_cache:
from app.core.feature_gate import get_vendor_features
from app.services.feature_service import feature_service
try:
self._feature_cache[vendor_id] = get_vendor_features(self.db, vendor_id)
features = feature_service.get_vendor_features(self.db, vendor_id)
# Convert to set of feature codes
self._feature_cache[vendor_id] = {f.code for f in features.features}
except Exception:
self._feature_cache[vendor_id] = set()
@@ -1161,10 +1163,10 @@ class EmailService:
# Whitelabel: use vendor branding throughout
return BrandingContext(
platform_name=vendor.name,
platform_logo_url=vendor.logo_url,
platform_logo_url=vendor.get_logo_url(),
support_email=vendor.support_email or PLATFORM_SUPPORT_EMAIL,
vendor_name=vendor.name,
vendor_logo_url=vendor.logo_url,
vendor_logo_url=vendor.get_logo_url(),
is_whitelabel=True,
)
else:
@@ -1174,7 +1176,7 @@ class EmailService:
platform_logo_url=None, # Use default platform logo
support_email=PLATFORM_SUPPORT_EMAIL,
vendor_name=vendor.name if vendor else None,
vendor_logo_url=vendor.logo_url if vendor else None,
vendor_logo_url=vendor.get_logo_url() if vendor else None,
is_whitelabel=False,
)

11
app/services/inventory_service.py
View File

@@ -755,8 +755,17 @@ class InventoryService:
) -> AdminVendorsWithInventoryResponse:
"""Get list of vendors that have inventory entries (admin only)."""
# noqa: SVC-005 - Admin function, intentionally cross-vendor
# Use subquery to avoid DISTINCT on JSON columns (PostgreSQL can't compare JSON)
vendor_ids_subquery = (
db.query(Inventory.vendor_id)
.distinct()
.subquery()
)
vendors = (
db.query(Vendor).join(Inventory).distinct().order_by(Vendor.name).all()
db.query(Vendor)
.filter(Vendor.id.in_(db.query(vendor_ids_subquery.c.vendor_id)))
.order_by(Vendor.name)
.all()
)
return AdminVendorsWithInventoryResponse(

41
app/services/marketplace_product_service.py
View File

@@ -246,20 +246,27 @@ class MarketplaceProductService:
if search:
# Search in marketplace, vendor_name, brand, and translations
search_term = f"%{search}%"
# Join with translations for title/description search
query = query.outerjoin(MarketplaceProductTranslation).filter(
or_(
MarketplaceProduct.marketplace.ilike(search_term),
MarketplaceProduct.vendor_name.ilike(search_term),
MarketplaceProduct.brand.ilike(search_term),
MarketplaceProduct.gtin.ilike(search_term),
MarketplaceProduct.marketplace_product_id.ilike(search_term),
MarketplaceProductTranslation.title.ilike(search_term),
MarketplaceProductTranslation.description.ilike(search_term),
# Use subquery to get distinct IDs (PostgreSQL can't compare JSON for DISTINCT)
id_subquery = (
db.query(MarketplaceProduct.id)
.outerjoin(MarketplaceProductTranslation)
.filter(
or_(
MarketplaceProduct.marketplace.ilike(search_term),
MarketplaceProduct.vendor_name.ilike(search_term),
MarketplaceProduct.brand.ilike(search_term),
MarketplaceProduct.gtin.ilike(search_term),
MarketplaceProduct.marketplace_product_id.ilike(search_term),
MarketplaceProductTranslation.title.ilike(search_term),
MarketplaceProductTranslation.description.ilike(search_term),
)
)
.distinct()
.subquery()
)
# Remove duplicates from join
query = query.distinct()
query = query.filter(MarketplaceProduct.id.in_(
db.query(id_subquery.c.id)
))
total = query.count()
products = query.offset(skip).limit(limit).all()
@@ -634,8 +641,10 @@ class MarketplaceProductService:
if search:
search_term = f"%{search}%"
query = (
query.outerjoin(MarketplaceProductTranslation)
# Use subquery to get distinct IDs (PostgreSQL can't compare JSON for DISTINCT)
id_subquery = (
db.query(MarketplaceProduct.id)
.outerjoin(MarketplaceProductTranslation)
.filter(
or_(
MarketplaceProductTranslation.title.ilike(search_term),
@@ -647,7 +656,11 @@ class MarketplaceProductService:
)
)
.distinct()
.subquery()
)
query = query.filter(MarketplaceProduct.id.in_(
db.query(id_subquery.c.id)
))
if marketplace:
query = query.filter(MarketplaceProduct.marketplace == marketplace)

10
app/services/onboarding_service.py
View File

@@ -203,6 +203,11 @@ class OnboardingService:
Returns response with next step information.
"""
# Check vendor exists BEFORE creating onboarding record (FK constraint)
vendor = self.db.query(Vendor).filter(Vendor.id == vendor_id).first()
if not vendor:
raise VendorNotFoundException(vendor_id)
onboarding = self.get_or_create_onboarding(vendor_id)
# Update onboarding status if this is the first step
@@ -210,11 +215,6 @@ class OnboardingService:
onboarding.status = OnboardingStatus.IN_PROGRESS.value
onboarding.started_at = datetime.now(UTC)
# Get vendor and company
vendor = self.db.query(Vendor).filter(Vendor.id == vendor_id).first()
if not vendor:
raise VendorNotFoundException(vendor_id)
company = vendor.company
# Update company name if provided

11
app/services/product_service.py
View File

@@ -278,9 +278,9 @@ class ProductService:
# Prepare search pattern for LIKE queries
search_pattern = f"%{query}%"
# Build base query with translation join
base_query = (
db.query(Product)
# Use subquery to get distinct IDs (PostgreSQL can't compare JSON for DISTINCT)
id_subquery = (
db.query(Product.id)
.outerjoin(
ProductTranslation,
(Product.id == ProductTranslation.product_id)
@@ -303,6 +303,11 @@ class ProductService:
)
)
.distinct()
.subquery()
)
base_query = db.query(Product).filter(
Product.id.in_(db.query(id_subquery.c.id))
)
# Get total count

12
app/services/subscription_service.py
View File

@@ -207,6 +207,18 @@ class SubscriptionService:
)
return subscription
def get_current_tier(
self, db: Session, vendor_id: int
) -> TierCode | None:
"""Get vendor's current subscription tier code."""
subscription = self.get_subscription(db, vendor_id)
if subscription:
try:
return TierCode(subscription.tier)
except ValueError:
return None
return None
def get_or_create_subscription(
self,
db: Session,

2
app/services/vendor_email_settings_service.py
View File

@@ -141,7 +141,7 @@ class VendorEmailSettingsService:
raise AuthorizationException(
message=f"Provider '{provider}' requires Business or Enterprise tier. "
"Upgrade your plan to use advanced email providers.",
required_permission="business_tier",
details={"required_permission": "business_tier"},
)
settings = self.get_settings(vendor_id)

44
app/utils/database.py
View File

@@ -2,8 +2,10 @@
"""Database utilities for database operations.
This module provides utility functions and classes to interact with a database using SQLAlchemy. It includes:
- Creating a database engine with connection pooling.
- Creating a PostgreSQL database engine with connection pooling.
- Generating a session factory for creating sessions.
Note: This project uses PostgreSQL only. SQLite is not supported.
"""
import logging
@@ -16,31 +18,35 @@ logger = logging.getLogger(__name__)
def get_db_engine(database_url: str):
"""Create a database engine with connection pooling.
"""Create a PostgreSQL database engine with connection pooling.
Args:
database_url (str): The URL string to connect to the database. It can be for SQLite or PostgreSQL databases.
database_url (str): The PostgreSQL URL string to connect to the database.
Returns:
sqlalchemy.engine.Engine: A SQLAlchemy Engine instance configured according to the provided database URL.
sqlalchemy.engine.Engine: A SQLAlchemy Engine instance configured for PostgreSQL.
Raises:
ValueError: If database_url is not a PostgreSQL URL.
"""
if database_url.startswith("sqlite"):
# Configuration for SQLite database
engine = create_engine(
database_url, connect_args={"check_same_thread": False}, echo=False
)
else:
# Configuration for PostgreSQL databases with connection pooling
engine = create_engine(
database_url,
poolclass=QueuePool,
pool_size=10,
max_overflow=20,
pool_pre_ping=True,
echo=False,
if not database_url.startswith("postgresql"):
raise ValueError(
f"Unsupported database: {database_url.split(':')[0]}. "
"Only PostgreSQL is supported."
)
logger.info(f"Database engine created for: {database_url.split('@')[0]}@...")
engine = create_engine(
database_url,
poolclass=QueuePool,
pool_size=10,
max_overflow=20,
pool_pre_ping=True,
echo=False,
)
# Log URL without password
safe_url = database_url.split("@")[0] + "@..." if "@" in database_url else database_url
logger.info(f"Database engine created for: {safe_url}")
return engine