fix: improve SQLite concurrency and database logging reliability

Database improvements:
- Enable WAL mode for better concurrent read/write access
- Add busy_timeout (30s) to wait for locked database
- Add synchronous=NORMAL for balanced safety/performance
- Configure check_same_thread=False for thread safety

Logging improvements:
- Add retry logic (3 attempts) for database locked errors
- Silently skip logging on persistent failures to avoid spam
- Properly rollback failed transactions

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2025-12-12 22:36:27 +01:00
parent da9e0b7c64
commit 942f3722f5
2 changed files with 95 additions and 52 deletions

View File

@@ -10,12 +10,39 @@ This module provides classes and functions for:
import logging import logging
from sqlalchemy import create_engine from sqlalchemy import create_engine, event
from sqlalchemy.orm import declarative_base, sessionmaker from sqlalchemy.orm import declarative_base, sessionmaker
from .config import settings from .config import settings
engine = create_engine(settings.database_url)
def _configure_sqlite_connection(dbapi_connection, connection_record):
"""Configure SQLite connection for better concurrency.
- WAL mode: Allows concurrent reads during writes
- busy_timeout: Wait up to 30 seconds if database is locked
- synchronous=NORMAL: Balance between safety and performance
"""
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA busy_timeout=30000")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.close()
# Create engine with SQLite-specific configuration
engine_kwargs = {}
# Add SQLite-specific settings for better concurrent access
if settings.database_url.startswith("sqlite"):
engine_kwargs["connect_args"] = {"check_same_thread": False}
engine = create_engine(settings.database_url, **engine_kwargs)
# Configure SQLite pragmas on connection
if settings.database_url.startswith("sqlite"):
event.listen(engine, "connect", _configure_sqlite_connection)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base() Base = declarative_base()

View File

@@ -23,14 +23,21 @@ class DatabaseLogHandler(logging.Handler):
Custom logging handler that stores WARNING, ERROR, and CRITICAL logs in database. Custom logging handler that stores WARNING, ERROR, and CRITICAL logs in database.
Runs asynchronously to avoid blocking application performance. Runs asynchronously to avoid blocking application performance.
Uses retry logic for SQLite database locking issues.
""" """
MAX_RETRIES = 3
RETRY_DELAY = 0.1 # 100ms delay between retries
def __init__(self): def __init__(self):
super().__init__() super().__init__()
self.setLevel(logging.WARNING) # Only log WARNING and above to database self.setLevel(logging.WARNING) # Only log WARNING and above to database
def emit(self, record): def emit(self, record):
"""Emit a log record to the database.""" """Emit a log record to the database with retry logic for SQLite locking."""
import time
for attempt in range(self.MAX_RETRIES):
try: try:
from app.core.database import SessionLocal from app.core.database import SessionLocal
from models.database.admin import ApplicationLog from models.database.admin import ApplicationLog
@@ -77,11 +84,18 @@ class DatabaseLogHandler(logging.Handler):
db.add(log_entry) db.add(log_entry)
db.commit() db.commit()
return # Success, exit retry loop
except Exception as e: except Exception as e:
# If database logging fails, don't crash the app db.rollback()
# Just print to stderr # Check if it's a database locked error
print(f"Failed to write log to database: {e}", file=sys.stderr) if "database is locked" in str(e).lower():
if attempt < self.MAX_RETRIES - 1:
time.sleep(self.RETRY_DELAY * (attempt + 1))
continue
# For other errors or final attempt, silently skip
# Don't print to stderr to avoid log spam during imports
pass
finally: finally:
db.close() db.close()
@@ -89,6 +103,8 @@ class DatabaseLogHandler(logging.Handler):
# Silently fail - logging should never crash the app # Silently fail - logging should never crash the app
pass pass
break # Exit retry loop on non-recoverable errors
def get_log_level_from_db(): def get_log_level_from_db():
""" """