fix: improve SQLite concurrency and database logging reliability
Database improvements: - Enable WAL mode for better concurrent read/write access - Add busy_timeout (30s) to wait for locked database - Add synchronous=NORMAL for balanced safety/performance - Configure check_same_thread=False for thread safety Logging improvements: - Add retry logic (3 attempts) for database locked errors - Silently skip logging on persistent failures to avoid spam - Properly rollback failed transactions 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -10,12 +10,39 @@ This module provides classes and functions for:
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine, event
|
||||||
from sqlalchemy.orm import declarative_base, sessionmaker
|
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||||
|
|
||||||
from .config import settings
|
from .config import settings
|
||||||
|
|
||||||
engine = create_engine(settings.database_url)
|
|
||||||
|
def _configure_sqlite_connection(dbapi_connection, connection_record):
|
||||||
|
"""Configure SQLite connection for better concurrency.
|
||||||
|
|
||||||
|
- WAL mode: Allows concurrent reads during writes
|
||||||
|
- busy_timeout: Wait up to 30 seconds if database is locked
|
||||||
|
- synchronous=NORMAL: Balance between safety and performance
|
||||||
|
"""
|
||||||
|
cursor = dbapi_connection.cursor()
|
||||||
|
cursor.execute("PRAGMA journal_mode=WAL")
|
||||||
|
cursor.execute("PRAGMA busy_timeout=30000")
|
||||||
|
cursor.execute("PRAGMA synchronous=NORMAL")
|
||||||
|
cursor.close()
|
||||||
|
|
||||||
|
|
||||||
|
# Create engine with SQLite-specific configuration
|
||||||
|
engine_kwargs = {}
|
||||||
|
|
||||||
|
# Add SQLite-specific settings for better concurrent access
|
||||||
|
if settings.database_url.startswith("sqlite"):
|
||||||
|
engine_kwargs["connect_args"] = {"check_same_thread": False}
|
||||||
|
|
||||||
|
engine = create_engine(settings.database_url, **engine_kwargs)
|
||||||
|
|
||||||
|
# Configure SQLite pragmas on connection
|
||||||
|
if settings.database_url.startswith("sqlite"):
|
||||||
|
event.listen(engine, "connect", _configure_sqlite_connection)
|
||||||
|
|
||||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||||
|
|
||||||
Base = declarative_base()
|
Base = declarative_base()
|
||||||
|
|||||||
@@ -23,14 +23,21 @@ class DatabaseLogHandler(logging.Handler):
|
|||||||
Custom logging handler that stores WARNING, ERROR, and CRITICAL logs in database.
|
Custom logging handler that stores WARNING, ERROR, and CRITICAL logs in database.
|
||||||
|
|
||||||
Runs asynchronously to avoid blocking application performance.
|
Runs asynchronously to avoid blocking application performance.
|
||||||
|
Uses retry logic for SQLite database locking issues.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
MAX_RETRIES = 3
|
||||||
|
RETRY_DELAY = 0.1 # 100ms delay between retries
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.setLevel(logging.WARNING) # Only log WARNING and above to database
|
self.setLevel(logging.WARNING) # Only log WARNING and above to database
|
||||||
|
|
||||||
def emit(self, record):
|
def emit(self, record):
|
||||||
"""Emit a log record to the database."""
|
"""Emit a log record to the database with retry logic for SQLite locking."""
|
||||||
|
import time
|
||||||
|
|
||||||
|
for attempt in range(self.MAX_RETRIES):
|
||||||
try:
|
try:
|
||||||
from app.core.database import SessionLocal
|
from app.core.database import SessionLocal
|
||||||
from models.database.admin import ApplicationLog
|
from models.database.admin import ApplicationLog
|
||||||
@@ -77,11 +84,18 @@ class DatabaseLogHandler(logging.Handler):
|
|||||||
|
|
||||||
db.add(log_entry)
|
db.add(log_entry)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
return # Success, exit retry loop
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# If database logging fails, don't crash the app
|
db.rollback()
|
||||||
# Just print to stderr
|
# Check if it's a database locked error
|
||||||
print(f"Failed to write log to database: {e}", file=sys.stderr)
|
if "database is locked" in str(e).lower():
|
||||||
|
if attempt < self.MAX_RETRIES - 1:
|
||||||
|
time.sleep(self.RETRY_DELAY * (attempt + 1))
|
||||||
|
continue
|
||||||
|
# For other errors or final attempt, silently skip
|
||||||
|
# Don't print to stderr to avoid log spam during imports
|
||||||
|
pass
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
@@ -89,6 +103,8 @@ class DatabaseLogHandler(logging.Handler):
|
|||||||
# Silently fail - logging should never crash the app
|
# Silently fail - logging should never crash the app
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
break # Exit retry loop on non-recoverable errors
|
||||||
|
|
||||||
|
|
||||||
def get_log_level_from_db():
|
def get_log_level_from_db():
|
||||||
"""
|
"""
|
||||||
|
|||||||
Reference in New Issue
Block a user