feat: add Celery task infrastructure for module system
Phase 4 of module migration plan: - Add ScheduledTask dataclass for declaring Celery Beat tasks - Add tasks_path and scheduled_tasks fields to ModuleDefinition - Create ModuleTask base class with database session management - Create task discovery utilities (discover_module_tasks, build_beat_schedule) - Update celery_config.py to discover and register module tasks - Maintain backward compatibility with legacy task modules Modules can now define tasks in their tasks/ directory and scheduled tasks in their definition. The infrastructure supports gradual migration of existing tasks from app/tasks/ to their respective modules. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,8 +8,17 @@ It includes:
|
||||
- Celery Beat schedule for periodic tasks
|
||||
- Task retry policies
|
||||
- Sentry integration for error tracking
|
||||
- Module-based task discovery (discovers tasks from app/modules/*/tasks/)
|
||||
|
||||
Task Discovery:
|
||||
- Legacy tasks: Explicitly listed in the 'include' parameter
|
||||
- Module tasks: Auto-discovered via discover_module_tasks()
|
||||
|
||||
As modules are migrated, their tasks will move from the legacy include list
|
||||
to automatic discovery from the module's tasks/ directory.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
import sentry_sdk
|
||||
@@ -17,6 +26,8 @@ from celery import Celery
|
||||
from celery.schedules import crontab
|
||||
from sentry_sdk.integrations.celery import CeleryIntegration
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Redis URL from environment or default
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
|
||||
|
||||
@@ -34,19 +45,49 @@ if SENTRY_DSN:
|
||||
send_default_pii=True,
|
||||
)
|
||||
|
||||
# =============================================================================
|
||||
# TASK DISCOVERY
|
||||
# =============================================================================
|
||||
# Legacy tasks (will be migrated to modules over time)
|
||||
LEGACY_TASK_MODULES = [
|
||||
"app.tasks.celery_tasks.marketplace",
|
||||
"app.tasks.celery_tasks.letzshop",
|
||||
"app.tasks.celery_tasks.subscription",
|
||||
"app.tasks.celery_tasks.export",
|
||||
"app.tasks.celery_tasks.code_quality",
|
||||
"app.tasks.celery_tasks.test_runner",
|
||||
]
|
||||
|
||||
|
||||
def get_all_task_modules() -> list[str]:
|
||||
"""
|
||||
Get all task modules (legacy + module-based).
|
||||
|
||||
Returns:
|
||||
Combined list of legacy task modules and discovered module tasks
|
||||
"""
|
||||
all_modules = list(LEGACY_TASK_MODULES)
|
||||
|
||||
try:
|
||||
from app.modules.tasks import discover_module_tasks
|
||||
|
||||
module_tasks = discover_module_tasks()
|
||||
all_modules.extend(module_tasks)
|
||||
logger.info(f"Discovered {len(module_tasks)} module task packages")
|
||||
except ImportError as e:
|
||||
logger.warning(f"Could not import module task discovery: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error discovering module tasks: {e}")
|
||||
|
||||
return all_modules
|
||||
|
||||
|
||||
# Create Celery application
|
||||
celery_app = Celery(
|
||||
"wizamart",
|
||||
broker=REDIS_URL,
|
||||
backend=REDIS_URL,
|
||||
include=[
|
||||
"app.tasks.celery_tasks.marketplace",
|
||||
"app.tasks.celery_tasks.letzshop",
|
||||
"app.tasks.celery_tasks.subscription",
|
||||
"app.tasks.celery_tasks.export",
|
||||
"app.tasks.celery_tasks.code_quality",
|
||||
"app.tasks.celery_tasks.test_runner",
|
||||
],
|
||||
include=get_all_task_modules(),
|
||||
)
|
||||
|
||||
# =============================================================================
|
||||
@@ -98,7 +139,9 @@ celery_app.conf.task_routes = {
|
||||
# =============================================================================
|
||||
# CELERY BEAT SCHEDULE - Periodic tasks
|
||||
# =============================================================================
|
||||
celery_app.conf.beat_schedule = {
|
||||
|
||||
# Legacy scheduled tasks (will be migrated to module definitions)
|
||||
LEGACY_BEAT_SCHEDULE = {
|
||||
# Reset usage counters at start of each period
|
||||
"reset-period-counters-daily": {
|
||||
"task": "app.tasks.celery_tasks.subscription.reset_period_counters",
|
||||
@@ -131,6 +174,33 @@ celery_app.conf.beat_schedule = {
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_full_beat_schedule() -> dict:
|
||||
"""
|
||||
Get complete beat schedule (legacy + module-defined).
|
||||
|
||||
Returns:
|
||||
Merged beat schedule from legacy tasks and module definitions
|
||||
"""
|
||||
schedule = dict(LEGACY_BEAT_SCHEDULE)
|
||||
|
||||
try:
|
||||
from app.modules.tasks import build_beat_schedule
|
||||
|
||||
module_schedule = build_beat_schedule()
|
||||
schedule.update(module_schedule)
|
||||
if module_schedule:
|
||||
logger.info(f"Added {len(module_schedule)} scheduled tasks from modules")
|
||||
except ImportError as e:
|
||||
logger.warning(f"Could not import module beat schedule builder: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error building module beat schedule: {e}")
|
||||
|
||||
return schedule
|
||||
|
||||
|
||||
celery_app.conf.beat_schedule = get_full_beat_schedule()
|
||||
|
||||
# =============================================================================
|
||||
# QUEUE CONFIGURATION
|
||||
# =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user