feat: add Celery task infrastructure for module system

Phase 4 of module migration plan:
- Add ScheduledTask dataclass for declaring Celery Beat tasks
- Add tasks_path and scheduled_tasks fields to ModuleDefinition
- Create ModuleTask base class with database session management
- Create task discovery utilities (discover_module_tasks, build_beat_schedule)
- Update celery_config.py to discover and register module tasks
- Maintain backward compatibility with legacy task modules

Modules can now define tasks in their tasks/ directory and scheduled
tasks in their definition. The infrastructure supports gradual migration
of existing tasks from app/tasks/ to their respective modules.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
2026-01-27 22:52:01 +01:00
parent 7dbdbd4c7e
commit f1f91abe51
5 changed files with 556 additions and 11 deletions

View File

@@ -8,8 +8,17 @@ It includes:
- Celery Beat schedule for periodic tasks
- Task retry policies
- Sentry integration for error tracking
- Module-based task discovery (discovers tasks from app/modules/*/tasks/)
Task Discovery:
- Legacy tasks: Explicitly listed in the 'include' parameter
- Module tasks: Auto-discovered via discover_module_tasks()
As modules are migrated, their tasks will move from the legacy include list
to automatic discovery from the module's tasks/ directory.
"""
import logging
import os
import sentry_sdk
@@ -17,6 +26,8 @@ from celery import Celery
from celery.schedules import crontab
from sentry_sdk.integrations.celery import CeleryIntegration
logger = logging.getLogger(__name__)
# Redis URL from environment or default
REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0")
@@ -34,19 +45,49 @@ if SENTRY_DSN:
send_default_pii=True,
)
# =============================================================================
# TASK DISCOVERY
# =============================================================================
# Legacy tasks (will be migrated to modules over time)
LEGACY_TASK_MODULES = [
"app.tasks.celery_tasks.marketplace",
"app.tasks.celery_tasks.letzshop",
"app.tasks.celery_tasks.subscription",
"app.tasks.celery_tasks.export",
"app.tasks.celery_tasks.code_quality",
"app.tasks.celery_tasks.test_runner",
]
def get_all_task_modules() -> list[str]:
"""
Get all task modules (legacy + module-based).
Returns:
Combined list of legacy task modules and discovered module tasks
"""
all_modules = list(LEGACY_TASK_MODULES)
try:
from app.modules.tasks import discover_module_tasks
module_tasks = discover_module_tasks()
all_modules.extend(module_tasks)
logger.info(f"Discovered {len(module_tasks)} module task packages")
except ImportError as e:
logger.warning(f"Could not import module task discovery: {e}")
except Exception as e:
logger.error(f"Error discovering module tasks: {e}")
return all_modules
# Create Celery application
celery_app = Celery(
"wizamart",
broker=REDIS_URL,
backend=REDIS_URL,
include=[
"app.tasks.celery_tasks.marketplace",
"app.tasks.celery_tasks.letzshop",
"app.tasks.celery_tasks.subscription",
"app.tasks.celery_tasks.export",
"app.tasks.celery_tasks.code_quality",
"app.tasks.celery_tasks.test_runner",
],
include=get_all_task_modules(),
)
# =============================================================================
@@ -98,7 +139,9 @@ celery_app.conf.task_routes = {
# =============================================================================
# CELERY BEAT SCHEDULE - Periodic tasks
# =============================================================================
celery_app.conf.beat_schedule = {
# Legacy scheduled tasks (will be migrated to module definitions)
LEGACY_BEAT_SCHEDULE = {
# Reset usage counters at start of each period
"reset-period-counters-daily": {
"task": "app.tasks.celery_tasks.subscription.reset_period_counters",
@@ -131,6 +174,33 @@ celery_app.conf.beat_schedule = {
},
}
def get_full_beat_schedule() -> dict:
"""
Get complete beat schedule (legacy + module-defined).
Returns:
Merged beat schedule from legacy tasks and module definitions
"""
schedule = dict(LEGACY_BEAT_SCHEDULE)
try:
from app.modules.tasks import build_beat_schedule
module_schedule = build_beat_schedule()
schedule.update(module_schedule)
if module_schedule:
logger.info(f"Added {len(module_schedule)} scheduled tasks from modules")
except ImportError as e:
logger.warning(f"Could not import module beat schedule builder: {e}")
except Exception as e:
logger.error(f"Error building module beat schedule: {e}")
return schedule
celery_app.conf.beat_schedule = get_full_beat_schedule()
# =============================================================================
# QUEUE CONFIGURATION
# =============================================================================