feat: add Celery/Redis task queue with feature flag support
Migrate background tasks from FastAPI BackgroundTasks to Celery with Redis for persistent task queuing, retries, and scheduled jobs. Key changes: - Add Celery configuration with Redis broker/backend - Create task dispatcher with USE_CELERY feature flag for gradual rollout - Add Celery task wrappers for all background operations: - Marketplace imports - Letzshop historical imports - Product exports - Code quality scans - Test runs - Subscription scheduled tasks (via Celery Beat) - Add celery_task_id column to job tables for Flower integration - Add Flower dashboard link to admin background tasks page - Update docker-compose.yml with worker, beat, and flower services - Add Makefile targets: celery-worker, celery-beat, celery-dev, flower When USE_CELERY=false (default), system falls back to FastAPI BackgroundTasks for development without Redis dependency. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
142
Makefile
142
Makefile
@@ -1,7 +1,7 @@
|
||||
# Wizamart Multi-Tenant E-Commerce Platform Makefile
|
||||
# Cross-platform compatible (Windows & Linux)
|
||||
|
||||
.PHONY: install install-dev install-docs install-all dev test test-coverage lint format check docker-build docker-up docker-down clean help tailwind-install tailwind-dev tailwind-build tailwind-watch arch-check arch-check-file arch-check-object
|
||||
.PHONY: install install-dev install-docs install-all dev test test-coverage lint format check docker-build docker-up docker-down clean help tailwind-install tailwind-dev tailwind-build tailwind-watch arch-check arch-check-file arch-check-object test-db-up test-db-down test-db-reset test-db-status celery-worker celery-beat celery-dev flower celery-status celery-purge
|
||||
|
||||
# Detect OS
|
||||
ifeq ($(OS),Windows_NT)
|
||||
@@ -89,6 +89,10 @@ migrate-status:
|
||||
@echo "Migration history:"
|
||||
$(PYTHON) -m alembic history --verbose
|
||||
|
||||
migrate-squash:
|
||||
@echo "Squashing all migrations into a single initial migration..."
|
||||
$(PYTHON) scripts/squash_migrations.py
|
||||
|
||||
# =============================================================================
|
||||
# DATABASE INITIALIZATION & SEEDING
|
||||
# =============================================================================
|
||||
@@ -158,15 +162,16 @@ db-setup: migrate-up init-prod seed-demo
|
||||
|
||||
db-reset:
|
||||
@echo "⚠️ WARNING: This will DELETE ALL existing data!"
|
||||
@echo "Rolling back all migrations..."
|
||||
$(PYTHON) -m alembic downgrade base || true
|
||||
@echo "Applying all migrations..."
|
||||
$(PYTHON) -m alembic upgrade head
|
||||
@echo "Initializing production data..."
|
||||
$(PYTHON) scripts/init_production.py
|
||||
@echo "Seeding demo data..."
|
||||
ifeq ($(DETECTED_OS),Windows)
|
||||
@set SEED_MODE=reset&& set FORCE_RESET=true&& $(PYTHON) -m alembic downgrade -1
|
||||
@set SEED_MODE=reset&& set FORCE_RESET=true&& $(PYTHON) -m alembic upgrade head
|
||||
@set SEED_MODE=reset&& set FORCE_RESET=true&& $(PYTHON) scripts/init_production.py
|
||||
@set SEED_MODE=reset&& set FORCE_RESET=true&& $(PYTHON) scripts/seed_demo.py
|
||||
else
|
||||
$(PYTHON) -m alembic downgrade -1
|
||||
$(PYTHON) -m alembic upgrade head
|
||||
$(PYTHON) scripts/init_production.py
|
||||
SEED_MODE=reset FORCE_RESET=true $(PYTHON) scripts/seed_demo.py
|
||||
endif
|
||||
@echo ""
|
||||
@@ -192,36 +197,88 @@ init-logging:
|
||||
$(PYTHON) scripts/init_log_settings.py
|
||||
@echo "✅ Log settings initialized"
|
||||
|
||||
# =============================================================================
|
||||
# TEST DATABASE (PostgreSQL in Docker)
|
||||
# =============================================================================
|
||||
|
||||
test-db-up:
|
||||
@echo "Starting test database (PostgreSQL on port 5433)..."
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
@sleep 3
|
||||
@echo "✅ Test database ready"
|
||||
|
||||
test-db-down:
|
||||
@echo "Stopping test database..."
|
||||
docker compose -f docker-compose.test.yml down
|
||||
@echo "✅ Test database stopped"
|
||||
|
||||
test-db-reset:
|
||||
@echo "Resetting test database..."
|
||||
docker compose -f docker-compose.test.yml down -v
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
@sleep 3
|
||||
@echo "✅ Test database reset"
|
||||
|
||||
test-db-status:
|
||||
@docker compose -f docker-compose.test.yml ps
|
||||
|
||||
# =============================================================================
|
||||
# TESTING
|
||||
# =============================================================================
|
||||
|
||||
test:
|
||||
$(PYTHON) -m pytest tests/ -v
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/ -v
|
||||
|
||||
test-unit:
|
||||
$(PYTHON) -m pytest tests/ -v -m unit
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/ -v -m unit
|
||||
|
||||
test-integration:
|
||||
$(PYTHON) -m pytest tests/ -v -m integration
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/ -v -m integration
|
||||
|
||||
test-coverage:
|
||||
$(PYTHON) -m pytest tests/ --cov=app --cov=models --cov=utils --cov=middleware --cov-report=html --cov-report=term-missing
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/ --cov=app --cov=models --cov=utils --cov=middleware --cov-report=html --cov-report=term-missing
|
||||
|
||||
test-fast:
|
||||
$(PYTHON) -m pytest tests/ -v -m "not slow"
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/ -v -m "not slow"
|
||||
|
||||
test-slow:
|
||||
$(PYTHON) -m pytest tests/ -v -m slow
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/ -v -m slow
|
||||
|
||||
test-auth:
|
||||
$(PYTHON) -m pytest tests/test_auth.py -v
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/test_auth.py -v
|
||||
|
||||
test-products:
|
||||
$(PYTHON) -m pytest tests/test_products.py -v
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/test_products.py -v
|
||||
|
||||
test-inventory:
|
||||
$(PYTHON) -m pytest tests/test_inventory.py -v
|
||||
@docker compose -f docker-compose.test.yml up -d 2>/dev/null || true
|
||||
@sleep 2
|
||||
TEST_DATABASE_URL="postgresql://test_user:test_password@localhost:5433/wizamart_test" \
|
||||
$(PYTHON) -m pytest tests/test_inventory.py -v
|
||||
|
||||
# =============================================================================
|
||||
# CODE QUALITY
|
||||
@@ -344,18 +401,46 @@ tailwind-watch:
|
||||
@echo "Note: This watches admin CSS only. Run in separate terminal."
|
||||
$(TAILWIND_CLI) -i static/admin/css/tailwind.css -o static/admin/css/tailwind.output.css --watch
|
||||
|
||||
# =============================================================================
|
||||
# CELERY / TASK QUEUE
|
||||
# =============================================================================
|
||||
|
||||
celery-worker:
|
||||
@echo "Starting Celery worker..."
|
||||
celery -A app.core.celery_config worker --loglevel=info -Q default,long_running,scheduled
|
||||
|
||||
celery-beat:
|
||||
@echo "Starting Celery beat scheduler..."
|
||||
celery -A app.core.celery_config beat --loglevel=info
|
||||
|
||||
celery-dev:
|
||||
@echo "Starting Celery worker + beat (development mode)..."
|
||||
celery -A app.core.celery_config worker --loglevel=info -B -Q default,long_running,scheduled
|
||||
|
||||
flower:
|
||||
@echo "Starting Flower monitoring dashboard on http://localhost:5555..."
|
||||
celery -A app.core.celery_config flower --port=5555 --basic-auth=admin:admin
|
||||
|
||||
celery-status:
|
||||
@echo "Celery worker status:"
|
||||
celery -A app.core.celery_config inspect active
|
||||
|
||||
celery-purge:
|
||||
@echo "Purging all pending tasks..."
|
||||
celery -A app.core.celery_config purge -f
|
||||
|
||||
# =============================================================================
|
||||
# DOCKER
|
||||
# =============================================================================
|
||||
|
||||
docker-build:
|
||||
docker-compose build
|
||||
docker compose build
|
||||
|
||||
docker-up:
|
||||
docker-compose up -d
|
||||
docker compose up -d
|
||||
|
||||
docker-down:
|
||||
docker-compose down
|
||||
docker compose down
|
||||
|
||||
docker-restart: docker-down docker-up
|
||||
|
||||
@@ -364,10 +449,10 @@ docker-restart: docker-down docker-up
|
||||
# =============================================================================
|
||||
|
||||
deploy-staging: migrate-up
|
||||
docker-compose -f docker-compose.staging.yml up -d
|
||||
docker compose -f docker-compose.staging.yml up -d
|
||||
|
||||
deploy-prod: migrate-up
|
||||
docker-compose -f docker-compose.prod.yml up -d
|
||||
docker compose -f docker-compose.prod.yml up -d
|
||||
|
||||
# =============================================================================
|
||||
# UTILITIES
|
||||
@@ -443,8 +528,11 @@ help:
|
||||
@echo " db-setup - Full dev setup (migrate + init-prod + seed-demo)"
|
||||
@echo " backup-db - Backup database"
|
||||
@echo ""
|
||||
@echo "=== TESTING ==="
|
||||
@echo " test - Run all tests"
|
||||
@echo "=== TESTING (PostgreSQL) ==="
|
||||
@echo " test-db-up - Start test PostgreSQL database"
|
||||
@echo " test-db-down - Stop test database"
|
||||
@echo " test-db-reset - Reset test database"
|
||||
@echo " test - Run all tests (auto-starts DB)"
|
||||
@echo " test-coverage - Run tests with coverage"
|
||||
@echo " test-fast - Run fast tests only"
|
||||
@echo ""
|
||||
@@ -470,6 +558,14 @@ help:
|
||||
@echo " tailwind-build - Build Tailwind CSS (production, minified)"
|
||||
@echo " tailwind-watch - Watch and rebuild on changes"
|
||||
@echo ""
|
||||
@echo "=== CELERY / TASK QUEUE ==="
|
||||
@echo " celery-worker - Start Celery worker"
|
||||
@echo " celery-beat - Start Celery beat scheduler"
|
||||
@echo " celery-dev - Start worker + beat together (dev)"
|
||||
@echo " flower - Start Flower monitoring (localhost:5555)"
|
||||
@echo " celery-status - Show active worker tasks"
|
||||
@echo " celery-purge - Purge all pending tasks"
|
||||
@echo ""
|
||||
@echo "=== DOCKER ==="
|
||||
@echo " docker-build - Build Docker containers"
|
||||
@echo " docker-up - Start Docker containers"
|
||||
|
||||
Reference in New Issue
Block a user