From 9a13aee8ed2750c2bb3bd13d5a2a95516fc590b6 Mon Sep 17 00:00:00 2001 From: Samir Boulahtit Date: Thu, 12 Mar 2026 22:29:31 +0100 Subject: [PATCH] feat: add module-aware test impact analysis and fix CI test scope Add run_affected_tests.py script that uses module dependency graph to run only tests for changed modules and their dependents. Fix CI and Makefile to use pyproject.toml testpaths (was missing 9 of 18 modules). Co-Authored-By: Claude Opus 4.6 --- .gitea/workflows/ci.yml | 2 +- Makefile | 32 ++-- scripts/tests/run_affected_tests.py | 276 ++++++++++++++++++++++++++++ 3 files changed, 297 insertions(+), 13 deletions(-) create mode 100755 scripts/tests/run_affected_tests.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 57d4ce71..bef710f0 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: run: uv pip install --system -r requirements.txt -r requirements-test.txt - name: Run tests - run: python -m pytest tests/ -v --tb=short + run: python -m pytest -v --tb=short validate: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index db12faae..ab686d7e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Orion Multi-Tenant E-Commerce Platform Makefile # Cross-platform compatible (Windows & Linux) -.PHONY: install install-dev install-docs install-all dev test test-coverage lint format check docker-build docker-up docker-down clean help tailwind-install tailwind-dev tailwind-build tailwind-watch arch-check arch-check-file arch-check-object test-db-up test-db-down test-db-reset test-db-status celery-worker celery-beat celery-dev flower celery-status celery-purge urls infra-check +.PHONY: install install-dev install-docs install-all dev test test-coverage lint format check docker-build docker-up docker-down clean help tailwind-install tailwind-dev tailwind-build tailwind-watch arch-check arch-check-file arch-check-object test-db-up test-db-down test-db-reset test-db-status celery-worker celery-beat celery-dev flower celery-status celery-purge urls infra-check test-affected test-affected-dry # Detect OS ifeq ($(OS),Windows_NT) @@ -249,24 +249,21 @@ ifdef frontend endif endif -# All testpaths (central + module tests) -TEST_PATHS := tests/ app/modules/tenancy/tests/ app/modules/catalog/tests/ app/modules/billing/tests/ app/modules/messaging/tests/ app/modules/orders/tests/ app/modules/customers/tests/ app/modules/marketplace/tests/ app/modules/inventory/tests/ app/modules/loyalty/tests/ - test: @docker compose -f docker-compose.test.yml up -d 2>/dev/null || true @sleep 2 TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v $(MARKER_EXPR) + $(PYTHON) -m pytest -v $(MARKER_EXPR) test-unit: @docker compose -f docker-compose.test.yml up -d 2>/dev/null || true @sleep 2 ifdef module TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v -m "unit and $(module)" + $(PYTHON) -m pytest -v -m "unit and $(module)" else TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v -m unit + $(PYTHON) -m pytest -v -m unit endif test-integration: @@ -274,29 +271,38 @@ test-integration: @sleep 2 ifdef module TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v -m "integration and $(module)" + $(PYTHON) -m pytest -v -m "integration and $(module)" else TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v -m integration + $(PYTHON) -m pytest -v -m integration endif test-coverage: @docker compose -f docker-compose.test.yml up -d 2>/dev/null || true @sleep 2 TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) --cov=app --cov=models --cov=utils --cov=middleware --cov-report=html --cov-report=term-missing $(MARKER_EXPR) + $(PYTHON) -m pytest --cov=app --cov=models --cov=utils --cov=middleware --cov-report=html --cov-report=term-missing $(MARKER_EXPR) + +test-affected: + @docker compose -f docker-compose.test.yml up -d 2>/dev/null || true + @sleep 2 + TEST_DATABASE_URL="$(TEST_DB_URL)" \ + $(PYTHON) scripts/tests/run_affected_tests.py $(AFFECTED_ARGS) + +test-affected-dry: + @$(PYTHON) scripts/tests/run_affected_tests.py --dry-run $(AFFECTED_ARGS) test-fast: @docker compose -f docker-compose.test.yml up -d 2>/dev/null || true @sleep 2 TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v -m "not slow" $(MARKER_EXPR) + $(PYTHON) -m pytest -v -m "not slow" $(MARKER_EXPR) test-slow: @docker compose -f docker-compose.test.yml up -d 2>/dev/null || true @sleep 2 TEST_DATABASE_URL="$(TEST_DB_URL)" \ - $(PYTHON) -m pytest $(TEST_PATHS) -v -m slow + $(PYTHON) -m pytest -v -m slow # ============================================================================= # CODE QUALITY @@ -569,6 +575,8 @@ help: @echo " test-unit module=X - Run unit tests for module X" @echo " test-integration - Run integration tests only" @echo " test-coverage - Run tests with coverage" + @echo " test-affected - Run tests for modules affected by changes" + @echo " test-affected-dry - Show affected modules without running tests" @echo " test-fast - Run fast tests only" @echo " test frontend=storefront - Run storefront tests" @echo "" diff --git a/scripts/tests/run_affected_tests.py b/scripts/tests/run_affected_tests.py new file mode 100755 index 00000000..afbec51f --- /dev/null +++ b/scripts/tests/run_affected_tests.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +""" +Module-Aware Test Impact Analysis + +Determines which modules are affected by recent changes and runs only their +tests. Uses the explicit `requires=[]` dependency declarations in each +module's definition.py to build a reverse dependency graph. + +Usage: + python scripts/tests/run_affected_tests.py # diff against gitea/master + python scripts/tests/run_affected_tests.py --base origin/master # diff against specific ref + python scripts/tests/run_affected_tests.py --dry-run # show what would run + python scripts/tests/run_affected_tests.py --all # run all tests +""" + +import argparse +import re +import subprocess +import sys +from collections import deque +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +MODULES_PATH = PROJECT_ROOT / "app" / "modules" + +# Changes to these paths trigger a full test suite run +SHARED_PATHS = ( + "app/core/", + "app/api/", + "app/utils/", + "app/exceptions/", + "app/handlers/", + "app/tasks/", + "tests/conftest.py", + "tests/fixtures/", + "conftest.py", + "static/shared/", + "pyproject.toml", + "requirements", # matches requirements*.txt + "middleware/", + "models/", +) + +MODULE_PATH_RE = re.compile(r"^app/modules/([^/]+)/") +REQUIRES_RE = re.compile(r"requires\s*=\s*\[([^\]]*)\]") + + +def get_changed_files(base_ref: str) -> list[str]: + """Get list of files changed between base_ref and HEAD.""" + try: + result = subprocess.run( + ["git", "diff", "--name-only", f"{base_ref}..HEAD"], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + check=True, + ) + return [f for f in result.stdout.strip().splitlines() if f] + except subprocess.CalledProcessError: + # If the ref doesn't exist, try without ..HEAD (uncommitted changes) + try: + result = subprocess.run( + ["git", "diff", "--name-only", base_ref], + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + check=True, + ) + return [f for f in result.stdout.strip().splitlines() if f] + except subprocess.CalledProcessError as e: + print(f"Error: Could not get diff against '{base_ref}': {e.stderr.strip()}", file=sys.stderr) + sys.exit(1) + + +def map_files_to_modules(changed_files: list[str]) -> tuple[set[str], bool]: + """Map changed files to module names. Returns (modules, shared_changed).""" + modules = set() + shared_changed = False + + for filepath in changed_files: + # Check if it's a shared path + if any(filepath.startswith(sp) for sp in SHARED_PATHS): + shared_changed = True + continue + + # Check if it's a module path + match = MODULE_PATH_RE.match(filepath) + if match: + modules.add(match.group(1)) + + return modules, shared_changed + + +def parse_module_dependencies() -> dict[str, list[str]]: + """Parse requires=[] from each module's definition.py.""" + deps: dict[str, list[str]] = {} + + for module_dir in MODULES_PATH.iterdir(): + if not module_dir.is_dir() or module_dir.name.startswith((".", "__")): + continue + + module_name = module_dir.name + definition = module_dir / "definition.py" + + if not definition.exists(): + deps[module_name] = [] + continue + + content = definition.read_text() + match = REQUIRES_RE.search(content) + if match: + raw = match.group(1) + requires = [s.strip().strip("\"'") for s in raw.split(",") if s.strip().strip("\"'")] + deps[module_name] = requires + else: + deps[module_name] = [] + + return deps + + +def build_reverse_deps(deps: dict[str, list[str]]) -> dict[str, set[str]]: + """Build reverse dependency graph: if A requires B, then B -> A.""" + reverse: dict[str, set[str]] = {mod: set() for mod in deps} + for module, requires in deps.items(): + for req in requires: + if req in reverse: + reverse[req].add(module) + return reverse + + +def expand_affected(changed_modules: set[str], reverse_deps: dict[str, set[str]]) -> set[str]: + """BFS to find all transitively affected modules.""" + affected = set(changed_modules) + queue = deque(changed_modules) + + while queue: + module = queue.popleft() + for dependent in reverse_deps.get(module, set()): + if dependent not in affected: + affected.add(dependent) + queue.append(dependent) + + return affected + + +def get_all_modules() -> set[str]: + """Get all module names from the filesystem.""" + return { + d.name + for d in MODULES_PATH.iterdir() + if d.is_dir() and not d.name.startswith((".", "__")) + } + + +def get_test_dirs(modules: set[str]) -> list[Path]: + """Get test directories for the given modules.""" + test_dirs = [] + for mod in sorted(modules): + test_dir = MODULES_PATH / mod / "tests" + if test_dir.exists(): + test_dirs.append(test_dir) + return test_dirs + + +def run_pytest(test_dirs: list[Path], extra_args: list[str] | None = None) -> int: + """Run pytest on the given test directories.""" + cmd = [ + sys.executable, "-m", "pytest", + "--override-ini=addopts=", + "-v", "--tb=short", "--strict-markers", "-q", + ] + + if extra_args: + cmd.extend(extra_args) + + cmd.extend(str(d) for d in test_dirs) + + result = subprocess.run(cmd, cwd=PROJECT_ROOT) + return result.returncode + + +def main(): + parser = argparse.ArgumentParser( + description="Run tests only for modules affected by recent changes" + ) + parser.add_argument( + "--base", + default="gitea/master", + help="Base ref to diff against (default: gitea/master)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would run without executing tests", + ) + parser.add_argument( + "--all", + action="store_true", + help="Run all tests regardless of changes", + ) + parser.add_argument( + "pytest_args", + nargs="*", + help="Extra arguments to pass to pytest", + ) + + args = parser.parse_args() + + all_modules = get_all_modules() + deps = parse_module_dependencies() + reverse_deps = build_reverse_deps(deps) + + # --all: run everything + if args.all: + test_dirs = get_test_dirs(all_modules) + print(f"Running ALL tests ({len(all_modules)} modules, {len(test_dirs)} test dirs)") + if args.dry_run: + for d in test_dirs: + print(f" {d.relative_to(PROJECT_ROOT)}") + return 0 + return run_pytest(test_dirs, args.pytest_args or None) + + # Analyze changes + print(f"Analyzing changes ({args.base}..HEAD)...") + changed_files = get_changed_files(args.base) + + if not changed_files: + print(" No changes detected. Nothing to test.") + return 0 + + print(f" Changed files: {len(changed_files)}") + + changed_modules, shared_changed = map_files_to_modules(changed_files) + + # Shared code changed → run all tests + if shared_changed: + print(" Shared code changed → running ALL tests") + test_dirs = get_test_dirs(all_modules) + if args.dry_run: + for d in test_dirs: + print(f" {d.relative_to(PROJECT_ROOT)}") + return 0 + return run_pytest(test_dirs, args.pytest_args or None) + + if not changed_modules: + print(" No module changes detected (only non-module files changed).") + print(" Consider running full suite if changes affect test infrastructure.") + return 0 + + print(f" Modules changed: {', '.join(sorted(changed_modules))}") + + affected = expand_affected(changed_modules, reverse_deps) + skipped = all_modules - affected + + print(f" Affected (with deps): {', '.join(sorted(affected))}") + if skipped: + print(f" Skipped: {len(skipped)} modules") + + test_dirs = get_test_dirs(affected) + + if not test_dirs: + print(" No test directories found for affected modules.") + return 0 + + if args.dry_run: + print("\nWould run tests for:") + for d in test_dirs: + print(f" {d.relative_to(PROJECT_ROOT)}") + return 0 + + print(f"\nRunning tests for: {', '.join(sorted(affected))}") + return run_pytest(test_dirs, args.pytest_args or None) + + +if __name__ == "__main__": + sys.exit(main())