diff --git a/.audit-rules/change_management.yaml b/.audit-rules/change_management.yaml index 32723a21..a18ae9dc 100644 --- a/.audit-rules/change_management.yaml +++ b/.audit-rules/change_management.yaml @@ -55,6 +55,7 @@ rules: type: file_exists paths: - ".github/PULL_REQUEST_TEMPLATE.md" + - ".gitlab/merge_request_templates/*.md" message: "Pull request template recommended" - id: CHANGE-REV-002 @@ -73,6 +74,7 @@ rules: type: file_exists paths: - ".github/CODEOWNERS" + - "CODEOWNERS" # GitLab uses root CODEOWNERS or .gitlab/CODEOWNERS - "CODEOWNERS" message: "Consider defining code owners for critical paths" @@ -89,6 +91,7 @@ rules: paths: - ".github/workflows/ci.yml" - ".github/workflows/test.yml" + - ".gitlab-ci.yml" message: "CI workflow for automated testing required" - id: CHANGE-CI-002 @@ -99,6 +102,7 @@ rules: type: pattern_recommended paths: - ".github/workflows/*.yml" + - ".gitlab-ci.yml" patterns: - "security|bandit|safety|snyk|trivy" message: "Consider security scanning in CI pipeline" @@ -111,6 +115,7 @@ rules: type: pattern_required paths: - ".github/workflows/*.yml" + - ".gitlab-ci.yml" patterns: - "ruff|flake8|pylint|mypy|lint" message: "Code quality checks required in CI" @@ -141,6 +146,7 @@ rules: paths: - ".github/workflows/release.yml" - ".github/workflows/deploy.yml" + - ".gitlab-ci.yml" - "Dockerfile" message: "Automated deployment process recommended" @@ -193,6 +199,7 @@ rules: paths: - "Dockerfile" - ".github/workflows/*.yml" + - ".gitlab-ci.yml" patterns: - "tag|version|:v" message: "Container image versioning recommended" diff --git a/.audit-rules/compliance.yaml b/.audit-rules/compliance.yaml index c305d26c..61e2006b 100644 --- a/.audit-rules/compliance.yaml +++ b/.audit-rules/compliance.yaml @@ -122,8 +122,10 @@ rules: type: file_exists paths: - ".github/PULL_REQUEST_TEMPLATE.md" + - ".gitlab/merge_request_templates/*.md" - "CONTRIBUTING.md" - ".github/workflows/*.yml" + - ".gitlab-ci.yml" message: "Code review process must be documented/enforced" - id: COMP-POL-002 @@ -134,7 +136,10 @@ rules: type: file_exists paths: - ".github/CODEOWNERS" + - "CODEOWNERS" - ".github/workflows/*.yml" + - ".gitlab-ci.yml" + - ".gitlab-ci.yml" message: "Document change approval requirements" - id: COMP-POL-003 @@ -161,6 +166,7 @@ rules: type: file_exists paths: - ".github/workflows/ci.yml" + - ".gitlab-ci.yml" - "pytest.ini" - "pyproject.toml" patterns: @@ -175,6 +181,7 @@ rules: type: file_exists paths: - ".github/workflows/*.yml" + - ".gitlab-ci.yml" patterns: - "deploy|release" message: "Deployment process must be automated and logged" diff --git a/.audit-rules/documentation.yaml b/.audit-rules/documentation.yaml index 56112aff..a1b236d0 100644 --- a/.audit-rules/documentation.yaml +++ b/.audit-rules/documentation.yaml @@ -94,6 +94,7 @@ rules: paths: - "SECURITY.md" - ".github/SECURITY.md" + - ".gitlab/SECURITY.md" message: "Security policy (SECURITY.md) required" - id: DOC-SEC-002 diff --git a/.audit-rules/third_party.yaml b/.audit-rules/third_party.yaml index c6310d12..e3e7347f 100644 --- a/.audit-rules/third_party.yaml +++ b/.audit-rules/third_party.yaml @@ -57,6 +57,7 @@ rules: type: file_exists paths: - ".github/workflows/*.yml" + - ".gitlab-ci.yml" patterns: - "safety|pip-audit|snyk|dependabot" message: "Dependency vulnerability scanning required" @@ -69,6 +70,7 @@ rules: type: file_exists paths: - ".github/dependabot.yml" + - ".gitlab-ci.yml" # GitLab uses built-in dependency scanning message: "Consider enabling Dependabot for security updates" - id: THIRD-VULN-003 @@ -79,6 +81,7 @@ rules: type: pattern_recommended paths: - ".github/workflows/*.yml" + - ".gitlab-ci.yml" patterns: - "trivy|grype|snyk.*container" message: "Consider container image vulnerability scanning" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..e7e30815 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,112 @@ +# GitLab CI/CD Configuration +# ========================= + +stages: + - lint + - test + - security + - build + +variables: + PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + PYTHON_VERSION: "3.11" + +# Cache dependencies between jobs +cache: + paths: + - .cache/pip + - .venv/ + +# Lint Stage +# ---------- + +ruff: + stage: lint + image: python:${PYTHON_VERSION} + before_script: + - pip install uv + - uv sync --frozen + script: + - .venv/bin/ruff check . + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +# Test Stage +# ---------- + +pytest: + stage: test + image: python:${PYTHON_VERSION} + before_script: + - pip install uv + - uv sync --frozen + script: + - .venv/bin/python -m pytest tests/ -v --tb=short + coverage: '/TOTAL.*\s+(\d+%)/' + artifacts: + reports: + junit: report.xml + coverage_report: + coverage_format: cobertura + path: coverage.xml + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +architecture: + stage: test + image: python:${PYTHON_VERSION} + before_script: + - pip install uv + - uv sync --frozen + script: + - .venv/bin/python scripts/validate_architecture.py + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +# Security Stage +# -------------- + +dependency_scanning: + stage: security + image: python:${PYTHON_VERSION} + before_script: + - pip install pip-audit + script: + - pip-audit --requirement requirements.txt || true + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +audit: + stage: security + image: python:${PYTHON_VERSION} + before_script: + - pip install uv + - uv sync --frozen + script: + - .venv/bin/python scripts/validate_audit.py + allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +# Build Stage +# ----------- + +docs: + stage: build + image: python:${PYTHON_VERSION} + before_script: + - pip install uv + - uv sync --frozen + script: + - .venv/bin/mkdocs build + artifacts: + paths: + - site/ + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/.gitlab/merge_request_templates/default.md b/.gitlab/merge_request_templates/default.md new file mode 100644 index 00000000..e4b7d3ee --- /dev/null +++ b/.gitlab/merge_request_templates/default.md @@ -0,0 +1,41 @@ +## Summary + + + +## Changes + + +- + +## Type of Change + + +- [ ] Bug fix (non-breaking change that fixes an issue) +- [ ] New feature (non-breaking change that adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Documentation update +- [ ] Refactoring (no functional changes) + +## Testing + + +- [ ] Unit tests pass (`pytest tests/`) +- [ ] Architecture validation passes (`python scripts/validate_architecture.py`) +- [ ] Manual testing performed + +## Checklist + +- [ ] My code follows the project's coding standards +- [ ] I have updated the documentation accordingly +- [ ] I have added tests that prove my fix/feature works +- [ ] All new and existing tests pass +- [ ] No new warnings are introduced + +## Related Issues + + +Closes # + +## Screenshots + + diff --git a/app/api/v1/vendor/onboarding.py b/app/api/v1/vendor/onboarding.py index 583566e2..3bd48acc 100644 --- a/app/api/v1/vendor/onboarding.py +++ b/app/api/v1/vendor/onboarding.py @@ -13,12 +13,13 @@ Vendor Context: Uses token_vendor_id from JWT token (authenticated vendor API pa import logging -from fastapi import APIRouter, Depends +from fastapi import APIRouter, BackgroundTasks, Depends from sqlalchemy.orm import Session from app.api.deps import get_current_vendor_api from app.core.database import get_db from app.services.onboarding_service import OnboardingService +from app.tasks.letzshop_tasks import process_historical_import from models.database.user import User from models.schema.onboarding import ( CompanyProfileRequest, @@ -206,6 +207,7 @@ def save_product_import_config( @router.post("/step/order-sync/trigger", response_model=OrderSyncTriggerResponse) def trigger_order_sync( request: OrderSyncTriggerRequest, + background_tasks: BackgroundTasks, current_user: User = Depends(get_current_vendor_api), db: Session = Depends(get_db), ): @@ -222,6 +224,16 @@ def trigger_order_sync( include_products=request.include_products, ) db.commit() # Commit at API level for transaction control + + # Queue background task to process the import + if result.get("success") and result.get("job_id"): + background_tasks.add_task( + process_historical_import, + result["job_id"], + current_user.token_vendor_id, + ) + logger.info(f"Queued historical import task for job {result['job_id']}") + return result diff --git a/scripts/validate_audit.py b/scripts/validate_audit.py index d13989cc..9b0884e9 100644 --- a/scripts/validate_audit.py +++ b/scripts/validate_audit.py @@ -265,22 +265,27 @@ class AuditValidator(BaseValidator): str(self.project_root), ) - # Check CI/CD exists - ci_workflow = self.project_root / ".github" / "workflows" / "ci.yml" - if not ci_workflow.exists(): + # Check CI/CD exists (GitHub or GitLab) + github_ci = self.project_root / ".github" / "workflows" / "ci.yml" + gitlab_ci = self.project_root / ".gitlab-ci.yml" + if not github_ci.exists() and not gitlab_ci.exists(): self.add_warning( "COMP-EVID-001", "CI workflow for automated testing recommended", - ".github/workflows/ci.yml", + ".gitlab-ci.yml or .github/workflows/ci.yml", ) - # Check code review process - pr_template = self.project_root / ".github" / "PULL_REQUEST_TEMPLATE.md" - if not pr_template.exists(): + # Check code review process (GitHub or GitLab) + github_pr_template = self.project_root / ".github" / "PULL_REQUEST_TEMPLATE.md" + gitlab_mr_templates = self.project_root / ".gitlab" / "merge_request_templates" + has_mr_template = github_pr_template.exists() or ( + gitlab_mr_templates.exists() and any(gitlab_mr_templates.iterdir()) + ) + if not has_mr_template: self.add_warning( "COMP-POL-001", - "Pull request template recommended for code review", - ".github/PULL_REQUEST_TEMPLATE.md", + "Merge request template recommended for code review", + ".gitlab/merge_request_templates/ or .github/PULL_REQUEST_TEMPLATE.md", ) # ================== @@ -367,13 +372,19 @@ class AuditValidator(BaseValidator): "pyproject.toml", ) - # Check for Dependabot + # Check for dependency scanning (GitHub Dependabot or GitLab) dependabot = self.project_root / ".github" / "dependabot.yml" - if not dependabot.exists(): + gitlab_ci = self.project_root / ".gitlab-ci.yml" + has_dep_scanning = dependabot.exists() + if not has_dep_scanning and gitlab_ci.exists(): + # Check if GitLab CI includes dependency scanning + ci_content = gitlab_ci.read_text() + has_dep_scanning = "dependency_scanning" in ci_content.lower() + if not has_dep_scanning: self.add_info( "THIRD-VULN-002", - "Consider enabling Dependabot for security updates", - ".github/dependabot.yml", + "Consider enabling dependency scanning for security updates", + ".gitlab-ci.yml (include dependency_scanning) or .github/dependabot.yml", ) # Check for insecure package sources