diff --git a/app/tasks/letzshop_tasks.py b/app/tasks/letzshop_tasks.py new file mode 100644 index 00000000..b6bfc963 --- /dev/null +++ b/app/tasks/letzshop_tasks.py @@ -0,0 +1,235 @@ +# app/tasks/letzshop_tasks.py +"""Background tasks for Letzshop historical order imports.""" + +import logging +from datetime import UTC, datetime +from typing import Callable + +from app.core.database import SessionLocal +from app.services.letzshop import LetzshopClientError +from app.services.letzshop.credentials_service import LetzshopCredentialsService +from app.services.letzshop.order_service import LetzshopOrderService +from models.database.letzshop import LetzshopHistoricalImportJob + +logger = logging.getLogger(__name__) + + +def _get_credentials_service(db) -> LetzshopCredentialsService: + """Create a credentials service instance.""" + return LetzshopCredentialsService(db) + + +def _get_order_service(db) -> LetzshopOrderService: + """Create an order service instance.""" + return LetzshopOrderService(db) + + +def process_historical_import(job_id: int, vendor_id: int): + """ + Background task for historical order import with progress tracking. + + Imports both confirmed and declined orders from Letzshop API, + updating job progress in the database for frontend polling. + + Args: + job_id: ID of the LetzshopHistoricalImportJob record + vendor_id: ID of the vendor to import orders for + """ + db = SessionLocal() + job = None + + try: + # Get the import job + job = ( + db.query(LetzshopHistoricalImportJob) + .filter(LetzshopHistoricalImportJob.id == job_id) + .first() + ) + if not job: + logger.error(f"Historical import job {job_id} not found") + return + + # Mark as started + job.status = "fetching" + job.started_at = datetime.now(UTC) + db.commit() + + creds_service = _get_credentials_service(db) + order_service = _get_order_service(db) + + # Create progress callback for fetching + def fetch_progress_callback(page: int, total_fetched: int): + """Update fetch progress in database.""" + job.current_page = page + job.shipments_fetched = total_fetched + db.commit() + + # Create progress callback for processing + def create_processing_callback( + phase: str, + ) -> Callable[[int, int, int, int], None]: + """Create a processing progress callback for a phase.""" + + def callback(processed: int, imported: int, updated: int, skipped: int): + job.orders_processed = processed + job.orders_imported = imported + job.orders_updated = updated + job.orders_skipped = skipped + db.commit() + + return callback + + with creds_service.create_client(vendor_id) as client: + # ================================================================ + # Phase 1: Import confirmed orders + # ================================================================ + job.current_phase = "confirmed" + job.current_page = 0 + job.shipments_fetched = 0 + db.commit() + + logger.info(f"Job {job_id}: Fetching confirmed shipments for vendor {vendor_id}") + + confirmed_shipments = client.get_all_shipments_paginated( + state="confirmed", + page_size=50, + progress_callback=fetch_progress_callback, + ) + + logger.info(f"Job {job_id}: Fetched {len(confirmed_shipments)} confirmed shipments") + + # Process confirmed shipments + job.status = "processing" + job.orders_processed = 0 + job.orders_imported = 0 + job.orders_updated = 0 + job.orders_skipped = 0 + db.commit() + + confirmed_stats = order_service.import_historical_shipments( + vendor_id=vendor_id, + shipments=confirmed_shipments, + match_products=True, + progress_callback=create_processing_callback("confirmed"), + ) + + # Store confirmed stats + job.confirmed_stats = { + "total": confirmed_stats["total"], + "imported": confirmed_stats["imported"], + "updated": confirmed_stats["updated"], + "skipped": confirmed_stats["skipped"], + "products_matched": confirmed_stats["products_matched"], + "products_not_found": confirmed_stats["products_not_found"], + } + job.products_matched = confirmed_stats["products_matched"] + job.products_not_found = confirmed_stats["products_not_found"] + db.commit() + + logger.info( + f"Job {job_id}: Confirmed phase complete - " + f"imported={confirmed_stats['imported']}, " + f"updated={confirmed_stats['updated']}, " + f"skipped={confirmed_stats['skipped']}" + ) + + # ================================================================ + # Phase 2: Import unconfirmed (pending) orders + # Note: Letzshop API has no "declined" state. Declined items + # are tracked at the inventory unit level, not shipment level. + # Valid states: unconfirmed, confirmed, completed, accepted + # ================================================================ + job.current_phase = "unconfirmed" + job.status = "fetching" + job.current_page = 0 + job.shipments_fetched = 0 + db.commit() + + logger.info(f"Job {job_id}: Fetching unconfirmed shipments for vendor {vendor_id}") + + unconfirmed_shipments = client.get_all_shipments_paginated( + state="unconfirmed", + page_size=50, + progress_callback=fetch_progress_callback, + ) + + logger.info(f"Job {job_id}: Fetched {len(unconfirmed_shipments)} unconfirmed shipments") + + # Process unconfirmed shipments + job.status = "processing" + job.orders_processed = 0 + db.commit() + + unconfirmed_stats = order_service.import_historical_shipments( + vendor_id=vendor_id, + shipments=unconfirmed_shipments, + match_products=True, + progress_callback=create_processing_callback("unconfirmed"), + ) + + # Store unconfirmed stats (in declined_stats field for compatibility) + job.declined_stats = { + "total": unconfirmed_stats["total"], + "imported": unconfirmed_stats["imported"], + "updated": unconfirmed_stats["updated"], + "skipped": unconfirmed_stats["skipped"], + "products_matched": unconfirmed_stats["products_matched"], + "products_not_found": unconfirmed_stats["products_not_found"], + } + + # Add to cumulative product matching stats + job.products_matched += unconfirmed_stats["products_matched"] + job.products_not_found += unconfirmed_stats["products_not_found"] + + logger.info( + f"Job {job_id}: Unconfirmed phase complete - " + f"imported={unconfirmed_stats['imported']}, " + f"updated={unconfirmed_stats['updated']}, " + f"skipped={unconfirmed_stats['skipped']}" + ) + + # ================================================================ + # Complete + # ================================================================ + job.status = "completed" + job.completed_at = datetime.now(UTC) + db.commit() + + # Update credentials sync status + creds_service.update_sync_status(vendor_id, "success", None) + + logger.info(f"Job {job_id}: Historical import completed successfully") + + except LetzshopClientError as e: + logger.error(f"Job {job_id}: Letzshop API error: {e}") + if job is not None: + try: + job.status = "failed" + job.error_message = f"Letzshop API error: {e}" + job.completed_at = datetime.now(UTC) + db.commit() + + creds_service = _get_credentials_service(db) + creds_service.update_sync_status(vendor_id, "failed", str(e)) + except Exception as commit_error: + logger.error(f"Job {job_id}: Failed to update job status: {commit_error}") + db.rollback() + + except Exception as e: + logger.error(f"Job {job_id}: Unexpected error: {e}", exc_info=True) + if job is not None: + try: + job.status = "failed" + job.error_message = str(e) + job.completed_at = datetime.now(UTC) + db.commit() + except Exception as commit_error: + logger.error(f"Job {job_id}: Failed to update job status: {commit_error}") + db.rollback() + + finally: + if hasattr(db, "close") and callable(db.close): + try: + db.close() + except Exception as close_error: + logger.error(f"Job {job_id}: Error closing database session: {close_error}") diff --git a/docs/implementation/letzshop-order-import-improvements.md b/docs/implementation/letzshop-order-import-improvements.md index f735e5cb..2db034f0 100644 --- a/docs/implementation/letzshop-order-import-improvements.md +++ b/docs/implementation/letzshop-order-import-improvements.md @@ -463,39 +463,97 @@ From the Letzshop merchant interface: - Filter dropdown, status badges, and action buttons now use "Declined" - Added "Declined" stats card to orders dashboard -### Historical Import: Declined Orders ✅ -- Historical import now fetches both `confirmed` AND `declined` shipments +### Historical Import: Multiple Phases ✅ +- Historical import now fetches both `confirmed` AND `unconfirmed` (pending) shipments +- Note: "declined" is NOT a valid Letzshop shipment state - declined items are tracked at inventory unit level - Combined stats shown in import result --- +## Completed (2025-12-19) + +### Historical Import Progress Bar ✅ +Real-time progress feedback for historical import using background tasks with database polling. + +**Implementation:** +- Background task (`app/tasks/letzshop_tasks.py`) runs historical import asynchronously +- Progress stored in `LetzshopHistoricalImportJob` database model +- Frontend polls status endpoint every 2 seconds +- Two-phase import: confirmed orders first, then unconfirmed (pending) orders + +**Backend:** +- `LetzshopHistoricalImportJob` model tracks: status, current_phase, current_page, shipments_fetched, orders_processed, confirmed_stats, declined_stats +- `POST /vendors/{id}/import-history` starts background job, returns job_id immediately +- `GET /vendors/{id}/import-history/{job_id}/status` returns current progress + +**Frontend:** +- Progress panel shows: phase (confirmed/pending), page number, shipments fetched, orders processed +- Disabled "Import History" button during import with spinner +- Final result summary shows combined stats from both phases + +**Key Discovery:** +- Letzshop API has NO "declined" shipment state +- Valid states: `awaiting_order_completion`, `unconfirmed`, `completed`, `accepted`, `confirmed` +- Declined items are tracked at inventory unit level with state `confirmed_unavailable` + +### Filter for Declined Items ✅ +Added ability to filter orders that have at least one declined/unavailable item. + +**Backend:** +- `list_orders()` accepts `has_declined_items: bool` parameter +- Uses JSON string contains check: `inventory_units.cast(String).contains("confirmed_unavailable")` +- `get_order_stats()` returns `has_declined_items` count + +**Frontend:** +- "Has Declined Items" toggle button in filters section +- Shows count badge when there are orders with declined items +- Toggles between all orders and filtered view + +**API:** +- `GET /vendors/{id}/orders?has_declined_items=true` - filter orders + +### Order Date Display ✅ +Orders now display the actual order date from Letzshop instead of the import date. + +**Database:** +- Added `order_date` column to `LetzshopOrder` model +- Migration: `2362c2723a93_add_order_date_to_letzshop_orders.py` + +**Backend:** +- `create_order()` extracts `completedAt` from Letzshop order data and stores as `order_date` +- `update_order_from_shipment()` populates `order_date` if not already set +- Date parsing handles ISO format with timezone (including `Z` suffix) + +**Frontend:** +- Order table displays `order_date` with fallback to `created_at` for legacy orders +- Format: localized date/time string + +**Note:** Existing orders imported before this change will continue showing `created_at` until re-imported via historical import. + +### Search Filter ✅ +Added search functionality to find orders by order number, customer name, or email. + +**Backend:** +- `list_orders()` accepts `search: str` parameter +- Uses ILIKE for case-insensitive partial matching across: + - `letzshop_order_number` + - `customer_name` + - `customer_email` + +**Frontend:** +- Search input field with magnifying glass icon +- Debounced input (300ms) to avoid excessive API calls +- Clear button to reset search +- Resets to page 1 when search changes + +**API:** +- `GET /vendors/{id}/orders?search=query` - search orders + +--- + ## Next Steps (TODO) -### Priority 1: Historical Import Progress Bar -Add real-time progress feedback for historical import (currently no visibility into import progress). - -**Requirements:** -- Show progress indicator while import is running -- Display current page being fetched (e.g., "Fetching page 3 of 12...") -- Show running count of orders imported/updated -- Prevent user from thinking the process is stuck - -**Implementation options:** -1. **Polling approach**: Frontend polls a status endpoint every few seconds -2. **Server-Sent Events (SSE)**: Real-time updates pushed to frontend -3. **WebSocket**: Bi-directional real-time communication - -**Backend changes needed:** -- Store import progress in database or cache (Redis) -- Add endpoint `GET /api/v1/admin/letzshop/vendors/{id}/import-progress` -- Update `import_historical_shipments()` to report progress - -**Frontend changes needed:** -- Progress bar component in Orders tab -- Polling/SSE logic to fetch progress updates -- Disable "Import History" button while import is in progress - -### Priority 2: Stock Management +### Priority 1: Stock Management When an order is confirmed/imported: 1. Match EAN from order to local product catalog 2. Decrease stock quantity for matched products @@ -506,18 +564,12 @@ When an order is confirmed/imported: - Need rollback mechanism if order is rejected - Handle partial matches (some items found, some not) -### Priority 2: Order Detail View Enhancement -Improve the order detail modal to show: -- Product details (name, EAN, MPN, SKU) -- Match status per line item (found/not found in catalog) -- Link to local product if matched - -### Priority 3: Invoice Generation +### Priority 2: Invoice Generation Use `customer_locale` to generate invoices in customer's language: - Invoice template with multi-language support - PDF generation -### Priority 4: Analytics Dashboard +### Priority 3: Analytics Dashboard Build sales analytics based on imported orders: - Sales by product - Sales by time period @@ -526,20 +578,24 @@ Build sales analytics based on imported orders: --- -## Files Modified (2025-12-16 to 2025-12-18) +## Files Modified (2025-12-16 to 2025-12-19) | File | Changes | |------|---------| | `app/services/letzshop/client_service.py` | Added paginated query, updated all queries with EAN/locale/country | -| `app/services/letzshop/order_service.py` | Added historical import, EAN matching, summary endpoint, order stats | -| `models/database/letzshop.py` | Added locale and country columns | +| `app/services/letzshop/order_service.py` | Historical import, EAN matching, order stats, has_declined_items filter, search filter, order_date extraction | +| `models/database/letzshop.py` | Added locale/country/order_date columns, `LetzshopHistoricalImportJob` model | | `models/database/product.py` | Added `gtin` and `gtin_type` columns for EAN matching | -| `models/schema/letzshop.py` | Added `LetzshopOrderStats` schema, stats to order list response | -| `app/api/v1/admin/letzshop.py` | Added import-history and import-summary endpoints, stats in orders response | -| `app/templates/admin/partials/letzshop-orders-tab.html` | Added Import History button and result display | -| `static/admin/js/marketplace-letzshop.js` | Added importHistoricalOrders(), server-side stats | +| `models/schema/letzshop.py` | Added `LetzshopOrderStats`, `LetzshopHistoricalImportJobResponse`, `order_date` field | +| `app/api/v1/admin/letzshop.py` | Import-history endpoints, has_declined_items filter, search filter, order_date in response | +| `app/tasks/letzshop_tasks.py` | **NEW** - Background task for historical import with progress tracking | +| `app/templates/admin/partials/letzshop-orders-tab.html` | Import History button, progress panel, declined items filter, search input, order_date display | +| `static/admin/js/marketplace-letzshop.js` | Historical import polling, progress display, declined items filter, search functionality | | `tests/unit/services/test_letzshop_service.py` | Added tests for new functionality | | `scripts/test_historical_import.py` | Manual test script for historical import | +| `scripts/debug_historical_import.py` | **NEW** - Debug script for shipment states and declined items | | `scripts/letzshop_introspect.py` | GraphQL schema introspection tool, tracking workaround tests | | `alembic/versions/a9a86cef6cca_*.py` | Migration for locale/country columns | | `alembic/versions/cb88bc9b5f86_*.py` | Migration for gtin columns on Product table | +| `alembic/versions/*_add_historical_import_jobs.py` | **NEW** - Migration for LetzshopHistoricalImportJob table | +| `alembic/versions/2362c2723a93_*.py` | **NEW** - Migration for order_date column | diff --git a/mkdocs.yml b/mkdocs.yml index 67035bcc..934a0fa1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -135,6 +135,8 @@ nav: - Vendor Operations Expansion: development/migration/vendor-operations-expansion.md - Implementation Plans: - Admin Inventory Management: implementation/inventory-admin-migration.md + - Letzshop Order Import: implementation/letzshop-order-import-improvements.md + - Unified Order View: implementation/unified-order-view.md - Seed Scripts Audit: development/seed-scripts-audit.md - Database Seeder: - Documentation: development/database-seeder/database-seeder-documentation.md diff --git a/scripts/debug_historical_import.py b/scripts/debug_historical_import.py new file mode 100644 index 00000000..b01b177b --- /dev/null +++ b/scripts/debug_historical_import.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Debug script for historical import issues.""" + +import sys +sys.path.insert(0, ".") + +from app.core.database import SessionLocal +from app.services.letzshop.credentials_service import LetzshopCredentialsService +from models.database.letzshop import LetzshopHistoricalImportJob + + +def get_valid_shipment_states(vendor_id: int = 1): + """Query the GraphQL schema to find valid ShipmentStateEnum values.""" + db = SessionLocal() + try: + creds_service = LetzshopCredentialsService(db) + + # Introspection query to get enum values + introspection_query = """ + query { + __type(name: "ShipmentStateEnum") { + name + enumValues { + name + description + } + } + } + """ + + with creds_service.create_client(vendor_id) as client: + print("Querying ShipmentStateEnum values...") + result = client._execute(introspection_query) + + if result and "__type" in result: + type_info = result["__type"] + if type_info: + print(f"\nEnum: {type_info['name']}") + print("Valid values:") + for value in type_info.get("enumValues", []): + print(f" - {value['name']}: {value.get('description', 'No description')}") + else: + print("ShipmentStateEnum type not found") + else: + print(f"Unexpected result: {result}") + + finally: + db.close() + + +def check_import_job_status(): + """Check the status of historical import jobs.""" + db = SessionLocal() + try: + jobs = db.query(LetzshopHistoricalImportJob).order_by( + LetzshopHistoricalImportJob.id.desc() + ).limit(5).all() + + print("\n=== Recent Historical Import Jobs ===") + for job in jobs: + print(f"\nJob {job.id}:") + print(f" Status: {job.status}") + print(f" Phase: {job.current_phase}") + print(f" Page: {job.current_page}") + print(f" Shipments fetched: {job.shipments_fetched}") + print(f" Orders processed: {job.orders_processed}") + print(f" Orders imported: {job.orders_imported}") + print(f" Orders updated: {job.orders_updated}") + print(f" Orders skipped: {job.orders_skipped}") + print(f" Confirmed stats: {job.confirmed_stats}") + print(f" Declined stats: {job.declined_stats}") + print(f" Error: {job.error_message}") + print(f" Started: {job.started_at}") + print(f" Completed: {job.completed_at}") + finally: + db.close() + + +def test_fetch_states(vendor_id: int = 1): + """Test fetching shipments with different states.""" + db = SessionLocal() + try: + creds_service = LetzshopCredentialsService(db) + + # States to test + states_to_test = ["unconfirmed", "confirmed", "declined", "shipped", "rejected"] + + with creds_service.create_client(vendor_id) as client: + for state in states_to_test: + print(f"\nTesting state: {state}") + try: + # Just try to fetch first page + shipments = client.get_all_shipments_paginated( + state=state, + page_size=1, + max_pages=1, + ) + print(f" ✓ Success: {len(shipments)} shipments") + except Exception as e: + print(f" ✗ Error: {e}") + + finally: + db.close() + + +def check_declined_items(vendor_id: int = 1): + """Check for orders with declined inventory units.""" + db = SessionLocal() + try: + from models.database.letzshop import LetzshopOrder + from collections import Counter + + # Get all orders with inventory_units + orders = db.query(LetzshopOrder).filter( + LetzshopOrder.vendor_id == vendor_id, + LetzshopOrder.inventory_units.isnot(None), + ).all() + + # Count all states + state_counts = Counter() + total_units = 0 + + for order in orders: + units = order.inventory_units or [] + for unit in units: + state = unit.get("state", "unknown") + state_counts[state] += 1 + total_units += 1 + + print(f"\n=== Inventory Unit States (all {len(orders)} orders) ===") + print(f" Total units: {total_units}") + print(f"\n State breakdown:") + for state, count in sorted(state_counts.items(), key=lambda x: -x[1]): + print(f" {state}: {count}") + + # Show a sample order with its units + if orders: + sample = orders[0] + print(f"\nSample order #{sample.id} (shipment {sample.letzshop_shipment_id}):") + print(f" Shipment state: {sample.letzshop_state}") + print(f" Sync status: {sample.sync_status}") + if sample.inventory_units: + for i, unit in enumerate(sample.inventory_units[:5]): + print(f" Unit {i+1}: state={unit.get('state')}, id={unit.get('id')}") + + finally: + db.close() + + +if __name__ == "__main__": + print("=== Letzshop Historical Import Debug ===\n") + + print("1. Checking valid shipment states...") + get_valid_shipment_states() + + print("\n\n2. Testing different state values...") + test_fetch_states() + + print("\n\n3. Checking import job status...") + check_import_job_status() + + print("\n\n4. Checking declined items in inventory units...") + check_declined_items()