sboulahtit/orion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: update Letzshop order import documentation

Browse Source 
- Update implementation guide with unified order approach
- Add mkdocs navigation entry
- Add background task for order sync
- Add debug script for historical imports

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Samir Boulahtit
2025-12-19 21:18:55 +01:00
parent 8e8d1d1ac0
commit 6a10fbba10
4 changed files with 498 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
app/tasks/letzshop_tasks.py Normal file
View File

@@ -0,0 +1,235 @@
# app/tasks/letzshop_tasks.py
"""Background tasks for Letzshop historical order imports."""
import logging
from datetime import UTC, datetime
from typing import Callable
from app.core.database import SessionLocal
from app.services.letzshop import LetzshopClientError
from app.services.letzshop.credentials_service import LetzshopCredentialsService
from app.services.letzshop.order_service import LetzshopOrderService
from models.database.letzshop import LetzshopHistoricalImportJob
logger = logging.getLogger(__name__)
def _get_credentials_service(db) -> LetzshopCredentialsService:
"""Create a credentials service instance."""
return LetzshopCredentialsService(db)
def _get_order_service(db) -> LetzshopOrderService:
"""Create an order service instance."""
return LetzshopOrderService(db)
def process_historical_import(job_id: int, vendor_id: int):
"""
Background task for historical order import with progress tracking.
Imports both confirmed and declined orders from Letzshop API,
updating job progress in the database for frontend polling.
Args:
job_id: ID of the LetzshopHistoricalImportJob record
vendor_id: ID of the vendor to import orders for
"""
db = SessionLocal()
job = None
try:
# Get the import job
job = (
db.query(LetzshopHistoricalImportJob)
.filter(LetzshopHistoricalImportJob.id == job_id)
.first()
)
if not job:
logger.error(f"Historical import job {job_id} not found")
return
# Mark as started
job.status = "fetching"
job.started_at = datetime.now(UTC)
db.commit()
creds_service = _get_credentials_service(db)
order_service = _get_order_service(db)
# Create progress callback for fetching
def fetch_progress_callback(page: int, total_fetched: int):
"""Update fetch progress in database."""
job.current_page = page
job.shipments_fetched = total_fetched
db.commit()
# Create progress callback for processing
def create_processing_callback(
phase: str,
) -> Callable[[int, int, int, int], None]:
"""Create a processing progress callback for a phase."""
def callback(processed: int, imported: int, updated: int, skipped: int):
job.orders_processed = processed
job.orders_imported = imported
job.orders_updated = updated
job.orders_skipped = skipped
db.commit()
return callback
with creds_service.create_client(vendor_id) as client:
# ================================================================
# Phase 1: Import confirmed orders
# ================================================================
job.current_phase = "confirmed"
job.current_page = 0
job.shipments_fetched = 0
db.commit()
logger.info(f"Job {job_id}: Fetching confirmed shipments for vendor {vendor_id}")
confirmed_shipments = client.get_all_shipments_paginated(
state="confirmed",
page_size=50,
progress_callback=fetch_progress_callback,
)
logger.info(f"Job {job_id}: Fetched {len(confirmed_shipments)} confirmed shipments")
# Process confirmed shipments
job.status = "processing"
job.orders_processed = 0
job.orders_imported = 0
job.orders_updated = 0
job.orders_skipped = 0
db.commit()
confirmed_stats = order_service.import_historical_shipments(
vendor_id=vendor_id,
shipments=confirmed_shipments,
match_products=True,
progress_callback=create_processing_callback("confirmed"),
)
# Store confirmed stats
job.confirmed_stats = {
"total": confirmed_stats["total"],
"imported": confirmed_stats["imported"],
"updated": confirmed_stats["updated"],
"skipped": confirmed_stats["skipped"],
"products_matched": confirmed_stats["products_matched"],
"products_not_found": confirmed_stats["products_not_found"],
}
job.products_matched = confirmed_stats["products_matched"]
job.products_not_found = confirmed_stats["products_not_found"]
db.commit()
logger.info(
f"Job {job_id}: Confirmed phase complete - "
f"imported={confirmed_stats['imported']}, "
f"updated={confirmed_stats['updated']}, "
f"skipped={confirmed_stats['skipped']}"
)
# ================================================================
# Phase 2: Import unconfirmed (pending) orders
# Note: Letzshop API has no "declined" state. Declined items
# are tracked at the inventory unit level, not shipment level.
# Valid states: unconfirmed, confirmed, completed, accepted
# ================================================================
job.current_phase = "unconfirmed"
job.status = "fetching"
job.current_page = 0
job.shipments_fetched = 0
db.commit()
logger.info(f"Job {job_id}: Fetching unconfirmed shipments for vendor {vendor_id}")
unconfirmed_shipments = client.get_all_shipments_paginated(
state="unconfirmed",
page_size=50,
progress_callback=fetch_progress_callback,
)
logger.info(f"Job {job_id}: Fetched {len(unconfirmed_shipments)} unconfirmed shipments")
# Process unconfirmed shipments
job.status = "processing"
job.orders_processed = 0
db.commit()
unconfirmed_stats = order_service.import_historical_shipments(
vendor_id=vendor_id,
shipments=unconfirmed_shipments,
match_products=True,
progress_callback=create_processing_callback("unconfirmed"),
)
# Store unconfirmed stats (in declined_stats field for compatibility)
job.declined_stats = {
"total": unconfirmed_stats["total"],
"imported": unconfirmed_stats["imported"],
"updated": unconfirmed_stats["updated"],
"skipped": unconfirmed_stats["skipped"],
"products_matched": unconfirmed_stats["products_matched"],
"products_not_found": unconfirmed_stats["products_not_found"],
}
# Add to cumulative product matching stats
job.products_matched += unconfirmed_stats["products_matched"]
job.products_not_found += unconfirmed_stats["products_not_found"]
logger.info(
f"Job {job_id}: Unconfirmed phase complete - "
f"imported={unconfirmed_stats['imported']}, "
f"updated={unconfirmed_stats['updated']}, "
f"skipped={unconfirmed_stats['skipped']}"
)
# ================================================================
# Complete
# ================================================================
job.status = "completed"
job.completed_at = datetime.now(UTC)
db.commit()
# Update credentials sync status
creds_service.update_sync_status(vendor_id, "success", None)
logger.info(f"Job {job_id}: Historical import completed successfully")
except LetzshopClientError as e:
logger.error(f"Job {job_id}: Letzshop API error: {e}")
if job is not None:
try:
job.status = "failed"
job.error_message = f"Letzshop API error: {e}"
job.completed_at = datetime.now(UTC)
db.commit()
creds_service = _get_credentials_service(db)
creds_service.update_sync_status(vendor_id, "failed", str(e))
except Exception as commit_error:
logger.error(f"Job {job_id}: Failed to update job status: {commit_error}")
db.rollback()
except Exception as e:
logger.error(f"Job {job_id}: Unexpected error: {e}", exc_info=True)
if job is not None:
try:
job.status = "failed"
job.error_message = str(e)
job.completed_at = datetime.now(UTC)
db.commit()
except Exception as commit_error:
logger.error(f"Job {job_id}: Failed to update job status: {commit_error}")
db.rollback()
finally:
if hasattr(db, "close") and callable(db.close):
try:
db.close()
except Exception as close_error:
logger.error(f"Job {job_id}: Error closing database session: {close_error}")

140
docs/implementation/letzshop-order-import-improvements.md
View File

@@ -463,39 +463,97 @@ From the Letzshop merchant interface:
- Filter dropdown, status badges, and action buttons now use "Declined"
- Added "Declined" stats card to orders dashboard
### Historical Import: Declined Orders ✅
- Historical import now fetches both `confirmed` AND `declined` shipments
### Historical Import: Multiple Phases ✅
- Historical import now fetches both `confirmed` AND `unconfirmed` (pending) shipments
- Note: "declined" is NOT a valid Letzshop shipment state - declined items are tracked at inventory unit level
- Combined stats shown in import result
---
## Completed (2025-12-19)
### Historical Import Progress Bar ✅
Real-time progress feedback for historical import using background tasks with database polling.
**Implementation:**
- Background task (`app/tasks/letzshop_tasks.py`) runs historical import asynchronously
- Progress stored in `LetzshopHistoricalImportJob` database model
- Frontend polls status endpoint every 2 seconds
- Two-phase import: confirmed orders first, then unconfirmed (pending) orders
**Backend:**
- `LetzshopHistoricalImportJob` model tracks: status, current_phase, current_page, shipments_fetched, orders_processed, confirmed_stats, declined_stats
- `POST /vendors/{id}/import-history` starts background job, returns job_id immediately
- `GET /vendors/{id}/import-history/{job_id}/status` returns current progress
**Frontend:**
- Progress panel shows: phase (confirmed/pending), page number, shipments fetched, orders processed
- Disabled "Import History" button during import with spinner
- Final result summary shows combined stats from both phases
**Key Discovery:**
- Letzshop API has NO "declined" shipment state
- Valid states: `awaiting_order_completion`, `unconfirmed`, `completed`, `accepted`, `confirmed`
- Declined items are tracked at inventory unit level with state `confirmed_unavailable`
### Filter for Declined Items ✅
Added ability to filter orders that have at least one declined/unavailable item.
**Backend:**
- `list_orders()` accepts `has_declined_items: bool` parameter
- Uses JSON string contains check: `inventory_units.cast(String).contains("confirmed_unavailable")`
- `get_order_stats()` returns `has_declined_items` count
**Frontend:**
- "Has Declined Items" toggle button in filters section
- Shows count badge when there are orders with declined items
- Toggles between all orders and filtered view
**API:**
- `GET /vendors/{id}/orders?has_declined_items=true` - filter orders
### Order Date Display ✅
Orders now display the actual order date from Letzshop instead of the import date.
**Database:**
- Added `order_date` column to `LetzshopOrder` model
- Migration: `2362c2723a93_add_order_date_to_letzshop_orders.py`
**Backend:**
- `create_order()` extracts `completedAt` from Letzshop order data and stores as `order_date`
- `update_order_from_shipment()` populates `order_date` if not already set
- Date parsing handles ISO format with timezone (including `Z` suffix)
**Frontend:**
- Order table displays `order_date` with fallback to `created_at` for legacy orders
- Format: localized date/time string
**Note:** Existing orders imported before this change will continue showing `created_at` until re-imported via historical import.
### Search Filter ✅
Added search functionality to find orders by order number, customer name, or email.
**Backend:**
- `list_orders()` accepts `search: str` parameter
- Uses ILIKE for case-insensitive partial matching across:
- `letzshop_order_number`
- `customer_name`
- `customer_email`
**Frontend:**
- Search input field with magnifying glass icon
- Debounced input (300ms) to avoid excessive API calls
- Clear button to reset search
- Resets to page 1 when search changes
**API:**
- `GET /vendors/{id}/orders?search=query` - search orders
---
## Next Steps (TODO)
### Priority 1: Historical Import Progress Bar
Add real-time progress feedback for historical import (currently no visibility into import progress).
**Requirements:**
- Show progress indicator while import is running
- Display current page being fetched (e.g., "Fetching page 3 of 12...")
- Show running count of orders imported/updated
- Prevent user from thinking the process is stuck
**Implementation options:**
1. **Polling approach**: Frontend polls a status endpoint every few seconds
2. **Server-Sent Events (SSE)**: Real-time updates pushed to frontend
3. **WebSocket**: Bi-directional real-time communication
**Backend changes needed:**
- Store import progress in database or cache (Redis)
- Add endpoint `GET /api/v1/admin/letzshop/vendors/{id}/import-progress`
- Update `import_historical_shipments()` to report progress
**Frontend changes needed:**
- Progress bar component in Orders tab
- Polling/SSE logic to fetch progress updates
- Disable "Import History" button while import is in progress
### Priority 2: Stock Management
### Priority 1: Stock Management
When an order is confirmed/imported:
1. Match EAN from order to local product catalog
2. Decrease stock quantity for matched products
@@ -506,18 +564,12 @@ When an order is confirmed/imported:
- Need rollback mechanism if order is rejected
- Handle partial matches (some items found, some not)
### Priority 2: Order Detail View Enhancement
Improve the order detail modal to show:
- Product details (name, EAN, MPN, SKU)
- Match status per line item (found/not found in catalog)
- Link to local product if matched
### Priority 3: Invoice Generation
### Priority 2: Invoice Generation
Use `customer_locale` to generate invoices in customer's language:
- Invoice template with multi-language support
- PDF generation
### Priority 4: Analytics Dashboard
### Priority 3: Analytics Dashboard
Build sales analytics based on imported orders:
- Sales by product
- Sales by time period
@@ -526,20 +578,24 @@ Build sales analytics based on imported orders:
---
## Files Modified (2025-12-16 to 2025-12-18)
## Files Modified (2025-12-16 to 2025-12-19)
| File | Changes |
|------|---------|
| `app/services/letzshop/client_service.py` | Added paginated query, updated all queries with EAN/locale/country |
| `app/services/letzshop/order_service.py` | Added historical import, EAN matching, summary endpoint, order stats |
| `models/database/letzshop.py` | Added locale and country columns |
| `app/services/letzshop/order_service.py` | Historical import, EAN matching, order stats, has_declined_items filter, search filter, order_date extraction |
| `models/database/letzshop.py` | Added locale/country/order_date columns, `LetzshopHistoricalImportJob` model |
| `models/database/product.py` | Added `gtin` and `gtin_type` columns for EAN matching |
| `models/schema/letzshop.py` | Added `LetzshopOrderStats` schema, stats to order list response |
| `app/api/v1/admin/letzshop.py` | Added import-history and import-summary endpoints, stats in orders response |
| `app/templates/admin/partials/letzshop-orders-tab.html` | Added Import History button and result display |
| `static/admin/js/marketplace-letzshop.js` | Added importHistoricalOrders(), server-side stats |
| `models/schema/letzshop.py` | Added `LetzshopOrderStats`, `LetzshopHistoricalImportJobResponse`, `order_date` field |
| `app/api/v1/admin/letzshop.py` | Import-history endpoints, has_declined_items filter, search filter, order_date in response |
| `app/tasks/letzshop_tasks.py` | **NEW** - Background task for historical import with progress tracking |
| `app/templates/admin/partials/letzshop-orders-tab.html` | Import History button, progress panel, declined items filter, search input, order_date display |
| `static/admin/js/marketplace-letzshop.js` | Historical import polling, progress display, declined items filter, search functionality |
| `tests/unit/services/test_letzshop_service.py` | Added tests for new functionality |
| `scripts/test_historical_import.py` | Manual test script for historical import |
| `scripts/debug_historical_import.py` | **NEW** - Debug script for shipment states and declined items |
| `scripts/letzshop_introspect.py` | GraphQL schema introspection tool, tracking workaround tests |
| `alembic/versions/a9a86cef6cca_*.py` | Migration for locale/country columns |
| `alembic/versions/cb88bc9b5f86_*.py` | Migration for gtin columns on Product table |
| `alembic/versions/*_add_historical_import_jobs.py` | **NEW** - Migration for LetzshopHistoricalImportJob table |
| `alembic/versions/2362c2723a93_*.py` | **NEW** - Migration for order_date column |

2
mkdocs.yml
View File

@@ -135,6 +135,8 @@ nav:
- Vendor Operations Expansion: development/migration/vendor-operations-expansion.md
- Implementation Plans:
- Admin Inventory Management: implementation/inventory-admin-migration.md
- Letzshop Order Import: implementation/letzshop-order-import-improvements.md
- Unified Order View: implementation/unified-order-view.md
- Seed Scripts Audit: development/seed-scripts-audit.md
- Database Seeder:
- Documentation: development/database-seeder/database-seeder-documentation.md

163
scripts/debug_historical_import.py Normal file
View File

@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""Debug script for historical import issues."""
import sys
sys.path.insert(0, ".")
from app.core.database import SessionLocal
from app.services.letzshop.credentials_service import LetzshopCredentialsService
from models.database.letzshop import LetzshopHistoricalImportJob
def get_valid_shipment_states(vendor_id: int = 1):
"""Query the GraphQL schema to find valid ShipmentStateEnum values."""
db = SessionLocal()
try:
creds_service = LetzshopCredentialsService(db)
# Introspection query to get enum values
introspection_query = """
query {
__type(name: "ShipmentStateEnum") {
name
enumValues {
name
description
}
}
}
"""
with creds_service.create_client(vendor_id) as client:
print("Querying ShipmentStateEnum values...")
result = client._execute(introspection_query)
if result and "__type" in result:
type_info = result["__type"]
if type_info:
print(f"\nEnum: {type_info['name']}")
print("Valid values:")
for value in type_info.get("enumValues", []):
print(f" - {value['name']}: {value.get('description', 'No description')}")
else:
print("ShipmentStateEnum type not found")
else:
print(f"Unexpected result: {result}")
finally:
db.close()
def check_import_job_status():
"""Check the status of historical import jobs."""
db = SessionLocal()
try:
jobs = db.query(LetzshopHistoricalImportJob).order_by(
LetzshopHistoricalImportJob.id.desc()
).limit(5).all()
print("\n=== Recent Historical Import Jobs ===")
for job in jobs:
print(f"\nJob {job.id}:")
print(f" Status: {job.status}")
print(f" Phase: {job.current_phase}")
print(f" Page: {job.current_page}")
print(f" Shipments fetched: {job.shipments_fetched}")
print(f" Orders processed: {job.orders_processed}")
print(f" Orders imported: {job.orders_imported}")
print(f" Orders updated: {job.orders_updated}")
print(f" Orders skipped: {job.orders_skipped}")
print(f" Confirmed stats: {job.confirmed_stats}")
print(f" Declined stats: {job.declined_stats}")
print(f" Error: {job.error_message}")
print(f" Started: {job.started_at}")
print(f" Completed: {job.completed_at}")
finally:
db.close()
def test_fetch_states(vendor_id: int = 1):
"""Test fetching shipments with different states."""
db = SessionLocal()
try:
creds_service = LetzshopCredentialsService(db)
# States to test
states_to_test = ["unconfirmed", "confirmed", "declined", "shipped", "rejected"]
with creds_service.create_client(vendor_id) as client:
for state in states_to_test:
print(f"\nTesting state: {state}")
try:
# Just try to fetch first page
shipments = client.get_all_shipments_paginated(
state=state,
page_size=1,
max_pages=1,
)
print(f" ✓ Success: {len(shipments)} shipments")
except Exception as e:
print(f" ✗ Error: {e}")
finally:
db.close()
def check_declined_items(vendor_id: int = 1):
"""Check for orders with declined inventory units."""
db = SessionLocal()
try:
from models.database.letzshop import LetzshopOrder
from collections import Counter
# Get all orders with inventory_units
orders = db.query(LetzshopOrder).filter(
LetzshopOrder.vendor_id == vendor_id,
LetzshopOrder.inventory_units.isnot(None),
).all()
# Count all states
state_counts = Counter()
total_units = 0
for order in orders:
units = order.inventory_units or []
for unit in units:
state = unit.get("state", "unknown")
state_counts[state] += 1
total_units += 1
print(f"\n=== Inventory Unit States (all {len(orders)} orders) ===")
print(f" Total units: {total_units}")
print(f"\n State breakdown:")
for state, count in sorted(state_counts.items(), key=lambda x: -x[1]):
print(f" {state}: {count}")
# Show a sample order with its units
if orders:
sample = orders[0]
print(f"\nSample order #{sample.id} (shipment {sample.letzshop_shipment_id}):")
print(f" Shipment state: {sample.letzshop_state}")
print(f" Sync status: {sample.sync_status}")
if sample.inventory_units:
for i, unit in enumerate(sample.inventory_units[:5]):
print(f" Unit {i+1}: state={unit.get('state')}, id={unit.get('id')}")
finally:
db.close()
if __name__ == "__main__":
print("=== Letzshop Historical Import Debug ===\n")
print("1. Checking valid shipment states...")
get_valid_shipment_states()
print("\n\n2. Testing different state values...")
test_fetch_states()
print("\n\n3. Checking import job status...")
check_import_job_status()
print("\n\n4. Checking declined items in inventory units...")
check_declined_items()