Refactoring code for modular approach

app/api/deps.py

@@ -0,0 +1,40 @@
+from fastapi import Depends, HTTPException
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from models.database_models import User, Shop
+from middleware.auth import AuthManager
+from middleware.rate_limiter import RateLimiter
+
+security = HTTPBearer()
+auth_manager = AuthManager()
+rate_limiter = RateLimiter()
+
+
+def get_current_user(
+        credentials: HTTPAuthorizationCredentials = Depends(security),
+        db: Session = Depends(get_db)
+):
+    """Get current authenticated user"""
+    return auth_manager.get_current_user(db, credentials)
+
+
+def get_current_admin_user(current_user: User = Depends(get_current_user)):
+    """Require admin user"""
+    return auth_manager.require_admin(current_user)
+
+
+def get_user_shop(
+        shop_code: str,
+        current_user: User = Depends(get_current_user),
+        db: Session = Depends(get_db)
+):
+    """Get shop and verify user ownership"""
+    shop = db.query(Shop).filter(Shop.shop_code == shop_code.upper()).first()
+    if not shop:
+        raise HTTPException(status_code=404, detail="Shop not found")
+
+    if current_user.role != "admin" and shop.owner_id != current_user.id:
+        raise HTTPException(status_code=403, detail="Access denied to this shop")
+
+    return shop

app/api/main.py

@@ -0,0 +1,13 @@
+from fastapi import APIRouter
+from app.api.v1 import auth, products, stock, shops, marketplace, admin, stats
+
+api_router = APIRouter()
+
+# Include all route modules
+api_router.include_router(auth.router, prefix="/auth", tags=["authentication"])
+api_router.include_router(products.router, prefix="/products", tags=["products"])
+api_router.include_router(stock.router, prefix="/stock", tags=["stock"])
+api_router.include_router(shops.router, prefix="/shops", tags=["shops"])
+api_router.include_router(marketplace.router, prefix="/marketplace", tags=["marketplace"])
+api_router.include_router(admin.router, prefix="/admin", tags=["admin"])
+api_router.include_router(stats.router, prefix="/stats", tags=["statistics"])

app/api/v1/admin.py

@@ -0,0 +1,153 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from app.tasks.background_tasks import process_marketplace_import
+from middleware.decorators import rate_limit
+from models.api_models import MarketplaceImportJobResponse, MarketplaceImportRequest
+from models.database_models import User, MarketplaceImportJob, Shop
+from datetime import datetime
+import logging
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# Admin-only routes
+@router.get("/admin/users", response_model=List[UserResponse])
+def get_all_users(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Get all users (Admin only)"""
+    users = db.query(User).offset(skip).limit(limit).all()
+    return [UserResponse.model_validate(user) for user in users]
+
+
+@router.put("/admin/users/{user_id}/status")
+def toggle_user_status(
+        user_id: int,
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Toggle user active status (Admin only)"""
+    user = db.query(User).filter(User.id == user_id).first()
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+
+    if user.id == current_admin.id:
+        raise HTTPException(status_code=400, detail="Cannot deactivate your own account")
+
+    user.is_active = not user.is_active
+    user.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(user)
+
+    status = "activated" if user.is_active else "deactivated"
+    return {"message": f"User {user.username} has been {status}"}
+
+
+@router.get("/admin/shops", response_model=ShopListResponse)
+def get_all_shops_admin(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Get all shops with admin view (Admin only)"""
+    total = db.query(Shop).count()
+    shops = db.query(Shop).offset(skip).limit(limit).all()
+
+    return ShopListResponse(
+        shops=shops,
+        total=total,
+        skip=skip,
+        limit=limit
+    )
+
+
+@router.put("/admin/shops/{shop_id}/verify")
+def verify_shop(
+        shop_id: int,
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Verify/unverify shop (Admin only)"""
+    shop = db.query(Shop).filter(Shop.id == shop_id).first()
+    if not shop:
+        raise HTTPException(status_code=404, detail="Shop not found")
+
+    shop.is_verified = not shop.is_verified
+    shop.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(shop)
+
+    status = "verified" if shop.is_verified else "unverified"
+    return {"message": f"Shop {shop.shop_code} has been {status}"}
+
+
+@router.put("/admin/shops/{shop_id}/status")
+def toggle_shop_status(
+        shop_id: int,
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Toggle shop active status (Admin only)"""
+    shop = db.query(Shop).filter(Shop.id == shop_id).first()
+    if not shop:
+        raise HTTPException(status_code=404, detail="Shop not found")
+
+    shop.is_active = not shop.is_active
+    shop.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(shop)
+
+    status = "activated" if shop.is_active else "deactivated"
+    return {"message": f"Shop {shop.shop_code} has been {status}"}
+
+
+@router.get("/admin/marketplace-import-jobs", response_model=List[MarketplaceImportJobResponse])
+def get_all_marketplace_import_jobs(
+        marketplace: Optional[str] = Query(None),
+        shop_name: Optional[str] = Query(None),
+        status: Optional[str] = Query(None),
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=100),
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Get all marketplace import jobs (Admin only)"""
+
+    query = db.query(MarketplaceImportJob)
+
+    # Apply filters
+    if marketplace:
+        query = query.filter(MarketplaceImportJob.marketplace.ilike(f"%{marketplace}%"))
+    if shop_name:
+        query = query.filter(MarketplaceImportJob.shop_name.ilike(f"%{shop_name}%"))
+    if status:
+        query = query.filter(MarketplaceImportJob.status == status)
+
+    # Order by creation date and apply pagination
+    jobs = query.order_by(MarketplaceImportJob.created_at.desc()).offset(skip).limit(limit).all()
+
+    return [
+        MarketplaceImportJobResponse(
+            job_id=job.id,
+            status=job.status,
+            marketplace=job.marketplace,
+            shop_name=job.shop_name,
+            imported=job.imported_count or 0,
+            updated=job.updated_count or 0,
+            total_processed=job.total_processed or 0,
+            error_count=job.error_count or 0,
+            error_message=job.error_message,
+            created_at=job.created_at,
+            started_at=job.started_at,
+            completed_at=job.completed_at
+        ) for job in jobs
+    ]

app/api/v1/auth.py

@@ -0,0 +1,70 @@
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from models.api_models import UserRegister, UserLogin, UserResponse, LoginResponse
+from models.database_models import User
+from middleware.auth import AuthManager
+import logging
+
+router = APIRouter()
+auth_manager = AuthManager()
+logger = logging.getLogger(__name__)
+
+
+# Authentication Routes
+@router.post("/register", response_model=UserResponse)
+def register_user(user_data: UserRegister, db: Session = Depends(get_db)):
+    """Register a new user"""
+    # Check if email already exists
+    existing_email = db.query(User).filter(User.email == user_data.email).first()
+    if existing_email:
+        raise HTTPException(status_code=400, detail="Email already registered")
+
+    # Check if username already exists
+    existing_username = db.query(User).filter(User.username == user_data.username).first()
+    if existing_username:
+        raise HTTPException(status_code=400, detail="Username already taken")
+
+    # Hash password and create user
+    hashed_password = auth_manager.hash_password(user_data.password)
+    new_user = User(
+        email=user_data.email,
+        username=user_data.username,
+        hashed_password=hashed_password,
+        role="user",
+        is_active=True
+    )
+
+    db.add(new_user)
+    db.commit()
+    db.refresh(new_user)
+
+    logger.info(f"New user registered: {new_user.username}")
+    return new_user
+
+
+@router.post("/login", response_model=LoginResponse)
+def login_user(user_credentials: UserLogin, db: Session = Depends(get_db)):
+    """Login user and return JWT token"""
+    user = auth_manager.authenticate_user(db, user_credentials.username, user_credentials.password)
+    if not user:
+        raise HTTPException(status_code=401, detail="Incorrect username or password")
+
+    # Create access token
+    token_data = auth_manager.create_access_token(user)
+
+    logger.info(f"User logged in: {user.username}")
+
+    return LoginResponse(
+        access_token=token_data["access_token"],
+        token_type=token_data["token_type"],
+        expires_in=token_data["expires_in"],
+        user=UserResponse.model_validate(user)
+    )
+
+
+@router.get("/me", response_model=UserResponse)
+def get_current_user_info(current_user: User = Depends(get_current_user)):
+    """Get current user information"""
+    return UserResponse.model_validate(current_user)

app/api/v1/marketplace.py

@@ -0,0 +1,146 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from app.tasks.background_tasks import process_marketplace_import
+from middleware.decorators import rate_limit
+from models.api_models import MarketplaceImportJobResponse, MarketplaceImportRequest
+from models.database_models import User, MarketplaceImportJob, Shop
+from datetime import datetime
+import logging
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# Marketplace Import Routes (Protected)
+@router.post("/import-from-marketplace", response_model=MarketplaceImportJobResponse)
+@rate_limit(max_requests=10, window_seconds=3600)  # Limit marketplace imports
+async def import_products_from_marketplace(
+        request: MarketplaceImportRequest,
+        background_tasks: BackgroundTasks,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Import products from marketplace CSV with background processing (Protected)"""
+
+    logger.info(
+        f"Starting marketplace import: {request.marketplace} -> {request.shop_code} by user {current_user.username}")
+
+    # Verify shop exists and user has access
+    shop = db.query(Shop).filter(Shop.shop_code == request.shop_code).first()
+    if not shop:
+        raise HTTPException(status_code=404, detail="Shop not found")
+
+    # Check permissions: admin can import for any shop, others only for their own
+    if current_user.role != "admin" and shop.owner_id != current_user.id:
+        raise HTTPException(status_code=403, detail="Access denied to this shop")
+
+    # Create marketplace import job record
+    import_job = MarketplaceImportJob(
+        status="pending",
+        source_url=request.url,
+        marketplace=request.marketplace,
+        shop_code=request.shop_code,
+        user_id=current_user.id,
+        created_at=datetime.utcnow()
+    )
+    db.add(import_job)
+    db.commit()
+    db.refresh(import_job)
+
+    # Process in background
+    background_tasks.add_task(
+        process_marketplace_import,
+        import_job.id,
+        request.url,
+        request.marketplace,
+        request.shop_code,
+        request.batch_size or 1000
+    )
+
+    return MarketplaceImportJobResponse(
+        job_id=import_job.id,
+        status="pending",
+        marketplace=request.marketplace,
+        shop_code=request.shop_code,
+        message=f"Marketplace import started from {request.marketplace}. Check status with "
+                f"/marketplace-import-status/{import_job.id}"
+    )
+
+
+@router.get("/marketplace-import-status/{job_id}", response_model=MarketplaceImportJobResponse)
+def get_marketplace_import_status(
+        job_id: int,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get status of marketplace import job (Protected)"""
+    job = db.query(MarketplaceImportJob).filter(MarketplaceImportJob.id == job_id).first()
+    if not job:
+        raise HTTPException(status_code=404, detail="Marketplace import job not found")
+
+    # Users can only see their own jobs, admins can see all
+    if current_user.role != "admin" and job.user_id != current_user.id:
+        raise HTTPException(status_code=403, detail="Access denied to this import job")
+
+    return MarketplaceImportJobResponse(
+        job_id=job.id,
+        status=job.status,
+        marketplace=job.marketplace,
+        shop_name=job.shop_name,
+        imported=job.imported_count or 0,
+        updated=job.updated_count or 0,
+        total_processed=job.total_processed or 0,
+        error_count=job.error_count or 0,
+        error_message=job.error_message,
+        created_at=job.created_at,
+        started_at=job.started_at,
+        completed_at=job.completed_at
+    )
+
+
+@router.get("/marketplace-import-jobs", response_model=List[MarketplaceImportJobResponse])
+def get_marketplace_import_jobs(
+        marketplace: Optional[str] = Query(None, description="Filter by marketplace"),
+        shop_name: Optional[str] = Query(None, description="Filter by shop name"),
+        skip: int = Query(0, ge=0),
+        limit: int = Query(50, ge=1, le=100),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get marketplace import jobs with filtering (Protected)"""
+
+    query = db.query(MarketplaceImportJob)
+
+    # Users can only see their own jobs, admins can see all
+    if current_user.role != "admin":
+        query = query.filter(MarketplaceImportJob.user_id == current_user.id)
+
+    # Apply filters
+    if marketplace:
+        query = query.filter(MarketplaceImportJob.marketplace.ilike(f"%{marketplace}%"))
+    if shop_name:
+        query = query.filter(MarketplaceImportJob.shop_name.ilike(f"%{shop_name}%"))
+
+    # Order by creation date (newest first) and apply pagination
+    jobs = query.order_by(MarketplaceImportJob.created_at.desc()).offset(skip).limit(limit).all()
+
+    return [
+        MarketplaceImportJobResponse(
+            job_id=job.id,
+            status=job.status,
+            marketplace=job.marketplace,
+            shop_name=job.shop_name,
+            imported=job.imported_count or 0,
+            updated=job.updated_count or 0,
+            total_processed=job.total_processed or 0,
+            error_count=job.error_count or 0,
+            error_message=job.error_message,
+            created_at=job.created_at,
+            started_at=job.started_at,
+            completed_at=job.completed_at
+        ) for job in jobs
+    ]

app/api/v1/products.py

@@ -0,0 +1,261 @@
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.responses import StreamingResponse
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from models.api_models import (ProductListResponse, ProductResponse, ProductCreate, ProductDetailResponse,
+                               StockLocationResponse, StockSummaryResponse, ProductUpdate)
+from models.database_models import User, Product, Stock
+from datetime import datetime
+import logging
+
+from utils.data_processing import GTINProcessor, PriceProcessor
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+# Initialize processors
+gtin_processor = GTINProcessor()
+price_processor = PriceProcessor()
+
+
+# Enhanced Product Routes with Marketplace Support
+@router.get("/products", response_model=ProductListResponse)
+def get_products(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        brand: Optional[str] = Query(None),
+        category: Optional[str] = Query(None),
+        availability: Optional[str] = Query(None),
+        marketplace: Optional[str] = Query(None, description="Filter by marketplace"),
+        shop_name: Optional[str] = Query(None, description="Filter by shop name"),
+        search: Optional[str] = Query(None),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get products with advanced filtering including marketplace and shop (Protected)"""
+
+    query = db.query(Product)
+
+    # Apply filters
+    if brand:
+        query = query.filter(Product.brand.ilike(f"%{brand}%"))
+    if category:
+        query = query.filter(Product.google_product_category.ilike(f"%{category}%"))
+    if availability:
+        query = query.filter(Product.availability == availability)
+    if marketplace:
+        query = query.filter(Product.marketplace.ilike(f"%{marketplace}%"))
+    if shop_name:
+        query = query.filter(Product.shop_name.ilike(f"%{shop_name}%"))
+    if search:
+        # Search in title, description, and marketplace
+        search_term = f"%{search}%"
+        query = query.filter(
+            (Product.title.ilike(search_term)) |
+            (Product.description.ilike(search_term)) |
+            (Product.marketplace.ilike(search_term)) |
+            (Product.shop_name.ilike(search_term))
+        )
+
+    # Get total count for pagination
+    total = query.count()
+
+    # Apply pagination
+    products = query.offset(skip).limit(limit).all()
+
+    return ProductListResponse(
+        products=products,
+        total=total,
+        skip=skip,
+        limit=limit
+    )
+
+
+@router.post("/products", response_model=ProductResponse)
+def create_product(
+        product: ProductCreate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Create a new product with validation and marketplace support (Protected)"""
+
+    # Check if product_id already exists
+    existing = db.query(Product).filter(Product.product_id == product.product_id).first()
+    if existing:
+        raise HTTPException(status_code=400, detail="Product with this ID already exists")
+
+    # Process and validate GTIN if provided
+    if product.gtin:
+        normalized_gtin = gtin_processor.normalize(product.gtin)
+        if not normalized_gtin:
+            raise HTTPException(status_code=400, detail="Invalid GTIN format")
+        product.gtin = normalized_gtin
+
+    # Process price if provided
+    if product.price:
+        parsed_price, currency = price_processor.parse_price_currency(product.price)
+        if parsed_price:
+            product.price = parsed_price
+            product.currency = currency
+
+    # Set default marketplace if not provided
+    if not product.marketplace:
+        product.marketplace = "Letzshop"
+
+    db_product = Product(**product.dict())
+    db.add(db_product)
+    db.commit()
+    db.refresh(db_product)
+
+    logger.info(
+        f"Created product {db_product.product_id} for marketplace {db_product.marketplace}, "
+        f"shop {db_product.shop_name}")
+    return db_product
+
+
+@router.get("/products/{product_id}", response_model=ProductDetailResponse)
+def get_product(product_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get product with stock information (Protected)"""
+
+    product = db.query(Product).filter(Product.product_id == product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found")
+
+    # Get stock information if GTIN exists
+    stock_info = None
+    if product.gtin:
+        stock_entries = db.query(Stock).filter(Stock.gtin == product.gtin).all()
+        if stock_entries:
+            total_quantity = sum(entry.quantity for entry in stock_entries)
+            locations = [
+                StockLocationResponse(location=entry.location, quantity=entry.quantity)
+                for entry in stock_entries
+            ]
+            stock_info = StockSummaryResponse(
+                gtin=product.gtin,
+                total_quantity=total_quantity,
+                locations=locations
+            )
+
+    return ProductDetailResponse(
+        product=product,
+        stock_info=stock_info
+    )
+
+
+@router.put("/products/{product_id}", response_model=ProductResponse)
+def update_product(
+        product_id: str,
+        product_update: ProductUpdate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Update product with validation and marketplace support (Protected)"""
+
+    product = db.query(Product).filter(Product.product_id == product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found")
+
+    # Update fields
+    update_data = product_update.dict(exclude_unset=True)
+
+    # Validate GTIN if being updated
+    if "gtin" in update_data and update_data["gtin"]:
+        normalized_gtin = gtin_processor.normalize(update_data["gtin"])
+        if not normalized_gtin:
+            raise HTTPException(status_code=400, detail="Invalid GTIN format")
+        update_data["gtin"] = normalized_gtin
+
+    # Process price if being updated
+    if "price" in update_data and update_data["price"]:
+        parsed_price, currency = price_processor.parse_price_currency(update_data["price"])
+        if parsed_price:
+            update_data["price"] = parsed_price
+            update_data["currency"] = currency
+
+    for key, value in update_data.items():
+        setattr(product, key, value)
+
+    product.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(product)
+
+    return product
+
+
+@router.delete("/products/{product_id}")
+def delete_product(
+        product_id: str,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Delete product and associated stock (Protected)"""
+
+    product = db.query(Product).filter(Product.product_id == product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found")
+
+    # Delete associated stock entries if GTIN exists
+    if product.gtin:
+        db.query(Stock).filter(Stock.gtin == product.gtin).delete()
+
+    db.delete(product)
+    db.commit()
+
+    return {"message": "Product and associated stock deleted successfully"}
+
+# Export with streaming for large datasets (Protected)
+@router.get("/export-csv")
+async def export_csv(
+        marketplace: Optional[str] = Query(None, description="Filter by marketplace"),
+        shop_name: Optional[str] = Query(None, description="Filter by shop name"),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Export products as CSV with streaming and marketplace filtering (Protected)"""
+
+    def generate_csv():
+        # Stream CSV generation for memory efficiency
+        yield "product_id,title,description,link,image_link,availability,price,currency,brand,gtin,marketplace,shop_name\n"
+
+        batch_size = 1000
+        offset = 0
+
+        while True:
+            query = db.query(Product)
+
+            # Apply marketplace filters
+            if marketplace:
+                query = query.filter(Product.marketplace.ilike(f"%{marketplace}%"))
+            if shop_name:
+                query = query.filter(Product.shop_name.ilike(f"%{shop_name}%"))
+
+            products = query.offset(offset).limit(batch_size).all()
+            if not products:
+                break
+
+            for product in products:
+                # Create CSV row with marketplace fields
+                row = (f'"{product.product_id}","{product.title or ""}","{product.description or ""}",'
+                       f'"{product.link or ""}","{product.image_link or ""}","{product.availability or ""}",'
+                       f'"{product.price or ""}","{product.currency or ""}","{product.brand or ""}",'
+                       f'"{product.gtin or ""}","{product.marketplace or ""}","{product.shop_name or ""}"\n')
+                yield row
+
+            offset += batch_size
+
+    filename = "products_export"
+    if marketplace:
+        filename += f"_{marketplace}"
+    if shop_name:
+        filename += f"_{shop_name}"
+    filename += ".csv"
+
+    return StreamingResponse(
+        generate_csv(),
+        media_type="text/csv",
+        headers={"Content-Disposition": f"attachment; filename={filename}"}
+    )

app/api/v1/shops.py

@@ -0,0 +1,188 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from app.tasks.background_tasks import process_marketplace_import
+from middleware.decorators import rate_limit
+from models.api_models import MarketplaceImportJobResponse, MarketplaceImportRequest
+from models.database_models import User, MarketplaceImportJob, Shop
+from datetime import datetime
+import logging
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# Shop Management Routes
+@router.post("/shops", response_model=ShopResponse)
+def create_shop(
+        shop_data: ShopCreate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Create a new shop (Protected)"""
+    # Check if shop code already exists
+    existing_shop = db.query(Shop).filter(Shop.shop_code == shop_data.shop_code).first()
+    if existing_shop:
+        raise HTTPException(status_code=400, detail="Shop code already exists")
+
+    # Create shop
+    new_shop = Shop(
+        **shop_data.dict(),
+        owner_id=current_user.id,
+        is_active=True,
+        is_verified=(current_user.role == "admin")  # Auto-verify if admin creates shop
+    )
+
+    db.add(new_shop)
+    db.commit()
+    db.refresh(new_shop)
+
+    logger.info(f"New shop created: {new_shop.shop_code} by {current_user.username}")
+    return new_shop
+
+
+@router.get("/shops", response_model=ShopListResponse)
+def get_shops(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        active_only: bool = Query(True),
+        verified_only: bool = Query(False),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get shops with filtering (Protected)"""
+    query = db.query(Shop)
+
+    # Non-admin users can only see active and verified shops, plus their own
+    if current_user.role != "admin":
+        query = query.filter(
+            (Shop.is_active == True) &
+            ((Shop.is_verified == True) | (Shop.owner_id == current_user.id))
+        )
+    else:
+        # Admin can apply filters
+        if active_only:
+            query = query.filter(Shop.is_active == True)
+        if verified_only:
+            query = query.filter(Shop.is_verified == True)
+
+    total = query.count()
+    shops = query.offset(skip).limit(limit).all()
+
+    return ShopListResponse(
+        shops=shops,
+        total=total,
+        skip=skip,
+        limit=limit
+    )
+
+
+@router.get("/shops/{shop_code}", response_model=ShopResponse)
+def get_shop(shop_code: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get shop details (Protected)"""
+    shop = db.query(Shop).filter(Shop.shop_code == shop_code.upper()).first()
+    if not shop:
+        raise HTTPException(status_code=404, detail="Shop not found")
+
+    # Non-admin users can only see active verified shops or their own shops
+    if current_user.role != "admin":
+        if not shop.is_active or (not shop.is_verified and shop.owner_id != current_user.id):
+            raise HTTPException(status_code=404, detail="Shop not found")
+
+    return shop
+
+
+# Shop Product Management
+@router.post("/shops/{shop_code}/products", response_model=ShopProductResponse)
+def add_product_to_shop(
+        shop_code: str,
+        shop_product: ShopProductCreate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Add existing product to shop catalog with shop-specific settings (Protected)"""
+
+    # Get and verify shop
+    shop = get_user_shop(shop_code, current_user, db)
+
+    # Check if product exists
+    product = db.query(Product).filter(Product.product_id == shop_product.product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found in marketplace catalog")
+
+    # Check if product already in shop
+    existing_shop_product = db.query(ShopProduct).filter(
+        ShopProduct.shop_id == shop.id,
+        ShopProduct.product_id == product.id
+    ).first()
+
+    if existing_shop_product:
+        raise HTTPException(status_code=400, detail="Product already in shop catalog")
+
+    # Create shop-product association
+    new_shop_product = ShopProduct(
+        shop_id=shop.id,
+        product_id=product.id,
+        **shop_product.dict(exclude={'product_id'})
+    )
+
+    db.add(new_shop_product)
+    db.commit()
+    db.refresh(new_shop_product)
+
+    # Return with product details
+    response = ShopProductResponse.model_validate(new_shop_product)
+    response.product = product
+    return response
+
+
+@router.get("/shops/{shop_code}/products")
+def get_shop_products(
+        shop_code: str,
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        active_only: bool = Query(True),
+        featured_only: bool = Query(False),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get products in shop catalog (Protected)"""
+
+    # Get shop (public can view active/verified shops)
+    shop = db.query(Shop).filter(Shop.shop_code == shop_code.upper()).first()
+    if not shop:
+        raise HTTPException(status_code=404, detail="Shop not found")
+
+    # Non-owners can only see active verified shops
+    if current_user.role != "admin" and shop.owner_id != current_user.id:
+        if not shop.is_active or not shop.is_verified:
+            raise HTTPException(status_code=404, detail="Shop not found")
+
+    # Query shop products
+    query = db.query(ShopProduct).filter(ShopProduct.shop_id == shop.id)
+
+    if active_only:
+        query = query.filter(ShopProduct.is_active == True)
+    if featured_only:
+        query = query.filter(ShopProduct.is_featured == True)
+
+    total = query.count()
+    shop_products = query.offset(skip).limit(limit).all()
+
+    # Format response
+    products = []
+    for sp in shop_products:
+        product_response = ShopProductResponse.model_validate(sp)
+        product_response.product = sp.product
+        products.append(product_response)
+
+    return {
+        "products": products,
+        "total": total,
+        "skip": skip,
+        "limit": limit,
+        "shop": ShopResponse.model_validate(shop)
+    }

app/api/v1/stats.py

@@ -0,0 +1,84 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from app.tasks.background_tasks import process_marketplace_import
+from middleware.decorators import rate_limit
+from models.api_models import MarketplaceImportJobResponse, MarketplaceImportRequest
+from models.database_models import User, MarketplaceImportJob, Shop
+from datetime import datetime
+import logging
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# Enhanced Statistics with Marketplace Support
+@router.get("/stats", response_model=StatsResponse)
+def get_stats(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get comprehensive statistics with marketplace data (Protected)"""
+
+    # Use more efficient queries with proper indexes
+    total_products = db.query(Product).count()
+
+    unique_brands = db.query(Product.brand).filter(
+        Product.brand.isnot(None),
+        Product.brand != ""
+    ).distinct().count()
+
+    unique_categories = db.query(Product.google_product_category).filter(
+        Product.google_product_category.isnot(None),
+        Product.google_product_category != ""
+    ).distinct().count()
+
+    # New marketplace statistics
+    unique_marketplaces = db.query(Product.marketplace).filter(
+        Product.marketplace.isnot(None),
+        Product.marketplace != ""
+    ).distinct().count()
+
+    unique_shops = db.query(Product.shop_name).filter(
+        Product.shop_name.isnot(None),
+        Product.shop_name != ""
+    ).distinct().count()
+
+    # Stock statistics
+    total_stock_entries = db.query(Stock).count()
+    total_inventory = db.query(func.sum(Stock.quantity)).scalar() or 0
+
+    return StatsResponse(
+        total_products=total_products,
+        unique_brands=unique_brands,
+        unique_categories=unique_categories,
+        unique_marketplaces=unique_marketplaces,
+        unique_shops=unique_shops,
+        total_stock_entries=total_stock_entries,
+        total_inventory_quantity=total_inventory
+    )
+
+
+@router.get("/marketplace-stats", response_model=List[MarketplaceStatsResponse])
+def get_marketplace_stats(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get statistics broken down by marketplace (Protected)"""
+
+    # Query to get stats per marketplace
+    marketplace_stats = db.query(
+        Product.marketplace,
+        func.count(Product.id).label('total_products'),
+        func.count(func.distinct(Product.shop_name)).label('unique_shops'),
+        func.count(func.distinct(Product.brand)).label('unique_brands')
+    ).filter(
+        Product.marketplace.isnot(None)
+    ).group_by(Product.marketplace).all()
+
+    return [
+        MarketplaceStatsResponse(
+            marketplace=stat.marketplace,
+            total_products=stat.total_products,
+            unique_shops=stat.unique_shops,
+            unique_brands=stat.unique_brands
+        ) for stat in marketplace_stats
+    ]
+

app/api/v1/stock.py

@@ -0,0 +1,315 @@
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
+from sqlalchemy.orm import Session
+from app.core.database import get_db
+from app.api.deps import get_current_user
+from app.tasks.background_tasks import process_marketplace_import
+from middleware.decorators import rate_limit
+from models.api_models import MarketplaceImportJobResponse, MarketplaceImportRequest, StockResponse, \
+    StockSummaryResponse
+from models.database_models import User, MarketplaceImportJob, Shop
+from datetime import datetime
+import logging
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+# Stock Management Routes (Protected)
+
+@router.post("/stock", response_model=StockResponse)
+def set_stock(stock: StockCreate, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Set exact stock quantity for a GTIN at a specific location (replaces existing quantity)"""
+
+    # Normalize GTIN
+    def normalize_gtin(gtin_value):
+        if not gtin_value:
+            return None
+        gtin_str = str(gtin_value).strip()
+        if '.' in gtin_str:
+            gtin_str = gtin_str.split('.')[0]
+        gtin_clean = ''.join(filter(str.isdigit, gtin_str))
+        if len(gtin_clean) in [8, 12, 13, 14]:
+            return gtin_clean.zfill(13) if len(gtin_clean) == 13 else gtin_clean.zfill(12)
+        return gtin_clean if gtin_clean else None
+
+    normalized_gtin = normalize_gtin(stock.gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    # Check if stock entry already exists for this GTIN and location
+    existing_stock = db.query(Stock).filter(
+        Stock.gtin == normalized_gtin,
+        Stock.location == stock.location.strip().upper()
+    ).first()
+
+    if existing_stock:
+        # Update existing stock (SET to exact quantity)
+        old_quantity = existing_stock.quantity
+        existing_stock.quantity = stock.quantity
+        existing_stock.updated_at = datetime.utcnow()
+        db.commit()
+        db.refresh(existing_stock)
+        logger.info(f"Updated stock for GTIN {normalized_gtin} at {stock.location}: {old_quantity} → {stock.quantity}")
+        return existing_stock
+    else:
+        # Create new stock entry
+        new_stock = Stock(
+            gtin=normalized_gtin,
+            location=stock.location.strip().upper(),
+            quantity=stock.quantity
+        )
+        db.add(new_stock)
+        db.commit()
+        db.refresh(new_stock)
+        logger.info(f"Created new stock for GTIN {normalized_gtin} at {stock.location}: {stock.quantity}")
+        return new_stock
+
+
+@router.post("/stock/add", response_model=StockResponse)
+def add_stock(stock: StockAdd, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Add quantity to existing stock for a GTIN at a specific location (adds to existing quantity)"""
+
+    # Normalize GTIN
+    def normalize_gtin(gtin_value):
+        if not gtin_value:
+            return None
+        gtin_str = str(gtin_value).strip()
+        if '.' in gtin_str:
+            gtin_str = gtin_str.split('.')[0]
+        gtin_clean = ''.join(filter(str.isdigit, gtin_str))
+        if len(gtin_clean) in [8, 12, 13, 14]:
+            return gtin_clean.zfill(13) if len(gtin_clean) == 13 else gtin_clean.zfill(12)
+        return gtin_clean if gtin_clean else None
+
+    normalized_gtin = normalize_gtin(stock.gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    # Check if stock entry already exists for this GTIN and location
+    existing_stock = db.query(Stock).filter(
+        Stock.gtin == normalized_gtin,
+        Stock.location == stock.location.strip().upper()
+    ).first()
+
+    if existing_stock:
+        # Add to existing stock
+        old_quantity = existing_stock.quantity
+        existing_stock.quantity += stock.quantity
+        existing_stock.updated_at = datetime.utcnow()
+        db.commit()
+        db.refresh(existing_stock)
+        logger.info(
+            f"Added stock for GTIN {normalized_gtin} at {stock.location}: {old_quantity} + {stock.quantity} = {existing_stock.quantity}")
+        return existing_stock
+    else:
+        # Create new stock entry with the quantity
+        new_stock = Stock(
+            gtin=normalized_gtin,
+            location=stock.location.strip().upper(),
+            quantity=stock.quantity
+        )
+        db.add(new_stock)
+        db.commit()
+        db.refresh(new_stock)
+        logger.info(f"Created new stock for GTIN {normalized_gtin} at {stock.location}: {stock.quantity}")
+        return new_stock
+
+
+@router.post("/stock/remove", response_model=StockResponse)
+def remove_stock(stock: StockAdd, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Remove quantity from existing stock for a GTIN at a specific location"""
+
+    # Normalize GTIN
+    def normalize_gtin(gtin_value):
+        if not gtin_value:
+            return None
+        gtin_str = str(gtin_value).strip()
+        if '.' in gtin_str:
+            gtin_str = gtin_str.split('.')[0]
+        gtin_clean = ''.join(filter(str.isdigit, gtin_str))
+        if len(gtin_clean) in [8, 12, 13, 14]:
+            return gtin_clean.zfill(13) if len(gtin_clean) == 13 else gtin_clean.zfill(12)
+        return gtin_clean if gtin_clean else None
+
+    normalized_gtin = normalize_gtin(stock.gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    # Find existing stock entry
+    existing_stock = db.query(Stock).filter(
+        Stock.gtin == normalized_gtin,
+        Stock.location == stock.location.strip().upper()
+    ).first()
+
+    if not existing_stock:
+        raise HTTPException(
+            status_code=404,
+            detail=f"No stock found for GTIN {normalized_gtin} at location {stock.location}"
+        )
+
+    # Check if we have enough stock to remove
+    if existing_stock.quantity < stock.quantity:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Insufficient stock. Available: {existing_stock.quantity}, Requested to remove: {stock.quantity}"
+        )
+
+    # Remove from existing stock
+    old_quantity = existing_stock.quantity
+    existing_stock.quantity -= stock.quantity
+    existing_stock.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(existing_stock)
+    logger.info(
+        f"Removed stock for GTIN {normalized_gtin} at {stock.location}: {old_quantity} - {stock.quantity} = {existing_stock.quantity}")
+    return existing_stock
+
+
+@router.get("/stock/{gtin}", response_model=StockSummaryResponse)
+def get_stock_by_gtin(gtin: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get all stock locations and total quantity for a specific GTIN"""
+
+    # Normalize GTIN
+    def normalize_gtin(gtin_value):
+        if not gtin_value:
+            return None
+        gtin_str = str(gtin_value).strip()
+        if '.' in gtin_str:
+            gtin_str = gtin_str.split('.')[0]
+        gtin_clean = ''.join(filter(str.isdigit, gtin_str))
+        if len(gtin_clean) in [8, 12, 13, 14]:
+            return gtin_clean.zfill(13) if len(gtin_clean) == 13 else gtin_clean.zfill(12)
+        return gtin_clean if gtin_clean else None
+
+    normalized_gtin = normalize_gtin(gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    # Get all stock entries for this GTIN
+    stock_entries = db.query(Stock).filter(Stock.gtin == normalized_gtin).all()
+
+    if not stock_entries:
+        raise HTTPException(status_code=404, detail=f"No stock found for GTIN: {gtin}")
+
+    # Calculate total quantity and build locations list
+    total_quantity = 0
+    locations = []
+
+    for entry in stock_entries:
+        total_quantity += entry.quantity
+        locations.append(StockLocationResponse(
+            location=entry.location,
+            quantity=entry.quantity
+        ))
+
+    # Try to get product title for reference
+    product = db.query(Product).filter(Product.gtin == normalized_gtin).first()
+    product_title = product.title if product else None
+
+    return StockSummaryResponse(
+        gtin=normalized_gtin,
+        total_quantity=total_quantity,
+        locations=locations,
+        product_title=product_title
+    )
+
+
+@router.get("/stock/{gtin}/total")
+def get_total_stock(gtin: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get total quantity in stock for a specific GTIN"""
+
+    # Normalize GTIN
+    def normalize_gtin(gtin_value):
+        if not gtin_value:
+            return None
+        gtin_str = str(gtin_value).strip()
+        if '.' in gtin_str:
+            gtin_str = gtin_str.split('.')[0]
+        gtin_clean = ''.join(filter(str.isdigit, gtin_str))
+        if len(gtin_clean) in [8, 12, 13, 14]:
+            return gtin_clean.zfill(13) if len(gtin_clean) == 13 else gtin_clean.zfill(12)
+        return gtin_clean if gtin_clean else None
+
+    normalized_gtin = normalize_gtin(gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    # Calculate total stock
+    total_stock = db.query(Stock).filter(Stock.gtin == normalized_gtin).all()
+    total_quantity = sum(entry.quantity for entry in total_stock)
+
+    # Get product info for context
+    product = db.query(Product).filter(Product.gtin == normalized_gtin).first()
+
+    return {
+        "gtin": normalized_gtin,
+        "total_quantity": total_quantity,
+        "product_title": product.title if product else None,
+        "locations_count": len(total_stock)
+    }
+
+
+@router.get("/stock", response_model=List[StockResponse])
+def get_all_stock(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        location: Optional[str] = Query(None, description="Filter by location"),
+        gtin: Optional[str] = Query(None, description="Filter by GTIN"),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get all stock entries with optional filtering"""
+    query = db.query(Stock)
+
+    if location:
+        query = query.filter(Stock.location.ilike(f"%{location}%"))
+
+    if gtin:
+        # Normalize GTIN for search
+        def normalize_gtin(gtin_value):
+            if not gtin_value:
+                return None
+            gtin_str = str(gtin_value).strip()
+            if '.' in gtin_str:
+                gtin_str = gtin_str.split('.')[0]
+            gtin_clean = ''.join(filter(str.isdigit, gtin_str))
+            if len(gtin_clean) in [8, 12, 13, 14]:
+                return gtin_clean.zfill(13) if len(gtin_clean) == 13 else gtin_clean.zfill(12)
+            return gtin_clean if gtin_clean else None
+
+        normalized_gtin = normalize_gtin(gtin)
+        if normalized_gtin:
+            query = query.filter(Stock.gtin == normalized_gtin)
+
+    stock_entries = query.offset(skip).limit(limit).all()
+    return stock_entries
+
+
+@router.put("/stock/{stock_id}", response_model=StockResponse)
+def update_stock(stock_id: int, stock_update: StockUpdate, db: Session = Depends(get_db),
+                 current_user: User = Depends(get_current_user)):
+    """Update stock quantity for a specific stock entry"""
+    stock_entry = db.query(Stock).filter(Stock.id == stock_id).first()
+    if not stock_entry:
+        raise HTTPException(status_code=404, detail="Stock entry not found")
+
+    stock_entry.quantity = stock_update.quantity
+    stock_entry.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(stock_entry)
+    return stock_entry
+
+
+@router.delete("/stock/{stock_id}")
+def delete_stock(stock_id: int, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Delete a stock entry"""
+    stock_entry = db.query(Stock).filter(Stock.id == stock_id).first()
+    if not stock_entry:
+        raise HTTPException(status_code=404, detail="Stock entry not found")
+
+    db.delete(stock_entry)
+    db.commit()
+    return {"message": "Stock entry deleted successfully"}
+