Initial commit

main.py

@@ -0,0 +1,712 @@
+from fastapi import FastAPI, HTTPException, Query, Depends, BackgroundTasks
+from fastapi.responses import StreamingResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from pydantic import BaseModel, Field, validator
+from typing import Optional, List, Dict, Any
+from datetime import datetime, timedelta
+from sqlalchemy import create_engine, Column, Integer, String, DateTime, text, ForeignKey, Index
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker, Session, relationship
+from contextlib import asynccontextmanager
+import pandas as pd
+import requests
+from io import StringIO, BytesIO
+import logging
+import asyncio
+import time
+from functools import wraps
+import os
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+# Import utility modules
+from utils.data_processing import GTINProcessor, PriceProcessor
+from utils.csv_processor import CSVProcessor
+from utils.database import get_db_engine, get_session_local
+from models.database_models import Base, Product, Stock, ImportJob, User
+from models.api_models import *
+from middleware.rate_limiter import RateLimiter
+from middleware.auth import AuthManager
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Initialize processors
+gtin_processor = GTINProcessor()
+price_processor = PriceProcessor()
+csv_processor = CSVProcessor()
+
+# Database setup
+DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://user:password@localhost/ecommerce")
+engine = get_db_engine(DATABASE_URL)
+SessionLocal = get_session_local(engine)
+
+# Rate limiter and auth manager
+rate_limiter = RateLimiter()
+auth_manager = AuthManager()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Application lifespan events"""
+    # Startup
+    logger.info("Starting up ecommerce API with authentication")
+
+    # Create tables
+    Base.metadata.create_all(bind=engine)
+
+    # Create default admin user
+    db = SessionLocal()
+    try:
+        auth_manager.create_default_admin_user(db)
+    except Exception as e:
+        logger.error(f"Failed to create default admin user: {e}")
+    finally:
+        db.close()
+
+    # Add indexes
+    with engine.connect() as conn:
+        try:
+            # User indexes
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_user_email ON users(email)"))
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_user_username ON users(username)"))
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_user_role ON users(role)"))
+
+            # Product indexes
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_product_gtin ON products(gtin)"))
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_product_brand ON products(brand)"))
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_product_category ON products(google_product_category)"))
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_product_availability ON products(availability)"))
+
+            # Stock indexes
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_stock_gtin_location ON stock(gtin, location)"))
+            conn.execute(text("CREATE INDEX IF NOT EXISTS idx_stock_location ON stock(location)"))
+
+            conn.commit()
+            logger.info("Database indexes created successfully")
+        except Exception as e:
+            logger.warning(f"Index creation warning: {e}")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down ecommerce API")
+
+
+# FastAPI app with lifespan
+app = FastAPI(
+    title="Ecommerce Backend API",
+    description="Advanced product management system with JWT authentication, CSV import/export and stock management",
+    version="2.1.0",
+    lifespan=lifespan
+)
+
+# Add CORS middleware
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # Configure appropriately for production
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Security
+security = HTTPBearer()
+
+
+# Database dependency with connection pooling
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    except Exception as e:
+        db.rollback()
+        logger.error(f"Database error: {e}")
+        raise
+    finally:
+        db.close()
+
+
+# Authentication dependencies
+def get_current_user(credentials: HTTPAuthorizationCredentials = Depends(security), db: Session = Depends(get_db)):
+    """Get current authenticated user"""
+    return auth_manager.get_current_user(db, credentials)
+
+
+def get_current_admin_user(current_user: User = Depends(get_current_user)):
+    """Require admin user"""
+    return auth_manager.require_admin(current_user)
+
+
+# Rate limiting decorator
+def rate_limit(max_requests: int = 100, window_seconds: int = 3600):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            # Extract client IP or user ID for rate limiting
+            client_id = "anonymous"  # In production, extract from request
+
+            if not rate_limiter.allow_request(client_id, max_requests, window_seconds):
+                raise HTTPException(
+                    status_code=429,
+                    detail="Rate limit exceeded"
+                )
+
+            return await func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+# Authentication Routes
+@app.post("/register", response_model=UserResponse)
+def register_user(user_data: UserRegister, db: Session = Depends(get_db)):
+    """Register a new user"""
+
+    # Check if email already exists
+    existing_email = db.query(User).filter(User.email == user_data.email).first()
+    if existing_email:
+        raise HTTPException(status_code=400, detail="Email already registered")
+
+    # Check if username already exists
+    existing_username = db.query(User).filter(User.username == user_data.username).first()
+    if existing_username:
+        raise HTTPException(status_code=400, detail="Username already taken")
+
+    # Hash password and create user
+    hashed_password = auth_manager.hash_password(user_data.password)
+    new_user = User(
+        email=user_data.email,
+        username=user_data.username,
+        hashed_password=hashed_password,
+        role="user",  # Default role
+        is_active=True
+    )
+
+    db.add(new_user)
+    db.commit()
+    db.refresh(new_user)
+
+    logger.info(f"New user registered: {new_user.username}")
+    return new_user
+
+
+@app.post("/login", response_model=LoginResponse)
+def login_user(user_credentials: UserLogin, db: Session = Depends(get_db)):
+    """Login user and return JWT token"""
+
+    user = auth_manager.authenticate_user(db, user_credentials.username, user_credentials.password)
+    if not user:
+        raise HTTPException(
+            status_code=401,
+            detail="Incorrect username or password"
+        )
+
+    # Create access token
+    token_data = auth_manager.create_access_token(user)
+
+    logger.info(f"User logged in: {user.username}")
+
+    return LoginResponse(
+        access_token=token_data["access_token"],
+        token_type=token_data["token_type"],
+        expires_in=token_data["expires_in"],
+        user=UserResponse.model_validate(user)
+    )
+
+
+@app.get("/me", response_model=UserResponse)
+def get_current_user_info(current_user: User = Depends(get_current_user)):
+    """Get current user information"""
+    return UserResponse.model_validate(current_user)
+
+
+# Public Routes (no authentication required)
+@app.get("/")
+def root():
+    return {
+        "message": "Ecommerce Backend API v2.1 with JWT Authentication",
+        "status": "operational",
+        "auth_required": "Most endpoints require Bearer token authentication"
+    }
+
+
+@app.get("/health")
+def health_check(db: Session = Depends(get_db)):
+    """Health check endpoint"""
+    try:
+        # Test database connection
+        db.execute(text("SELECT 1"))
+        return {"status": "healthy", "timestamp": datetime.utcnow()}
+    except Exception as e:
+        logger.error(f"Health check failed: {e}")
+        raise HTTPException(status_code=503, detail="Service unhealthy")
+
+
+# Protected Routes (authentication required)
+@app.post("/import-csv", response_model=ImportJobResponse)
+@rate_limit(max_requests=10, window_seconds=3600)  # Limit CSV imports
+async def import_csv_from_url(
+        request: CSVImportRequest,
+        background_tasks: BackgroundTasks,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Import products from CSV URL with background processing (Protected)"""
+
+    # Create import job record
+    import_job = ImportJob(
+        status="pending",
+        source_url=request.url,
+        user_id=current_user.id,
+        created_at=datetime.utcnow()
+    )
+    db.add(import_job)
+    db.commit()
+    db.refresh(import_job)
+
+    # Process in background
+    background_tasks.add_task(
+        process_csv_import,
+        import_job.id,
+        request.url,
+        request.batch_size or 1000
+    )
+
+    return ImportJobResponse(
+        job_id=import_job.id,
+        status="pending",
+        message="CSV import started. Check status with /import-status/{job_id}"
+    )
+
+
+async def process_csv_import(job_id: int, url: str, batch_size: int = 1000):
+    """Background task to process CSV import with batching"""
+    db = SessionLocal()
+
+    try:
+        # Update job status
+        job = db.query(ImportJob).filter(ImportJob.id == job_id).first()
+        if not job:
+            logger.error(f"Import job {job_id} not found")
+            return
+
+        job.status = "processing"
+        job.started_at = datetime.utcnow()
+        db.commit()
+
+        # Process CSV
+        result = await csv_processor.process_csv_from_url(url, batch_size, db)
+
+        # Update job with results
+        job.status = "completed"
+        job.completed_at = datetime.utcnow()
+        job.imported_count = result["imported"]
+        job.updated_count = result["updated"]
+        job.error_count = result.get("errors", 0)
+        job.total_processed = result["total_processed"]
+
+        if result.get("errors", 0) > 0:
+            job.status = "completed_with_errors"
+            job.error_message = f"{result['errors']} rows had errors"
+
+        db.commit()
+        logger.info(f"Import job {job_id} completed successfully")
+
+    except Exception as e:
+        logger.error(f"Import job {job_id} failed: {e}")
+        job.status = "failed"
+        job.completed_at = datetime.utcnow()
+        job.error_message = str(e)
+        db.commit()
+
+    finally:
+        db.close()
+
+
+@app.get("/import-status/{job_id}", response_model=ImportJobResponse)
+def get_import_status(job_id: int, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get status of CSV import job (Protected)"""
+    job = db.query(ImportJob).filter(ImportJob.id == job_id).first()
+    if not job:
+        raise HTTPException(status_code=404, detail="Import job not found")
+
+    # Users can only see their own jobs, admins can see all
+    if current_user.role != "admin" and job.user_id != current_user.id:
+        raise HTTPException(status_code=403, detail="Access denied to this import job")
+
+    return ImportJobResponse(
+        job_id=job.id,
+        status=job.status,
+        imported=job.imported_count or 0,
+        updated=job.updated_count or 0,
+        total_processed=job.total_processed or 0,
+        error_count=job.error_count or 0,
+        error_message=job.error_message,
+        created_at=job.created_at,
+        started_at=job.started_at,
+        completed_at=job.completed_at
+    )
+
+
+@app.get("/products", response_model=ProductListResponse)
+def get_products(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        brand: Optional[str] = Query(None),
+        category: Optional[str] = Query(None),
+        availability: Optional[str] = Query(None),
+        search: Optional[str] = Query(None),
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Get products with advanced filtering and search (Protected)"""
+
+    query = db.query(Product)
+
+    # Apply filters
+    if brand:
+        query = query.filter(Product.brand.ilike(f"%{brand}%"))
+    if category:
+        query = query.filter(Product.google_product_category.ilike(f"%{category}%"))
+    if availability:
+        query = query.filter(Product.availability == availability)
+    if search:
+        # Search in title and description
+        search_term = f"%{search}%"
+        query = query.filter(
+            (Product.title.ilike(search_term)) |
+            (Product.description.ilike(search_term))
+        )
+
+    # Get total count for pagination
+    total = query.count()
+
+    # Apply pagination
+    products = query.offset(skip).limit(limit).all()
+
+    return ProductListResponse(
+        products=products,
+        total=total,
+        skip=skip,
+        limit=limit
+    )
+
+
+@app.post("/products", response_model=ProductResponse)
+def create_product(
+        product: ProductCreate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Create a new product with validation (Protected)"""
+
+    # Check if product_id already exists
+    existing = db.query(Product).filter(Product.product_id == product.product_id).first()
+    if existing:
+        raise HTTPException(status_code=400, detail="Product with this ID already exists")
+
+    # Process and validate GTIN if provided
+    if product.gtin:
+        normalized_gtin = gtin_processor.normalize(product.gtin)
+        if not normalized_gtin:
+            raise HTTPException(status_code=400, detail="Invalid GTIN format")
+        product.gtin = normalized_gtin
+
+    # Process price if provided
+    if product.price:
+        parsed_price, currency = price_processor.parse_price_currency(product.price)
+        if parsed_price:
+            product.price = parsed_price
+            product.currency = currency
+
+    db_product = Product(**product.dict())
+    db.add(db_product)
+    db.commit()
+    db.refresh(db_product)
+
+    return db_product
+
+
+@app.get("/products/{product_id}", response_model=ProductDetailResponse)
+def get_product(product_id: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get product with stock information (Protected)"""
+
+    product = db.query(Product).filter(Product.product_id == product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found")
+
+    # Get stock information if GTIN exists
+    stock_info = None
+    if product.gtin:
+        stock_entries = db.query(Stock).filter(Stock.gtin == product.gtin).all()
+        if stock_entries:
+            total_quantity = sum(entry.quantity for entry in stock_entries)
+            locations = [
+                StockLocationResponse(location=entry.location, quantity=entry.quantity)
+                for entry in stock_entries
+            ]
+            stock_info = StockSummaryResponse(
+                gtin=product.gtin,
+                total_quantity=total_quantity,
+                locations=locations
+            )
+
+    return ProductDetailResponse(
+        product=product,
+        stock_info=stock_info
+    )
+
+
+@app.put("/products/{product_id}", response_model=ProductResponse)
+def update_product(
+        product_id: str,
+        product_update: ProductUpdate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Update product with validation (Protected)"""
+
+    product = db.query(Product).filter(Product.product_id == product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found")
+
+    # Update fields
+    update_data = product_update.dict(exclude_unset=True)
+
+    # Validate GTIN if being updated
+    if "gtin" in update_data and update_data["gtin"]:
+        normalized_gtin = gtin_processor.normalize(update_data["gtin"])
+        if not normalized_gtin:
+            raise HTTPException(status_code=400, detail="Invalid GTIN format")
+        update_data["gtin"] = normalized_gtin
+
+    # Process price if being updated
+    if "price" in update_data and update_data["price"]:
+        parsed_price, currency = price_processor.parse_price_currency(update_data["price"])
+        if parsed_price:
+            update_data["price"] = parsed_price
+            update_data["currency"] = currency
+
+    for key, value in update_data.items():
+        setattr(product, key, value)
+
+    product.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(product)
+
+    return product
+
+
+@app.delete("/products/{product_id}")
+def delete_product(
+        product_id: str,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Delete product and associated stock (Protected)"""
+
+    product = db.query(Product).filter(Product.product_id == product_id).first()
+    if not product:
+        raise HTTPException(status_code=404, detail="Product not found")
+
+    # Delete associated stock entries if GTIN exists
+    if product.gtin:
+        db.query(Stock).filter(Stock.gtin == product.gtin).delete()
+
+    db.delete(product)
+    db.commit()
+
+    return {"message": "Product and associated stock deleted successfully"}
+
+
+# Stock Management Routes (Protected)
+@app.post("/stock", response_model=StockResponse)
+def set_stock(
+        stock: StockCreate,
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Set stock with GTIN validation (Protected)"""
+
+    # Normalize and validate GTIN
+    normalized_gtin = gtin_processor.normalize(stock.gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    # Verify GTIN exists in products
+    product = db.query(Product).filter(Product.gtin == normalized_gtin).first()
+    if not product:
+        logger.warning(f"Setting stock for GTIN {normalized_gtin} without corresponding product")
+
+    # Check existing stock
+    existing_stock = db.query(Stock).filter(
+        Stock.gtin == normalized_gtin,
+        Stock.location == stock.location.strip().upper()
+    ).first()
+
+    if existing_stock:
+        existing_stock.quantity = stock.quantity
+        existing_stock.updated_at = datetime.utcnow()
+        db.commit()
+        db.refresh(existing_stock)
+        return existing_stock
+    else:
+        new_stock = Stock(
+            gtin=normalized_gtin,
+            location=stock.location.strip().upper(),
+            quantity=stock.quantity
+        )
+        db.add(new_stock)
+        db.commit()
+        db.refresh(new_stock)
+        return new_stock
+
+
+@app.get("/stock/{gtin}", response_model=StockSummaryResponse)
+def get_stock_by_gtin(gtin: str, db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get stock summary with product validation (Protected)"""
+
+    normalized_gtin = gtin_processor.normalize(gtin)
+    if not normalized_gtin:
+        raise HTTPException(status_code=400, detail="Invalid GTIN format")
+
+    stock_entries = db.query(Stock).filter(Stock.gtin == normalized_gtin).all()
+    if not stock_entries:
+        raise HTTPException(status_code=404, detail=f"No stock found for GTIN: {gtin}")
+
+    total_quantity = sum(entry.quantity for entry in stock_entries)
+    locations = [
+        StockLocationResponse(location=entry.location, quantity=entry.quantity)
+        for entry in stock_entries
+    ]
+
+    # Get product info
+    product = db.query(Product).filter(Product.gtin == normalized_gtin).first()
+
+    return StockSummaryResponse(
+        gtin=normalized_gtin,
+        total_quantity=total_quantity,
+        locations=locations,
+        product_title=product.title if product else None
+    )
+
+
+@app.get("/stats", response_model=StatsResponse)
+def get_stats(db: Session = Depends(get_db), current_user: User = Depends(get_current_user)):
+    """Get comprehensive statistics (Protected)"""
+
+    # Use more efficient queries with proper indexes
+    total_products = db.query(Product).count()
+
+    unique_brands = db.query(Product.brand).filter(
+        Product.brand.isnot(None),
+        Product.brand != ""
+    ).distinct().count()
+
+    unique_categories = db.query(Product.google_product_category).filter(
+        Product.google_product_category.isnot(None),
+        Product.google_product_category != ""
+    ).distinct().count()
+
+    # Additional stock statistics
+    total_stock_entries = db.query(Stock).count()
+    total_inventory = db.query(Stock.quantity).scalar() or 0
+
+    return StatsResponse(
+        total_products=total_products,
+        unique_brands=unique_brands,
+        unique_categories=unique_categories,
+        total_stock_entries=total_stock_entries,
+        total_inventory_quantity=total_inventory
+    )
+
+
+# Export with streaming for large datasets (Protected)
+@app.get("/export-csv")
+async def export_csv(
+        db: Session = Depends(get_db),
+        current_user: User = Depends(get_current_user)
+):
+    """Export products as CSV with streaming (Protected)"""
+
+    def generate_csv():
+        # Stream CSV generation for memory efficiency
+        yield "product_id,title,description,link,image_link,availability,price,currency,brand,gtin\n"
+
+        batch_size = 1000
+        offset = 0
+
+        while True:
+            products = db.query(Product).offset(offset).limit(batch_size).all()
+            if not products:
+                break
+
+            for product in products:
+                # Create CSV row
+                row = f'"{product.product_id}","{product.title or ""}","{product.description or ""}","{product.link or ""}","{product.image_link or ""}","{product.availability or ""}","{product.price or ""}","{product.currency or ""}","{product.brand or ""}","{product.gtin or ""}"\n'
+                yield row
+
+            offset += batch_size
+
+    return StreamingResponse(
+        generate_csv(),
+        media_type="text/csv",
+        headers={"Content-Disposition": "attachment; filename=products_export.csv"}
+    )
+
+
+# Admin-only routes
+@app.get("/admin/users", response_model=List[UserResponse])
+def get_all_users(
+        skip: int = Query(0, ge=0),
+        limit: int = Query(100, ge=1, le=1000),
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Get all users (Admin only)"""
+    users = db.query(User).offset(skip).limit(limit).all()
+    return [UserResponse.model_validate(user) for user in users]
+
+
+@app.put("/admin/users/{user_id}/status")
+def toggle_user_status(
+        user_id: int,
+        db: Session = Depends(get_db),
+        current_admin: User = Depends(get_current_admin_user)
+):
+    """Toggle user active status (Admin only)"""
+    user = db.query(User).filter(User.id == user_id).first()
+    if not user:
+        raise HTTPException(status_code=404, detail="User not found")
+
+    if user.id == current_admin.id:
+        raise HTTPException(status_code=400, detail="Cannot deactivate your own account")
+
+    user.is_active = not user.is_active
+    user.updated_at = datetime.utcnow()
+    db.commit()
+    db.refresh(user)
+
+    status = "activated" if user.is_active else "deactivated"
+    return {"message": f"User {user.username} has been {status}"}
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(
+        "main:app",
+        host="0.0.0.0",
+        port=8000,
+        reload=True,
+        log_level="info"
+    )