diff --git a/utils/csv_processor.py b/utils/csv_processor.py index 5b8c0065..559fff83 100644 --- a/utils/csv_processor.py +++ b/utils/csv_processor.py @@ -2,7 +2,9 @@ import pandas as pd import requests from io import StringIO -from typing import Dict, Any, Optional +from typing import Dict, Any + +from sqlalchemy import literal from sqlalchemy.orm import Session from models.database_models import Product from datetime import datetime @@ -16,6 +18,17 @@ class CSVProcessor: ENCODINGS = ['utf-8', 'latin-1', 'iso-8859-1', 'cp1252', 'utf-8-sig'] + PARSING_CONFIGS = [ + # Try auto-detection first + {'sep': None, 'engine': 'python'}, + # Try semicolon (common in European CSVs) + {'sep': ';', 'engine': 'python'}, + # Try comma + {'sep': ',', 'engine': 'python'}, + # Try tab + {'sep': '\t', 'engine': 'python'}, + ] + COLUMN_MAPPING = { # Standard variations 'id': 'product_id', @@ -95,18 +108,8 @@ class CSVProcessor: def parse_csv(self, csv_content: str) -> pd.DataFrame: """Parse CSV with multiple separator attempts""" - parsing_configs = [ - # Try auto-detection first - {'sep': None, 'engine': 'python'}, - # Try semicolon (common in European CSVs) - {'sep': ';', 'engine': 'python'}, - # Try comma - {'sep': ',', 'engine': 'python'}, - # Try tab - {'sep': '\t', 'engine': 'python'}, - ] - for config in parsing_configs: + for config in self.PARSING_CONFIGS: try: df = pd.read_csv( StringIO(csv_content), @@ -226,12 +229,12 @@ class CSVProcessor: } async def _process_marketplace_batch( - self, - batch_df: pd.DataFrame, - marketplace: str, - shop_name: str, - db: Session, - batch_num: int + self, + batch_df: pd.DataFrame, + marketplace: str, + shop_name: str, + db: Session, + batch_num: int ) -> Dict[str, int]: """Process a batch of CSV rows with marketplace information""" imported = 0 @@ -262,7 +265,7 @@ class CSVProcessor: # Check if product exists existing_product = db.query(Product).filter( - Product.product_id == product_data['product_id'] + Product.product_id == literal(product_data['product_id']) ).first() if existing_product: