Fix warnings
This commit is contained in:
@@ -2,7 +2,9 @@
|
||||
import pandas as pd
|
||||
import requests
|
||||
from io import StringIO
|
||||
from typing import Dict, Any, Optional
|
||||
from typing import Dict, Any
|
||||
|
||||
from sqlalchemy import literal
|
||||
from sqlalchemy.orm import Session
|
||||
from models.database_models import Product
|
||||
from datetime import datetime
|
||||
@@ -16,6 +18,17 @@ class CSVProcessor:
|
||||
|
||||
ENCODINGS = ['utf-8', 'latin-1', 'iso-8859-1', 'cp1252', 'utf-8-sig']
|
||||
|
||||
PARSING_CONFIGS = [
|
||||
# Try auto-detection first
|
||||
{'sep': None, 'engine': 'python'},
|
||||
# Try semicolon (common in European CSVs)
|
||||
{'sep': ';', 'engine': 'python'},
|
||||
# Try comma
|
||||
{'sep': ',', 'engine': 'python'},
|
||||
# Try tab
|
||||
{'sep': '\t', 'engine': 'python'},
|
||||
]
|
||||
|
||||
COLUMN_MAPPING = {
|
||||
# Standard variations
|
||||
'id': 'product_id',
|
||||
@@ -95,18 +108,8 @@ class CSVProcessor:
|
||||
|
||||
def parse_csv(self, csv_content: str) -> pd.DataFrame:
|
||||
"""Parse CSV with multiple separator attempts"""
|
||||
parsing_configs = [
|
||||
# Try auto-detection first
|
||||
{'sep': None, 'engine': 'python'},
|
||||
# Try semicolon (common in European CSVs)
|
||||
{'sep': ';', 'engine': 'python'},
|
||||
# Try comma
|
||||
{'sep': ',', 'engine': 'python'},
|
||||
# Try tab
|
||||
{'sep': '\t', 'engine': 'python'},
|
||||
]
|
||||
|
||||
for config in parsing_configs:
|
||||
for config in self.PARSING_CONFIGS:
|
||||
try:
|
||||
df = pd.read_csv(
|
||||
StringIO(csv_content),
|
||||
@@ -226,12 +229,12 @@ class CSVProcessor:
|
||||
}
|
||||
|
||||
async def _process_marketplace_batch(
|
||||
self,
|
||||
batch_df: pd.DataFrame,
|
||||
marketplace: str,
|
||||
shop_name: str,
|
||||
db: Session,
|
||||
batch_num: int
|
||||
self,
|
||||
batch_df: pd.DataFrame,
|
||||
marketplace: str,
|
||||
shop_name: str,
|
||||
db: Session,
|
||||
batch_num: int
|
||||
) -> Dict[str, int]:
|
||||
"""Process a batch of CSV rows with marketplace information"""
|
||||
imported = 0
|
||||
@@ -262,7 +265,7 @@ class CSVProcessor:
|
||||
|
||||
# Check if product exists
|
||||
existing_product = db.query(Product).filter(
|
||||
Product.product_id == product_data['product_id']
|
||||
Product.product_id == literal(product_data['product_id'])
|
||||
).first()
|
||||
|
||||
if existing_product:
|
||||
|
||||
Reference in New Issue
Block a user