Fix warnings

This commit is contained in:
2025-09-06 14:55:18 +02:00
parent 4fb67e594d
commit 3342f1ab45

View File

@@ -2,7 +2,9 @@
import pandas as pd
import requests
from io import StringIO
from typing import Dict, Any, Optional
from typing import Dict, Any
from sqlalchemy import literal
from sqlalchemy.orm import Session
from models.database_models import Product
from datetime import datetime
@@ -16,6 +18,17 @@ class CSVProcessor:
ENCODINGS = ['utf-8', 'latin-1', 'iso-8859-1', 'cp1252', 'utf-8-sig']
PARSING_CONFIGS = [
# Try auto-detection first
{'sep': None, 'engine': 'python'},
# Try semicolon (common in European CSVs)
{'sep': ';', 'engine': 'python'},
# Try comma
{'sep': ',', 'engine': 'python'},
# Try tab
{'sep': '\t', 'engine': 'python'},
]
COLUMN_MAPPING = {
# Standard variations
'id': 'product_id',
@@ -95,18 +108,8 @@ class CSVProcessor:
def parse_csv(self, csv_content: str) -> pd.DataFrame:
"""Parse CSV with multiple separator attempts"""
parsing_configs = [
# Try auto-detection first
{'sep': None, 'engine': 'python'},
# Try semicolon (common in European CSVs)
{'sep': ';', 'engine': 'python'},
# Try comma
{'sep': ',', 'engine': 'python'},
# Try tab
{'sep': '\t', 'engine': 'python'},
]
for config in parsing_configs:
for config in self.PARSING_CONFIGS:
try:
df = pd.read_csv(
StringIO(csv_content),
@@ -226,12 +229,12 @@ class CSVProcessor:
}
async def _process_marketplace_batch(
self,
batch_df: pd.DataFrame,
marketplace: str,
shop_name: str,
db: Session,
batch_num: int
self,
batch_df: pd.DataFrame,
marketplace: str,
shop_name: str,
db: Session,
batch_num: int
) -> Dict[str, int]:
"""Process a batch of CSV rows with marketplace information"""
imported = 0
@@ -262,7 +265,7 @@ class CSVProcessor:
# Check if product exists
existing_product = db.query(Product).filter(
Product.product_id == product_data['product_id']
Product.product_id == literal(product_data['product_id'])
).first()
if existing_product: