sboulahtit/orion
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix warnings

Browse Source
This commit is contained in:
Samir Boulahtit
2025-09-06 14:55:18 +02:00
parent 4fb67e594d
commit 3342f1ab45
1 changed files with 22 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
utils/csv_processor.py
View File

@@ -2,7 +2,9 @@
import pandas as pd import pandas as pd
import requests import requests
from io import StringIO from io import StringIO
from typing import Dict, Any, Optional from typing import Dict, Any
from sqlalchemy import literal
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from models.database_models import Product from models.database_models import Product
from datetime import datetime from datetime import datetime
@@ -16,6 +18,17 @@ class CSVProcessor:
ENCODINGS = ['utf-8', 'latin-1', 'iso-8859-1', 'cp1252', 'utf-8-sig'] ENCODINGS = ['utf-8', 'latin-1', 'iso-8859-1', 'cp1252', 'utf-8-sig']
PARSING_CONFIGS = [
# Try auto-detection first
{'sep': None, 'engine': 'python'},
# Try semicolon (common in European CSVs)
{'sep': ';', 'engine': 'python'},
# Try comma
{'sep': ',', 'engine': 'python'},
# Try tab
{'sep': '\t', 'engine': 'python'},
]
COLUMN_MAPPING = { COLUMN_MAPPING = {
# Standard variations # Standard variations
'id': 'product_id', 'id': 'product_id',
@@ -95,18 +108,8 @@ class CSVProcessor:
def parse_csv(self, csv_content: str) -> pd.DataFrame: def parse_csv(self, csv_content: str) -> pd.DataFrame:
"""Parse CSV with multiple separator attempts""" """Parse CSV with multiple separator attempts"""
parsing_configs = [
# Try auto-detection first
{'sep': None, 'engine': 'python'},
# Try semicolon (common in European CSVs)
{'sep': ';', 'engine': 'python'},
# Try comma
{'sep': ',', 'engine': 'python'},
# Try tab
{'sep': '\t', 'engine': 'python'},
]
for config in parsing_configs: for config in self.PARSING_CONFIGS:
try: try:
df = pd.read_csv( df = pd.read_csv(
StringIO(csv_content), StringIO(csv_content),
@@ -226,12 +229,12 @@ class CSVProcessor:
} }
async def _process_marketplace_batch( async def _process_marketplace_batch(
self, self,
batch_df: pd.DataFrame, batch_df: pd.DataFrame,
marketplace: str, marketplace: str,
shop_name: str, shop_name: str,
db: Session, db: Session,
batch_num: int batch_num: int
) -> Dict[str, int]: ) -> Dict[str, int]:
"""Process a batch of CSV rows with marketplace information""" """Process a batch of CSV rows with marketplace information"""
imported = 0 imported = 0
@@ -262,7 +265,7 @@ class CSVProcessor:
# Check if product exists # Check if product exists
existing_product = db.query(Product).filter( existing_product = db.query(Product).filter(
Product.product_id == product_data['product_id'] Product.product_id == literal(product_data['product_id'])
).first() ).first()
if existing_product: if existing_product: