Refactoring code for modular approach

tests/test_csv_processor.py

@@ -1,5 +1,7 @@
 # tests/test_csv_processor.py
 import pytest
+import requests
+import requests.exceptions
 from unittest.mock import Mock, patch, AsyncMock
 from io import StringIO
 import pandas as pd
@@ -11,18 +13,61 @@ class TestCSVProcessor:
         self.processor = CSVProcessor()
 
     @patch('requests.get')
-    def test_download_csv_success(self, mock_get):
-        """Test successful CSV download"""
-        # Mock successful HTTP response
+    def test_download_csv_encoding_fallback(self, mock_get):
+        """Test CSV download with encoding fallback"""
+        # Create content with special characters that would fail UTF-8 if not properly encoded
+        special_content = "product_id,title,price\nTEST001,Café Product,10.99"
+
         mock_response = Mock()
         mock_response.status_code = 200
-        mock_response.text = "product_id,title,price\nTEST001,Test Product,10.99"
+        # Use latin-1 encoding which your method should try
+        mock_response.content = special_content.encode('latin-1')
+        mock_response.raise_for_status.return_value = None
         mock_get.return_value = mock_response
 
-        csv_content = self.processor._download_csv("http://example.com/test.csv")
+        csv_content = self.processor.download_csv("http://example.com/test.csv")
 
+        mock_get.assert_called_once_with("http://example.com/test.csv", timeout=30)
+        assert isinstance(csv_content, str)
+        assert "Café Product" in csv_content
+
+    @patch('requests.get')
+    def test_download_csv_encoding_ignore_fallback(self, mock_get):
+        """Test CSV download falls back to UTF-8 with error ignoring"""
+        # Create problematic bytes that would fail most encoding attempts
+        mock_response = Mock()
+        mock_response.status_code = 200
+        # Create bytes that will fail most encodings
+        mock_response.content = b"product_id,title,price\nTEST001,\xff\xfe Product,10.99"
+        mock_response.raise_for_status.return_value = None
+        mock_get.return_value = mock_response
+
+        csv_content = self.processor.download_csv("http://example.com/test.csv")
+
+        mock_get.assert_called_once_with("http://example.com/test.csv", timeout=30)
+        assert isinstance(csv_content, str)
+        # Should still contain basic content even with ignored errors
         assert "product_id,title,price" in csv_content
-        assert "TEST001,Test Product,10.99" in csv_content
+        assert "TEST001" in csv_content
+
+    @patch('requests.get')
+    def test_download_csv_request_exception(self, mock_get):
+        """Test CSV download with request exception"""
+        mock_get.side_effect = requests.exceptions.RequestException("Connection error")
+
+        with pytest.raises(requests.exceptions.RequestException):
+            self.processor.download_csv("http://example.com/test.csv")
+
+    @patch('requests.get')
+    def test_download_csv_http_error(self, mock_get):
+        """Test CSV download with HTTP error"""
+        mock_response = Mock()
+        mock_response.status_code = 404
+        mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("404 Not Found")
+        mock_get.return_value = mock_response
+
+        with pytest.raises(requests.exceptions.HTTPError):
+            self.processor.download_csv("http://example.com/nonexistent.csv")
 
     @patch('requests.get')
     def test_download_csv_failure(self, mock_get):
@@ -41,30 +86,18 @@ class TestCSVProcessor:
 TEST001,Test Product 1,10.99,TestMarket
 TEST002,Test Product 2,15.99,TestMarket"""
 
-        df = self.processor._parse_csv_content(csv_content)
+        df = self.processor.parse_csv(csv_content)
 
         assert len(df) == 2
         assert "product_id" in df.columns
         assert df.iloc[0]["product_id"] == "TEST001"
-        assert df.iloc[1]["price"] == "15.99"
-
-    def test_validate_csv_headers(self):
-        """Test CSV header validation"""
-        # Valid headers
-        valid_df = pd.DataFrame({
-            "product_id": ["TEST001"],
-            "title": ["Test"],
-            "price": ["10.99"]
-        })
-
-        assert self.processor._validate_csv_headers(invalid_df) == False
+        assert df.iloc[1]["price"] == 15.99
 
     @pytest.mark.asyncio
     async def test_process_marketplace_csv_from_url(self, db):
         """Test complete marketplace CSV processing"""
-        with patch.object(self.processor, '_download_csv') as mock_download, \
-                patch.object(self.processor, '_parse_csv_content') as mock_parse, \
-                patch.object(self.processor, '_validate_csv_headers') as mock_validate:
+        with patch.object(self.processor, 'download_csv') as mock_download, \
+                patch.object(self.processor, 'parse_csv') as mock_parse:
             # Mock successful download and parsing
             mock_download.return_value = "csv_content"
             mock_df = pd.DataFrame({
@@ -75,7 +108,7 @@ TEST002,Test Product 2,15.99,TestMarket"""
                 "shop_name": ["TestShop", "TestShop"]
             })
             mock_parse.return_value = mock_df
-            mock_validate.return_value = True
+
 
             result = await self.processor.process_marketplace_csv_from_url(
                 "http://example.com/test.csv",