Samir Boulahtit
50a4fc38a7
- Add PROSPECTING_BATCH_DELAY_SECONDS config (default 1.0s) — polite delay between prospects in batch scans to avoid rate limiting - Apply delay to all 5 batch API endpoints and all Celery tasks - Fix Celery tasks: error_message → error_log (matches model field) - Add batch-scanning.md docs with rate limiting guide, scaling estimates for 70k+ URL imports, and pipeline order recommendations Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
38 lines
990 B
Python
38 lines
990 B
Python
# app/modules/prospecting/config.py
|
|
"""
|
|
Module configuration.
|
|
|
|
Environment-based configuration using Pydantic Settings.
|
|
Settings are loaded from environment variables with PROSPECTING_ prefix.
|
|
|
|
Example:
|
|
PROSPECTING_PAGESPEED_API_KEY=your_key
|
|
"""
|
|
from pydantic_settings import BaseSettings
|
|
|
|
|
|
class ModuleConfig(BaseSettings):
|
|
"""Configuration for prospecting module."""
|
|
|
|
# PageSpeed Insights API key (optional, free 25k/day without key)
|
|
pagespeed_api_key: str = ""
|
|
|
|
# HTTP request timeout in seconds
|
|
http_timeout: int = 10
|
|
|
|
# Batch operation limits
|
|
batch_size: int = 100
|
|
|
|
# Max concurrent HTTP requests for batch scanning
|
|
max_concurrent_requests: int = 10
|
|
|
|
# Delay between prospects in batch scans (seconds) — be polite to target sites
|
|
batch_delay_seconds: float = 1.0
|
|
|
|
model_config = {"env_prefix": "PROSPECTING_", "env_file": ".env", "extra": "ignore"}
|
|
|
|
|
|
# Export for auto-discovery
|
|
config_class = ModuleConfig
|
|
config = ModuleConfig()
|