Some checks failed
Phase 3 of the production launch plan: task reliability improvements to prevent DB lock issues at scale and handle transient wallet API failures gracefully. - 3.1 Batched point expiration: rewrite per-card Python loop to chunked processing (LIMIT 500 FOR UPDATE SKIP LOCKED). Each chunk commits independently, releasing row locks before processing the next batch. Notifications sent after commit (outside lock window). Warning emails also chunked with same pattern. - 3.2 Wallet sync exponential backoff: replace time.sleep(2) single retry with 4 attempts using [1s, 4s, 16s] backoff delays. Per-card try/except ensures one failing card doesn't block the batch. Failed card IDs logged for observability. 342 tests pass. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
147 lines
4.3 KiB
Python
147 lines
4.3 KiB
Python
# app/modules/loyalty/tasks/wallet_sync.py
|
|
"""
|
|
Wallet synchronization task.
|
|
|
|
Handles syncing loyalty card data to Google Wallet and Apple Wallet
|
|
for cards that may have missed real-time updates.
|
|
|
|
Uses exponential backoff (1s, 4s, 16s) per card to handle transient
|
|
API failures without blocking the entire batch.
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
|
|
from celery import shared_task
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Exponential backoff delays in seconds: 1s, 4s, 16s
|
|
_RETRY_DELAYS = [1, 4, 16]
|
|
_MAX_ATTEMPTS = len(_RETRY_DELAYS) + 1 # 4 total attempts
|
|
|
|
|
|
@shared_task(name="loyalty.sync_wallet_passes")
|
|
def sync_wallet_passes() -> dict:
|
|
"""
|
|
Sync wallet passes for cards that may be out of sync.
|
|
|
|
This catches any cards that missed real-time updates due to
|
|
errors or network issues.
|
|
|
|
Returns:
|
|
Summary of synced passes
|
|
"""
|
|
from datetime import UTC, datetime, timedelta
|
|
|
|
from app.core.database import SessionLocal
|
|
from app.modules.loyalty.models import LoyaltyCard, LoyaltyTransaction
|
|
from app.modules.loyalty.services import wallet_service
|
|
|
|
db = SessionLocal()
|
|
try:
|
|
# Find cards with transactions in the last hour that have wallet IDs
|
|
one_hour_ago = datetime.now(UTC) - timedelta(hours=1)
|
|
|
|
recent_tx_card_ids = (
|
|
db.query(LoyaltyTransaction.card_id)
|
|
.filter(LoyaltyTransaction.transaction_at >= one_hour_ago)
|
|
.distinct()
|
|
.all()
|
|
)
|
|
card_ids = [row[0] for row in recent_tx_card_ids]
|
|
|
|
if not card_ids:
|
|
logger.info("No cards with recent transactions to sync")
|
|
return {
|
|
"status": "success",
|
|
"cards_checked": 0,
|
|
"google_synced": 0,
|
|
"apple_synced": 0,
|
|
"failed_card_ids": [],
|
|
}
|
|
|
|
cards = (
|
|
db.query(LoyaltyCard)
|
|
.filter(
|
|
LoyaltyCard.id.in_(card_ids),
|
|
(LoyaltyCard.google_object_id.isnot(None))
|
|
| (LoyaltyCard.apple_serial_number.isnot(None)),
|
|
)
|
|
.all()
|
|
)
|
|
|
|
google_synced = 0
|
|
apple_synced = 0
|
|
failed_card_ids = []
|
|
|
|
for card in cards:
|
|
success, google, apple = _sync_card_with_backoff(
|
|
wallet_service, db, card
|
|
)
|
|
if success:
|
|
google_synced += google
|
|
apple_synced += apple
|
|
else:
|
|
failed_card_ids.append(card.id)
|
|
|
|
if failed_card_ids:
|
|
logger.error(
|
|
f"Wallet sync: {len(failed_card_ids)} cards failed after "
|
|
f"{_MAX_ATTEMPTS} attempts each: {failed_card_ids}"
|
|
)
|
|
|
|
logger.info(
|
|
f"Wallet sync complete: {len(cards)} cards checked, "
|
|
f"{google_synced} Google, {apple_synced} Apple, "
|
|
f"{len(failed_card_ids)} failed"
|
|
)
|
|
|
|
return {
|
|
"status": "success",
|
|
"cards_checked": len(cards),
|
|
"google_synced": google_synced,
|
|
"apple_synced": apple_synced,
|
|
"failed_card_ids": failed_card_ids,
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.error(f"Wallet sync task failed: {e}")
|
|
return {
|
|
"status": "error",
|
|
"error": str(e),
|
|
"failed_card_ids": [],
|
|
}
|
|
finally:
|
|
db.close()
|
|
|
|
|
|
def _sync_card_with_backoff(wallet_service, db, card) -> tuple[bool, int, int]:
|
|
"""Sync a single card with exponential backoff.
|
|
|
|
Returns (success, google_count, apple_count).
|
|
"""
|
|
last_error = None
|
|
|
|
for attempt in range(_MAX_ATTEMPTS):
|
|
try:
|
|
results = wallet_service.sync_card_to_wallets(db, card)
|
|
google = 1 if results.get("google_wallet") else 0
|
|
apple = 1 if results.get("apple_wallet") else 0
|
|
return True, google, apple
|
|
except Exception as e:
|
|
last_error = e
|
|
if attempt < len(_RETRY_DELAYS):
|
|
delay = _RETRY_DELAYS[attempt]
|
|
logger.warning(
|
|
f"Card {card.id} sync failed (attempt {attempt + 1}/"
|
|
f"{_MAX_ATTEMPTS}), retrying in {delay}s: {e}"
|
|
)
|
|
time.sleep(delay)
|
|
|
|
logger.error(
|
|
f"Card {card.id} sync failed after {_MAX_ATTEMPTS} attempts: "
|
|
f"{last_error}"
|
|
)
|
|
return False, 0, 0
|