feat(workers): update workers

7project/backend/app/services/bank_scraper.py

@@ -7,7 +7,7 @@ from uuid import UUID
 import httpx
 from sqlalchemy import select
 
-from app.core.db import async_session_maker
+from app.core.db import sync_session_maker
 from app.models.transaction import Transaction
 from app.models.user import User
 
@@ -20,26 +20,78 @@ CERTS = (
 )
 
 
-async def aload_ceska_sporitelna_transactions(user_id: str) -> None:
+def load_mock_bank_transactions(user_id: str) -> None:
     try:
         uid = UUID(str(user_id))
     except Exception:
-        logger.error("Invalid user_id provided to bank_scraper (async): %r", user_id)
+        logger.error("Invalid user_id provided to bank_scraper (sync): %r", user_id)
         return
 
-    await _aload_ceska_sporitelna_transactions(uid)
+    _load_mock_bank_transactions(uid)
 
 
-async def aload_all_ceska_sporitelna_transactions() -> None:
-    async with async_session_maker() as session:
-        result = await session.execute(select(User))
-        users = result.unique().scalars().all()
+def load_all_mock_bank_transactions() -> None:
+    with sync_session_maker() as session:
+        users = session.execute(select(User)).unique().scalars().all()
+        logger.info("[BankScraper] Starting Mock Bank scrape for all users | count=%d", len(users))
+
+    processed = 0
+    for user in users:
+        try:
+            _load_mock_bank_transactions(user.id)
+            processed += 1
+        except Exception:
+            logger.exception("[BankScraper] Error scraping for user id=%s email=%s", user.id,
+                             getattr(user, 'email', None))
+    logger.info("[BankScraper] Finished Mock Bank scrape for all users | processed=%d", processed)
+
+
+def _load_mock_bank_transactions(user_id: UUID) -> None:
+    with sync_session_maker() as session:
+        user: User | None = session.execute(select(User).where(User.id == user_id)).unique().scalar_one_or_none()
+        if user is None:
+            logger.warning("User not found for id=%s", user_id)
+            return
+
+        transactions = []
+        with httpx.Client() as client:
+            response = client.get("http://127.0.0.1:8000/mock-bank/scrape")
+            if response.status_code != httpx.codes.OK:
+                return
+            for transaction in response.json():
+                transactions.append(
+                    Transaction(
+                        amount=transaction["amount"],
+                        description=transaction.get("description"),
+                        date=strptime(transaction["date"], "%Y-%m-%d"),
+                        user_id=user_id,
+                    )
+                )
+
+        for transaction in transactions:
+            session.add(transaction)
+            session.commit()
+
+
+def load_ceska_sporitelna_transactions(user_id: str) -> None:
+    try:
+        uid = UUID(str(user_id))
+    except Exception:
+        logger.error("Invalid user_id provided to bank_scraper (sync): %r", user_id)
+        return
+
+    _load_ceska_sporitelna_transactions(uid)
+
+
+def load_all_ceska_sporitelna_transactions() -> None:
+    with sync_session_maker() as session:
+        users = session.execute(select(User)).unique().scalars().all()
         logger.info("[BankScraper] Starting CSAS scrape for all users | count=%d", len(users))
 
     processed = 0
     for user in users:
         try:
-            await _aload_ceska_sporitelna_transactions(user.id)
+            _load_ceska_sporitelna_transactions(user.id)
             processed += 1
         except Exception:
             logger.exception("[BankScraper] Error scraping for user id=%s email=%s", user.id,
@@ -47,10 +99,9 @@ async def aload_all_ceska_sporitelna_transactions() -> None:
     logger.info("[BankScraper] Finished CSAS scrape for all users | processed=%d", processed)
 
 
-async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
-    async with (async_session_maker() as session):
-        result = await session.execute(select(User).where(User.id == user_id))
-        user: User = result.unique().scalar_one_or_none()
+def _load_ceska_sporitelna_transactions(user_id: UUID) -> None:
+    with sync_session_maker() as session:
+        user: User | None = session.execute(select(User).where(User.id == user_id)).unique().scalar_one_or_none()
         if user is None:
             logger.warning("User not found for id=%s", user_id)
             return
@@ -65,8 +116,8 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
 
         accounts = []
         try:
-            async with httpx.AsyncClient(cert=CERTS, timeout=httpx.Timeout(20.0)) as client:
-                response = await client.get(
+            with httpx.Client(cert=CERTS, timeout=httpx.Timeout(20.0)) as client:
+                response = client.get(
                     "https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts?size=10&page=0&sort=iban&order=desc",
                     headers={
                         "Authorization": f"Bearer {cfg['access_token']}",
@@ -77,7 +128,7 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
                 if response.status_code != httpx.codes.OK:
                     return
 
-                for account in response.json()["accounts"]:
+                for account in response.json().get("accounts", []):
                     accounts.append(account)
 
         except (httpx.HTTPError,) as e:
@@ -85,11 +136,13 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
             return
 
         for account in accounts:
-            id = account["id"]
+            acc_id = account.get("id")
+            if not acc_id:
+                continue
 
-            url = f"https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts/{id}/transactions?size=100&page=0&sort=bookingdate&order=desc"
-            async with httpx.AsyncClient(cert=CERTS) as client:
-                response = await client.get(
+            url = f"https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts/{acc_id}/transactions?size=100&page=0&sort=bookingdate&order=desc"
+            with httpx.Client(cert=CERTS) as client:
+                response = client.get(
                     url,
                     headers={
                         "Authorization": f"Bearer {cfg['access_token']}",
@@ -100,7 +153,7 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
                 if response.status_code != httpx.codes.OK:
                     continue
 
-                transactions = response.json()["transactions"]
+                transactions = response.json().get("transactions", [])
 
             for transaction in transactions:
                 description = transaction.get("entryDetails", {}).get("transactionDetails", {}).get(
@@ -108,9 +161,12 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
                 date_str = transaction.get("bookingDate", {}).get("date")
                 date = strptime(date_str, "%Y-%m-%d") if date_str else None
                 amount = transaction.get("amount", {}).get("value")
-                if transaction.get("creditDebitIndicator") == "DBIT":
+                if transaction.get("creditDebitIndicator") == "DBIT" and amount is not None:
                     amount = -abs(amount)
 
+                if amount is None:
+                    continue
+
                 obj = Transaction(
                     amount=amount,
                     description=description,
@@ -118,7 +174,4 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
                     user_id=user_id,
                 )
                 session.add(obj)
-                await session.commit()
-
-                pass
-            pass
+                session.commit()