mirror of
https://github.com/dat515-2025/Group-8.git
synced 2026-03-22 06:57:47 +01:00
Compare commits
17 Commits
c53e314b2a
...
merge/upda
| Author | SHA1 | Date | |
|---|---|---|---|
| ed723d1d13 | |||
| b0dee5e289 | |||
| 640da2ee04 | |||
| ab9aefd140 | |||
|
|
4eaf46e77e | ||
|
|
a30ae4d010 | ||
|
|
ef26e88713 | ||
|
|
2e1dddb4f8 | ||
|
|
25e587cea8 | ||
|
|
3cdefc33fc | ||
|
|
5954e56956 | ||
|
|
8575ef8ff5 | ||
|
|
8b92b9bd18 | ||
|
|
3d26ed6a62 | ||
|
|
67b44539f2 | ||
|
|
ff9cc712db | ||
|
|
1da927dc07 |
4
.github/workflows/deploy-pr.yaml
vendored
4
.github/workflows/deploy-pr.yaml
vendored
@@ -85,6 +85,7 @@ jobs:
|
|||||||
DOMAIN_SCHEME: "${{ needs.get_urls.outputs.backend_url_scheme }}"
|
DOMAIN_SCHEME: "${{ needs.get_urls.outputs.backend_url_scheme }}"
|
||||||
FRONTEND_DOMAIN: "${{ needs.get_urls.outputs.frontend_url }}"
|
FRONTEND_DOMAIN: "${{ needs.get_urls.outputs.frontend_url }}"
|
||||||
FRONTEND_DOMAIN_SCHEME: "${{ needs.get_urls.outputs.frontend_url_scheme }}"
|
FRONTEND_DOMAIN_SCHEME: "${{ needs.get_urls.outputs.frontend_url_scheme }}"
|
||||||
|
UNIRATE_API_KEY: ${{ secrets.UNIRATE_API_KEY }}
|
||||||
run: |
|
run: |
|
||||||
PR=${{ github.event.pull_request.number }}
|
PR=${{ github.event.pull_request.number }}
|
||||||
RELEASE=myapp-pr-$PR
|
RELEASE=myapp-pr-$PR
|
||||||
@@ -102,7 +103,8 @@ jobs:
|
|||||||
--set-string rabbitmq.password="$RABBITMQ_PASSWORD" \
|
--set-string rabbitmq.password="$RABBITMQ_PASSWORD" \
|
||||||
--set-string database.password="$DB_PASSWORD" \
|
--set-string database.password="$DB_PASSWORD" \
|
||||||
--set-string database.encryptionSecret="$PR" \
|
--set-string database.encryptionSecret="$PR" \
|
||||||
--set-string app.name="finance-tracker-pr-$PR"
|
--set-string app.name="finance-tracker-pr-$PR" \
|
||||||
|
--set-string unirate.key="$UNIRATE_API_KEY"
|
||||||
|
|
||||||
- name: Post preview URLs as PR comment
|
- name: Post preview URLs as PR comment
|
||||||
uses: actions/github-script@v7
|
uses: actions/github-script@v7
|
||||||
|
|||||||
4
.github/workflows/deploy-prod.yaml
vendored
4
.github/workflows/deploy-prod.yaml
vendored
@@ -99,6 +99,7 @@ jobs:
|
|||||||
SMTP_USE_TLS: ${{ secrets.SMTP_USE_TLS }}
|
SMTP_USE_TLS: ${{ secrets.SMTP_USE_TLS }}
|
||||||
SMTP_USE_SSL: ${{ secrets.SMTP_USE_SSL }}
|
SMTP_USE_SSL: ${{ secrets.SMTP_USE_SSL }}
|
||||||
SMTP_FROM: ${{ secrets.SMTP_FROM }}
|
SMTP_FROM: ${{ secrets.SMTP_FROM }}
|
||||||
|
UNIRATE_API_KEY: ${{ secrets.UNIRATE_API_KEY }}
|
||||||
run: |
|
run: |
|
||||||
helm upgrade --install myapp ./7project/charts/myapp-chart \
|
helm upgrade --install myapp ./7project/charts/myapp-chart \
|
||||||
-n prod --create-namespace \
|
-n prod --create-namespace \
|
||||||
@@ -125,4 +126,5 @@ jobs:
|
|||||||
--set-string smtp.password="$SMTP_PASSWORD" \
|
--set-string smtp.password="$SMTP_PASSWORD" \
|
||||||
--set-string smtp.tls="$SMTP_USE_TLS" \
|
--set-string smtp.tls="$SMTP_USE_TLS" \
|
||||||
--set-string smtp.ssl="$SMTP_USE_SSL" \
|
--set-string smtp.ssl="$SMTP_USE_SSL" \
|
||||||
--set-string smtp.from="$SMTP_FROM"
|
--set-string smtp.from="$SMTP_FROM" \
|
||||||
|
--set-string unirate.key="$UNIRATE_API_KEY"
|
||||||
3
.github/workflows/run-tests.yml
vendored
3
.github/workflows/run-tests.yml
vendored
@@ -31,6 +31,9 @@ jobs:
|
|||||||
MARIADB_DB: group_project
|
MARIADB_DB: group_project
|
||||||
MARIADB_USER: appuser
|
MARIADB_USER: appuser
|
||||||
MARIADB_PASSWORD: apppass
|
MARIADB_PASSWORD: apppass
|
||||||
|
# Ensure the application uses MariaDB (async) during tests
|
||||||
|
DATABASE_URL: mysql+asyncmy://appuser:apppass@127.0.0.1:3306/group_project
|
||||||
|
DISABLE_METRICS: "1"
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Check out repository code
|
- name: Check out repository code
|
||||||
|
|||||||
66
7project/backend/app/api/exchange_rates.py
Normal file
66
7project/backend/app/api/exchange_rates.py
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
import os
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from fastapi import APIRouter, HTTPException, Query, status
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/exchange-rates", tags=["exchange-rates"])
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("", status_code=status.HTTP_200_OK)
|
||||||
|
async def get_exchange_rates(symbols: str = Query("EUR,USD,NOK", description="Comma-separated currency codes to fetch vs CZK")):
|
||||||
|
"""
|
||||||
|
Fetch exchange rates from UniRate API on the backend and return CZK-per-target rates.
|
||||||
|
- Always requests CZK in addition to requested symbols to compute conversion from USD-base.
|
||||||
|
- Returns a list of {currencyCode, rate} where rate is CZK per 1 unit of the target currency.
|
||||||
|
"""
|
||||||
|
api_key = os.getenv("UNIRATE_API_KEY")
|
||||||
|
if not api_key:
|
||||||
|
raise HTTPException(status_code=500, detail="Server is not configured with UNIRATE_API_KEY")
|
||||||
|
|
||||||
|
# Ensure CZK is included for conversion
|
||||||
|
requested = [s.strip().upper() for s in symbols.split(",") if s.strip()]
|
||||||
|
if "CZK" not in requested:
|
||||||
|
requested.append("CZK")
|
||||||
|
query_symbols = ",".join(sorted(set(requested)))
|
||||||
|
|
||||||
|
url = f"https://unirateapi.com/api/rates?api_key={api_key}&symbols={query_symbols}"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=httpx.Timeout(15.0)) as client:
|
||||||
|
resp = await client.get(url)
|
||||||
|
if resp.status_code != httpx.codes.OK:
|
||||||
|
raise HTTPException(status_code=502, detail=f"Upstream UniRate error: HTTP {resp.status_code}")
|
||||||
|
data = resp.json()
|
||||||
|
except httpx.HTTPError as e:
|
||||||
|
raise HTTPException(status_code=502, detail=f"Failed to contact UniRate: {str(e)}")
|
||||||
|
|
||||||
|
# Validate response structure
|
||||||
|
rates = data.get("rates") if isinstance(data, dict) else None
|
||||||
|
base = data.get("base") if isinstance(data, dict) else None
|
||||||
|
if not rates or base != "USD" or "CZK" not in rates:
|
||||||
|
# Prefer upstream message when available
|
||||||
|
detail = data.get("message") if isinstance(data, dict) else None
|
||||||
|
if not detail and isinstance(data, dict):
|
||||||
|
err = data.get("error")
|
||||||
|
if isinstance(err, dict):
|
||||||
|
detail = err.get("info")
|
||||||
|
raise HTTPException(status_code=502, detail=detail or "Invalid response from UniRate API")
|
||||||
|
|
||||||
|
czk_per_usd = rates["CZK"]
|
||||||
|
|
||||||
|
# Build result excluding CZK itself
|
||||||
|
result = []
|
||||||
|
for code in requested:
|
||||||
|
if code == "CZK":
|
||||||
|
continue
|
||||||
|
target_per_usd = rates.get(code)
|
||||||
|
if target_per_usd in (None, 0):
|
||||||
|
# Skip unavailable or invalid
|
||||||
|
continue
|
||||||
|
czk_per_target = czk_per_usd / target_per_usd
|
||||||
|
result.append({"currencyCode": code, "rate": czk_per_target})
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
116
7project/backend/app/api/mock_bank.py
Normal file
116
7project/backend/app/api/mock_bank.py
Normal file
@@ -0,0 +1,116 @@
|
|||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import List, Optional
|
||||||
|
import random
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from pydantic import BaseModel, Field, conint, confloat, validator
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.services.db import get_async_session
|
||||||
|
from app.services.user_service import current_active_user
|
||||||
|
from app.models.user import User
|
||||||
|
from app.models.transaction import Transaction
|
||||||
|
from app.models.categories import Category
|
||||||
|
from app.schemas.transaction import TransactionRead
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/mock-bank", tags=["mock-bank"])
|
||||||
|
|
||||||
|
|
||||||
|
class GenerateOptions(BaseModel):
|
||||||
|
count: conint(strict=True, gt=0) = Field(default=10, description="Number of transactions to generate")
|
||||||
|
minAmount: confloat(strict=True) = Field(default=-200.0, description="Minimum transaction amount")
|
||||||
|
maxAmount: confloat(strict=True) = Field(default=200.0, description="Maximum transaction amount")
|
||||||
|
startDate: Optional[str] = Field(None, description="Earliest date (YYYY-MM-DD)")
|
||||||
|
endDate: Optional[str] = Field(None, description="Latest date (YYYY-MM-DD)")
|
||||||
|
categoryIds: List[int] = Field(default_factory=list, description="Optional category IDs to assign randomly")
|
||||||
|
|
||||||
|
@validator("maxAmount")
|
||||||
|
def _validate_amounts(cls, v, values):
|
||||||
|
min_amt = values.get("minAmount")
|
||||||
|
if min_amt is not None and v < min_amt:
|
||||||
|
raise ValueError("maxAmount must be greater than or equal to minAmount")
|
||||||
|
return v
|
||||||
|
|
||||||
|
@validator("endDate")
|
||||||
|
def _validate_dates(cls, v, values):
|
||||||
|
sd = values.get("startDate")
|
||||||
|
if v and sd:
|
||||||
|
try:
|
||||||
|
ed = datetime.strptime(v, "%Y-%m-%d").date()
|
||||||
|
st = datetime.strptime(sd, "%Y-%m-%d").date()
|
||||||
|
except ValueError:
|
||||||
|
raise ValueError("Invalid date format, expected YYYY-MM-DD")
|
||||||
|
if ed < st:
|
||||||
|
raise ValueError("endDate must be greater than or equal to startDate")
|
||||||
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
class GeneratedTransaction(BaseModel):
|
||||||
|
amount: float
|
||||||
|
date: str # YYYY-MM-DD
|
||||||
|
category_ids: List[int] = []
|
||||||
|
description: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/generate", response_model=List[GeneratedTransaction])
|
||||||
|
async def generate_mock_transactions(
|
||||||
|
options: GenerateOptions,
|
||||||
|
user: User = Depends(current_active_user),
|
||||||
|
):
|
||||||
|
# Seed randomness per user to make results less erratic across multiple calls in quick succession
|
||||||
|
seed = int(datetime.utcnow().timestamp()) ^ int(user.id)
|
||||||
|
rnd = random.Random(seed)
|
||||||
|
|
||||||
|
# Determine date range
|
||||||
|
if options.startDate:
|
||||||
|
start_date = datetime.strptime(options.startDate, "%Y-%m-%d").date()
|
||||||
|
else:
|
||||||
|
start_date = (datetime.utcnow() - timedelta(days=365)).date()
|
||||||
|
if options.endDate:
|
||||||
|
end_date = datetime.strptime(options.endDate, "%Y-%m-%d").date()
|
||||||
|
else:
|
||||||
|
end_date = datetime.utcnow().date()
|
||||||
|
|
||||||
|
span_days = max(0, (end_date - start_date).days)
|
||||||
|
|
||||||
|
results: List[GeneratedTransaction] = []
|
||||||
|
for _ in range(options.count):
|
||||||
|
amount = round(rnd.uniform(options.minAmount, options.maxAmount), 2)
|
||||||
|
# Pick a random date in the inclusive range
|
||||||
|
rand_day = rnd.randint(0, span_days) if span_days > 0 else 0
|
||||||
|
tx_date = start_date + timedelta(days=rand_day)
|
||||||
|
# Pick category randomly from provided list, or empty
|
||||||
|
if options.categoryIds:
|
||||||
|
cat = [rnd.choice(options.categoryIds)]
|
||||||
|
else:
|
||||||
|
cat = []
|
||||||
|
# Optional simple description for flavor
|
||||||
|
desc = None
|
||||||
|
# Assemble
|
||||||
|
results.append(GeneratedTransaction(
|
||||||
|
amount=amount,
|
||||||
|
date=tx_date.isoformat(),
|
||||||
|
category_ids=cat,
|
||||||
|
description=desc,
|
||||||
|
))
|
||||||
|
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scrape")
|
||||||
|
async def scrape_mock_bank():
|
||||||
|
# 80% of the time: nothing to scrape
|
||||||
|
if random.random() < 0.8:
|
||||||
|
return []
|
||||||
|
|
||||||
|
transactions = []
|
||||||
|
count = random.randint(1, 10)
|
||||||
|
for _ in range(count):
|
||||||
|
transactions.append({
|
||||||
|
"amount": round(random.uniform(-200.0, 200.0), 2),
|
||||||
|
"date": (datetime.utcnow().date() - timedelta(days=random.randint(0, 30))).isoformat(),
|
||||||
|
"description": "Mock transaction",
|
||||||
|
})
|
||||||
|
|
||||||
|
return transactions
|
||||||
@@ -21,6 +21,7 @@ from app.api.auth import router as auth_router
|
|||||||
from app.api.csas import router as csas_router
|
from app.api.csas import router as csas_router
|
||||||
from app.api.categories import router as categories_router
|
from app.api.categories import router as categories_router
|
||||||
from app.api.transactions import router as transactions_router
|
from app.api.transactions import router as transactions_router
|
||||||
|
from app.api.exchange_rates import router as exchange_rates_router
|
||||||
from app.services.user_service import auth_backend, current_active_verified_user, fastapi_users, get_oauth_provider, \
|
from app.services.user_service import auth_backend, current_active_verified_user, fastapi_users, get_oauth_provider, \
|
||||||
UserManager, get_jwt_strategy
|
UserManager, get_jwt_strategy
|
||||||
from app.core.security import extract_bearer_token, is_token_revoked, decode_and_verify_jwt
|
from app.core.security import extract_bearer_token, is_token_revoked, decode_and_verify_jwt
|
||||||
@@ -29,7 +30,8 @@ from app.services.user_service import SECRET
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
import sentry_sdk
|
import sentry_sdk
|
||||||
from fastapi_users.db import SQLAlchemyUserDatabase
|
from fastapi_users.db import SQLAlchemyUserDatabase
|
||||||
from app.core.db import async_session_maker
|
from app.core.db import async_session_maker, engine
|
||||||
|
from app.core.base import Base
|
||||||
|
|
||||||
sentry_sdk.init(
|
sentry_sdk.init(
|
||||||
dsn=os.getenv("SENTRY_DSN"),
|
dsn=os.getenv("SENTRY_DSN"),
|
||||||
@@ -65,6 +67,9 @@ if not os.getenv("PYTEST_RUN_CONFIG"):
|
|||||||
fastApi.include_router(auth_router)
|
fastApi.include_router(auth_router)
|
||||||
fastApi.include_router(categories_router)
|
fastApi.include_router(categories_router)
|
||||||
fastApi.include_router(transactions_router)
|
fastApi.include_router(transactions_router)
|
||||||
|
fastApi.include_router(exchange_rates_router)
|
||||||
|
from app.api.mock_bank import router as mock_bank_router
|
||||||
|
fastApi.include_router(mock_bank_router)
|
||||||
|
|
||||||
for h in list(logging.root.handlers):
|
for h in list(logging.root.handlers):
|
||||||
logging.root.removeHandler(h)
|
logging.root.removeHandler(h)
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
from app.core.base import Base
|
from app.core.base import Base
|
||||||
|
|
||||||
DATABASE_URL = os.getenv("DATABASE_URL")
|
DATABASE_URL = os.getenv("DATABASE_URL")
|
||||||
@@ -23,6 +25,7 @@ host_env = os.getenv("MARIADB_HOST", "localhost")
|
|||||||
ssl_enabled = host_env not in {"localhost", "127.0.0.1"}
|
ssl_enabled = host_env not in {"localhost", "127.0.0.1"}
|
||||||
connect_args = {"ssl": {"ssl": True}} if ssl_enabled else {}
|
connect_args = {"ssl": {"ssl": True}} if ssl_enabled else {}
|
||||||
|
|
||||||
|
# Async engine/session for the async parts of the app
|
||||||
engine = create_async_engine(
|
engine = create_async_engine(
|
||||||
DATABASE_URL,
|
DATABASE_URL,
|
||||||
pool_pre_ping=True,
|
pool_pre_ping=True,
|
||||||
@@ -30,3 +33,13 @@ engine = create_async_engine(
|
|||||||
connect_args=connect_args,
|
connect_args=connect_args,
|
||||||
)
|
)
|
||||||
async_session_maker = async_sessionmaker(engine, expire_on_commit=False)
|
async_session_maker = async_sessionmaker(engine, expire_on_commit=False)
|
||||||
|
|
||||||
|
# Synchronous engine/session for sync utilities (e.g., bank_scraper)
|
||||||
|
SYNC_DATABASE_URL = DATABASE_URL.replace("+asyncmy", "+pymysql")
|
||||||
|
engine_sync = create_engine(
|
||||||
|
SYNC_DATABASE_URL,
|
||||||
|
pool_pre_ping=True,
|
||||||
|
echo=os.getenv("SQL_ECHO", "0") == "1",
|
||||||
|
connect_args=connect_args,
|
||||||
|
)
|
||||||
|
sync_session_maker = sessionmaker(bind=engine_sync, expire_on_commit=False)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from uuid import UUID
|
|||||||
import httpx
|
import httpx
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from app.core.db import async_session_maker
|
from app.core.db import sync_session_maker
|
||||||
from app.models.transaction import Transaction
|
from app.models.transaction import Transaction
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
|
|
||||||
@@ -20,26 +20,78 @@ CERTS = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
async def aload_ceska_sporitelna_transactions(user_id: str) -> None:
|
def load_mock_bank_transactions(user_id: str) -> None:
|
||||||
try:
|
try:
|
||||||
uid = UUID(str(user_id))
|
uid = UUID(str(user_id))
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.error("Invalid user_id provided to bank_scraper (async): %r", user_id)
|
logger.error("Invalid user_id provided to bank_scraper (sync): %r", user_id)
|
||||||
return
|
return
|
||||||
|
|
||||||
await _aload_ceska_sporitelna_transactions(uid)
|
_load_mock_bank_transactions(uid)
|
||||||
|
|
||||||
|
|
||||||
async def aload_all_ceska_sporitelna_transactions() -> None:
|
def load_all_mock_bank_transactions() -> None:
|
||||||
async with async_session_maker() as session:
|
with sync_session_maker() as session:
|
||||||
result = await session.execute(select(User))
|
users = session.execute(select(User)).unique().scalars().all()
|
||||||
users = result.unique().scalars().all()
|
logger.info("[BankScraper] Starting Mock Bank scrape for all users | count=%d", len(users))
|
||||||
|
|
||||||
|
processed = 0
|
||||||
|
for user in users:
|
||||||
|
try:
|
||||||
|
_load_mock_bank_transactions(user.id)
|
||||||
|
processed += 1
|
||||||
|
except Exception:
|
||||||
|
logger.exception("[BankScraper] Error scraping for user id=%s email=%s", user.id,
|
||||||
|
getattr(user, 'email', None))
|
||||||
|
logger.info("[BankScraper] Finished Mock Bank scrape for all users | processed=%d", processed)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_mock_bank_transactions(user_id: UUID) -> None:
|
||||||
|
with sync_session_maker() as session:
|
||||||
|
user: User | None = session.execute(select(User).where(User.id == user_id)).unique().scalar_one_or_none()
|
||||||
|
if user is None:
|
||||||
|
logger.warning("User not found for id=%s", user_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
transactions = []
|
||||||
|
with httpx.Client() as client:
|
||||||
|
response = client.get("http://127.0.0.1:8000/mock-bank/scrape")
|
||||||
|
if response.status_code != httpx.codes.OK:
|
||||||
|
return
|
||||||
|
for transaction in response.json():
|
||||||
|
transactions.append(
|
||||||
|
Transaction(
|
||||||
|
amount=transaction["amount"],
|
||||||
|
description=transaction.get("description"),
|
||||||
|
date=strptime(transaction["date"], "%Y-%m-%d"),
|
||||||
|
user_id=user_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
for transaction in transactions:
|
||||||
|
session.add(transaction)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def load_ceska_sporitelna_transactions(user_id: str) -> None:
|
||||||
|
try:
|
||||||
|
uid = UUID(str(user_id))
|
||||||
|
except Exception:
|
||||||
|
logger.error("Invalid user_id provided to bank_scraper (sync): %r", user_id)
|
||||||
|
return
|
||||||
|
|
||||||
|
_load_ceska_sporitelna_transactions(uid)
|
||||||
|
|
||||||
|
|
||||||
|
def load_all_ceska_sporitelna_transactions() -> None:
|
||||||
|
with sync_session_maker() as session:
|
||||||
|
users = session.execute(select(User)).unique().scalars().all()
|
||||||
logger.info("[BankScraper] Starting CSAS scrape for all users | count=%d", len(users))
|
logger.info("[BankScraper] Starting CSAS scrape for all users | count=%d", len(users))
|
||||||
|
|
||||||
processed = 0
|
processed = 0
|
||||||
for user in users:
|
for user in users:
|
||||||
try:
|
try:
|
||||||
await _aload_ceska_sporitelna_transactions(user.id)
|
_load_ceska_sporitelna_transactions(user.id)
|
||||||
processed += 1
|
processed += 1
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("[BankScraper] Error scraping for user id=%s email=%s", user.id,
|
logger.exception("[BankScraper] Error scraping for user id=%s email=%s", user.id,
|
||||||
@@ -47,10 +99,9 @@ async def aload_all_ceska_sporitelna_transactions() -> None:
|
|||||||
logger.info("[BankScraper] Finished CSAS scrape for all users | processed=%d", processed)
|
logger.info("[BankScraper] Finished CSAS scrape for all users | processed=%d", processed)
|
||||||
|
|
||||||
|
|
||||||
async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
def _load_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
||||||
async with (async_session_maker() as session):
|
with sync_session_maker() as session:
|
||||||
result = await session.execute(select(User).where(User.id == user_id))
|
user: User | None = session.execute(select(User).where(User.id == user_id)).unique().scalar_one_or_none()
|
||||||
user: User = result.unique().scalar_one_or_none()
|
|
||||||
if user is None:
|
if user is None:
|
||||||
logger.warning("User not found for id=%s", user_id)
|
logger.warning("User not found for id=%s", user_id)
|
||||||
return
|
return
|
||||||
@@ -65,8 +116,8 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
|||||||
|
|
||||||
accounts = []
|
accounts = []
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient(cert=CERTS, timeout=httpx.Timeout(20.0)) as client:
|
with httpx.Client(cert=CERTS, timeout=httpx.Timeout(20.0)) as client:
|
||||||
response = await client.get(
|
response = client.get(
|
||||||
"https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts?size=10&page=0&sort=iban&order=desc",
|
"https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts?size=10&page=0&sort=iban&order=desc",
|
||||||
headers={
|
headers={
|
||||||
"Authorization": f"Bearer {cfg['access_token']}",
|
"Authorization": f"Bearer {cfg['access_token']}",
|
||||||
@@ -77,7 +128,7 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
|||||||
if response.status_code != httpx.codes.OK:
|
if response.status_code != httpx.codes.OK:
|
||||||
return
|
return
|
||||||
|
|
||||||
for account in response.json()["accounts"]:
|
for account in response.json().get("accounts", []):
|
||||||
accounts.append(account)
|
accounts.append(account)
|
||||||
|
|
||||||
except (httpx.HTTPError,) as e:
|
except (httpx.HTTPError,) as e:
|
||||||
@@ -85,11 +136,13 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
for account in accounts:
|
for account in accounts:
|
||||||
id = account["id"]
|
acc_id = account.get("id")
|
||||||
|
if not acc_id:
|
||||||
|
continue
|
||||||
|
|
||||||
url = f"https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts/{id}/transactions?size=100&page=0&sort=bookingdate&order=desc"
|
url = f"https://webapi.developers.erstegroup.com/api/csas/sandbox/v4/account-information/my/accounts/{acc_id}/transactions?size=100&page=0&sort=bookingdate&order=desc"
|
||||||
async with httpx.AsyncClient(cert=CERTS) as client:
|
with httpx.Client(cert=CERTS) as client:
|
||||||
response = await client.get(
|
response = client.get(
|
||||||
url,
|
url,
|
||||||
headers={
|
headers={
|
||||||
"Authorization": f"Bearer {cfg['access_token']}",
|
"Authorization": f"Bearer {cfg['access_token']}",
|
||||||
@@ -100,7 +153,7 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
|||||||
if response.status_code != httpx.codes.OK:
|
if response.status_code != httpx.codes.OK:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
transactions = response.json()["transactions"]
|
transactions = response.json().get("transactions", [])
|
||||||
|
|
||||||
for transaction in transactions:
|
for transaction in transactions:
|
||||||
description = transaction.get("entryDetails", {}).get("transactionDetails", {}).get(
|
description = transaction.get("entryDetails", {}).get("transactionDetails", {}).get(
|
||||||
@@ -108,9 +161,12 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
|||||||
date_str = transaction.get("bookingDate", {}).get("date")
|
date_str = transaction.get("bookingDate", {}).get("date")
|
||||||
date = strptime(date_str, "%Y-%m-%d") if date_str else None
|
date = strptime(date_str, "%Y-%m-%d") if date_str else None
|
||||||
amount = transaction.get("amount", {}).get("value")
|
amount = transaction.get("amount", {}).get("value")
|
||||||
if transaction.get("creditDebitIndicator") == "DBIT":
|
if transaction.get("creditDebitIndicator") == "DBIT" and amount is not None:
|
||||||
amount = -abs(amount)
|
amount = -abs(amount)
|
||||||
|
|
||||||
|
if amount is None:
|
||||||
|
continue
|
||||||
|
|
||||||
obj = Transaction(
|
obj = Transaction(
|
||||||
amount=amount,
|
amount=amount,
|
||||||
description=description,
|
description=description,
|
||||||
@@ -118,7 +174,4 @@ async def _aload_ceska_sporitelna_transactions(user_id: UUID) -> None:
|
|||||||
user_id=user_id,
|
user_id=user_id,
|
||||||
)
|
)
|
||||||
session.add(obj)
|
session.add(obj)
|
||||||
await session.commit()
|
session.commit()
|
||||||
|
|
||||||
pass
|
|
||||||
pass
|
|
||||||
|
|||||||
@@ -1,12 +1,10 @@
|
|||||||
import logging
|
import logging
|
||||||
import asyncio
|
|
||||||
import os
|
import os
|
||||||
import smtplib
|
import smtplib
|
||||||
from email.message import EmailMessage
|
from email.message import EmailMessage
|
||||||
|
|
||||||
from celery import shared_task
|
|
||||||
|
|
||||||
import app.services.bank_scraper
|
import app.services.bank_scraper
|
||||||
|
from app.celery_app import celery_app
|
||||||
|
|
||||||
logger = logging.getLogger("celery_tasks")
|
logger = logging.getLogger("celery_tasks")
|
||||||
if not logger.handlers:
|
if not logger.handlers:
|
||||||
@@ -15,73 +13,7 @@ if not logger.handlers:
|
|||||||
logger.setLevel(logging.INFO)
|
logger.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
def run_coro(coro) -> None:
|
@celery_app.task(name="workers.send_email")
|
||||||
"""Run an async coroutine in a fresh event loop without using run_until_complete.
|
|
||||||
Primary strategy runs in a new loop in the current thread. If that fails due to
|
|
||||||
debugger patches (e.g., Bad file descriptor from pydevd_nest_asyncio), fall back
|
|
||||||
to running in a dedicated thread with its own event loop.
|
|
||||||
"""
|
|
||||||
import threading
|
|
||||||
|
|
||||||
def _cleanup_loop(loop):
|
|
||||||
try:
|
|
||||||
pending = [t for t in asyncio.all_tasks(loop) if not t.done()]
|
|
||||||
for t in pending:
|
|
||||||
t.cancel()
|
|
||||||
if pending:
|
|
||||||
loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
finally:
|
|
||||||
try:
|
|
||||||
loop.close()
|
|
||||||
finally:
|
|
||||||
asyncio.set_event_loop(None)
|
|
||||||
|
|
||||||
# First attempt: Run in current thread with a fresh event loop
|
|
||||||
try:
|
|
||||||
loop = asyncio.get_event_loop_policy().new_event_loop()
|
|
||||||
try:
|
|
||||||
asyncio.set_event_loop(loop)
|
|
||||||
task = loop.create_task(coro)
|
|
||||||
task.add_done_callback(lambda _t: loop.stop())
|
|
||||||
loop.run_forever()
|
|
||||||
exc = task.exception()
|
|
||||||
if exc:
|
|
||||||
raise exc
|
|
||||||
return
|
|
||||||
finally:
|
|
||||||
_cleanup_loop(loop)
|
|
||||||
except OSError as e:
|
|
||||||
logger.warning("run_coro primary strategy failed (%s). Falling back to thread runner.", e)
|
|
||||||
except Exception:
|
|
||||||
# For any other unexpected errors, try thread fallback as well
|
|
||||||
logger.exception("run_coro primary strategy raised; attempting thread fallback")
|
|
||||||
|
|
||||||
# Fallback: Run in a dedicated thread with its own event loop
|
|
||||||
error = {"exc": None}
|
|
||||||
|
|
||||||
def _thread_target():
|
|
||||||
loop = asyncio.new_event_loop()
|
|
||||||
try:
|
|
||||||
asyncio.set_event_loop(loop)
|
|
||||||
task = loop.create_task(coro)
|
|
||||||
task.add_done_callback(lambda _t: loop.stop())
|
|
||||||
loop.run_forever()
|
|
||||||
exc = task.exception()
|
|
||||||
if exc:
|
|
||||||
error["exc"] = exc
|
|
||||||
finally:
|
|
||||||
_cleanup_loop(loop)
|
|
||||||
|
|
||||||
th = threading.Thread(target=_thread_target, name="celery-async-runner", daemon=True)
|
|
||||||
th.start()
|
|
||||||
th.join()
|
|
||||||
if error["exc"] is not None:
|
|
||||||
raise error["exc"]
|
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name="workers.send_email")
|
|
||||||
def send_email(to: str, subject: str, body: str) -> None:
|
def send_email(to: str, subject: str, body: str) -> None:
|
||||||
if not (to and subject and body):
|
if not (to and subject and body):
|
||||||
logger.error("Email task missing fields. to=%r subject=%r body_len=%r", to, subject, len(body) if body else 0)
|
logger.error("Email task missing fields. to=%r subject=%r body_len=%r", to, subject, len(body) if body else 0)
|
||||||
@@ -128,20 +60,27 @@ def send_email(to: str, subject: str, body: str) -> None:
|
|||||||
host, port, use_tls, use_ssl)
|
host, port, use_tls, use_ssl)
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name="workers.load_transactions")
|
@celery_app.task(name="workers.load_transactions")
|
||||||
def load_transactions(user_id: str) -> None:
|
def load_transactions(user_id: str) -> None:
|
||||||
if not user_id:
|
if not user_id:
|
||||||
logger.error("Load transactions task missing user_id.")
|
logger.error("Load transactions task missing user_id.")
|
||||||
return
|
return
|
||||||
|
|
||||||
run_coro(app.services.bank_scraper.aload_ceska_sporitelna_transactions(user_id))
|
logger.info("[Celery] Starting load_transactions | user_id=%s", user_id)
|
||||||
|
try:
|
||||||
# Placeholder for real transaction loading logic
|
# Use synchronous bank scraper functions directly, mirroring load_all_transactions
|
||||||
logger.info("[Celery] Transactions loaded for user_id=%s", user_id)
|
app.services.bank_scraper.load_mock_bank_transactions(user_id)
|
||||||
|
app.services.bank_scraper.load_ceska_sporitelna_transactions(user_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to load transactions for user_id=%s", user_id)
|
||||||
|
else:
|
||||||
|
logger.info("[Celery] Finished load_transactions | user_id=%s", user_id)
|
||||||
|
|
||||||
|
|
||||||
@shared_task(name="workers.load_all_transactions")
|
@celery_app.task(name="workers.load_all_transactions")
|
||||||
def load_all_transactions() -> None:
|
def load_all_transactions() -> None:
|
||||||
logger.info("[Celery] Starting load_all_transactions")
|
logger.info("[Celery] Starting load_all_transactions")
|
||||||
run_coro(app.services.bank_scraper.aload_all_ceska_sporitelna_transactions())
|
# Now use synchronous bank scraper functions directly
|
||||||
|
app.services.bank_scraper.load_all_mock_bank_transactions()
|
||||||
|
app.services.bank_scraper.load_all_ceska_sporitelna_transactions()
|
||||||
logger.info("[Celery] Finished load_all_transactions")
|
logger.info("[Celery] Finished load_all_transactions")
|
||||||
|
|||||||
@@ -90,6 +90,11 @@ spec:
|
|||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
name: prod
|
name: prod
|
||||||
key: CSAS_CLIENT_SECRET
|
key: CSAS_CLIENT_SECRET
|
||||||
|
- name: UNIRATE_API_KEY
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: prod
|
||||||
|
key: UNIRATE_API_KEY
|
||||||
- name: DOMAIN
|
- name: DOMAIN
|
||||||
value: {{ required "Set .Values.domain" .Values.domain | quote }}
|
value: {{ required "Set .Values.domain" .Values.domain | quote }}
|
||||||
- name: DOMAIN_SCHEME
|
- name: DOMAIN_SCHEME
|
||||||
|
|||||||
@@ -26,3 +26,4 @@ stringData:
|
|||||||
SMTP_USE_TLS: {{ .Values.smtp.tls | default false | quote }}
|
SMTP_USE_TLS: {{ .Values.smtp.tls | default false | quote }}
|
||||||
SMTP_USE_SSL: {{ .Values.smtp.ssl | default false | quote }}
|
SMTP_USE_SSL: {{ .Values.smtp.ssl | default false | quote }}
|
||||||
SMTP_FROM: {{ .Values.smtp.from | default "" | quote }}
|
SMTP_FROM: {{ .Values.smtp.from | default "" | quote }}
|
||||||
|
UNIRATE_API_KEY: {{ .Values.unirate.key | default "" | quote }}
|
||||||
|
|||||||
@@ -13,6 +13,9 @@ deployment: ""
|
|||||||
domain: ""
|
domain: ""
|
||||||
domain_scheme: ""
|
domain_scheme: ""
|
||||||
|
|
||||||
|
unirate:
|
||||||
|
key: ""
|
||||||
|
|
||||||
frontend_domain: ""
|
frontend_domain: ""
|
||||||
frontend_domain_scheme: ""
|
frontend_domain_scheme: ""
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import BalanceChart from './BalanceChart';
|
|||||||
import ManualManagement from './ManualManagement';
|
import ManualManagement from './ManualManagement';
|
||||||
import CategoryPieChart from './CategoryPieChart';
|
import CategoryPieChart from './CategoryPieChart';
|
||||||
import MockBankModal, { type MockGenerationOptions } from './MockBankModal';
|
import MockBankModal, { type MockGenerationOptions } from './MockBankModal';
|
||||||
import { BACKEND_URL, VITE_UNIRATE_API_KEY } from '../config';
|
import { BACKEND_URL } from '../config';
|
||||||
|
|
||||||
function formatAmount(n: number) {
|
function formatAmount(n: number) {
|
||||||
return new Intl.NumberFormat(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(n);
|
return new Intl.NumberFormat(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(n);
|
||||||
@@ -21,17 +21,6 @@ type RateData = {
|
|||||||
rate: number;
|
rate: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
// The part of the API response structure we need
|
|
||||||
type UnirateApiResponse = {
|
|
||||||
base: string;
|
|
||||||
rates: { [key: string]: number };
|
|
||||||
// We'll also check for error formats just in case
|
|
||||||
message?: string;
|
|
||||||
error?: {
|
|
||||||
info: string;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
// The currencies you want to display
|
// The currencies you want to display
|
||||||
const TARGET_CURRENCIES = ['EUR', 'USD', 'NOK'];
|
const TARGET_CURRENCIES = ['EUR', 'USD', 'NOK'];
|
||||||
|
|
||||||
@@ -45,49 +34,20 @@ function CurrencyRates() {
|
|||||||
setLoading(true);
|
setLoading(true);
|
||||||
setError(null);
|
setError(null);
|
||||||
|
|
||||||
const API_KEY = VITE_UNIRATE_API_KEY;
|
|
||||||
|
|
||||||
// We need to get the CZK rate as well, to use it for conversion
|
|
||||||
const allSymbols = [...TARGET_CURRENCIES, 'CZK'].join(',');
|
|
||||||
|
|
||||||
// We remove the `base` param, as the API seems to force base=USD
|
|
||||||
const UNIRATE_API_URL = `https://unirateapi.com/api/rates?api_key=${API_KEY}&symbols=${allSymbols}`;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch(UNIRATE_API_URL);
|
const base = BACKEND_URL.replace(/\/$/, '');
|
||||||
const data: UnirateApiResponse = await res.json();
|
const url = `${base}/exchange-rates?symbols=${TARGET_CURRENCIES.join(',')}`;
|
||||||
|
const token = localStorage.getItem('token');
|
||||||
// --- THIS IS THE NEW, CORRECTED LOGIC ---
|
const res = await fetch(url, {
|
||||||
|
headers: token ? { Authorization: `Bearer ${token}` } : undefined,
|
||||||
// 1. Check if the 'rates' object exists. If not, it's an error.
|
credentials: 'include',
|
||||||
if (!data.rates) {
|
|
||||||
let errorMessage = data.message || (data.error ? data.error.info : 'Invalid API response');
|
|
||||||
throw new Error(errorMessage || 'Could not load rates');
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Check that we got the base currency (USD) and our conversion currency (CZK)
|
|
||||||
if (data.base !== 'USD' || !data.rates.CZK) {
|
|
||||||
throw new Error('API response is missing required data for conversion (USD or CZK)');
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Get our main conversion factor
|
|
||||||
const czkPerUsd = data.rates.CZK; // e.g., 23.0
|
|
||||||
|
|
||||||
// 4. Calculate the rates for our target currencies
|
|
||||||
const formattedRates = TARGET_CURRENCIES.map(code => {
|
|
||||||
const targetPerUsd = data.rates[code]; // e.g., 0.9 for EUR
|
|
||||||
|
|
||||||
// This calculates: (CZK per USD) / (TARGET per USD) = CZK per TARGET
|
|
||||||
// e.g. (23.0 CZK / 1 USD) / (0.9 EUR / 1 USD) = 25.55 CZK / 1 EUR
|
|
||||||
const rate = czkPerUsd / targetPerUsd;
|
|
||||||
|
|
||||||
return {
|
|
||||||
currencyCode: code,
|
|
||||||
rate: rate,
|
|
||||||
};
|
|
||||||
});
|
});
|
||||||
|
if (!res.ok) {
|
||||||
setRates(formattedRates);
|
const text = await res.text();
|
||||||
|
throw new Error(text || `Failed to load rates (${res.status})`);
|
||||||
|
}
|
||||||
|
const data: RateData[] = await res.json();
|
||||||
|
setRates(data);
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
setError(err.message || 'Could not load rates');
|
setError(err.message || 'Could not load rates');
|
||||||
} finally {
|
} finally {
|
||||||
@@ -235,44 +195,50 @@ export default function Dashboard({ onLogout }: { onLogout: () => void }) {
|
|||||||
setIsGenerating(true);
|
setIsGenerating(true);
|
||||||
setMockModalOpen(false);
|
setMockModalOpen(false);
|
||||||
|
|
||||||
const { count, minAmount, maxAmount, startDate, endDate, categoryIds } = options;
|
try {
|
||||||
const newTransactions: Transaction[] = [];
|
const base = BACKEND_URL.replace(/\/$/, '');
|
||||||
|
const url = `${base}/mock-bank/generate`;
|
||||||
const startDateTime = new Date(startDate).getTime();
|
const token = localStorage.getItem('token');
|
||||||
const endDateTime = new Date(endDate).getTime();
|
const res = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
for (let i = 0; i < count; i++) {
|
headers: {
|
||||||
// Generate random data based on user input
|
'Content-Type': 'application/json',
|
||||||
const amount = parseFloat((Math.random() * (maxAmount - minAmount) + minAmount).toFixed(2));
|
...(token ? { Authorization: `Bearer ${token}` } : {}),
|
||||||
|
},
|
||||||
const randomTime = Math.random() * (endDateTime - startDateTime) + startDateTime;
|
credentials: 'include',
|
||||||
const date = new Date(randomTime);
|
body: JSON.stringify(options),
|
||||||
const dateString = date.toISOString().split('T')[0];
|
});
|
||||||
|
if (!res.ok) {
|
||||||
const randomCategory = categoryIds.length > 0
|
const text = await res.text();
|
||||||
? [categoryIds[Math.floor(Math.random() * categoryIds.length)]]
|
throw new Error(text || `Failed to generate mock transactions (${res.status})`);
|
||||||
: [];
|
|
||||||
|
|
||||||
const payload = {
|
|
||||||
amount,
|
|
||||||
date: dateString,
|
|
||||||
category_ids: randomCategory,
|
|
||||||
};
|
|
||||||
|
|
||||||
try {
|
|
||||||
const created = await createTransaction(payload);
|
|
||||||
newTransactions.push(created);
|
|
||||||
} catch (err) {
|
|
||||||
console.error("Failed to create mock transaction:", err);
|
|
||||||
alert('An error occurred while generating transactions. Check the console.');
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
const generated: Array<{ amount: number; date: string; category_ids: number[]; description?: string | null }>
|
||||||
|
= await res.json();
|
||||||
|
|
||||||
|
const newTransactions: Transaction[] = [];
|
||||||
|
for (const g of generated) {
|
||||||
|
try {
|
||||||
|
const created = await createTransaction({
|
||||||
|
amount: g.amount,
|
||||||
|
date: g.date,
|
||||||
|
category_ids: g.category_ids || [],
|
||||||
|
description: g.description || undefined,
|
||||||
|
});
|
||||||
|
newTransactions.push(created);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to create mock transaction:', err);
|
||||||
|
// continue creating others
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
alert(`${newTransactions.length} mock transactions were successfully generated!`);
|
||||||
|
} catch (err: any) {
|
||||||
|
console.error(err);
|
||||||
|
alert(err?.message || 'Failed to generate mock transactions');
|
||||||
|
} finally {
|
||||||
|
setIsGenerating(false);
|
||||||
|
await loadAll();
|
||||||
}
|
}
|
||||||
|
|
||||||
setIsGenerating(false);
|
|
||||||
alert(`${newTransactions.length} mock transactions were successfully generated!`);
|
|
||||||
|
|
||||||
await loadAll();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
useEffect(() => { loadAll(); }, [startDate, endDate]);
|
useEffect(() => { loadAll(); }, [startDate, endDate]);
|
||||||
|
|||||||
Reference in New Issue
Block a user