Compare commits

..

14 Commits

Author SHA1 Message Date
c53e314b2a fix(tests): set pytest env
Some checks are pending
Deploy Prod / Run Python Tests (push) Waiting to run
Deploy Prod / Build and push image (reusable) (push) Waiting to run
Deploy Prod / Generate Production URLs (push) Waiting to run
Deploy Prod / Frontend - Build and Deploy to Cloudflare Pages (prod) (push) Blocked by required conditions
Deploy Prod / Helm upgrade/install (prod) (push) Blocked by required conditions
2025-11-11 15:36:12 +01:00
c0bc44622f fix(tests): set pytest env 2025-11-11 15:34:11 +01:00
3d31ff4631 fix(tests): do not include prometheus in test env 2025-11-11 15:29:47 +01:00
dc7ce9e6a1 Merge pull request #49 from dat515-2025/merge/email_sender
feat(infrastructure): add email sender
2025-11-11 15:04:40 +01:00
188cdf5727 Update .github/workflows/deploy-prod.yaml
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-11 15:03:53 +01:00
4cf0d2a981 Update 7project/charts/myapp-chart/templates/prod.yaml
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-11 15:03:43 +01:00
9986cce8f9 Update 7project/charts/myapp-chart/values.yaml
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2025-11-11 15:03:36 +01:00
b3b5717e9e feat(infrastructure): add email sender 2025-11-11 14:59:28 +01:00
537d050080 feat(deployment): add 404 for public access 2025-11-11 14:16:08 +01:00
1e4f342176 feat(deployment): add cron support 2025-11-11 14:07:33 +01:00
c62e0adcf3 feat(deployment): add cron support 2025-11-11 14:03:31 +01:00
24d86abfc4 feat(deployment): add cron support 2025-11-11 13:58:36 +01:00
21305f18e2 feat(deployment): add cron support 2025-11-11 13:54:45 +01:00
e708f7b18b feat(deployment): add cron support 2025-11-11 13:52:17 +01:00
9 changed files with 164 additions and 26 deletions

View File

@@ -92,6 +92,13 @@ jobs:
CSAS_CLIENT_ID: ${{ secrets.CSAS_CLIENT_ID }} CSAS_CLIENT_ID: ${{ secrets.CSAS_CLIENT_ID }}
CSAS_CLIENT_SECRET: ${{ secrets.CSAS_CLIENT_SECRET }} CSAS_CLIENT_SECRET: ${{ secrets.CSAS_CLIENT_SECRET }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }} SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
SMTP_HOST: ${{ secrets.SMTP_HOST }}
SMTP_PORT: ${{ secrets.SMTP_PORT }}
SMTP_USERNAME: ${{ secrets.SMTP_USERNAME }}
SMTP_PASSWORD: ${{ secrets.SMTP_PASSWORD }}
SMTP_USE_TLS: ${{ secrets.SMTP_USE_TLS }}
SMTP_USE_SSL: ${{ secrets.SMTP_USE_SSL }}
SMTP_FROM: ${{ secrets.SMTP_FROM }}
run: | run: |
helm upgrade --install myapp ./7project/charts/myapp-chart \ helm upgrade --install myapp ./7project/charts/myapp-chart \
-n prod --create-namespace \ -n prod --create-namespace \
@@ -111,4 +118,11 @@ jobs:
--set-string oauth.csas.clientId="$CSAS_CLIENT_ID" \ --set-string oauth.csas.clientId="$CSAS_CLIENT_ID" \
--set-string oauth.csas.clientSecret="$CSAS_CLIENT_SECRET" \ --set-string oauth.csas.clientSecret="$CSAS_CLIENT_SECRET" \
--set-string sentry_dsn="$SENTRY_DSN" \ --set-string sentry_dsn="$SENTRY_DSN" \
--set-string database.encryptionSecret="${{ secrets.PROD_DB_ENCRYPTION_KEY }}" --set-string database.encryptionSecret="${{ secrets.PROD_DB_ENCRYPTION_KEY }}" \
--set-string smtp.host="$SMTP_HOST" \
--set smtp.port="$SMTP_PORT" \
--set-string smtp.username="$SMTP_USERNAME" \
--set-string smtp.password="$SMTP_PASSWORD" \
--set-string smtp.tls="$SMTP_USE_TLS" \
--set-string smtp.ssl="$SMTP_USE_SSL" \
--set-string smtp.from="$SMTP_FROM"

View File

@@ -57,5 +57,7 @@ jobs:
working-directory: ./7project/backend working-directory: ./7project/backend
- name: Run tests with pytest - name: Run tests with pytest
env:
PYTEST_RUN_CONFIG: "True"
run: pytest run: pytest
working-directory: ./7project/backend working-directory: ./7project/backend

View File

@@ -1,10 +1,11 @@
import json
import logging import logging
import os import os
import sys import sys
from datetime import datetime from datetime import datetime
from pythonjsonlogger import jsonlogger from pythonjsonlogger import jsonlogger
from fastapi import Depends, FastAPI from fastapi import Depends, FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.cors import CORSMiddleware
from prometheus_fastapi_instrumentator import Instrumentator, metrics from prometheus_fastapi_instrumentator import Instrumentator, metrics
from starlette.requests import Request from starlette.requests import Request
@@ -50,22 +51,21 @@ fastApi.add_middleware(
allow_headers=["*"], allow_headers=["*"],
) )
prometheus = Instrumentator().instrument(fastApi)
# Register custom metrics if not os.getenv("PYTEST_RUN_CONFIG"):
prometheus.add(number_of_users()).add(number_of_transactions()) prometheus = Instrumentator().instrument(fastApi)
# Register custom metrics
prometheus.expose( prometheus.add(number_of_users()).add(number_of_transactions())
fastApi, prometheus.expose(
endpoint="/metrics", fastApi,
include_in_schema=True, endpoint="/metrics",
) include_in_schema=True,
)
fastApi.include_router(auth_router) fastApi.include_router(auth_router)
fastApi.include_router(categories_router) fastApi.include_router(categories_router)
fastApi.include_router(transactions_router) fastApi.include_router(transactions_router)
for h in list(logging.root.handlers): for h in list(logging.root.handlers):
logging.root.removeHandler(h) logging.root.removeHandler(h)
@@ -78,7 +78,6 @@ _log_handler.setFormatter(_formatter)
logging.root.setLevel(logging.INFO) logging.root.setLevel(logging.INFO)
logging.root.addHandler(_log_handler) logging.root.addHandler(_log_handler)
for _name in ("uvicorn", "uvicorn.error", "uvicorn.access"): for _name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
_logger = logging.getLogger(_name) _logger = logging.getLogger(_name)
_logger.handlers = [_log_handler] _logger.handlers = [_log_handler]
@@ -161,16 +160,12 @@ async def authenticated_route(user: User = Depends(current_active_verified_user)
return {"message": f"Hello {user.email}!"} return {"message": f"Hello {user.email}!"}
@fastApi.get("/debug/scrape/csas/all", tags=["debug"]) @fastApi.get("/_cron", include_in_schema=False)
async def debug_scrape_csas_all(): async def handle_cron(request: Request):
logging.info("[Debug] Queueing CSAS scrape for all users via HTTP endpoint (Celery)") # endpoint accessed by Clodflare => return 404
if request.headers.get("cf-connecting-ip"):
raise HTTPException(status_code=404)
logging.info("[Cron] Triggering scheduled tasks via HTTP endpoint")
task = load_all_transactions.delay() task = load_all_transactions.delay()
return {"status": "queued", "action": "csas_scrape_all", "task_id": getattr(task, 'id', None)} return {"status": "queued", "action": "csas_scrape_all", "task_id": getattr(task, 'id', None)}
@fastApi.post("/debug/scrape/csas/{user_id}", tags=["debug"])
async def debug_scrape_csas_user(user_id: str, user: User = Depends(current_active_verified_user)):
logging.info("[Debug] Queueing CSAS scrape for single user via HTTP endpoint (Celery) | user_id=%s", user_id)
task = load_transactions.delay(user_id)
return {"status": "queued", "action": "csas_scrape_single", "user_id": user_id,
"task_id": getattr(task, 'id', None)}

View File

@@ -1,5 +1,8 @@
import logging import logging
import asyncio import asyncio
import os
import smtplib
from email.message import EmailMessage
from celery import shared_task from celery import shared_task
@@ -84,8 +87,45 @@ def send_email(to: str, subject: str, body: str) -> None:
logger.error("Email task missing fields. to=%r subject=%r body_len=%r", to, subject, len(body) if body else 0) logger.error("Email task missing fields. to=%r subject=%r body_len=%r", to, subject, len(body) if body else 0)
return return
# Placeholder for real email sending logic host = os.getenv("SMTP_HOST")
logger.info("[Celery] Email sent | to=%s | subject=%s | body_len=%d", to, subject, len(body)) if not host:
logger.error("SMTP_HOST is not configured; cannot send email")
return
# Configuration
port = int(os.getenv("SMTP_PORT", "25"))
username = os.getenv("SMTP_USERNAME")
password = os.getenv("SMTP_PASSWORD")
use_tls = os.getenv("SMTP_USE_TLS", "0").lower() in {"1", "true", "yes"}
use_ssl = os.getenv("SMTP_USE_SSL", "0").lower() in {"1", "true", "yes"}
timeout = int(os.getenv("SMTP_TIMEOUT", "10"))
mail_from = os.getenv("SMTP_FROM") or username or "noreply@localhost"
# Build message
msg = EmailMessage()
msg["To"] = to
msg["From"] = mail_from
msg["Subject"] = subject
msg.set_content(body)
try:
if use_ssl:
with smtplib.SMTP_SSL(host=host, port=port, timeout=timeout) as smtp:
if username and password:
smtp.login(username, password)
smtp.send_message(msg)
else:
with smtplib.SMTP(host=host, port=port, timeout=timeout) as smtp:
# STARTTLS if requested
if use_tls:
smtp.starttls()
if username and password:
smtp.login(username, password)
smtp.send_message(msg)
logger.info("[Celery] Email sent | to=%s | subject=%s | body_len=%d", to, subject, len(body))
except Exception:
logger.exception("Failed to send email via SMTP to=%s subject=%s host=%s port=%s tls=%s ssl=%s", to, subject,
host, port, use_tls, use_ssl)
@shared_task(name="workers.load_transactions") @shared_task(name="workers.load_transactions")

View File

@@ -0,0 +1,25 @@
{{- if .Values.cron.enabled }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: cronjob
spec:
schedule: {{ .Values.cron.schedule | quote }}
concurrencyPolicy: {{ .Values.cron.concurrencyPolicy | quote }}
jobTemplate:
spec:
template:
spec:
containers:
- name: cronjob
image: curlimages/curl:latest
imagePullPolicy: IfNotPresent
args:
- -sS
- -o
- /dev/null
- -w
- "%{http_code}"
- {{ printf "%s://%s.%s.svc.cluster.local%s" .Values.cron.scheme .Values.app.name .Release.Namespace .Values.cron.endpoint | quote }}
restartPolicy: OnFailure
{{- end }}

View File

@@ -19,3 +19,10 @@ stringData:
RABBITMQ_USERNAME: {{ .Values.rabbitmq.username | quote }} RABBITMQ_USERNAME: {{ .Values.rabbitmq.username | quote }}
SENTRY_DSN: {{ .Values.sentry_dsn | quote }} SENTRY_DSN: {{ .Values.sentry_dsn | quote }}
DB_ENCRYPTION_KEY: {{ required "Set .Values.database.encryptionSecret" .Values.database.encryptionSecret | quote }} DB_ENCRYPTION_KEY: {{ required "Set .Values.database.encryptionSecret" .Values.database.encryptionSecret | quote }}
SMTP_HOST: {{ .Values.smtp.host | default "" | quote }}
SMTP_PORT: {{ .Values.smtp.port | default 587 | quote }}
SMTP_USERNAME: {{ .Values.smtp.username | default "" | quote }}
SMTP_PASSWORD: {{ .Values.smtp.password | default "" | quote }}
SMTP_USE_TLS: {{ .Values.smtp.tls | default false | quote }}
SMTP_USE_SSL: {{ .Values.smtp.ssl | default false | quote }}
SMTP_FROM: {{ .Values.smtp.from | default "" | quote }}

View File

@@ -85,3 +85,38 @@ spec:
secretKeyRef: secretKeyRef:
name: prod name: prod
key: DB_ENCRYPTION_KEY key: DB_ENCRYPTION_KEY
- name: SMTP_HOST
valueFrom:
secretKeyRef:
name: prod
key: SMTP_HOST
- name: SMTP_PORT
valueFrom:
secretKeyRef:
name: prod
key: SMTP_PORT
- name: SMTP_USERNAME
valueFrom:
secretKeyRef:
name: prod
key: SMTP_USERNAME
- name: SMTP_PASSWORD
valueFrom:
secretKeyRef:
name: prod
key: SMTP_PASSWORD
- name: SMTP_USE_TLS
valueFrom:
secretKeyRef:
name: prod
key: SMTP_USE_TLS
- name: SMTP_USE_SSL
valueFrom:
secretKeyRef:
name: prod
key: SMTP_USE_SSL
- name: SMTP_FROM
valueFrom:
secretKeyRef:
name: prod
key: SMTP_FROM

View File

@@ -5,3 +5,6 @@ app:
worker: worker:
replicas: 3 replicas: 3
cron:
enabled: true

View File

@@ -35,6 +35,23 @@ worker:
# Queue name for Celery worker and for CRD Queue # Queue name for Celery worker and for CRD Queue
mailQueueName: "mail_queue" mailQueueName: "mail_queue"
cron:
enabled: false
schedule: "*/5 * * * *" # every 5 minutes
scheme: "http"
endpoint: "/_cron"
concurrencyPolicy: "Forbid"
smtp:
host:
port: 587
username: ""
password: ""
tls: false
ssl: false
from: ""
service: service:
port: 80 port: 80