Compare commits

...

10 Commits

Author SHA1 Message Date
ribardej
f0c28ba9e1 feat(docs): codebase refactor - added src directory 2025-11-13 13:55:40 +01:00
ribardej
b560c07d62 feat(docs): codebase refactor - added src directory 2025-11-13 13:52:27 +01:00
ribardej
f0b1452e30 feat(docs): codebase refactor - added src directory 2025-11-13 13:45:41 +01:00
6effb2793a update report 2025-11-13 13:24:24 +01:00
ribardej
ba7798259c feat(docs): report.md update 2025-11-13 12:36:05 +01:00
deb67f421e Create README.md 2025-11-13 12:24:29 +01:00
74557eeea8 update report 2025-11-13 12:06:15 +01:00
2e0619d03f update report 2025-11-13 11:52:07 +01:00
31add42d6d update report 2025-11-13 11:13:11 +01:00
4de79169a2 update report 2025-11-13 11:11:16 +01:00
165 changed files with 119 additions and 100 deletions

View File

@@ -15,7 +15,7 @@ on:
context: context:
description: "Docker build context path" description: "Docker build context path"
required: false required: false
default: "7project/backend" default: "7project/src/backend"
type: string type: string
pr_number: pr_number:
description: "PR number (required when mode=pr)" description: "PR number (required when mode=pr)"

View File

@@ -21,7 +21,7 @@ jobs:
with: with:
mode: pr mode: pr
image_repo: lukastrkan/cc-app-demo image_repo: lukastrkan/cc-app-demo
context: 7project/backend context: 7project/src/backend
pr_number: ${{ github.event.pull_request.number }} pr_number: ${{ github.event.pull_request.number }}
secrets: inherit secrets: inherit
@@ -90,9 +90,9 @@ jobs:
PR=${{ github.event.pull_request.number }} PR=${{ github.event.pull_request.number }}
RELEASE=myapp-pr-$PR RELEASE=myapp-pr-$PR
NAMESPACE=pr-$PR NAMESPACE=pr-$PR
helm upgrade --install "$RELEASE" ./7project/charts/myapp-chart \ helm upgrade --install "$RELEASE" ./7project/src/charts/myapp-chart \
-n "$NAMESPACE" --create-namespace \ -n "$NAMESPACE" --create-namespace \
-f 7project/charts/myapp-chart/values-dev.yaml \ -f 7project/src/charts/myapp-chart/values-dev.yaml \
--set prNumber="$PR" \ --set prNumber="$PR" \
--set deployment="pr-$PR" \ --set deployment="pr-$PR" \
--set domain="$DOMAIN" \ --set domain="$DOMAIN" \

View File

@@ -4,9 +4,9 @@ on:
push: push:
branches: [ "main" ] branches: [ "main" ]
paths: paths:
- 7project/backend/** - ../../7project/src/backend/**
- 7project/frontend/** - ../../7project/src/frontend/**
- 7project/charts/myapp-chart/** - ../../7project/src/charts/myapp-chart/**
- .github/workflows/deploy-prod.yaml - .github/workflows/deploy-prod.yaml
- .github/workflows/build-image.yaml - .github/workflows/build-image.yaml
- .github/workflows/frontend-pages.yml - .github/workflows/frontend-pages.yml
@@ -32,7 +32,7 @@ jobs:
with: with:
mode: prod mode: prod
image_repo: lukastrkan/cc-app-demo image_repo: lukastrkan/cc-app-demo
context: 7project/backend context: 7project/src/backend
secrets: inherit secrets: inherit
get_urls: get_urls:
@@ -103,9 +103,9 @@ jobs:
SMTP_FROM: ${{ secrets.SMTP_FROM }} SMTP_FROM: ${{ secrets.SMTP_FROM }}
UNIRATE_API_KEY: ${{ secrets.UNIRATE_API_KEY }} UNIRATE_API_KEY: ${{ secrets.UNIRATE_API_KEY }}
run: | run: |
helm upgrade --install myapp ./7project/charts/myapp-chart \ helm upgrade --install myapp ./7project/src/charts/myapp-chart \
-n prod --create-namespace \ -n prod --create-namespace \
-f 7project/charts/myapp-chart/values-prod.yaml \ -f 7project/src/charts/myapp-chart/values-prod.yaml \
--set deployment="prod" \ --set deployment="prod" \
--set domain="$DOMAIN" \ --set domain="$DOMAIN" \
--set domain_scheme="$DOMAIN_SCHEME" \ --set domain_scheme="$DOMAIN_SCHEME" \

View File

@@ -35,7 +35,7 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
defaults: defaults:
run: run:
working-directory: 7project/frontend working-directory: 7project/src/frontend
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -45,7 +45,7 @@ jobs:
with: with:
node-version: '20' node-version: '20'
cache: 'npm' cache: 'npm'
cache-dependency-path: 7project/frontend/package-lock.json cache-dependency-path: 7project/src/frontend/package-lock.json
- name: Install dependencies - name: Install dependencies
run: npm ci run: npm ci
@@ -61,7 +61,7 @@ jobs:
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
name: frontend-dist name: frontend-dist
path: 7project/frontend/dist path: 7project/src/frontend/dist
deploy: deploy:
name: Deploy to Cloudflare Pages name: Deploy to Cloudflare Pages

View File

@@ -46,21 +46,21 @@ jobs:
- name: Add test dependencies to requirements - name: Add test dependencies to requirements
run: | run: |
echo "pytest==8.4.2" >> ./7project/backend/requirements.txt echo "pytest==8.4.2" >> ./7project/src/backend/requirements.txt
echo "pytest-asyncio==1.2.0" >> ./7project/backend/requirements.txt echo "pytest-asyncio==1.2.0" >> ./7project/src/backend/requirements.txt
- name: Install dependencies - name: Install dependencies
run: | run: |
python -m pip install --upgrade pip python -m pip install --upgrade pip
pip install -r ./7project/backend/requirements.txt pip install -r ./7project/src/backend/requirements.txt
- name: Run Alembic migrations - name: Run Alembic migrations
run: | run: |
alembic upgrade head alembic upgrade head
working-directory: ./7project/backend working-directory: ./7project/src/backend
- name: Run tests with pytest - name: Run tests with pytest
env: env:
PYTEST_RUN_CONFIG: "True" PYTEST_RUN_CONFIG: "True"
run: pytest run: pytest
working-directory: ./7project/backend working-directory: ./7project/src/backend

8
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

16
7project/.gitignore vendored
View File

@@ -1,8 +1,8 @@
/tofu/controlplane.yaml /src/tofu/controlplane.yaml
/tofu/kubeconfig /src/tofu/kubeconfig
/tofu/talosconfig /src/tofu/talosconfig
/tofu/terraform.tfstate /src/tofu/terraform.tfstate
/tofu/terraform.tfstate.backup /src/tofu/terraform.tfstate.backup
/tofu/worker.yaml /src/tofu/worker.yaml
/tofu/.terraform.lock.hcl /src/tofu/.terraform.lock.hcl
/tofu/.terraform/ /src/tofu/.terraform/

8
7project/.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

View File

@@ -22,17 +22,17 @@ filtering and visualization. New transactions are automatically fetched in the b
## Architecture Overview ## Architecture Overview
Our system is a fullstack web application composed of a React frontend, a FastAPI backend, Our system is a fullstack web application composed of a React frontend, a FastAPI backend,
a MariaDB database with Maxscale, and asynchronous background workers powered by Celery with RabbitMQ. a asynchronousMariaDB database with Maxscale, and background workers powered by Celery with RabbitMQ.
Redis is available for caching/kv and may be used by Celery as a result backend. The backend The backend exposes REST endpoints for authentication (email/password and OAuth), users, categories,
exposes REST endpoints for authentication (email/password and OAuth), users, categories, transactions, exchange rates and bank APIs. Infrastructure for Kubernetes is managed via Terraform/OpenTofu and
transactions, exchange rates and bank APIs. A thin controller layer (FastAPI routers) lives under app/api. the application is packaged via a Helm chart. This all is deployed on private TalosOS cluster running on Proxmox VE with
Infrastructure for Kubernetes is provided via OpenTofu (Terraformcompatible) modules and CI/CD and with public access over Cloudflare tunnels. Static files for frontend are served via Cloudflare pages.
the application is packaged via a Helm chart. Other services deployed in the cluster includes Longhorn for persistent storage, Prometheus with Grafana for monitoring.
### High-Level Architecture ### High-Level Architecture
```mermaid ```mermaid
flowchart LR flowchart TB
n3(("User")) <--> client["Frontend"] n3(("User")) <--> client["Frontend"]
proc_queue["Message Queue"] --> proc_queue_worker["Worker Service"] proc_queue["Message Queue"] --> proc_queue_worker["Worker Service"]
proc_queue_worker -- SMTP --> ext_mail[("Email Service")] proc_queue_worker -- SMTP --> ext_mail[("Email Service")]
@@ -50,11 +50,9 @@ The workflow works in the following way:
- Client connects to the frontend. After login, frontend automatically fetches the stored transactions from - Client connects to the frontend. After login, frontend automatically fetches the stored transactions from
the database via the backend API and currency rates from UniRate API. the database via the backend API and currency rates from UniRate API.
- When the client opts for fetching new transactions via the Bank API, the backend delegates the task - When the client opts for fetching new transactions via the Bank API, cron will trigger periodic fetching
to a background worker service via the Message queue. using background worker.
- After successful load, these transactions are stored to the database and displayed to the client - After successful load, these transactions are stored to the database and displayed to the client
- There is also a Task planner, that executes periodic tasks, like fetching new transactions automatically from the Bank
APIs
### Features ### Features
@@ -62,6 +60,9 @@ The workflow works in the following way:
- For every pull request the full APP is deployed on a separate URL and the tests are run by github CI/CD - For every pull request the full APP is deployed on a separate URL and the tests are run by github CI/CD
- On every push to main, the production app is automatically updated - On every push to main, the production app is automatically updated
- UI is responsive for mobile devices - UI is responsive for mobile devices
- Slow operations (emails, transactions fetching) are handled
in the background by Celery workers.
- App is monitored using prometheus metrics endpoint and metrics are shown in Grafana dashboard.
### Components ### Components
@@ -69,13 +70,11 @@ The workflow works in the following way:
login/registration, shows latest transactions, filtering, and allows adding transactions. login/registration, shows latest transactions, filtering, and allows adding transactions.
- Backend API (backend/app): FastAPI app with routers under app/api for auth, users, categories, transactions, exchange - Backend API (backend/app): FastAPI app with routers under app/api for auth, users, categories, transactions, exchange
rates and bankAPI. Uses FastAPI Users for auth (JWT + OAuth), SQLAlchemy ORM, and Pydantic v2 schemas. rates and bankAPI. Uses FastAPI Users for auth (JWT + OAuth), SQLAlchemy ORM, and Pydantic v2 schemas.
- Worker service (backend/app/workers): Celery worker handling asynchronous tasks (e.g., sending verification emails, - Worker service (backend/app/workers): Celery worker handling background tasks (emails, transactions fetching).
future background processing). - Database (MariaDB with Maxscale): Persists users, categories, transactions; schema managed by Alembic migrations.
- Database (PostgreSQL): Persists users, categories, transactions; schema managed by Alembic migrations. - Message Queue (RabbitMQ): Queues background tasks for Celery workers.
- Message Queue (RabbitMQ): Transports background jobs from the API to the worker. - Infrastructure as Code (tofu/): OpenTofu modules provisioning cluster services (RabbitMQ, Redis, Cloudflare tunnel,
- Cache/Result Store (Redis): Available for caching or Celery result backend. etc.).
- Infrastructure as Code (tofu/): OpenTofu modules provisioning cluster services (RabbitMQ, Redis, Argo CD,
cert-manager, Cloudflare tunnel, etc.).
- Deployment Chart (charts/myapp-chart/): Helm chart to deploy the application to Kubernetes. - Deployment Chart (charts/myapp-chart/): Helm chart to deploy the application to Kubernetes.
### Technologies Used ### Technologies Used
@@ -167,7 +166,7 @@ You can run the project with Docker Compose and Python virtual environment for t
```bash ```bash
git clone https://github.com/dat515-2025/Group-8.git git clone https://github.com/dat515-2025/Group-8.git
cd Group-8/7project cd Group-8/7project/src
``` ```
### 2) Install dependencies ### 2) Install dependencies
@@ -418,45 +417,25 @@ tofu apply
- UNIRATE_API_KEY - UNIRATE_API_KEY
5) On Github open Actions tab, select "Deploy Prod" and run workflow manually 5) On Github open Actions tab, select "Deploy Prod" and run workflow manually
# TODO: REMOVE I guess
2) Deploy the app using Helm
```bash
# Set the namespace
kubectl create namespace myapp || true
# Install/upgrade the chart with required values
helm upgrade --install myapp charts/myapp-chart \
-n myapp \
-f charts/myapp-chart/values.yaml \
--set image.backend.repository=myorg/myapp-backend \
--set image.backend.tag=latest \
--set env.BACKEND_URL="https://myapp.example.com" \
--set env.FRONTEND_URL="https://myapp.example.com" \
--set env.SECRET="CHANGE_ME_SECRET"
```
## Testing Instructions ## Testing Instructions
The tests are located in 7project/backend/tests directory. All tests are run by GitHub actions on every pull request and The tests are located in 7project/backend/tests directory. All tests are run by GitHub actions on every pull request and
push to main. push to main.
See the workflow [here](../.github/workflows/run-tests.yml). See the workflow [here](../.github/workflows/run-tests.yml).
If you want to run the tests locally, the preferred is to use a [bash script](backend/test-with-ephemeral-mariadb.sh) If you want to run the tests locally, the preferred way is to use a [bash script](src/backend/test_locally.sh)
that will start a [test DB container](backend/docker-compose.test.yml) and remove it afterward. that will start a test DB container with [docker compose](src/backend/docker-compose.test.yml) and remove it afterwards.
```bash ```bash
cd 7project/backend cd 7project/backend
bash test-with-ephemeral-mariadb.sh bash test_locally.sh
``` ```
### Unit Tests ### Unit Tests
There are only 5 basic unit tests, since our services logic is very simple There are 5 basic unit tests, since our services logic is very simple
```bash ```bash
bash test-with-ephemeral-mariadb.sh --only-unit bash test_locally.sh --only-unit
``` ```
### Integration Tests ### Integration Tests
@@ -464,7 +443,7 @@ bash test-with-ephemeral-mariadb.sh --only-unit
There are 9 basic unit tests, testing the individual backend API logic There are 9 basic unit tests, testing the individual backend API logic
```bash ```bash
bash test-with-ephemeral-mariadb.sh --only-integration bash test_locally.sh --only-integration
``` ```
### End-to-End Tests ### End-to-End Tests
@@ -472,7 +451,7 @@ bash test-with-ephemeral-mariadb.sh --only-integration
There are 7 e2e tests, testing more complex app logic There are 7 e2e tests, testing more complex app logic
```bash ```bash
bash test-with-ephemeral-mariadb.sh --only-e2e bash test_locally.sh --only-e2e
``` ```
## Usage Examples ## Usage Examples
@@ -600,18 +579,22 @@ curl -H "Authorization: Bearer $TOKEN" http://127.0.0.1:8000/authenticated-route
### Dejan ### Dejan
| Date | Activity | Hours | Description | | Date | Activity | Hours | Description | Representative Commit / PR |
|-----------------|----------------------|--------|---------------------------------------------------------------| |:----------------|:-------------------------|:-------|:--------------------------------------------------------------|:---------------------------------------------------------|
| 25.9. | Design | 2 | 6design | | 25.9. | Design | 2 | 6design | |
| 9.10 to 11.10. | Backend APIs | 12 | Implemented Backend APIs | | 9.10 to 11.10. | Backend APIs | 14 | Implemented Backend APIs | `PR #26`, `20-create-a-controller-layer-on-backend-side` |
| 13.10 to 15.10. | Frontend Development | 8 | Created user interface mockups | | 13.10 to 15.10. | Frontend Development | 8 | Created user interface mockups | `PR #28`, `frontend basics` |
| Continually | Documentation | 6 | Documenting the dev process | | Continually | Documentation | 8 | Documenting the dev process | |
| 21.10 to 23.10 | Tests, frontend | 10 | Test basics, balance charts, and frontend improvement | | 21.10 to 23.10 | Tests, frontend | 10 | Test basics, balance charts, and frontend improvement | `PR #31`, `30 create tests and set up a GitHub pipeline` |
| 28.10 to 30.10 | CI | 6 | Integrated tests with test database setup on github workflows | | 28.10 to 30.10 | CI | 6 | Integrated tests with test database setup on github workflows | `PR #28`, `frontend basics` |
| 28.10 to 30.10 | Frontend | 7 | UI improvements and exchange rate API integration | | 28.10 to 30.10 | Frontend | 8 | UI improvements and exchange rate API integration | `PR #28`, `frontend basics` |
| 4.11 to 6.11 | Tests | 6 | Test fixes improvement, more integration and e2e | | 4.11 to 6.11 | Tests | 6 | Test fixes improvement, more integration and e2e | `PR #28`, `frontend basics` |
| 4.11 to 6.11 | Frontend | 6 | Fixes, Improved UI, added support for mobile devices | | 4.11 to 6.11 | Frontend | 6 | Fixes, Improved UI, added support for mobile devices | `PR #28`, `frontend basics` |
| **Total** | | **63** | | | 11.11 | Backend APIs | 4 | Moved rates API, mock bank to Backend, few fixes | `PR #28`, `frontend basics` |
| 11.11 to 12.11 | Tests | 3 | Local testing DB container, few fixes | `PR #28`, `frontend basics` |
| 12.11 | Frontend | 3 | Enabled multiple transaction edits at once, CSAS button state | `PR #28`, `frontend basics` |
| 13.11 | Video | 3 | Video | |
| **Total** | | **81** | | |
### Group Total: [XXX.X] hours ### Group Total: [XXX.X] hours
@@ -624,36 +607,47 @@ curl -H "Authorization: Bearer $TOKEN" http://127.0.0.1:8000/authenticated-route
[Reflect on the key technical and collaboration skills learned during this project] [Reflect on the key technical and collaboration skills learned during this project]
### Challenges Faced ### Challenges Faced
#### Slow cluster performance #### Slow cluster performance
This was caused by single SATA SSD disk running all VMs. This was solved by adding second NVMe disk just for Talos VMs. This was caused by single SATA SSD disk running all VMs. This was solved by adding second NVMe disk just for Talos VMs.
#### Stucked IaC deployment
[Describe the main challenges and how you overcame them] If the deployed module (helm chart for example) was not configured properly, it would get stuck and timeout resulting in
namespace that cannot be deleted.
This was solved by using snapshots in Proxmox and restoring if this happened.
### If We Did This Again ### If We Did This Again
#### Different framework #### Different framework
FastAPI lacks usable build in support for database migrations and implementing Alembic was a bit tricky.
FastAPI lacks usable build in support for database migrations and implementing Alembic was a bit tricky.
Tricky was also integrating FastAPI auth system with React frontend, since there is no official project template. Tricky was also integrating FastAPI auth system with React frontend, since there is no official project template.
Using .NET (which we considered initially) would probably solve these issues. Using .NET (which we considered initially) would probably solve these issues.
[What would you do differently? What worked well that you'd keep?] [What would you do differently? What worked well that you'd keep?]
### Individual Growth ### Individual Growth
#### [Lukas] #### [Lukas]
This course finally forced me to learn kubernetes (been on by TODO list for at least 3 years).
I had some prior experience with terraform/opentofu from work but this improved by understanding of it.
The biggest challenge for me was time tracking since I am used to tracking to projects, not to tasks. This course finally forced me to learn kubernetes (been on by TODO list for at least 3 years).
I had some prior experience with terraform/opentofu from work but this improved by understanding of it.
The biggest challenge for me was time tracking since I am used to tracking to projects, not to tasks.
(I am bad even at that :) ). (I am bad even at that :) ).
It was also interesting experience to be the one responsible for the initial project structure/design/setup It was also interesting experience to be the one responsible for the initial project structure/design/setup
used not only by myself. used not only by myself.
[Personal reflection on growth, challenges, and learning] [Personal reflection on growth, challenges, and learning]
#### [Team Member 2 Name] #### [Dejan]
Since I do not have a job, this project was probably the most complex one I have ever worked on.
It was also the first school project where I was encouraged to use AI.
Lukas
[Personal reflection on growth, challenges, and learning] [Personal reflection on growth, challenges, and learning]
@@ -661,4 +655,4 @@ used not only by myself.
--- ---
**Report Completion Date**: [Date] **Report Completion Date**: [Date]
**Last Updated**: 15.10.2025 **Last Updated**: 13.11.2025

8
7project/src/backend/.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

View File

@@ -4,13 +4,13 @@ set -euo pipefail
# Run tests against a disposable local MariaDB on host port 3307 using Docker Compose. # Run tests against a disposable local MariaDB on host port 3307 using Docker Compose.
# Requirements: Docker, docker compose plugin, Python, Alembic, pytest. # Requirements: Docker, docker compose plugin, Python, Alembic, pytest.
# Usage: # Usage:
# chmod +x ./test-with-ephemeral-mariadb.sh # chmod +x ./test_locally.sh
# # From 7project/backend directory # # From 7project/backend directory
# ./test-with-ephemeral-mariadb.sh [--only-unit|--only-integration|--only-e2e] [pytest-args...] # ./test_locally.sh [--only-unit|--only-integration|--only-e2e] [pytest-args...]
# # Examples: # # Examples:
# ./test-with-ephemeral-mariadb.sh --only-unit -q # ./test_locally.sh --only-unit -q
# ./test-with-ephemeral-mariadb.sh --only-integration -k "login" # ./test_locally.sh --only-integration -k "login"
# ./test-with-ephemeral-mariadb.sh --only-e2e -vv # ./test_locally.sh --only-e2e -vv
# #
# This script will: # This script will:
# 1) Start a MariaDB 11.4 container (ephemeral storage, port 3307) # 1) Start a MariaDB 11.4 container (ephemeral storage, port 3307)

View File

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

Some files were not shown because too many files have changed in this diff Show More