From 964b545bc5a5eb041dc30526a585bdd0ddd7b25f Mon Sep 17 00:00:00 2001 From: Schubert Ferenc Date: Fri, 3 Jul 2026 18:11:58 +0200 Subject: [PATCH] feat(storage): add local storage framework --- .env.example | 10 + ARCHITECTURE.md | 106 +++++++++- CONTRIBUTING.md | 33 +++- README-DEV.md | 185 ++++++++++++++++++ ROADMAP.md | 16 +- backend/hermes/.env.example | 3 + backend/hermes/app/api/audit.py | 2 + backend/hermes/app/api/knowledge.py | 12 +- backend/hermes/app/core/config.py | 20 ++ backend/hermes/app/services/audit_service.py | 13 ++ .../hermes/app/services/knowledge_service.py | 116 +++++------ backend/hermes/app/storage/__init__.py | 3 + backend/hermes/app/storage/base.py | 34 ++++ backend/hermes/app/storage/exceptions.py | 14 ++ backend/hermes/app/storage/local.py | 141 +++++++++++++ backend/hermes/app/storage/schemas.py | 23 +++ backend/hermes/app/storage/service.py | 103 ++++++++++ backend/hermes/docker-compose.yml | 8 +- docker-compose.yml | 8 +- scripts/backup.sh | 33 ++++ scripts/deploy.sh | 18 ++ scripts/healthcheck.sh | 26 +++ scripts/migrate.sh | 15 ++ scripts/restore.sh | 46 +++++ 24 files changed, 890 insertions(+), 98 deletions(-) create mode 100644 README-DEV.md create mode 100644 backend/hermes/app/storage/__init__.py create mode 100644 backend/hermes/app/storage/base.py create mode 100644 backend/hermes/app/storage/exceptions.py create mode 100644 backend/hermes/app/storage/local.py create mode 100644 backend/hermes/app/storage/schemas.py create mode 100644 backend/hermes/app/storage/service.py create mode 100755 scripts/backup.sh create mode 100755 scripts/deploy.sh create mode 100755 scripts/healthcheck.sh create mode 100755 scripts/migrate.sh create mode 100755 scripts/restore.sh diff --git a/.env.example b/.env.example index cd7b089..372e4a0 100644 --- a/.env.example +++ b/.env.example @@ -12,5 +12,15 @@ INITIAL_ADMIN_EMAIL= INITIAL_ADMIN_PASSWORD= INITIAL_ADMIN_FIRST_NAME= INITIAL_ADMIN_LAST_NAME= + +# Zentrales Storage Framework. +# Lokal kann STORAGE_HOST_PATH ein Repo-lokaler Ordner sein. +# Auf dem VPS wird empfohlen: STORAGE_HOST_PATH=/opt/olympus-storage +STORAGE_PROVIDER=local +STORAGE_BASE_PATH=/data/storage +STORAGE_MAX_UPLOAD_MB=50 +STORAGE_HOST_PATH=./storage + +# Legacy-Fallback fuer bestehende Knowledge-Installationen. KNOWLEDGE_STORAGE_PATH=/data/knowledge KNOWLEDGE_MAX_UPLOAD_MB=50 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 57ddb36..89a1283 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -244,6 +244,16 @@ Kunden: - `customers.read` - `customers.create` - `customers.update` +- `customers.delete` + +Knowledge: + +- `knowledge.read` +- `knowledge.create` +- `knowledge.update` +- `knowledge.delete` +- `knowledge.upload` +- `knowledge.download` ### Initial Admin Bootstrap @@ -294,7 +304,6 @@ customer_number;company_name;legal_name;customer_type;status;industry;website;em ``` Pflichtfeld ist `company_name`. Wenn `customer_number`, `customer_type` oder `status` fehlen, erzeugt der Import fuer neue Kunden eine Kundennummer bzw. nutzt produktive Defaults und weist in der Preview darauf hin. Vollstaendige CSV-Inhalte werden nicht im Audit Log gespeichert. -- `customers.delete` Projekte: @@ -527,14 +536,19 @@ Athena stellt die BFF-Routen unter `/api/knowledge/...` bereit. Der Browser spri ### Knowledge-Dateispeicherung -Dokumentuploads werden in v0.6.0 lokal im Hermes-Container oder in einem gemounteten Volume gespeichert. +Seit v0.7.0 laufen Knowledge-Dateien ueber das zentrale Storage Framework. KnowledgeService speichert, liest und loescht Dateien nicht mehr direkt ueber verstreute Dateioperationen, sondern ueber `StorageService`. + +Neue Uploads werden im Namespace `knowledge/documents/` gespeichert. Die Datenbank-Metadaten bleiben kompatibel: `knowledge_documents.file_path` enthaelt den Storage-Key fuer neue Dateien oder einen bestehenden Legacy-Pfad fuer alte Dateien. Konfiguration: -- `KNOWLEDGE_STORAGE_PATH`, Default `/data/knowledge` -- `KNOWLEDGE_MAX_UPLOAD_MB`, Default `50` +- `STORAGE_PROVIDER`, Default `local` +- `STORAGE_BASE_PATH`, Default `/data/storage` +- `STORAGE_MAX_UPLOAD_MB`, Default `50` +- `KNOWLEDGE_STORAGE_PATH`, Legacy-Fallback fuer bestehende Installationen +- `KNOWLEDGE_MAX_UPLOAD_MB`, Legacy-Fallback fuer bestehende Installationen -Docker Compose bindet das persistente Volume `knowledge-data` nach `/data/knowledge` ein. Dieses Volume darf nicht geloescht werden, wenn lokale Knowledge-Dateien erhalten bleiben sollen. +Docker Compose bindet den Host-Pfad aus `STORAGE_HOST_PATH` nach `/data/storage` ein. Fuer den VPS ist `/opt/olympus-storage` empfohlen. Bestehende Dateien unter `/data/knowledge` werden nicht automatisch verschoben; wenn Migration noetig ist, muss sie kontrolliert geplant und vorher gebackupt werden. Erlaubte Uploadtypen: @@ -545,7 +559,71 @@ Erlaubte Uploadtypen: - TXT - ZIP -Hermes normalisiert Dateinamen, validiert Extension und MIME-Type, begrenzt die Uploadgroesse und erzwingt, dass gespeicherte und heruntergeladene Dateien innerhalb von `KNOWLEDGE_STORAGE_PATH` liegen. +Hermes normalisiert Dateinamen, erzeugt eindeutige gespeicherte Dateinamen, validiert Extension und MIME-Type, begrenzt die Uploadgroesse, berechnet SHA256 und verhindert Path Traversal. + +## Storage Framework + +Das Storage Framework liegt unter `backend/hermes/app/storage`. + +Bestandteile: + +- `StorageProvider` als Interface +- `LocalDiskStorageProvider` als erste Implementierung +- `StorageService` als zentrale API fuer Fachmodule +- Storage-Schemas und eigene Storage-Exceptions + +Zentrale Operationen: + +- `save_file()` +- `open_file()` +- `delete_file()` +- `file_exists()` +- `get_file_metadata()` +- `calculate_checksum()` +- `safe_filename()` +- `validate_file_type()` +- `validate_file_size()` + +Der aktuelle Provider ist `local`. Die Architektur ist bewusst fuer spaetere Provider vorbereitet: + +- NAS +- S3 +- MinIO +- Paperless +- Azure Blob +- Backblaze / Wasabi + +Pfadstruktur im lokalen Provider: + +```text +/data/storage/ + knowledge/ + documents/ + thumbnails/ + customers/ + projects/ + tickets/ + imports/ + temp/ +``` + +Datei-Inhalte werden nicht geloggt. Browser greifen nie direkt auf Storage oder Hermes-Dateipfade zu; Downloads laufen ueber Athena-BFF und Hermes-Berechtigungspruefung. + +## Backup und Deployment + +Im Projektroot liegen robuste Bash-Skripte fuer Betrieb und Deployment: + +- `scripts/migrate.sh` +- `scripts/healthcheck.sh` +- `scripts/deploy.sh` +- `scripts/backup.sh` +- `scripts/restore.sh` + +`deploy.sh` baut Images, startet Docker Compose, fuehrt Migrationen aus und startet den Healthcheck. Es erzwingt kein `git pull`. + +`backup.sh` sichert PostgreSQL, wenn `POSTGRES_CONTAINER` oder `DATABASE_URL` mit lokalem `pg_dump` verfuegbar ist, und archiviert den Storage-Host-Pfad. `.env` wird bewusst nicht automatisch ins Backup kopiert und muss sicher separat verwaltet werden. + +`restore.sh` ist bewusst bestaetigungspflichtig und startet erst nach Eingabe von `RESTORE`. ### Knowledge-RBAC @@ -627,11 +705,23 @@ docker-compose.yml `LOG_LEVEL` : Runtime-Loglevel fuer Hermes, z. B. `INFO`, `WARNING` oder `ERROR`. +`STORAGE_PROVIDER` +: Storage Provider. Aktuell produktiv implementiert: `local`. + +`STORAGE_BASE_PATH` +: Interner Storage-Basispfad in Hermes. Default `/data/storage`. + +`STORAGE_MAX_UPLOAD_MB` +: Maximale Uploadgroesse fuer Storage-Dateien in MB. Default `50`. + +`STORAGE_HOST_PATH` +: Docker-Host-Pfad, der nach `/data/storage` gemountet wird. Lokal z. B. `./storage`, auf dem VPS empfohlen `/opt/olympus-storage`. + `KNOWLEDGE_STORAGE_PATH` -: Lokales Speicherverzeichnis fuer Knowledge-Dateien. Default `/data/knowledge`. +: Legacy-Fallback fuer bestehende Knowledge-Dateien. Neue Installationen sollen `STORAGE_*` verwenden. `KNOWLEDGE_MAX_UPLOAD_MB` -: Maximale Uploadgroesse fuer Knowledge-Dokumente in MB. Default `50`. +: Legacy-Fallback fuer die maximale Knowledge-Uploadgroesse. ### Athena diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9419823..a0b8f51 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,7 +73,7 @@ Eine Aenderung gilt erst als fertig, wenn diese Punkte erfuellt sind: - keine `.venv` im Git - keine toten Imports - keine ungenutzten Dateien -- keine Debug-Ausgaben wie `console.log`, `alert` oder `confirm` +- keine Debug-Ausgaben oder Browser-Dialoge im Anwendungscode Standard-Checks: @@ -103,7 +103,7 @@ uv run alembic upgrade head - Wiederverwendbare Komponenten bevorzugen. - UI-Zustaende immer abbilden: Loading, Error, Empty State. - Erfolg und Fehler in mutierenden CRUD-Flows ueber den Toast-Provider melden. -- Keine Browser-Dialoge wie `alert()` oder `confirm()`. +- Keine nativen Browser-Dialoge fuer produktive UI-Flows. - Keine Tokens in Browser-JavaScript speichern. - Datei-Uploads vom Browser laufen ueber Athena-BFF-Routen und werden serverseitig an Hermes weitergeleitet. @@ -120,10 +120,12 @@ uv run alembic upgrade head - Mutierende Kernaktionen mit Audit Logs erfassen, sofern fachlich relevant. - Sensible Felder vor Persistenz in Logs oder Audit-Daten maskieren. - Import- und Bootstrap-Flows duerfen keine Passwoerter, Tokens, Secrets oder vollstaendige CSV-Inhalte loggen. +- Dateiablagen laufen ueber `StorageService`; direkte Dateioperationen in Fachservices sind nur mit guter Begruendung zulaessig. +- Uploads muessen Dateityp, MIME-Type, Groesse, Dateiname und Storage-Pfad validieren. ### Allgemein -- Keine TODOs als Ersatz fuer fertige Implementierung. +- Keine Platzhalter-Kommentare als Ersatz fuer fertige Implementierung. - Keine Quickfixes. - Keine Workarounds. - Keine ungeprueften Annahmen bei Auth, Datenbank oder Docker. @@ -237,17 +239,35 @@ Upload-Regeln: - Browser sendet Dateien nur an Athena. - Athena leitet FormData serverseitig an Hermes weiter. - Hermes validiert Dateityp, MIME-Type, Dateigroesse und Speicherpfad. -- Lokale Dateien liegen unter `KNOWLEDGE_STORAGE_PATH`. +- Lokale Dateien liegen ueber Docker unter `STORAGE_BASE_PATH`; alte Knowledge-Pfade bleiben nur als Legacy-Fallback erhalten. - Audit Logs duerfen keine Datei-Inhalte enthalten. - Keine Pfade aus unvalidierten Benutzereingaben zusammensetzen. Paperless-ngx ist nur vorbereitet. `paperless_document_id` und `external_url` duerfen gepflegt werden, aber es werden keine Paperless-Secrets oder API-Keys eingefuehrt. +### Storage + +Neue Dateiablagen muessen das zentrale Storage Framework verwenden. + +Regeln: + +- Keine unvalidierten Pfade aus Benutzereingaben zusammensetzen. +- Keine absoluten User-Pfade akzeptieren. +- Keine Path-Traversal-Moeglichkeiten zulassen. +- Originaldateinamen und gespeicherte Dateinamen fachlich unterscheiden. +- SHA256 fuer gespeicherte Dateien berechnen, sofern das Modul Datei-Metadaten persistiert. +- Keine Datei-Inhalte loggen. +- Bestehende Dateien nicht automatisch verschieben oder loeschen. + +Der lokale Provider nutzt `STORAGE_BASE_PATH`; Docker mountet den Host-Pfad aus `STORAGE_HOST_PATH` nach `/data/storage`. + ### Deployment Neue Installationen koennen optional ueber `INITIAL_ADMIN_*` einen initialen Administrator anlegen. Diese Variablen werden nur von Hermes gelesen und duerfen nicht im Frontend oder in Logs erscheinen. -Knowledge-Dateien benoetigen ein persistentes Docker-Volume. Vor produktiven Deployments muessen `KNOWLEDGE_STORAGE_PATH` und `KNOWLEDGE_MAX_UPLOAD_MB` bewusst gesetzt oder die Defaults akzeptiert werden. +Dateien benoetigen einen persistenten Storage-Mount. Vor produktiven Deployments muessen `STORAGE_PROVIDER`, `STORAGE_BASE_PATH`, `STORAGE_MAX_UPLOAD_MB` und `STORAGE_HOST_PATH` bewusst gesetzt oder die Defaults akzeptiert werden. + +Die Skripte unter `scripts/` sind die bevorzugte Grundlage fuer Migration, Healthcheck, Deploy, Backup und Restore. ### Neue Permissions @@ -299,6 +319,9 @@ Vor Merge pruefen: - Audit Logs fuer relevante Aenderungen vorhanden - API-Fehlerantworten konsistent - Toasts fuer mutierende UI-Aktionen vorhanden +- Dateiablagen verwenden `StorageService` +- Storage-Pfade sind gegen Path Traversal geschuetzt +- Backup-/Restore-Auswirkungen fuer Dateiablagen dokumentiert ## Migrationsregeln diff --git a/README-DEV.md b/README-DEV.md new file mode 100644 index 0000000..5ab75b1 --- /dev/null +++ b/README-DEV.md @@ -0,0 +1,185 @@ +# Olympus CRM Entwicklungssetup + +Dieses Dokument beschreibt das lokale Setup fuer neue Rechner und parallele Entwicklung auf mehreren Macs. + +## Setup auf neuem Mac + +Voraussetzungen: + +- Git +- Docker Desktop +- Node.js passend zu Athena +- Python/uv fuer lokale Hermes-Checks + +Repository klonen: + +```bash +git clone Olympus +cd Olympus +``` + +Umgebung anlegen: + +```bash +cp .env.example .env +``` + +Wichtige lokale Werte: + +```env +AUTH_COOKIE_SECURE=false +ATHENA_PUBLIC_ORIGIN=http://localhost:3001 +STORAGE_PROVIDER=local +STORAGE_BASE_PATH=/data/storage +STORAGE_MAX_UPLOAD_MB=50 +STORAGE_HOST_PATH=./storage +``` + +Wenn `SECRET_KEY` Sonderzeichen wie `$` enthaelt, den Wert in der Shell oder Compose-Umgebung korrekt quoten. Secrets gehoeren nicht ins Git. + +## Docker Netzwerk + +Das gemeinsame Compose-File nutzt ein externes Docker-Netzwerk: + +```bash +docker network create olympus-network +``` + +Wenn das Netzwerk bereits existiert, meldet Docker das nur als Hinweis. + +## Docker Compose Start + +```bash +docker compose build +docker compose up -d +``` + +Hermes ist im gemeinsamen Stack nur intern im Docker-Netzwerk erreichbar. Athena ist lokal ueber den Browser erreichbar: + +- Athena: `http://localhost:3001` +- Hermes intern: `http://hermes:8000` + +Der Browser spricht nicht direkt mit Hermes. + +## Migrationen + +Migrationen ausfuehren: + +```bash +scripts/migrate.sh +``` + +Alternativ lokal im Backend: + +```bash +cd backend/hermes +uv run alembic upgrade head +``` + +## Initial Admin + +Fuer neue Installationen ohne aktive Benutzer kann Hermes beim Startup einen initialen Admin anlegen. + +```env +INITIAL_ADMIN_USERNAME=admin +INITIAL_ADMIN_EMAIL=admin@example.local +INITIAL_ADMIN_PASSWORD= +INITIAL_ADMIN_FIRST_NAME= +INITIAL_ADMIN_LAST_NAME= +``` + +Diese Werte nur lokal oder sicher auf dem Zielsystem setzen. Nach dem ersten produktiven Login sollte das Passwort geaendert und die Bootstrap-Variablen wieder entfernt werden. + +## Login + +Nach Start und Migration: + +1. Browser auf `http://localhost:3001` oeffnen. +2. Mit dem initialen Admin oder einem bestehenden Benutzer anmelden. +3. Athena setzt das HttpOnly-Cookie. Hermes setzt keine Browser-Cookies. + +## Storage + +Neue Dateiablagen laufen ueber das Storage Framework. + +Lokaler Standard: + +```env +STORAGE_HOST_PATH=./storage +STORAGE_BASE_PATH=/data/storage +``` + +VPS-Empfehlung: + +```env +STORAGE_HOST_PATH=/opt/olympus-storage +STORAGE_BASE_PATH=/data/storage +``` + +Bestehende Knowledge-Dateien aus alten Setups unter `/data/knowledge` werden nicht automatisch verschoben. Vor einer manuellen Migration immer Backup erstellen. + +## Typische Fehler + +Hermes restartet wegen fehlender Migration: + +```bash +scripts/migrate.sh +docker compose restart hermes +``` + +`SECRET_KEY` mit `$` wird falsch interpretiert: + +- Wert in `.env` korrekt escapen oder quoten. +- Keine Secrets in Commit oder Logs schreiben. + +Docker Container Name Conflict: + +```bash +docker compose down +docker ps -a +``` + +Danach blockierenden Altcontainer gezielt entfernen, wenn er wirklich nicht mehr gebraucht wird. + +External network fehlt: + +```bash +docker network create olympus-network +``` + +DB leer oder Admin fehlt: + +- `INITIAL_ADMIN_*` Werte setzen. +- `scripts/migrate.sh` ausfuehren. +- `docker compose restart hermes`. + +Athena nicht erreichbar: + +- `docker compose ps` pruefen. +- Port `3001` auf dem Rechner pruefen. +- `scripts/healthcheck.sh` ausfuehren. + +## MacBook und Mac mini parallel + +Empfehlung: + +- `.env` pro Rechner lokal pflegen und nicht committen. +- `STORAGE_HOST_PATH` pro Rechner bewusst setzen. +- Datenbank- und Storage-Backups nicht ungeprueft zwischen Rechnern ueberschreiben. +- Vor Branch-Wechseln Migrationen pruefen. +- Bei paralleler Arbeit keine Docker-Volumes loeschen, solange ungesicherte Uploads existieren. + +## Qualitaetschecks + +```bash +python3 -m compileall backend/hermes/app +cd frontend/athena +npm run lint +npx next build --webpack +``` + +Skripte pruefen: + +```bash +bash -n scripts/*.sh +``` diff --git a/ROADMAP.md b/ROADMAP.md index 8f5cddb..3a50679 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -55,7 +55,17 @@ Die Roadmap beschreibt die geplante fachliche Entwicklung von Olympus CRM. Archi - Paperless-ngx vorbereitet ueber `paperless_document_id` und `external_url` - Persistentes Docker-Volume fuer Knowledge-Dateien -## v0.7.0 - Projektmodul, geplant +## v0.7.0 - Storage Framework und Deployment-Grundlage + +- Zentrales Storage Framework in Hermes +- LocalDisk Storage Provider +- Knowledge-Dateilogik ueber StorageService +- Storage-Konfiguration fuer lokale Entwicklung und VPS +- Empfohlener VPS-Pfad `/opt/olympus-storage` +- Deployment-, Migrations-, Healthcheck-, Backup- und Restore-Skripte +- README-DEV fuer mehrere Entwicklungsrechner + +## v0.8.0 - Projektmodul, geplant - Projektstammdaten - Projektstatus und Verantwortliche @@ -63,7 +73,7 @@ Die Roadmap beschreibt die geplante fachliche Entwicklung von Olympus CRM. Archi - RBAC-Permissions fuer Projekte - Audit Logs fuer Projektaktionen -## v0.8.0 - Tickets, geplant +## v0.9.0 - Tickets, geplant - Ticketverwaltung - Status- und Prioritaetsmodell @@ -71,7 +81,7 @@ Die Roadmap beschreibt die geplante fachliche Entwicklung von Olympus CRM. Archi - RBAC-Permissions fuer Tickets - Audit Logs fuer Ticketaktionen -## v0.9.0 - Integrationen Paperless/Lexoffice, geplant +## v0.10.0 - Integrationen Paperless/Lexoffice, geplant - Paperless-ngx Connector fuer Wissensdokumente - Lexoffice-Vorbereitung fuer Kunden- und Projektdaten diff --git a/backend/hermes/.env.example b/backend/hermes/.env.example index fab2c0f..72e203e 100644 --- a/backend/hermes/.env.example +++ b/backend/hermes/.env.example @@ -13,5 +13,8 @@ INITIAL_ADMIN_EMAIL= INITIAL_ADMIN_PASSWORD= INITIAL_ADMIN_FIRST_NAME= INITIAL_ADMIN_LAST_NAME= +STORAGE_PROVIDER=local +STORAGE_BASE_PATH=/data/storage +STORAGE_MAX_UPLOAD_MB=50 KNOWLEDGE_STORAGE_PATH=/data/knowledge KNOWLEDGE_MAX_UPLOAD_MB=50 diff --git a/backend/hermes/app/api/audit.py b/backend/hermes/app/api/audit.py index 15d151c..6929337 100644 --- a/backend/hermes/app/api/audit.py +++ b/backend/hermes/app/api/audit.py @@ -54,6 +54,8 @@ def can_read_activity(action: str, permissions: set[str]) -> bool: return "customers.read" in permissions if action.startswith("roles."): return "roles.read" in permissions + if action.startswith("knowledge."): + return "knowledge.read" in permissions if action.startswith("audit_logs."): return "audit_logs.read" in permissions if action.startswith("auth."): diff --git a/backend/hermes/app/api/knowledge.py b/backend/hermes/app/api/knowledge.py index 1eebcc2..ca239c4 100644 --- a/backend/hermes/app/api/knowledge.py +++ b/backend/hermes/app/api/knowledge.py @@ -1,5 +1,4 @@ import logging -from pathlib import Path from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response, UploadFile, status from fastapi.responses import FileResponse @@ -30,7 +29,7 @@ from app.schemas.knowledge import ( normalize_tags, ) from app.services.audit_service import sanitize, write_audit_log -from app.services.knowledge_service import KnowledgeService, assert_safe_path +from app.services.knowledge_service import KnowledgeService logger = logging.getLogger(__name__) @@ -256,7 +255,10 @@ def delete_document(document_id: int, request: Request, db: Session = Depends(ge document = get_document_or_404(db, document_id) before_data = sanitize(document) label = document.title + file_path = document.file_path delete_or_conflict(db, document) + if file_path: + KnowledgeService.delete_storage_key(file_path) write_audit_log(db, action="knowledge.documents.delete", entity_type="knowledge_documents", entity_id=document_id, entity_label=label, actor=current_user, request=request, before_data=before_data) return Response(status_code=status.HTTP_204_NO_CONTENT) @@ -264,11 +266,7 @@ def delete_document(document_id: int, request: Request, db: Session = Depends(ge @router.get("/documents/{document_id:int}/download") def download_document(document_id: int, db: Session = Depends(get_db), current_user: User = Depends(require_permission("knowledge.download"))): document = get_document_or_404(db, document_id) - if not document.file_path: - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Dokument hat keine lokale Datei") - path = assert_safe_path(Path(document.file_path)) - if not path.exists() or not path.is_file(): - raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Datei nicht gefunden") + path = KnowledgeService.open_document_file(document) logger.info("knowledge.documents.download", extra={"actor_user_id": current_user.id, "target_document_id": document_id}) return FileResponse(path, media_type=document.mime_type or "application/octet-stream", filename=document.file_name) diff --git a/backend/hermes/app/core/config.py b/backend/hermes/app/core/config.py index f9fce54..c336295 100644 --- a/backend/hermes/app/core/config.py +++ b/backend/hermes/app/core/config.py @@ -1,3 +1,6 @@ +from typing import Any + +from pydantic import model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -15,6 +18,9 @@ class Settings(BaseSettings): initial_admin_password: str | None = None initial_admin_first_name: str = "" initial_admin_last_name: str = "" + storage_provider: str = "local" + storage_base_path: str = "/data/storage" + storage_max_upload_mb: int = 50 knowledge_storage_path: str = "/data/knowledge" knowledge_max_upload_mb: int = 50 @@ -23,5 +29,19 @@ class Settings(BaseSettings): extra="ignore", ) + @model_validator(mode="before") + @classmethod + def apply_legacy_storage_settings(cls, values: Any) -> Any: + if not isinstance(values, dict): + return values + + if "storage_base_path" not in values and "knowledge_storage_path" in values: + values["storage_base_path"] = values["knowledge_storage_path"] + + if "storage_max_upload_mb" not in values and "knowledge_max_upload_mb" in values: + values["storage_max_upload_mb"] = values["knowledge_max_upload_mb"] + + return values + settings = Settings() diff --git a/backend/hermes/app/services/audit_service.py b/backend/hermes/app/services/audit_service.py index cd0a7fb..772e8e1 100644 --- a/backend/hermes/app/services/audit_service.py +++ b/backend/hermes/app/services/audit_service.py @@ -130,6 +130,19 @@ def action_title(action: str) -> str: "customer_contacts.create": "Ansprechpartner erstellt", "customer_contacts.update": "Ansprechpartner bearbeitet", "customer_contacts.delete": "Ansprechpartner gelöscht", + "knowledge.manufacturers.create": "Hersteller erstellt", + "knowledge.manufacturers.update": "Hersteller bearbeitet", + "knowledge.manufacturers.delete": "Hersteller gelöscht", + "knowledge.devices.create": "Gerät erstellt", + "knowledge.devices.update": "Gerät bearbeitet", + "knowledge.devices.delete": "Gerät gelöscht", + "knowledge.documents.create": "Dokument erstellt", + "knowledge.documents.upload": "Dokument hochgeladen", + "knowledge.documents.update": "Dokument bearbeitet", + "knowledge.documents.delete": "Dokument gelöscht", + "knowledge.notes.create": "Notiz erstellt", + "knowledge.notes.update": "Notiz bearbeitet", + "knowledge.notes.delete": "Notiz gelöscht", "users.initial_admin_bootstrap": "Initialer Administrator erstellt", "knowledge.manufacturers.create": "Hersteller erstellt", "knowledge.manufacturers.update": "Hersteller bearbeitet", diff --git a/backend/hermes/app/services/knowledge_service.py b/backend/hermes/app/services/knowledge_service.py index 1b58441..1e4840c 100644 --- a/backend/hermes/app/services/knowledge_service.py +++ b/backend/hermes/app/services/knowledge_service.py @@ -1,13 +1,9 @@ -import hashlib -import mimetypes import re import uuid -from pathlib import Path from fastapi import HTTPException, UploadFile, status from sqlalchemy.orm import Session -from app.core.config import settings from app.models.knowledge import KnowledgeDevice, KnowledgeDocument, KnowledgeManufacturer, KnowledgeNote from app.repositories.knowledge_repository import KnowledgeRepository from app.schemas.knowledge import ( @@ -20,17 +16,8 @@ from app.schemas.knowledge import ( KnowledgeNoteCreate, KnowledgeNoteUpdate, ) - -ALLOWED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".webp", ".txt", ".zip"} -ALLOWED_MIME_TYPES = { - "application/pdf", - "image/jpeg", - "image/png", - "image/webp", - "text/plain", - "application/zip", - "application/x-zip-compressed", -} +from app.storage import get_storage_service +from app.storage.exceptions import StorageFileNotFoundError, StorageValidationError def slugify(value: str) -> str: @@ -51,27 +38,6 @@ def unique_slug(db: Session, base: str, exists) -> str: return candidate -def storage_root() -> Path: - root = Path(settings.knowledge_storage_path).resolve() - root.mkdir(parents=True, exist_ok=True) - return root - - -def assert_safe_path(path: Path) -> Path: - root = storage_root() - resolved = path.resolve() - if root != resolved and root not in resolved.parents: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Ungültiger Dateipfad") - return resolved - - -def safe_file_name(file_name: str) -> str: - name = Path(file_name).name.strip() - stem = slugify(Path(name).stem) - suffix = Path(name).suffix.lower() - return f"{stem}{suffix}" if suffix else stem - - def parse_tags(value: str) -> list[str]: seen: set[str] = set() tags: list[str] = [] @@ -84,25 +50,10 @@ def parse_tags(value: str) -> list[str]: return tags -async def read_upload(file: UploadFile) -> tuple[bytes, str, str]: - original_name = file.filename or "" - file_name = safe_file_name(original_name) - extension = Path(file_name).suffix.lower() - if extension not in ALLOWED_EXTENSIONS: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Dateityp ist nicht erlaubt") - - max_bytes = max(1, settings.knowledge_max_upload_mb) * 1024 * 1024 - content = await file.read(max_bytes + 1) - if not content: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload-Datei ist leer") - if len(content) > max_bytes: - raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Upload-Datei ist zu groß") - - mime_type = file.content_type or mimetypes.guess_type(file_name)[0] or "application/octet-stream" - if mime_type not in ALLOWED_MIME_TYPES: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="MIME-Type ist nicht erlaubt") - - return content, file_name, mime_type +def storage_validation_error(exc: StorageValidationError) -> HTTPException: + detail = str(exc) or "Ungültige Datei" + status_code = status.HTTP_413_REQUEST_ENTITY_TOO_LARGE if "groß" in detail else status.HTTP_400_BAD_REQUEST + return HTTPException(status_code=status_code, detail=detail) class KnowledgeService: @@ -175,22 +126,29 @@ class KnowledgeService: payload: KnowledgeDocumentCreate, ) -> KnowledgeDocument: KnowledgeService._validate_document_links(db, payload.manufacturer_id, payload.device_id) - content, file_name, mime_type = await read_upload(file) - checksum = hashlib.sha256(content).hexdigest() + storage_service = get_storage_service() + max_bytes = storage_service.max_upload_mb * 1024 * 1024 + content = await file.read(max_bytes + 1) + original_name = file.filename or "" + mime_type = file.content_type + try: + metadata = storage_service.save_file( + namespace=f"knowledge/documents/{payload.manufacturer_id}", + content=content, + original_filename=original_name, + mime_type=mime_type, + ) + except StorageValidationError as exc: + raise storage_validation_error(exc) from exc slug = unique_slug(db, payload.title, lambda session, value: KnowledgeRepository.get_document_by_slug(session, value) is not None) - target_dir = storage_root() / str(payload.manufacturer_id) - target_dir.mkdir(parents=True, exist_ok=True) - stored_name = f"{uuid.uuid4().hex}-{file_name}" - file_path = assert_safe_path(target_dir / stored_name) - file_path.write_bytes(content) document = KnowledgeDocument( slug=slug, external_url=str(payload.external_url or ""), - file_name=file_name, - file_path=str(file_path), - mime_type=mime_type, - file_size=len(content), - checksum_sha256=checksum, + file_name=metadata.original_filename, + file_path=metadata.storage_key, + mime_type=metadata.mime_type, + file_size=metadata.size, + checksum_sha256=metadata.checksum_sha256, **payload.model_dump(exclude={"external_url"}), ) db.add(document) @@ -216,6 +174,30 @@ class KnowledgeService: db.commit() return KnowledgeRepository.get_note(db, note.id) or note + @staticmethod + def open_document_file(document: KnowledgeDocument): + if not document.file_path: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Dokument hat keine lokale Datei") + try: + return get_storage_service().open_file(document.file_path) + except StorageFileNotFoundError as exc: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Datei nicht gefunden") from exc + except StorageValidationError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Ungültiger Dateipfad") from exc + + @staticmethod + def delete_document_file(document: KnowledgeDocument) -> None: + if not document.file_path: + return + KnowledgeService.delete_storage_key(document.file_path) + + @staticmethod + def delete_storage_key(storage_key: str) -> None: + try: + get_storage_service().delete_file(storage_key) + except StorageValidationError as exc: + raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Ungültiger Dateipfad") from exc + @staticmethod def update_note(db: Session, note: KnowledgeNote, payload: KnowledgeNoteUpdate) -> KnowledgeNote: KnowledgeService._validate_optional_links(db, payload.manufacturer_id, payload.device_id) diff --git a/backend/hermes/app/storage/__init__.py b/backend/hermes/app/storage/__init__.py new file mode 100644 index 0000000..2795312 --- /dev/null +++ b/backend/hermes/app/storage/__init__.py @@ -0,0 +1,3 @@ +from app.storage.service import StorageService, get_storage_service + +__all__ = ["StorageService", "get_storage_service"] diff --git a/backend/hermes/app/storage/base.py b/backend/hermes/app/storage/base.py new file mode 100644 index 0000000..8c12f4a --- /dev/null +++ b/backend/hermes/app/storage/base.py @@ -0,0 +1,34 @@ +from abc import ABC, abstractmethod +from pathlib import Path + +from app.storage.schemas import FileMetadata, StoredFileMetadata + + +class StorageProvider(ABC): + @abstractmethod + def save_file( + self, + *, + namespace: str, + content: bytes, + original_filename: str, + stored_filename: str, + mime_type: str, + ) -> StoredFileMetadata: + raise NotImplementedError + + @abstractmethod + def open_file(self, storage_key: str) -> Path: + raise NotImplementedError + + @abstractmethod + def delete_file(self, storage_key: str) -> None: + raise NotImplementedError + + @abstractmethod + def file_exists(self, storage_key: str) -> bool: + raise NotImplementedError + + @abstractmethod + def get_file_metadata(self, storage_key: str) -> FileMetadata: + raise NotImplementedError diff --git a/backend/hermes/app/storage/exceptions.py b/backend/hermes/app/storage/exceptions.py new file mode 100644 index 0000000..9056ef9 --- /dev/null +++ b/backend/hermes/app/storage/exceptions.py @@ -0,0 +1,14 @@ +class StorageError(Exception): + """Base exception for storage operations.""" + + +class StorageConfigurationError(StorageError): + pass + + +class StorageValidationError(StorageError): + pass + + +class StorageFileNotFoundError(StorageError): + pass diff --git a/backend/hermes/app/storage/local.py b/backend/hermes/app/storage/local.py new file mode 100644 index 0000000..9267a12 --- /dev/null +++ b/backend/hermes/app/storage/local.py @@ -0,0 +1,141 @@ +from pathlib import Path +import hashlib + +from app.storage.base import StorageProvider +from app.storage.exceptions import StorageFileNotFoundError, StorageValidationError +from app.storage.schemas import FileMetadata, StoredFileMetadata + + +class LocalDiskStorageProvider(StorageProvider): + def __init__( + self, + *, + base_path: str, + legacy_base_paths: list[str] | None = None, + ) -> None: + self.base_path = Path(base_path).expanduser().resolve() + self.legacy_base_paths = [ + Path(path).expanduser().resolve() + for path in legacy_base_paths or [] + if path + ] + self._ensure_layout() + + def save_file( + self, + *, + namespace: str, + content: bytes, + original_filename: str, + stored_filename: str, + mime_type: str, + ) -> StoredFileMetadata: + if Path(stored_filename).name != stored_filename or ".." in Path(stored_filename).parts: + raise StorageValidationError("Ungültiger gespeicherter Dateiname") + namespace_path = self._safe_namespace_path(namespace) + namespace_path.mkdir(parents=True, exist_ok=True) + absolute_path = self._safe_child(namespace_path / stored_filename) + absolute_path.write_bytes(content) + relative_path = absolute_path.relative_to(self.base_path).as_posix() + checksum = self._checksum(absolute_path) + return StoredFileMetadata( + storage_key=relative_path, + original_filename=original_filename, + stored_filename=stored_filename, + relative_path=relative_path, + absolute_path=absolute_path, + mime_type=mime_type, + size=len(content), + checksum_sha256=checksum, + ) + + def open_file(self, storage_key: str) -> Path: + path = self._resolve_storage_key(storage_key) + if not path.exists() or not path.is_file(): + raise StorageFileNotFoundError("Datei nicht gefunden") + return path + + def delete_file(self, storage_key: str) -> None: + path = self._resolve_storage_key(storage_key) + if not path.exists(): + return + if not path.is_file(): + raise StorageValidationError("Storage-Key verweist nicht auf eine Datei") + path.unlink() + + def file_exists(self, storage_key: str) -> bool: + try: + path = self._resolve_storage_key(storage_key) + except StorageValidationError: + return False + return path.exists() and path.is_file() + + def get_file_metadata(self, storage_key: str) -> FileMetadata: + path = self.open_file(storage_key) + return FileMetadata( + storage_key=storage_key, + relative_path=self._relative_storage_key(path), + absolute_path=path, + size=path.stat().st_size, + checksum_sha256=self._checksum(path), + ) + + def _ensure_layout(self) -> None: + for relative_path in [ + "knowledge/documents", + "knowledge/thumbnails", + "customers", + "projects", + "tickets", + "imports", + "temp", + ]: + (self.base_path / relative_path).mkdir(parents=True, exist_ok=True) + + def _safe_namespace_path(self, namespace: str) -> Path: + if namespace.startswith("/") or ".." in Path(namespace).parts: + raise StorageValidationError("Ungültiger Storage-Namespace") + return self._safe_child(self.base_path / namespace) + + def _safe_child(self, path: Path) -> Path: + resolved = path.resolve() + if self.base_path != resolved and self.base_path not in resolved.parents: + raise StorageValidationError("Ungültiger Storage-Pfad") + return resolved + + def _resolve_storage_key(self, storage_key: str) -> Path: + raw_path = Path(storage_key) + candidate_paths: list[Path] = [] + + if raw_path.is_absolute(): + candidate_paths.append(raw_path) + else: + candidate_paths.append(self.base_path / raw_path) + candidate_paths.extend(legacy_base / raw_path for legacy_base in self.legacy_base_paths) + + for candidate in candidate_paths: + resolved = candidate.expanduser().resolve() + if self._is_allowed_path(resolved): + return resolved + + raise StorageValidationError("Ungültiger Storage-Key") + + def _is_allowed_path(self, path: Path) -> bool: + roots = [self.base_path, *self.legacy_base_paths] + return any(root == path or root in path.parents for root in roots) + + def _relative_storage_key(self, path: Path) -> str: + for root in [self.base_path, *self.legacy_base_paths]: + try: + return path.relative_to(root).as_posix() + except ValueError: + continue + return path.name + + @staticmethod + def _checksum(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/backend/hermes/app/storage/schemas.py b/backend/hermes/app/storage/schemas.py new file mode 100644 index 0000000..0ed9556 --- /dev/null +++ b/backend/hermes/app/storage/schemas.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class StoredFileMetadata: + storage_key: str + original_filename: str + stored_filename: str + relative_path: str + absolute_path: Path + mime_type: str + size: int + checksum_sha256: str + + +@dataclass(frozen=True) +class FileMetadata: + storage_key: str + relative_path: str + absolute_path: Path + size: int + checksum_sha256: str diff --git a/backend/hermes/app/storage/service.py b/backend/hermes/app/storage/service.py new file mode 100644 index 0000000..47c45f3 --- /dev/null +++ b/backend/hermes/app/storage/service.py @@ -0,0 +1,103 @@ +from functools import lru_cache +from pathlib import Path +import hashlib +import mimetypes +import re +import uuid + +from app.core.config import settings +from app.storage.base import StorageProvider +from app.storage.exceptions import StorageConfigurationError, StorageValidationError +from app.storage.local import LocalDiskStorageProvider +from app.storage.schemas import FileMetadata, StoredFileMetadata + +ALLOWED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".webp", ".txt", ".zip"} +ALLOWED_MIME_TYPES = { + "application/pdf", + "image/jpeg", + "image/png", + "image/webp", + "text/plain", + "application/zip", + "application/x-zip-compressed", +} + + +class StorageService: + def __init__(self, provider: StorageProvider, *, max_upload_mb: int) -> None: + self.provider = provider + self.max_upload_mb = max(1, max_upload_mb) + + def save_file( + self, + *, + namespace: str, + content: bytes, + original_filename: str, + mime_type: str | None = None, + ) -> StoredFileMetadata: + self.validate_file_size(len(content)) + safe_name = self.safe_filename(original_filename) + resolved_mime_type = mime_type or mimetypes.guess_type(safe_name)[0] or "application/octet-stream" + self.validate_file_type(safe_name, resolved_mime_type) + stored_filename = f"{uuid.uuid4().hex}-{safe_name}" + return self.provider.save_file( + namespace=namespace, + content=content, + original_filename=original_filename, + stored_filename=stored_filename, + mime_type=resolved_mime_type, + ) + + def open_file(self, storage_key: str) -> Path: + return self.provider.open_file(storage_key) + + def delete_file(self, storage_key: str) -> None: + self.provider.delete_file(storage_key) + + def file_exists(self, storage_key: str) -> bool: + return self.provider.file_exists(storage_key) + + def get_file_metadata(self, storage_key: str) -> FileMetadata: + return self.provider.get_file_metadata(storage_key) + + @staticmethod + def calculate_checksum(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + @staticmethod + def safe_filename(file_name: str) -> str: + name = Path(file_name or "").name.strip() + if not name: + raise StorageValidationError("Dateiname fehlt") + stem = Path(name).stem.strip().lower() + stem = stem.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss") + stem = re.sub(r"[^a-z0-9]+", "-", stem).strip("-") + suffix = Path(name).suffix.lower() + safe_stem = stem or uuid.uuid4().hex[:10] + return f"{safe_stem}{suffix}" if suffix else safe_stem + + def validate_file_type(self, file_name: str, mime_type: str) -> None: + extension = Path(file_name).suffix.lower() + if extension not in ALLOWED_EXTENSIONS: + raise StorageValidationError("Dateityp ist nicht erlaubt") + if mime_type not in ALLOWED_MIME_TYPES: + raise StorageValidationError("MIME-Type ist nicht erlaubt") + + def validate_file_size(self, size: int) -> None: + if size <= 0: + raise StorageValidationError("Upload-Datei ist leer") + if size > self.max_upload_mb * 1024 * 1024: + raise StorageValidationError("Upload-Datei ist zu groß") + + +@lru_cache +def get_storage_service() -> StorageService: + if settings.storage_provider != "local": + raise StorageConfigurationError("Nur STORAGE_PROVIDER=local ist aktuell implementiert") + + provider = LocalDiskStorageProvider( + base_path=settings.storage_base_path, + legacy_base_paths=[settings.knowledge_storage_path], + ) + return StorageService(provider, max_upload_mb=settings.storage_max_upload_mb) diff --git a/backend/hermes/docker-compose.yml b/backend/hermes/docker-compose.yml index dfd941a..a2215df 100644 --- a/backend/hermes/docker-compose.yml +++ b/backend/hermes/docker-compose.yml @@ -17,11 +17,14 @@ services: ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60} JWT_ISSUER: ${JWT_ISSUER:-hermes} LOG_LEVEL: ${LOG_LEVEL:-INFO} + STORAGE_PROVIDER: ${STORAGE_PROVIDER:-local} + STORAGE_BASE_PATH: ${STORAGE_BASE_PATH:-/data/storage} + STORAGE_MAX_UPLOAD_MB: ${STORAGE_MAX_UPLOAD_MB:-50} KNOWLEDGE_STORAGE_PATH: ${KNOWLEDGE_STORAGE_PATH:-/data/knowledge} KNOWLEDGE_MAX_UPLOAD_MB: ${KNOWLEDGE_MAX_UPLOAD_MB:-50} volumes: - - knowledge-data:/data/knowledge + - ${STORAGE_HOST_PATH:-./storage}:/data/storage ports: - "8000:8000" @@ -33,6 +36,3 @@ services: networks: olympus-network: external: true - -volumes: - knowledge-data: diff --git a/docker-compose.yml b/docker-compose.yml index bada19e..3b1d3d4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,11 +20,14 @@ services: INITIAL_ADMIN_PASSWORD: ${INITIAL_ADMIN_PASSWORD:-} INITIAL_ADMIN_FIRST_NAME: ${INITIAL_ADMIN_FIRST_NAME:-} INITIAL_ADMIN_LAST_NAME: ${INITIAL_ADMIN_LAST_NAME:-} + STORAGE_PROVIDER: ${STORAGE_PROVIDER:-local} + STORAGE_BASE_PATH: ${STORAGE_BASE_PATH:-/data/storage} + STORAGE_MAX_UPLOAD_MB: ${STORAGE_MAX_UPLOAD_MB:-50} KNOWLEDGE_STORAGE_PATH: ${KNOWLEDGE_STORAGE_PATH:-/data/knowledge} KNOWLEDGE_MAX_UPLOAD_MB: ${KNOWLEDGE_MAX_UPLOAD_MB:-50} volumes: - - knowledge-data:/data/knowledge + - ${STORAGE_HOST_PATH:-./storage}:/data/storage expose: - "8000" @@ -58,6 +61,3 @@ services: networks: olympus-network: external: true - -volumes: - knowledge-data: diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100755 index 0000000..d72eea6 --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +BACKUP_DIR="${BACKUP_DIR:-./backups}" +STORAGE_HOST_PATH="${STORAGE_HOST_PATH:-./storage}" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +TARGET_DIR="${BACKUP_DIR}/${TIMESTAMP}" + +mkdir -p "${TARGET_DIR}" + +echo "==> Creating backup in ${TARGET_DIR}" + +if [[ -n "${POSTGRES_CONTAINER:-}" ]]; then + echo "==> Creating PostgreSQL dump from container ${POSTGRES_CONTAINER}" + docker exec "${POSTGRES_CONTAINER}" pg_dump -U "${POSTGRES_USER:-olympus}" "${POSTGRES_DB:-olympus}" > "${TARGET_DIR}/postgres.sql" +elif command -v pg_dump >/dev/null 2>&1 && [[ -n "${DATABASE_URL:-}" ]]; then + echo "==> Creating PostgreSQL dump from DATABASE_URL" + pg_dump "${DATABASE_URL}" > "${TARGET_DIR}/postgres.sql" +else + echo "WARN: PostgreSQL dump skipped. Set POSTGRES_CONTAINER or install pg_dump with DATABASE_URL." +fi + +if [[ -d "${STORAGE_HOST_PATH}" ]]; then + echo "==> Archiving storage directory" + tar -czf "${TARGET_DIR}/storage.tar.gz" -C "${STORAGE_HOST_PATH}" . +else + echo "WARN: Storage directory ${STORAGE_HOST_PATH} not found; storage backup skipped." +fi + +echo "INFO: .env is not copied automatically. Store production secrets separately and securely." +echo "==> Backup completed" diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..b6b8d3c --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "==> Building Docker images" +docker compose build + +echo "==> Starting services" +docker compose up -d + +echo "==> Running migrations" +scripts/migrate.sh + +echo "==> Running healthcheck" +scripts/healthcheck.sh + +echo "==> Deployment completed" diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh new file mode 100755 index 0000000..c07d1ee --- /dev/null +++ b/scripts/healthcheck.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +ATHENA_URL="${ATHENA_URL:-http://localhost:3001}" + +request_url() { + local url="$1" + if command -v curl >/dev/null 2>&1; then + curl -fsS "$url" >/dev/null + else + python3 -c "import urllib.request; urllib.request.urlopen('${url}', timeout=10).read()" + fi +} + +echo "==> Checking docker compose services" +docker compose ps + +echo "==> Checking Hermes health endpoint inside Docker network" +docker compose exec -T hermes python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=10).read()" + +echo "==> Checking Athena at ${ATHENA_URL}" +request_url "${ATHENA_URL}" + +echo "==> Healthcheck completed" diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..31820f8 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "==> Running database migrations" + +if docker compose ps --services --filter "status=running" | grep -qx "hermes"; then + docker compose exec -T hermes uv run alembic upgrade head +else + cd backend/hermes + uv run alembic upgrade head +fi + +echo "==> Migrations completed" diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 0000000..d15d5fe --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +BACKUP_SOURCE="${1:-}" +STORAGE_HOST_PATH="${STORAGE_HOST_PATH:-./storage}" + +if [[ -z "${BACKUP_SOURCE}" || ! -d "${BACKUP_SOURCE}" ]]; then + echo "Usage: scripts/restore.sh " >&2 + exit 1 +fi + +echo "This restore can overwrite database and storage state." +echo "Backup source: ${BACKUP_SOURCE}" +echo "Storage target: ${STORAGE_HOST_PATH}" +read -r -p "Type RESTORE to continue: " confirmation + +if [[ "${confirmation}" != "RESTORE" ]]; then + echo "Restore cancelled" + exit 0 +fi + +if [[ -f "${BACKUP_SOURCE}/postgres.sql" ]]; then + if [[ -n "${POSTGRES_CONTAINER:-}" ]]; then + echo "==> Restoring PostgreSQL dump into container ${POSTGRES_CONTAINER}" + docker exec -i "${POSTGRES_CONTAINER}" psql -U "${POSTGRES_USER:-olympus}" "${POSTGRES_DB:-olympus}" < "${BACKUP_SOURCE}/postgres.sql" + elif command -v psql >/dev/null 2>&1 && [[ -n "${DATABASE_URL:-}" ]]; then + echo "==> Restoring PostgreSQL dump from DATABASE_URL" + psql "${DATABASE_URL}" < "${BACKUP_SOURCE}/postgres.sql" + else + echo "WARN: PostgreSQL restore skipped. Set POSTGRES_CONTAINER or install psql with DATABASE_URL." + fi +else + echo "WARN: postgres.sql not found; database restore skipped." +fi + +if [[ -f "${BACKUP_SOURCE}/storage.tar.gz" ]]; then + mkdir -p "${STORAGE_HOST_PATH}" + echo "==> Restoring storage archive" + tar -xzf "${BACKUP_SOURCE}/storage.tar.gz" -C "${STORAGE_HOST_PATH}" +else + echo "WARN: storage.tar.gz not found; storage restore skipped." +fi + +echo "==> Restore completed"