feat(storage): add local storage framework
This commit is contained in:
parent
228da8f814
commit
964b545bc5
24 changed files with 890 additions and 98 deletions
|
|
@ -13,5 +13,8 @@ INITIAL_ADMIN_EMAIL=
|
|||
INITIAL_ADMIN_PASSWORD=
|
||||
INITIAL_ADMIN_FIRST_NAME=
|
||||
INITIAL_ADMIN_LAST_NAME=
|
||||
STORAGE_PROVIDER=local
|
||||
STORAGE_BASE_PATH=/data/storage
|
||||
STORAGE_MAX_UPLOAD_MB=50
|
||||
KNOWLEDGE_STORAGE_PATH=/data/knowledge
|
||||
KNOWLEDGE_MAX_UPLOAD_MB=50
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ def can_read_activity(action: str, permissions: set[str]) -> bool:
|
|||
return "customers.read" in permissions
|
||||
if action.startswith("roles."):
|
||||
return "roles.read" in permissions
|
||||
if action.startswith("knowledge."):
|
||||
return "knowledge.read" in permissions
|
||||
if action.startswith("audit_logs."):
|
||||
return "audit_logs.read" in permissions
|
||||
if action.startswith("auth."):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response, UploadFile, status
|
||||
from fastapi.responses import FileResponse
|
||||
|
|
@ -30,7 +29,7 @@ from app.schemas.knowledge import (
|
|||
normalize_tags,
|
||||
)
|
||||
from app.services.audit_service import sanitize, write_audit_log
|
||||
from app.services.knowledge_service import KnowledgeService, assert_safe_path
|
||||
from app.services.knowledge_service import KnowledgeService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -256,7 +255,10 @@ def delete_document(document_id: int, request: Request, db: Session = Depends(ge
|
|||
document = get_document_or_404(db, document_id)
|
||||
before_data = sanitize(document)
|
||||
label = document.title
|
||||
file_path = document.file_path
|
||||
delete_or_conflict(db, document)
|
||||
if file_path:
|
||||
KnowledgeService.delete_storage_key(file_path)
|
||||
write_audit_log(db, action="knowledge.documents.delete", entity_type="knowledge_documents", entity_id=document_id, entity_label=label, actor=current_user, request=request, before_data=before_data)
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
|
@ -264,11 +266,7 @@ def delete_document(document_id: int, request: Request, db: Session = Depends(ge
|
|||
@router.get("/documents/{document_id:int}/download")
|
||||
def download_document(document_id: int, db: Session = Depends(get_db), current_user: User = Depends(require_permission("knowledge.download"))):
|
||||
document = get_document_or_404(db, document_id)
|
||||
if not document.file_path:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Dokument hat keine lokale Datei")
|
||||
path = assert_safe_path(Path(document.file_path))
|
||||
if not path.exists() or not path.is_file():
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Datei nicht gefunden")
|
||||
path = KnowledgeService.open_document_file(document)
|
||||
logger.info("knowledge.documents.download", extra={"actor_user_id": current_user.id, "target_document_id": document_id})
|
||||
return FileResponse(path, media_type=document.mime_type or "application/octet-stream", filename=document.file_name)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
from typing import Any
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
|
|
@ -15,6 +18,9 @@ class Settings(BaseSettings):
|
|||
initial_admin_password: str | None = None
|
||||
initial_admin_first_name: str = ""
|
||||
initial_admin_last_name: str = ""
|
||||
storage_provider: str = "local"
|
||||
storage_base_path: str = "/data/storage"
|
||||
storage_max_upload_mb: int = 50
|
||||
knowledge_storage_path: str = "/data/knowledge"
|
||||
knowledge_max_upload_mb: int = 50
|
||||
|
||||
|
|
@ -23,5 +29,19 @@ class Settings(BaseSettings):
|
|||
extra="ignore",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def apply_legacy_storage_settings(cls, values: Any) -> Any:
|
||||
if not isinstance(values, dict):
|
||||
return values
|
||||
|
||||
if "storage_base_path" not in values and "knowledge_storage_path" in values:
|
||||
values["storage_base_path"] = values["knowledge_storage_path"]
|
||||
|
||||
if "storage_max_upload_mb" not in values and "knowledge_max_upload_mb" in values:
|
||||
values["storage_max_upload_mb"] = values["knowledge_max_upload_mb"]
|
||||
|
||||
return values
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
|
|
|||
|
|
@ -130,6 +130,19 @@ def action_title(action: str) -> str:
|
|||
"customer_contacts.create": "Ansprechpartner erstellt",
|
||||
"customer_contacts.update": "Ansprechpartner bearbeitet",
|
||||
"customer_contacts.delete": "Ansprechpartner gelöscht",
|
||||
"knowledge.manufacturers.create": "Hersteller erstellt",
|
||||
"knowledge.manufacturers.update": "Hersteller bearbeitet",
|
||||
"knowledge.manufacturers.delete": "Hersteller gelöscht",
|
||||
"knowledge.devices.create": "Gerät erstellt",
|
||||
"knowledge.devices.update": "Gerät bearbeitet",
|
||||
"knowledge.devices.delete": "Gerät gelöscht",
|
||||
"knowledge.documents.create": "Dokument erstellt",
|
||||
"knowledge.documents.upload": "Dokument hochgeladen",
|
||||
"knowledge.documents.update": "Dokument bearbeitet",
|
||||
"knowledge.documents.delete": "Dokument gelöscht",
|
||||
"knowledge.notes.create": "Notiz erstellt",
|
||||
"knowledge.notes.update": "Notiz bearbeitet",
|
||||
"knowledge.notes.delete": "Notiz gelöscht",
|
||||
"users.initial_admin_bootstrap": "Initialer Administrator erstellt",
|
||||
"knowledge.manufacturers.create": "Hersteller erstellt",
|
||||
"knowledge.manufacturers.update": "Hersteller bearbeitet",
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
import hashlib
|
||||
import mimetypes
|
||||
import re
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException, UploadFile, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.knowledge import KnowledgeDevice, KnowledgeDocument, KnowledgeManufacturer, KnowledgeNote
|
||||
from app.repositories.knowledge_repository import KnowledgeRepository
|
||||
from app.schemas.knowledge import (
|
||||
|
|
@ -20,17 +16,8 @@ from app.schemas.knowledge import (
|
|||
KnowledgeNoteCreate,
|
||||
KnowledgeNoteUpdate,
|
||||
)
|
||||
|
||||
ALLOWED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".webp", ".txt", ".zip"}
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"application/pdf",
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"text/plain",
|
||||
"application/zip",
|
||||
"application/x-zip-compressed",
|
||||
}
|
||||
from app.storage import get_storage_service
|
||||
from app.storage.exceptions import StorageFileNotFoundError, StorageValidationError
|
||||
|
||||
|
||||
def slugify(value: str) -> str:
|
||||
|
|
@ -51,27 +38,6 @@ def unique_slug(db: Session, base: str, exists) -> str:
|
|||
return candidate
|
||||
|
||||
|
||||
def storage_root() -> Path:
|
||||
root = Path(settings.knowledge_storage_path).resolve()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def assert_safe_path(path: Path) -> Path:
|
||||
root = storage_root()
|
||||
resolved = path.resolve()
|
||||
if root != resolved and root not in resolved.parents:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Ungültiger Dateipfad")
|
||||
return resolved
|
||||
|
||||
|
||||
def safe_file_name(file_name: str) -> str:
|
||||
name = Path(file_name).name.strip()
|
||||
stem = slugify(Path(name).stem)
|
||||
suffix = Path(name).suffix.lower()
|
||||
return f"{stem}{suffix}" if suffix else stem
|
||||
|
||||
|
||||
def parse_tags(value: str) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
tags: list[str] = []
|
||||
|
|
@ -84,25 +50,10 @@ def parse_tags(value: str) -> list[str]:
|
|||
return tags
|
||||
|
||||
|
||||
async def read_upload(file: UploadFile) -> tuple[bytes, str, str]:
|
||||
original_name = file.filename or ""
|
||||
file_name = safe_file_name(original_name)
|
||||
extension = Path(file_name).suffix.lower()
|
||||
if extension not in ALLOWED_EXTENSIONS:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Dateityp ist nicht erlaubt")
|
||||
|
||||
max_bytes = max(1, settings.knowledge_max_upload_mb) * 1024 * 1024
|
||||
content = await file.read(max_bytes + 1)
|
||||
if not content:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload-Datei ist leer")
|
||||
if len(content) > max_bytes:
|
||||
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Upload-Datei ist zu groß")
|
||||
|
||||
mime_type = file.content_type or mimetypes.guess_type(file_name)[0] or "application/octet-stream"
|
||||
if mime_type not in ALLOWED_MIME_TYPES:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="MIME-Type ist nicht erlaubt")
|
||||
|
||||
return content, file_name, mime_type
|
||||
def storage_validation_error(exc: StorageValidationError) -> HTTPException:
|
||||
detail = str(exc) or "Ungültige Datei"
|
||||
status_code = status.HTTP_413_REQUEST_ENTITY_TOO_LARGE if "groß" in detail else status.HTTP_400_BAD_REQUEST
|
||||
return HTTPException(status_code=status_code, detail=detail)
|
||||
|
||||
|
||||
class KnowledgeService:
|
||||
|
|
@ -175,22 +126,29 @@ class KnowledgeService:
|
|||
payload: KnowledgeDocumentCreate,
|
||||
) -> KnowledgeDocument:
|
||||
KnowledgeService._validate_document_links(db, payload.manufacturer_id, payload.device_id)
|
||||
content, file_name, mime_type = await read_upload(file)
|
||||
checksum = hashlib.sha256(content).hexdigest()
|
||||
storage_service = get_storage_service()
|
||||
max_bytes = storage_service.max_upload_mb * 1024 * 1024
|
||||
content = await file.read(max_bytes + 1)
|
||||
original_name = file.filename or ""
|
||||
mime_type = file.content_type
|
||||
try:
|
||||
metadata = storage_service.save_file(
|
||||
namespace=f"knowledge/documents/{payload.manufacturer_id}",
|
||||
content=content,
|
||||
original_filename=original_name,
|
||||
mime_type=mime_type,
|
||||
)
|
||||
except StorageValidationError as exc:
|
||||
raise storage_validation_error(exc) from exc
|
||||
slug = unique_slug(db, payload.title, lambda session, value: KnowledgeRepository.get_document_by_slug(session, value) is not None)
|
||||
target_dir = storage_root() / str(payload.manufacturer_id)
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
stored_name = f"{uuid.uuid4().hex}-{file_name}"
|
||||
file_path = assert_safe_path(target_dir / stored_name)
|
||||
file_path.write_bytes(content)
|
||||
document = KnowledgeDocument(
|
||||
slug=slug,
|
||||
external_url=str(payload.external_url or ""),
|
||||
file_name=file_name,
|
||||
file_path=str(file_path),
|
||||
mime_type=mime_type,
|
||||
file_size=len(content),
|
||||
checksum_sha256=checksum,
|
||||
file_name=metadata.original_filename,
|
||||
file_path=metadata.storage_key,
|
||||
mime_type=metadata.mime_type,
|
||||
file_size=metadata.size,
|
||||
checksum_sha256=metadata.checksum_sha256,
|
||||
**payload.model_dump(exclude={"external_url"}),
|
||||
)
|
||||
db.add(document)
|
||||
|
|
@ -216,6 +174,30 @@ class KnowledgeService:
|
|||
db.commit()
|
||||
return KnowledgeRepository.get_note(db, note.id) or note
|
||||
|
||||
@staticmethod
|
||||
def open_document_file(document: KnowledgeDocument):
|
||||
if not document.file_path:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Dokument hat keine lokale Datei")
|
||||
try:
|
||||
return get_storage_service().open_file(document.file_path)
|
||||
except StorageFileNotFoundError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Datei nicht gefunden") from exc
|
||||
except StorageValidationError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Ungültiger Dateipfad") from exc
|
||||
|
||||
@staticmethod
|
||||
def delete_document_file(document: KnowledgeDocument) -> None:
|
||||
if not document.file_path:
|
||||
return
|
||||
KnowledgeService.delete_storage_key(document.file_path)
|
||||
|
||||
@staticmethod
|
||||
def delete_storage_key(storage_key: str) -> None:
|
||||
try:
|
||||
get_storage_service().delete_file(storage_key)
|
||||
except StorageValidationError as exc:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Ungültiger Dateipfad") from exc
|
||||
|
||||
@staticmethod
|
||||
def update_note(db: Session, note: KnowledgeNote, payload: KnowledgeNoteUpdate) -> KnowledgeNote:
|
||||
KnowledgeService._validate_optional_links(db, payload.manufacturer_id, payload.device_id)
|
||||
|
|
|
|||
3
backend/hermes/app/storage/__init__.py
Normal file
3
backend/hermes/app/storage/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from app.storage.service import StorageService, get_storage_service
|
||||
|
||||
__all__ = ["StorageService", "get_storage_service"]
|
||||
34
backend/hermes/app/storage/base.py
Normal file
34
backend/hermes/app/storage/base.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from pathlib import Path
|
||||
|
||||
from app.storage.schemas import FileMetadata, StoredFileMetadata
|
||||
|
||||
|
||||
class StorageProvider(ABC):
|
||||
@abstractmethod
|
||||
def save_file(
|
||||
self,
|
||||
*,
|
||||
namespace: str,
|
||||
content: bytes,
|
||||
original_filename: str,
|
||||
stored_filename: str,
|
||||
mime_type: str,
|
||||
) -> StoredFileMetadata:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def open_file(self, storage_key: str) -> Path:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def delete_file(self, storage_key: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def file_exists(self, storage_key: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def get_file_metadata(self, storage_key: str) -> FileMetadata:
|
||||
raise NotImplementedError
|
||||
14
backend/hermes/app/storage/exceptions.py
Normal file
14
backend/hermes/app/storage/exceptions.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
class StorageError(Exception):
|
||||
"""Base exception for storage operations."""
|
||||
|
||||
|
||||
class StorageConfigurationError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class StorageValidationError(StorageError):
|
||||
pass
|
||||
|
||||
|
||||
class StorageFileNotFoundError(StorageError):
|
||||
pass
|
||||
141
backend/hermes/app/storage/local.py
Normal file
141
backend/hermes/app/storage/local.py
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
from pathlib import Path
|
||||
import hashlib
|
||||
|
||||
from app.storage.base import StorageProvider
|
||||
from app.storage.exceptions import StorageFileNotFoundError, StorageValidationError
|
||||
from app.storage.schemas import FileMetadata, StoredFileMetadata
|
||||
|
||||
|
||||
class LocalDiskStorageProvider(StorageProvider):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
base_path: str,
|
||||
legacy_base_paths: list[str] | None = None,
|
||||
) -> None:
|
||||
self.base_path = Path(base_path).expanduser().resolve()
|
||||
self.legacy_base_paths = [
|
||||
Path(path).expanduser().resolve()
|
||||
for path in legacy_base_paths or []
|
||||
if path
|
||||
]
|
||||
self._ensure_layout()
|
||||
|
||||
def save_file(
|
||||
self,
|
||||
*,
|
||||
namespace: str,
|
||||
content: bytes,
|
||||
original_filename: str,
|
||||
stored_filename: str,
|
||||
mime_type: str,
|
||||
) -> StoredFileMetadata:
|
||||
if Path(stored_filename).name != stored_filename or ".." in Path(stored_filename).parts:
|
||||
raise StorageValidationError("Ungültiger gespeicherter Dateiname")
|
||||
namespace_path = self._safe_namespace_path(namespace)
|
||||
namespace_path.mkdir(parents=True, exist_ok=True)
|
||||
absolute_path = self._safe_child(namespace_path / stored_filename)
|
||||
absolute_path.write_bytes(content)
|
||||
relative_path = absolute_path.relative_to(self.base_path).as_posix()
|
||||
checksum = self._checksum(absolute_path)
|
||||
return StoredFileMetadata(
|
||||
storage_key=relative_path,
|
||||
original_filename=original_filename,
|
||||
stored_filename=stored_filename,
|
||||
relative_path=relative_path,
|
||||
absolute_path=absolute_path,
|
||||
mime_type=mime_type,
|
||||
size=len(content),
|
||||
checksum_sha256=checksum,
|
||||
)
|
||||
|
||||
def open_file(self, storage_key: str) -> Path:
|
||||
path = self._resolve_storage_key(storage_key)
|
||||
if not path.exists() or not path.is_file():
|
||||
raise StorageFileNotFoundError("Datei nicht gefunden")
|
||||
return path
|
||||
|
||||
def delete_file(self, storage_key: str) -> None:
|
||||
path = self._resolve_storage_key(storage_key)
|
||||
if not path.exists():
|
||||
return
|
||||
if not path.is_file():
|
||||
raise StorageValidationError("Storage-Key verweist nicht auf eine Datei")
|
||||
path.unlink()
|
||||
|
||||
def file_exists(self, storage_key: str) -> bool:
|
||||
try:
|
||||
path = self._resolve_storage_key(storage_key)
|
||||
except StorageValidationError:
|
||||
return False
|
||||
return path.exists() and path.is_file()
|
||||
|
||||
def get_file_metadata(self, storage_key: str) -> FileMetadata:
|
||||
path = self.open_file(storage_key)
|
||||
return FileMetadata(
|
||||
storage_key=storage_key,
|
||||
relative_path=self._relative_storage_key(path),
|
||||
absolute_path=path,
|
||||
size=path.stat().st_size,
|
||||
checksum_sha256=self._checksum(path),
|
||||
)
|
||||
|
||||
def _ensure_layout(self) -> None:
|
||||
for relative_path in [
|
||||
"knowledge/documents",
|
||||
"knowledge/thumbnails",
|
||||
"customers",
|
||||
"projects",
|
||||
"tickets",
|
||||
"imports",
|
||||
"temp",
|
||||
]:
|
||||
(self.base_path / relative_path).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _safe_namespace_path(self, namespace: str) -> Path:
|
||||
if namespace.startswith("/") or ".." in Path(namespace).parts:
|
||||
raise StorageValidationError("Ungültiger Storage-Namespace")
|
||||
return self._safe_child(self.base_path / namespace)
|
||||
|
||||
def _safe_child(self, path: Path) -> Path:
|
||||
resolved = path.resolve()
|
||||
if self.base_path != resolved and self.base_path not in resolved.parents:
|
||||
raise StorageValidationError("Ungültiger Storage-Pfad")
|
||||
return resolved
|
||||
|
||||
def _resolve_storage_key(self, storage_key: str) -> Path:
|
||||
raw_path = Path(storage_key)
|
||||
candidate_paths: list[Path] = []
|
||||
|
||||
if raw_path.is_absolute():
|
||||
candidate_paths.append(raw_path)
|
||||
else:
|
||||
candidate_paths.append(self.base_path / raw_path)
|
||||
candidate_paths.extend(legacy_base / raw_path for legacy_base in self.legacy_base_paths)
|
||||
|
||||
for candidate in candidate_paths:
|
||||
resolved = candidate.expanduser().resolve()
|
||||
if self._is_allowed_path(resolved):
|
||||
return resolved
|
||||
|
||||
raise StorageValidationError("Ungültiger Storage-Key")
|
||||
|
||||
def _is_allowed_path(self, path: Path) -> bool:
|
||||
roots = [self.base_path, *self.legacy_base_paths]
|
||||
return any(root == path or root in path.parents for root in roots)
|
||||
|
||||
def _relative_storage_key(self, path: Path) -> str:
|
||||
for root in [self.base_path, *self.legacy_base_paths]:
|
||||
try:
|
||||
return path.relative_to(root).as_posix()
|
||||
except ValueError:
|
||||
continue
|
||||
return path.name
|
||||
|
||||
@staticmethod
|
||||
def _checksum(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
23
backend/hermes/app/storage/schemas.py
Normal file
23
backend/hermes/app/storage/schemas.py
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StoredFileMetadata:
|
||||
storage_key: str
|
||||
original_filename: str
|
||||
stored_filename: str
|
||||
relative_path: str
|
||||
absolute_path: Path
|
||||
mime_type: str
|
||||
size: int
|
||||
checksum_sha256: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FileMetadata:
|
||||
storage_key: str
|
||||
relative_path: str
|
||||
absolute_path: Path
|
||||
size: int
|
||||
checksum_sha256: str
|
||||
103
backend/hermes/app/storage/service.py
Normal file
103
backend/hermes/app/storage/service.py
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import mimetypes
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from app.core.config import settings
|
||||
from app.storage.base import StorageProvider
|
||||
from app.storage.exceptions import StorageConfigurationError, StorageValidationError
|
||||
from app.storage.local import LocalDiskStorageProvider
|
||||
from app.storage.schemas import FileMetadata, StoredFileMetadata
|
||||
|
||||
ALLOWED_EXTENSIONS = {".pdf", ".jpg", ".jpeg", ".png", ".webp", ".txt", ".zip"}
|
||||
ALLOWED_MIME_TYPES = {
|
||||
"application/pdf",
|
||||
"image/jpeg",
|
||||
"image/png",
|
||||
"image/webp",
|
||||
"text/plain",
|
||||
"application/zip",
|
||||
"application/x-zip-compressed",
|
||||
}
|
||||
|
||||
|
||||
class StorageService:
|
||||
def __init__(self, provider: StorageProvider, *, max_upload_mb: int) -> None:
|
||||
self.provider = provider
|
||||
self.max_upload_mb = max(1, max_upload_mb)
|
||||
|
||||
def save_file(
|
||||
self,
|
||||
*,
|
||||
namespace: str,
|
||||
content: bytes,
|
||||
original_filename: str,
|
||||
mime_type: str | None = None,
|
||||
) -> StoredFileMetadata:
|
||||
self.validate_file_size(len(content))
|
||||
safe_name = self.safe_filename(original_filename)
|
||||
resolved_mime_type = mime_type or mimetypes.guess_type(safe_name)[0] or "application/octet-stream"
|
||||
self.validate_file_type(safe_name, resolved_mime_type)
|
||||
stored_filename = f"{uuid.uuid4().hex}-{safe_name}"
|
||||
return self.provider.save_file(
|
||||
namespace=namespace,
|
||||
content=content,
|
||||
original_filename=original_filename,
|
||||
stored_filename=stored_filename,
|
||||
mime_type=resolved_mime_type,
|
||||
)
|
||||
|
||||
def open_file(self, storage_key: str) -> Path:
|
||||
return self.provider.open_file(storage_key)
|
||||
|
||||
def delete_file(self, storage_key: str) -> None:
|
||||
self.provider.delete_file(storage_key)
|
||||
|
||||
def file_exists(self, storage_key: str) -> bool:
|
||||
return self.provider.file_exists(storage_key)
|
||||
|
||||
def get_file_metadata(self, storage_key: str) -> FileMetadata:
|
||||
return self.provider.get_file_metadata(storage_key)
|
||||
|
||||
@staticmethod
|
||||
def calculate_checksum(content: bytes) -> str:
|
||||
return hashlib.sha256(content).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def safe_filename(file_name: str) -> str:
|
||||
name = Path(file_name or "").name.strip()
|
||||
if not name:
|
||||
raise StorageValidationError("Dateiname fehlt")
|
||||
stem = Path(name).stem.strip().lower()
|
||||
stem = stem.replace("ä", "ae").replace("ö", "oe").replace("ü", "ue").replace("ß", "ss")
|
||||
stem = re.sub(r"[^a-z0-9]+", "-", stem).strip("-")
|
||||
suffix = Path(name).suffix.lower()
|
||||
safe_stem = stem or uuid.uuid4().hex[:10]
|
||||
return f"{safe_stem}{suffix}" if suffix else safe_stem
|
||||
|
||||
def validate_file_type(self, file_name: str, mime_type: str) -> None:
|
||||
extension = Path(file_name).suffix.lower()
|
||||
if extension not in ALLOWED_EXTENSIONS:
|
||||
raise StorageValidationError("Dateityp ist nicht erlaubt")
|
||||
if mime_type not in ALLOWED_MIME_TYPES:
|
||||
raise StorageValidationError("MIME-Type ist nicht erlaubt")
|
||||
|
||||
def validate_file_size(self, size: int) -> None:
|
||||
if size <= 0:
|
||||
raise StorageValidationError("Upload-Datei ist leer")
|
||||
if size > self.max_upload_mb * 1024 * 1024:
|
||||
raise StorageValidationError("Upload-Datei ist zu groß")
|
||||
|
||||
|
||||
@lru_cache
|
||||
def get_storage_service() -> StorageService:
|
||||
if settings.storage_provider != "local":
|
||||
raise StorageConfigurationError("Nur STORAGE_PROVIDER=local ist aktuell implementiert")
|
||||
|
||||
provider = LocalDiskStorageProvider(
|
||||
base_path=settings.storage_base_path,
|
||||
legacy_base_paths=[settings.knowledge_storage_path],
|
||||
)
|
||||
return StorageService(provider, max_upload_mb=settings.storage_max_upload_mb)
|
||||
|
|
@ -17,11 +17,14 @@ services:
|
|||
ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES:-60}
|
||||
JWT_ISSUER: ${JWT_ISSUER:-hermes}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
STORAGE_PROVIDER: ${STORAGE_PROVIDER:-local}
|
||||
STORAGE_BASE_PATH: ${STORAGE_BASE_PATH:-/data/storage}
|
||||
STORAGE_MAX_UPLOAD_MB: ${STORAGE_MAX_UPLOAD_MB:-50}
|
||||
KNOWLEDGE_STORAGE_PATH: ${KNOWLEDGE_STORAGE_PATH:-/data/knowledge}
|
||||
KNOWLEDGE_MAX_UPLOAD_MB: ${KNOWLEDGE_MAX_UPLOAD_MB:-50}
|
||||
|
||||
volumes:
|
||||
- knowledge-data:/data/knowledge
|
||||
- ${STORAGE_HOST_PATH:-./storage}:/data/storage
|
||||
|
||||
ports:
|
||||
- "8000:8000"
|
||||
|
|
@ -33,6 +36,3 @@ services:
|
|||
networks:
|
||||
olympus-network:
|
||||
external: true
|
||||
|
||||
volumes:
|
||||
knowledge-data:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue