feat(backup): add enterprise backup & restore foundation
This commit is contained in:
Schubert Ferenc
parent
7835d3ca75
commit
6c917df515
30 changed files with 1538 additions and 61 deletions
150
scripts/backup.sh
Executable file → Normal file
150
scripts/backup.sh
Executable file → Normal file
|
|
@ -1,33 +1,139 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
BACKUP_DIR="${BACKUP_DIR:-./backups}"
|
||||
STORAGE_HOST_PATH="${STORAGE_HOST_PATH:-./storage}"
|
||||
BACKUP_DIR="${BACKUP_DIR:-${STORAGE_HOST_PATH%/}/backups}"
|
||||
APP_VERSION="${APP_VERSION:-0.9.1}"
|
||||
DATABASE_URL="${DATABASE_URL:-}"
|
||||
|
||||
if [[ -z "$DATABASE_URL" ]]; then
|
||||
echo "DATABASE_URL ist erforderlich." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v pg_dump >/dev/null 2>&1; then
|
||||
echo "pg_dump ist nicht verfuegbar." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$BACKUP_DIR" "$STORAGE_HOST_PATH"
|
||||
|
||||
TIMESTAMP="$(date +%Y%m%d-%H%M%S)"
|
||||
TARGET_DIR="${BACKUP_DIR}/${TIMESTAMP}"
|
||||
FILENAME="olympus-backup-${TIMESTAMP}.zip"
|
||||
WORK_DIR="$(mktemp -d "${BACKUP_DIR%/}/tmp.backup.XXXXXX")"
|
||||
trap 'rm -rf "$WORK_DIR"' EXIT
|
||||
|
||||
mkdir -p "${TARGET_DIR}"
|
||||
export STORAGE_HOST_PATH BACKUP_DIR WORK_DIR APP_VERSION DATABASE_URL FILENAME
|
||||
|
||||
echo "==> Creating backup in ${TARGET_DIR}"
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import os
|
||||
import shutil
|
||||
|
||||
if [[ -n "${POSTGRES_CONTAINER:-}" ]]; then
|
||||
echo "==> Creating PostgreSQL dump from container ${POSTGRES_CONTAINER}"
|
||||
docker exec "${POSTGRES_CONTAINER}" pg_dump -U "${POSTGRES_USER:-olympus}" "${POSTGRES_DB:-olympus}" > "${TARGET_DIR}/postgres.sql"
|
||||
elif command -v pg_dump >/dev/null 2>&1 && [[ -n "${DATABASE_URL:-}" ]]; then
|
||||
echo "==> Creating PostgreSQL dump from DATABASE_URL"
|
||||
pg_dump "${DATABASE_URL}" > "${TARGET_DIR}/postgres.sql"
|
||||
else
|
||||
echo "WARN: PostgreSQL dump skipped. Set POSTGRES_CONTAINER or install pg_dump with DATABASE_URL."
|
||||
fi
|
||||
source = Path(os.environ["STORAGE_HOST_PATH"]).resolve()
|
||||
backup_dir = Path(os.environ["BACKUP_DIR"]).resolve()
|
||||
target = Path(os.environ["WORK_DIR"]).resolve() / "storage"
|
||||
source.mkdir(parents=True, exist_ok=True)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if [[ -d "${STORAGE_HOST_PATH}" ]]; then
|
||||
echo "==> Archiving storage directory"
|
||||
tar -czf "${TARGET_DIR}/storage.tar.gz" -C "${STORAGE_HOST_PATH}" .
|
||||
else
|
||||
echo "WARN: Storage directory ${STORAGE_HOST_PATH} not found; storage backup skipped."
|
||||
fi
|
||||
for source_path in sorted(source.rglob("*")):
|
||||
if source_path == backup_dir or backup_dir in source_path.parents:
|
||||
continue
|
||||
relative = source_path.relative_to(source)
|
||||
destination = target / relative
|
||||
if source_path.is_dir():
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
if source_path.is_file():
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source_path, destination)
|
||||
PY
|
||||
|
||||
echo "INFO: .env is not copied automatically. Store production secrets separately and securely."
|
||||
echo "==> Backup completed"
|
||||
PG_DUMP_URL="$(python3 - <<'PY'
|
||||
import os
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
database_url = os.environ["DATABASE_URL"]
|
||||
parts = urlsplit(database_url)
|
||||
scheme = parts.scheme.split("+", 1)[0]
|
||||
print(urlunsplit((scheme, parts.netloc, parts.path, parts.query, parts.fragment)))
|
||||
PY
|
||||
)"
|
||||
|
||||
pg_dump \
|
||||
--format=custom \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
--file="$WORK_DIR/database.dump" \
|
||||
--dbname="$PG_DUMP_URL"
|
||||
|
||||
python3 - <<'PY'
|
||||
from datetime import datetime, UTC
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
import json
|
||||
import os
|
||||
from urllib.parse import urlsplit
|
||||
import zipfile
|
||||
|
||||
work_dir = Path(os.environ["WORK_DIR"]).resolve()
|
||||
storage_dir = work_dir / "storage"
|
||||
dump_path = work_dir / "database.dump"
|
||||
backup_dir = Path(os.environ["BACKUP_DIR"]).resolve()
|
||||
filename = os.environ["FILENAME"]
|
||||
archive_path = backup_dir / filename
|
||||
database_url = urlsplit(os.environ["DATABASE_URL"])
|
||||
|
||||
def update_digest_from_file(digest, file_path: Path) -> None:
|
||||
with file_path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
|
||||
checksum = sha256()
|
||||
checksum.update(b"database.dump")
|
||||
update_digest_from_file(checksum, dump_path)
|
||||
|
||||
file_count = 1
|
||||
total_size = dump_path.stat().st_size
|
||||
for file_path in sorted(storage_dir.rglob("*")):
|
||||
if file_path.is_dir():
|
||||
continue
|
||||
checksum.update(file_path.relative_to(work_dir).as_posix().encode("utf-8"))
|
||||
update_digest_from_file(checksum, file_path)
|
||||
file_count += 1
|
||||
total_size += file_path.stat().st_size
|
||||
|
||||
host = database_url.hostname or "unknown"
|
||||
manifest = {
|
||||
"backup_id": sha256(f"{filename}:{datetime.now(UTC).isoformat()}".encode("utf-8")).hexdigest()[:24],
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
"app_version": os.environ["APP_VERSION"],
|
||||
"backup_type": "full",
|
||||
"database_url_host_anonymized": f"sha256:{sha256(host.encode('utf-8')).hexdigest()[:12]}",
|
||||
"database_name": database_url.path.rsplit("/", 1)[-1] or "unknown",
|
||||
"storage_base_path": os.environ["STORAGE_HOST_PATH"],
|
||||
"included_sections": ["database", "storage"],
|
||||
"file_count": file_count,
|
||||
"total_size_bytes": total_size,
|
||||
"checksum_sha256": checksum.hexdigest(),
|
||||
"created_by_user_id": None,
|
||||
"created_by_username": "cli",
|
||||
}
|
||||
|
||||
manifest_path = work_dir / "manifest.json"
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=True), encoding="utf-8")
|
||||
|
||||
with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.writestr("storage/", "")
|
||||
archive.write(manifest_path, "manifest.json")
|
||||
archive.write(dump_path, "database.dump")
|
||||
for file_path in sorted(storage_dir.rglob("*")):
|
||||
if file_path.is_dir():
|
||||
continue
|
||||
archive.write(file_path, file_path.relative_to(work_dir).as_posix())
|
||||
PY
|
||||
|
||||
echo "Backup erstellt: ${BACKUP_DIR%/}/${FILENAME}"
|
||||
|
|
|
|||
161
scripts/restore.sh
Executable file → Normal file
161
scripts/restore.sh
Executable file → Normal file
|
|
@ -1,46 +1,151 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
ROOT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
BACKUP_SOURCE="${1:-}"
|
||||
STORAGE_HOST_PATH="${STORAGE_HOST_PATH:-./storage}"
|
||||
BACKUP_DIR="${BACKUP_DIR:-${STORAGE_HOST_PATH%/}/backups}"
|
||||
DATABASE_URL="${DATABASE_URL:-}"
|
||||
STOPPED_SERVICES=0
|
||||
|
||||
if [[ -z "${BACKUP_SOURCE}" || ! -d "${BACKUP_SOURCE}" ]]; then
|
||||
echo "Usage: scripts/restore.sh <backup-directory>" >&2
|
||||
if [[ -z "$BACKUP_SOURCE" || ! -f "$BACKUP_SOURCE" ]]; then
|
||||
echo "Usage: scripts/restore.sh <backup-zip>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "This restore can overwrite database and storage state."
|
||||
echo "Backup source: ${BACKUP_SOURCE}"
|
||||
echo "Storage target: ${STORAGE_HOST_PATH}"
|
||||
read -r -p "Type RESTORE to continue: " confirmation
|
||||
if [[ -z "$DATABASE_URL" ]]; then
|
||||
echo "DATABASE_URL ist erforderlich." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${confirmation}" != "RESTORE" ]]; then
|
||||
echo "Restore cancelled"
|
||||
if ! command -v pg_restore >/dev/null 2>&1; then
|
||||
echo "pg_restore ist nicht verfuegbar." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$STORAGE_HOST_PATH" "$BACKUP_DIR"
|
||||
|
||||
python3 - "$BACKUP_SOURCE" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
path = Path(sys.argv[1]).resolve()
|
||||
required = {"manifest.json", "database.dump"}
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
names = set(archive.namelist())
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise SystemExit(f"Ungueltige ZIP-Datei: {exc}") from exc
|
||||
|
||||
missing = [name for name in required if name not in names]
|
||||
if not any(name == "storage/" or name.startswith("storage/") for name in names):
|
||||
missing.append("storage/")
|
||||
if missing:
|
||||
raise SystemExit(f"Backup unvollstaendig: {', '.join(missing)}")
|
||||
PY
|
||||
|
||||
echo "WARNUNG: Dieser Restore kann Datenbank- und Storage-Daten dauerhaft ueberschreiben."
|
||||
echo "Backup: $BACKUP_SOURCE"
|
||||
read -r -p "Bitte exakt 'ICH VERSTEHE DAS RISIKO' eingeben: " confirmation
|
||||
|
||||
if [[ "$confirmation" != "ICH VERSTEHE DAS RISIKO" ]]; then
|
||||
echo "Restore abgebrochen."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -f "${BACKUP_SOURCE}/postgres.sql" ]]; then
|
||||
if [[ -n "${POSTGRES_CONTAINER:-}" ]]; then
|
||||
echo "==> Restoring PostgreSQL dump into container ${POSTGRES_CONTAINER}"
|
||||
docker exec -i "${POSTGRES_CONTAINER}" psql -U "${POSTGRES_USER:-olympus}" "${POSTGRES_DB:-olympus}" < "${BACKUP_SOURCE}/postgres.sql"
|
||||
elif command -v psql >/dev/null 2>&1 && [[ -n "${DATABASE_URL:-}" ]]; then
|
||||
echo "==> Restoring PostgreSQL dump from DATABASE_URL"
|
||||
psql "${DATABASE_URL}" < "${BACKUP_SOURCE}/postgres.sql"
|
||||
else
|
||||
echo "WARN: PostgreSQL restore skipped. Set POSTGRES_CONTAINER or install psql with DATABASE_URL."
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
read -r -p "Olympus-Dienste jetzt per docker compose stoppen? [y/N] " stop_reply
|
||||
if [[ "$stop_reply" =~ ^[Yy]$ ]]; then
|
||||
docker compose stop athena hermes
|
||||
STOPPED_SERVICES=1
|
||||
fi
|
||||
else
|
||||
echo "WARN: postgres.sql not found; database restore skipped."
|
||||
fi
|
||||
|
||||
if [[ -f "${BACKUP_SOURCE}/storage.tar.gz" ]]; then
|
||||
mkdir -p "${STORAGE_HOST_PATH}"
|
||||
echo "==> Restoring storage archive"
|
||||
tar -xzf "${BACKUP_SOURCE}/storage.tar.gz" -C "${STORAGE_HOST_PATH}"
|
||||
else
|
||||
echo "WARN: storage.tar.gz not found; storage restore skipped."
|
||||
fi
|
||||
WORK_DIR="$(mktemp -d "${BACKUP_DIR%/}/tmp.restore.XXXXXX")"
|
||||
trap 'rm -rf "$WORK_DIR"; if [[ "$STOPPED_SERVICES" -eq 1 ]]; then docker compose start hermes athena; fi' EXIT
|
||||
|
||||
echo "==> Restore completed"
|
||||
python3 - "$BACKUP_SOURCE" "$WORK_DIR" <<'PY'
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
archive_path = Path(sys.argv[1]).resolve()
|
||||
target_dir = Path(sys.argv[2]).resolve()
|
||||
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
archive.extractall(target_dir)
|
||||
PY
|
||||
|
||||
SNAPSHOT_PATH="${BACKUP_DIR%/}/pre-restore-storage-$(date +%Y%m%d-%H%M%S).tar.gz"
|
||||
export STORAGE_HOST_PATH SNAPSHOT_PATH
|
||||
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import os
|
||||
import tarfile
|
||||
|
||||
storage = Path(os.environ["STORAGE_HOST_PATH"]).resolve()
|
||||
snapshot = Path(os.environ["SNAPSHOT_PATH"]).resolve()
|
||||
storage.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with tarfile.open(snapshot, "w:gz") as archive:
|
||||
for path in sorted(storage.rglob("*")):
|
||||
if path.name == "backups" and path.is_dir():
|
||||
continue
|
||||
if "backups" in path.parts:
|
||||
continue
|
||||
archive.add(path, arcname=path.relative_to(storage))
|
||||
PY
|
||||
|
||||
PG_RESTORE_URL="$(python3 - <<'PY'
|
||||
import os
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
database_url = os.environ["DATABASE_URL"]
|
||||
parts = urlsplit(database_url)
|
||||
scheme = parts.scheme.split("+", 1)[0]
|
||||
print(urlunsplit((scheme, parts.netloc, parts.path, parts.query, parts.fragment)))
|
||||
PY
|
||||
)"
|
||||
|
||||
pg_restore \
|
||||
--clean \
|
||||
--if-exists \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
--dbname="$PG_RESTORE_URL" \
|
||||
"$WORK_DIR/database.dump"
|
||||
|
||||
export WORK_DIR
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import os
|
||||
import shutil
|
||||
|
||||
storage_target = Path(os.environ["STORAGE_HOST_PATH"]).resolve()
|
||||
storage_source = (Path(os.environ["WORK_DIR"]).resolve() / "storage")
|
||||
storage_target.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for path in storage_target.iterdir():
|
||||
if path.name == "backups":
|
||||
continue
|
||||
if path.is_dir():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink()
|
||||
|
||||
for source_path in sorted(storage_source.rglob("*")):
|
||||
relative = source_path.relative_to(storage_source)
|
||||
destination = storage_target / relative
|
||||
if source_path.is_dir():
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
continue
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source_path, destination)
|
||||
PY
|
||||
|
||||
echo "Restore abgeschlossen. Storage-Snapshot: ${SNAPSHOT_PATH}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue