# SPDX-FileCopyrightText: 2026 Helmholtz-Zentrum Dresden-Rossendorf e.V (HZDR)
# SPDX-License-Identifier: Apache-2.0
"""Keep entries safe when MongoDB cannot be reached.
The shotsheet competes with a spreadsheet, and a spreadsheet never tells an
operator that their work went nowhere. So a database that is unreachable must
not lose the entry and must not stall the page: the record is written to a
file on this machine, the operator is told exactly where it is, and it is
replayed into MongoDB when the connection comes back.
Two decisions matter here. The spool is plain JSON, one file per entry, in a
directory operators can open -- if every other part of this app failed, the
shot is still readable and could be retyped from what is on disk. And nothing
is ever deleted on replay: a synced file is moved into `synced/`, so an
operator who wants to check what happened can still see it.
"""
from __future__ import annotations
import datetime
import json
import logging
import re
import uuid
from pathlib import Path
from typing import Any
from flask import current_app, has_app_context
SPOOL_DIR_CONFIG_KEY = "LABFROG_OFFLINE_SPOOL_DIR"
DEFAULT_SPOOL_DIR_NAME = "offline-queue"
SYNCED_SUBDIR = "synced"
README_NAME = "READ-ME-FIRST.txt"
_SAFE_NAME = re.compile(r"[^A-Za-z0-9_.-]+")
# Driver errors routinely quote the whole connection URI, credentials included.
# The spool is a file on an operator's disk that may be copied or mailed, so
# anything that looks like a connection string is removed on the way in. The
# untouched detail stays in the server log, where it belongs.
_CONNECTION_URI = re.compile(r"\w+://\S+", re.IGNORECASE)
[docs]
def safe_reason(reason: str | None) -> str:
"""Return a reason safe to write to disk and show on screen."""
text = str(reason or "").strip()
if not text:
return "the database could not be reached"
return _CONNECTION_URI.sub("<connection details removed>", text)
READ_ME = """\
LabFrog could not reach the database, so your entries were saved here instead.
NOTHING HAS BEEN LOST. Each .json file in this folder is one entry exactly as
you typed it. You can open them in any text editor.
To put them back into the database:
1. Check that you are on the network and that the database is reachable.
2. Open LabFrog and go to the "Offline queue" page (link in the warning
banner), then press "Sync now".
Or, from a terminal in the LabFrog folder:
uv run poe sync-offline
3. Each entry that goes in is moved to the "synced" sub-folder, so you can
still see what happened. Nothing here is deleted.
If an entry refuses to sync, the reason is written next to it in a
.error.txt file. You can always retype the entry from the .json by hand --
the field names match the ones on the form.
"""
def _timestamp() -> str:
return datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%S")
[docs]
def spool_dir() -> Path:
"""Return the directory holding entries that could not be saved."""
configured = None
if has_app_context():
configured = current_app.config.get(SPOOL_DIR_CONFIG_KEY)
if not configured:
configured = Path(current_app.instance_path) / DEFAULT_SPOOL_DIR_NAME
if not configured:
configured = Path.cwd() / DEFAULT_SPOOL_DIR_NAME
return Path(configured)
def _ensure_dir() -> Path:
directory = spool_dir()
directory.mkdir(parents=True, exist_ok=True)
readme = directory / README_NAME
if not readme.exists():
readme.write_text(READ_ME, encoding="utf-8")
return directory
def _json_safe(value: Any) -> Any:
"""Convert a Mongo-bound document into something json.dump can write."""
if isinstance(value, dict):
return {str(key): _json_safe(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe(item) for item in value]
if isinstance(value, (str, int, float, bool)) or value is None:
return value
if hasattr(value, "isoformat"):
return value.isoformat()
return str(value)
def _entry_label(document: dict, mode: str) -> str:
number = document.get("set_number" if mode == "set" else "shot_number")
campaign = _SAFE_NAME.sub("-", str(document.get("Campaign") or "no-campaign"))
kind = "set" if mode == "set" else "shot"
return f"{campaign}_{kind}{number if number not in (None, '') else 'NA'}"
[docs]
def record_unsaved_entry(
document: dict,
*,
mode: str,
reason: str,
operation: str = "insert",
doc_id: str | None = None,
) -> Path:
"""Write one unsaved entry to the spool and return the file it landed in.
Input: the document the route was about to write, plus why it could not.
Output: the path to show the operator. Raising here would defeat the
purpose, so a spool failure is logged and re-raised only if even the
directory cannot be created -- at that point there is nowhere left to put
it and the caller must say so plainly.
"""
directory = _ensure_dir()
name = f"{_timestamp()}_{_entry_label(document, mode)}_{uuid.uuid4().hex[:8]}.json"
path = directory / name
payload = {
"saved_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"reason": safe_reason(reason),
"operation": operation,
"mode": mode,
"target_id": doc_id,
"entry": _json_safe(document),
}
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
logging.warning("Database unreachable; entry spooled to %s (%s)", path, reason)
return path
[docs]
def pending_files() -> list[Path]:
"""Return the spooled entries still waiting to reach MongoDB, oldest first."""
directory = spool_dir()
if not directory.is_dir():
return []
return sorted(p for p in directory.glob("*.json") if p.is_file())
[docs]
def pending_count() -> int:
return len(pending_files())
[docs]
def load_pending() -> list[dict]:
"""Return the pending entries as dictionaries, unreadable files included.
A file that will not parse is reported rather than skipped: silently
ignoring it would be the same failure this module exists to prevent.
"""
entries = []
for path in pending_files():
try:
payload = json.loads(path.read_text(encoding="utf-8"))
payload["file"] = path.name
entries.append(payload)
except (OSError, ValueError) as exc:
entries.append({
"file": path.name,
"unreadable": True,
"reason": f"could not be read: {exc}",
"entry": {},
})
return entries
def _mark_synced(path: Path) -> Path:
target_dir = path.parent / SYNCED_SUBDIR
target_dir.mkdir(parents=True, exist_ok=True)
destination = target_dir / path.name
path.replace(destination)
return destination
def _record_error(path: Path, message: str) -> None:
path.with_suffix(".error.txt").write_text(
f"{datetime.datetime.now(datetime.timezone.utc).isoformat()}\n{safe_reason(message)}\n",
encoding="utf-8",
)
[docs]
def replay_pending(shots_collection) -> dict:
"""Insert spooled entries into MongoDB, keeping whatever will not go in.
Output: counts plus per-file detail. Entries that fail stay in the spool
with the reason written beside them, so a partial recovery never quietly
drops the remainder.
"""
results = {"synced": [], "failed": [], "unreadable": []}
for path in pending_files():
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
results["unreadable"].append({
"file": path.name,
"reason": safe_reason(str(exc)),
})
continue
document = payload.get("entry") or {}
if not document:
results["unreadable"].append({
"file": path.name,
"reason": "no entry payload in file",
})
continue
try:
shots_collection.insert_one(dict(document))
except Exception as exc:
logging.exception("Offline entry %s could not be inserted", path.name)
_record_error(path, str(exc))
results["failed"].append({
"file": path.name,
"reason": safe_reason(str(exc)),
})
continue
_mark_synced(path)
results["synced"].append({"file": path.name})
results["synced_count"] = len(results["synced"])
results["failed_count"] = len(results["failed"]) + len(results["unreadable"])
return results
[docs]
def database_is_reachable(database) -> tuple[bool, str | None]:
"""Ping the database. Returns ``(reachable, reason)`` and never raises."""
try:
database.client.admin.command("ping")
except Exception as exc:
logging.warning("Database ping failed: %s", exc)
# Scrubbed here rather than at each call site: this reason is shown on
# a page, and a driver error quotes the connection URI with its password.
return False, safe_reason(str(exc))
return True, None