Kazeia-central/kazeia_central/core/store/db.py

558 lines
26 KiB
Python

"""Store local chiffré de Kazeia-central.
`sqlite3` (stdlib, zéro install) + chiffrement au niveau champ (`crypto.Vault`,
PyNaCl). Contient l'archive clinique rapatriée (tours de conversation, PII de santé
chiffrée), le mapping `serial → label patient` (§9), et la piste d'audit (§8).
Posture :
- Texte des tours et labels patients → **chiffrés** (colonnes `*_enc BLOB`).
- Métadonnées (serial, profile_id, session_id, timestamps) → **clair**, pour rester
requêtables (dashboard, collecte incrémentale, fan-out). Protégées au repos par le
chiffrement disque OS (§8).
- Tout accès sensible est journalisé (`audit`).
Le store s'ouvre verrouillé ; `unlock(password)` dérive la clé et la garde en mémoire
pour la session. Sans unlock, seules les métadonnées sont lisibles, jamais le contenu.
Thread-safety : les endpoints FastAPI synchrones tournent dans un threadpool →
connexion ouverte avec `check_same_thread=False` et accès sérialisé par un `RLock`.
"""
from __future__ import annotations
import hashlib
import json
import sqlite3
import threading
from pathlib import Path
from typing import Any, Iterable
from .crypto import KdfParams, Vault
_SCHEMA = """
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value BLOB
);
CREATE TABLE IF NOT EXISTS devices (
serial TEXT PRIMARY KEY,
label_enc BLOB, -- label patient, chiffré
created_at INTEGER NOT NULL,
last_seen_at INTEGER
);
CREATE TABLE IF NOT EXISTS turns (
serial TEXT NOT NULL,
profile_id TEXT,
session_id TEXT NOT NULL,
turn_id TEXT NOT NULL, -- `id` du tour côté provider
ts INTEGER, -- timestamp du tour (clair → tri/incrémental)
role TEXT, -- PATIENT | KAZEIA
text_enc BLOB, -- contenu, chiffré (PII)
ttft_ms INTEGER,
total_ms INTEGER,
voice_used TEXT,
model_used TEXT,
archived_at INTEGER NOT NULL,
PRIMARY KEY (serial, session_id, turn_id) -- dédoublonne les re-pulls
);
CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(serial, session_id, ts);
CREATE INDEX IF NOT EXISTS idx_turns_profile ON turns(serial, profile_id, ts);
CREATE TABLE IF NOT EXISTS audit (
id INTEGER PRIMARY KEY AUTOINCREMENT,
ts INTEGER NOT NULL,
actor TEXT,
action TEXT NOT NULL,
target TEXT,
detail TEXT -- métadonnée d'audit (pas de PII) → clair
);
-- Catalogue voix GLOBAL (flotte) : une voix = un asset central, PAS lié à une tablette.
-- La tablette d'enregistrement n'est qu'une métadonnée (origin_serial) ; le déploiement
-- est une relation séparée (voice_deployments).
CREATE TABLE IF NOT EXISTS voices (
voice_id TEXT PRIMARY KEY,
origin_serial TEXT, -- tablette d'enregistrement (métadonnée)
name TEXT,
transcription_enc BLOB, -- texte du .ovsp (ASR) → chiffré
source_wav_path TEXT, -- chemin device d'origine du WAV
wav_file TEXT, -- WAV chiffré archivé (relatif au dossier voices/)
wav_sha256 TEXT,
ovsp_enc BLOB, -- artefact .ovsp chiffré → déployable sans ré-enrôler
ovsp_bytes INTEGER,
archived_at INTEGER, -- WAV récupéré + archivé sur le PC
enrolled_at INTEGER, -- .ovsp produit
wav_deleted_at INTEGER, -- WAV supprimé de la tablette d'origine
locked_profile_id TEXT, -- exclusivité (= owner_profile_id ; NULL = généraliste)
scope TEXT, -- exclusive | global | pending
owner_name TEXT,
consent_text_enc BLOB -- phrase de consentement (PII) → chiffré
);
-- Déploiement many-to-many : quelles tablettes ont le .ovsp de quelle voix.
CREATE TABLE IF NOT EXISTS voice_deployments (
voice_id TEXT NOT NULL,
serial TEXT NOT NULL,
deployed_at INTEGER,
PRIMARY KEY (voice_id, serial)
);
-- Dossier patient (médical) : identité/contact/clinique CHIFFRÉS (PII santé, restent
-- sur le PC, jamais poussés sur la tablette). `language` = miroir de l'opérationnel poussé.
CREATE TABLE IF NOT EXISTS patient_fiche (
serial TEXT NOT NULL,
profile_id TEXT NOT NULL,
fiche_enc BLOB, -- JSON {civil, contact, clinical} chiffré
language TEXT, -- langue opérationnelle (miroir du profil on-device)
updated_at INTEGER,
PRIMARY KEY (serial, profile_id)
);
"""
class StoreLocked(Exception):
"""Opération sur du contenu chiffré alors que le store n'est pas déverrouillé."""
class Store:
def __init__(self, path: str | Path, *, kdf_ops: int | None = None,
kdf_mem: int | None = None) -> None:
# kdf_ops/kdf_mem : coût Argon2id à l'initialisation (1ᵉʳ unlock). None =
# défaut prod MODERATE. Les tests les baissent pour la vitesse ; ils sont
# persistés, donc les ré-ouvertures réutilisent les mêmes.
self.path = str(path)
self._kdf_ops = kdf_ops
self._kdf_mem = kdf_mem
self._lock = threading.RLock()
self._db = sqlite3.connect(self.path, check_same_thread=False)
self._db.row_factory = sqlite3.Row
self._db.execute("PRAGMA journal_mode=WAL")
self._db.execute("PRAGMA foreign_keys=ON")
self._db.executescript(_SCHEMA)
self._migrate()
self._db.commit()
self._vault: Vault | None = None
def _migrate(self) -> None:
"""Migrations légères idempotentes (CREATE TABLE IF NOT EXISTS ne fait pas
évoluer une table déjà créée)."""
cols = {r["name"] for r in self._db.execute("PRAGMA table_info(voices)")}
if "serial" in cols: # ANCIEN schéma per-tablette → mises à niveau puis refonte
if "locked_profile_id" not in cols:
self._db.execute("ALTER TABLE voices ADD COLUMN locked_profile_id TEXT")
if "cvps_bytes" in cols and "ovsp_bytes" not in cols:
self._db.execute("ALTER TABLE voices RENAME COLUMN cvps_bytes TO ovsp_bytes")
for col, decl in (("scope", "TEXT"), ("owner_name", "TEXT"),
("consent_text_enc", "BLOB")):
if col not in cols:
self._db.execute(f"ALTER TABLE voices ADD COLUMN {col} {decl}")
self._rebuild_voices_global()
def _rebuild_voices_global(self) -> None:
"""Refonte (serial, voice_id) → catalogue GLOBAL clé voice_id + table
voice_deployments. Les fichiers WAV chiffrés gardent leur chemin relatif
(`<serial>/<id>.wav.enc`) → pas de déplacement de fichier."""
db = self._db
db.execute("ALTER TABLE voices RENAME TO voices_old")
db.executescript(_SCHEMA) # recrée voices (nouveau) + voice_deployments
db.execute(
"INSERT OR IGNORE INTO voice_deployments(voice_id, serial, deployed_at) "
"SELECT voice_id, serial, deployed_at FROM voices_old WHERE deployed_at IS NOT NULL")
db.execute(
"INSERT OR IGNORE INTO voices(voice_id, origin_serial, transcription_enc, "
"source_wav_path, wav_file, wav_sha256, ovsp_bytes, archived_at, enrolled_at, "
"wav_deleted_at, locked_profile_id, scope, owner_name, consent_text_enc) "
"SELECT voice_id, serial, transcription_enc, source_wav_path, wav_file, "
"wav_sha256, ovsp_bytes, archived_at, enrolled_at, wav_deleted_at, "
"locked_profile_id, scope, owner_name, consent_text_enc FROM voices_old")
db.execute("DROP TABLE voices_old")
def close(self) -> None:
with self._lock:
self._db.close()
def __enter__(self) -> "Store":
return self
def __exit__(self, *exc: object) -> None:
self.close()
# ---- meta -------------------------------------------------------------
def _get_meta(self, key: str) -> bytes | None:
row = self._db.execute("SELECT value FROM meta WHERE key=?", (key,)).fetchone()
return row["value"] if row else None
def _set_meta(self, key: str, value: bytes) -> None:
self._db.execute(
"INSERT INTO meta(key, value) VALUES(?, ?) "
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
(key, value),
)
self._db.commit()
@property
def is_initialized(self) -> bool:
with self._lock:
return self._get_meta("kdf_salt") is not None
@property
def is_unlocked(self) -> bool:
return self._vault is not None
# ---- déverrouillage ---------------------------------------------------
def unlock(self, password: str) -> None:
"""Initialise le coffre au 1ᵉʳ appel, sinon valide le mot de passe.
Lève `crypto.BadPassword` si incorrect."""
with self._lock:
if not self._get_meta("kdf_salt"):
from .crypto import _DEFAULT_MEM, _DEFAULT_OPS
vault, params, verifier = Vault.setup(
password,
ops=self._kdf_ops if self._kdf_ops is not None else _DEFAULT_OPS,
mem=self._kdf_mem if self._kdf_mem is not None else _DEFAULT_MEM,
)
self._set_meta("kdf_salt", params.salt)
self._set_meta("kdf_ops", str(params.ops).encode())
self._set_meta("kdf_mem", str(params.mem).encode())
self._set_meta("verifier", verifier)
self._vault = vault
return
params = KdfParams(
salt=self._get_meta("kdf_salt"),
ops=int(self._get_meta("kdf_ops")),
mem=int(self._get_meta("kdf_mem")),
)
self._vault = Vault.unlock(password, params, self._get_meta("verifier"))
def lock(self) -> None:
"""Oublie la clé en mémoire (re-verrouille sans fermer la base)."""
self._vault = None
def _require_vault(self) -> Vault:
if self._vault is None:
raise StoreLocked("store verrouillé : appeler unlock(password) d'abord")
return self._vault
# ---- mapping serial → label patient (§9) ------------------------------
def set_device_label(self, serial: str, label: str, *, now: int) -> None:
with self._lock:
v = self._require_vault()
self._db.execute(
"INSERT INTO devices(serial, label_enc, created_at, last_seen_at) "
"VALUES(?,?,?,?) ON CONFLICT(serial) DO UPDATE SET "
"label_enc=excluded.label_enc, last_seen_at=excluded.last_seen_at",
(serial, v.seal(label), now, now),
)
self._db.commit()
def device_label(self, serial: str) -> str | None:
with self._lock:
v = self._require_vault()
row = self._db.execute(
"SELECT label_enc FROM devices WHERE serial=?", (serial,)
).fetchone()
return v.open(row["label_enc"]) if row else None
def device_labels(self) -> dict[str, str | None]:
"""Tous les labels {serial: label} — pour enrichir la liste du parc."""
with self._lock:
v = self._require_vault()
return {r["serial"]: v.open(r["label_enc"])
for r in self._db.execute("SELECT serial, label_enc FROM devices")}
def devices(self) -> list[dict[str, Any]]:
with self._lock:
v = self._require_vault()
return [{
"serial": r["serial"],
"label": v.open(r["label_enc"]),
"created_at": r["created_at"],
"last_seen_at": r["last_seen_at"],
} for r in self._db.execute("SELECT * FROM devices ORDER BY created_at")]
# ---- archive des tours (collecte) -------------------------------------
def archive_turns(self, serial: str, turns: Iterable[dict[str, Any]], *, now: int) -> int:
"""Insère/ignore des tours (idempotent par (serial, session_id, turn_id)).
Chaque `text` est chiffré. Renvoie le nombre de NOUVEAUX tours archivés."""
with self._lock:
v = self._require_vault()
before = self._db.total_changes
self._db.executemany(
"INSERT OR IGNORE INTO turns(serial, profile_id, session_id, turn_id, ts, "
"role, text_enc, ttft_ms, total_ms, voice_used, model_used, archived_at) "
"VALUES(:serial,:profile_id,:session_id,:turn_id,:ts,:role,:text_enc,"
":ttft_ms,:total_ms,:voice_used,:model_used,:archived_at)",
[{
"serial": serial,
"profile_id": t.get("profile_id"),
"session_id": t["session_id"],
"turn_id": str(t.get("id", t.get("turn_id"))),
"ts": t.get("timestamp", t.get("ts")),
"role": t.get("role"),
"text_enc": v.seal(t.get("text")),
"ttft_ms": t.get("ttft_ms"),
"total_ms": t.get("total_ms"),
"voice_used": t.get("voice_used"),
"model_used": t.get("model_used"),
"archived_at": now,
} for t in turns],
)
self._db.commit()
return self._db.total_changes - before
def session_turns(self, serial: str, session_id: str) -> list[dict[str, Any]]:
with self._lock:
v = self._require_vault()
rows = self._db.execute(
"SELECT * FROM turns WHERE serial=? AND session_id=? ORDER BY ts, turn_id",
(serial, session_id),
).fetchall()
out = []
for r in rows:
d = dict(r)
d["text"] = v.open(d.pop("text_enc"))
out.append(d)
return out
def last_turn_ts(self, serial: str, session_id: str | None = None) -> int | None:
"""Dernier timestamp archivé → borne `since` pour la collecte incrémentale
(autonomie). Pas besoin du vault (métadonnée en clair)."""
with self._lock:
if session_id is None:
row = self._db.execute(
"SELECT MAX(ts) m FROM turns WHERE serial=?", (serial,)
).fetchone()
else:
row = self._db.execute(
"SELECT MAX(ts) m FROM turns WHERE serial=? AND session_id=?",
(serial, session_id),
).fetchone()
return row["m"] if row else None
def archived_sessions(self, serial: str) -> list[dict[str, Any]]:
"""Sessions archivées (métadonnées, pas de vault) : id, profil, nb tours,
début/fin. Pour lister le rapatrié dans l'UI."""
with self._lock:
rows = self._db.execute(
"SELECT session_id, profile_id, COUNT(*) turns, MIN(ts) started_at, "
"MAX(ts) ended_at, MAX(archived_at) archived_at FROM turns WHERE serial=? "
"GROUP BY session_id ORDER BY started_at DESC", (serial,)).fetchall()
return [dict(r) for r in rows]
def purge_turns(self, serial: str, *, session_id: str | None = None,
profile_id: str | None = None, before: int | None = None) -> int:
"""Purge (RGPD) des tours archivés selon filtre. Renvoie le nombre supprimé."""
q = "DELETE FROM turns WHERE serial=?"
args: list = [serial]
if session_id:
q += " AND session_id=?"; args.append(session_id)
if profile_id:
q += " AND profile_id=?"; args.append(profile_id)
if before:
q += " AND ts<?"; args.append(before)
with self._lock:
before_ct = self._db.total_changes
self._db.execute(q, args)
self._db.commit()
return self._db.total_changes - before_ct
# ---- archive voix (enrôlement, §6 VOICE_ENROLLMENT_SPEC) --------------
def _voices_dir(self) -> Path:
d = Path(self.path).parent / "voices"
d.mkdir(parents=True, exist_ok=True)
return d
def _voice_upsert(self, voice_id: str, **cols: Any) -> None:
"""Upsert partiel d'une voix du catalogue global (colonnes données seulement)."""
keys = list(cols)
sets = ", ".join(f"{k}=excluded.{k}" for k in keys)
self._db.execute(
f"INSERT INTO voices(voice_id, {', '.join(keys)}) "
f"VALUES(?, {', '.join('?' for _ in keys)}) "
f"ON CONFLICT(voice_id) DO UPDATE SET {sets}",
(voice_id, *[cols[k] for k in keys]),
)
self._db.commit()
def archive_voice_wav(self, voice_id: str, wav_bytes: bytes, *, origin_serial: str | None,
source_wav_path: str | None, name: str | None = None, now: int) -> str:
"""Chiffre + archive le WAV d'origine sur le PC (asset global, pas lié à une
tablette). Renvoie le chemin du fichier chiffré."""
v = self._require_vault()
sha = hashlib.sha256(wav_bytes).hexdigest()
dest = self._voices_dir() / f"{voice_id}.wav.enc"
dest.write_bytes(v.seal_bytes(wav_bytes))
with self._lock:
self._voice_upsert(voice_id, origin_serial=origin_serial, name=name,
source_wav_path=source_wav_path,
wav_file=str(dest.relative_to(self._voices_dir())),
wav_sha256=sha, archived_at=now)
return str(dest)
def voice_wav_bytes(self, voice_id: str) -> bytes | None:
"""Relit le WAV archivé déchiffré (pour ré-enrôlement)."""
v = self._require_vault()
with self._lock:
row = self._db.execute(
"SELECT wav_file FROM voices WHERE voice_id=?", (voice_id,)).fetchone()
if not row or not row["wav_file"]:
return None
return v.open_bytes((self._voices_dir() / row["wav_file"]).read_bytes())
def set_voice_transcription(self, voice_id: str, text: str, *, now: int) -> None:
v = self._require_vault()
with self._lock:
self._voice_upsert(voice_id, transcription_enc=v.seal(text))
def store_voice_ovsp(self, voice_id: str, ovsp_bytes_data: bytes, *, now: int) -> None:
"""Stocke l'artefact `.ovsp` chiffré côté central → déployable sur N tablettes
sans ré-enrôler. Marque la voix enrôlée."""
v = self._require_vault()
with self._lock:
self._voice_upsert(voice_id, ovsp_enc=v.seal_bytes(ovsp_bytes_data),
ovsp_bytes=len(ovsp_bytes_data), enrolled_at=now)
def voice_ovsp_bytes(self, voice_id: str) -> bytes | None:
"""Relit l'artefact `.ovsp` déchiffré (pour déploiement)."""
v = self._require_vault()
with self._lock:
row = self._db.execute(
"SELECT ovsp_enc FROM voices WHERE voice_id=?", (voice_id,)).fetchone()
return v.open_bytes(row["ovsp_enc"]) if row and row["ovsp_enc"] else None
def mark_voice_wav_deleted(self, voice_id: str, *, now: int) -> None:
with self._lock:
self._voice_upsert(voice_id, wav_deleted_at=now)
def lock_voice(self, voice_id: str, profile_id: str, *, now: int) -> None:
"""Assigne une voix exclusivement à un profil patient (verrou)."""
with self._lock:
self._voice_upsert(voice_id, locked_profile_id=profile_id, scope="exclusive")
def unlock_voice(self, voice_id: str, *, now: int) -> None:
"""Lève le verrou → voix redevient généraliste."""
with self._lock:
self._voice_upsert(voice_id, locked_profile_id=None, scope="global")
def set_voice_manifest(self, voice_id: str, *, scope: str | None,
owner_name: str | None, consent_text: str | None, now: int) -> None:
"""Métadonnées du manifeste admin (§4.2) : portée + propriétaire + consentement
(chiffré). Le consentement = phrase nominative → PII."""
v = self._require_vault()
with self._lock:
self._voice_upsert(voice_id, scope=scope, owner_name=owner_name,
consent_text_enc=v.seal(consent_text))
def locked_profile(self, voice_id: str) -> str | None:
with self._lock:
row = self._db.execute(
"SELECT locked_profile_id FROM voices WHERE voice_id=?", (voice_id,)).fetchone()
return row["locked_profile_id"] if row else None
# ---- déploiements voix↔tablette (many-to-many) -----------------------
def add_deployment(self, voice_id: str, serial: str, *, now: int) -> None:
with self._lock:
self._db.execute(
"INSERT INTO voice_deployments(voice_id, serial, deployed_at) VALUES(?,?,?) "
"ON CONFLICT(voice_id, serial) DO UPDATE SET deployed_at=excluded.deployed_at",
(voice_id, serial, now))
self._db.commit()
def remove_deployment(self, voice_id: str, serial: str) -> None:
with self._lock:
self._db.execute("DELETE FROM voice_deployments WHERE voice_id=? AND serial=?",
(voice_id, serial))
self._db.commit()
def deployments_of(self, voice_id: str) -> list[str]:
with self._lock:
return [r["serial"] for r in self._db.execute(
"SELECT serial FROM voice_deployments WHERE voice_id=?", (voice_id,))]
def voices_on(self, serial: str) -> set[str]:
with self._lock:
return {r["voice_id"] for r in self._db.execute(
"SELECT voice_id FROM voice_deployments WHERE serial=?", (serial,))}
# ---- lecture catalogue (global) --------------------------------------
def _voice_dict(self, row, v) -> dict[str, Any]:
d = dict(row)
d["transcription"] = v.open(d.pop("transcription_enc", None))
d["consent_text"] = v.open(d.pop("consent_text_enc", None))
d.pop("ovsp_enc", None)
d["has_wav"] = bool(d.get("wav_file"))
d["has_ovsp"] = bool(d.get("ovsp_bytes"))
d["deployed_serials"] = self.deployments_of(d["voice_id"])
return d
def voice_record(self, voice_id: str) -> dict[str, Any] | None:
v = self._require_vault()
with self._lock:
row = self._db.execute(
"SELECT * FROM voices WHERE voice_id=?", (voice_id,)).fetchone()
return self._voice_dict(row, v) if row else None
def voices(self) -> list[dict[str, Any]]:
"""Catalogue voix GLOBAL (toutes tablettes confondues)."""
v = self._require_vault()
with self._lock:
rows = self._db.execute("SELECT * FROM voices ORDER BY voice_id").fetchall()
return [self._voice_dict(r, v) for r in rows]
# ---- dossier patient (médical, chiffré) -------------------------------
def save_fiche(self, serial: str, profile_id: str, fiche: dict[str, Any],
language: str | None, *, now: int) -> None:
"""Enregistre la fiche clinique chiffrée (identité/contact/clinique). Le
contenu = PII santé → chiffré ; `language` en clair (miroir opérationnel)."""
v = self._require_vault()
with self._lock:
self._db.execute(
"INSERT INTO patient_fiche(serial, profile_id, fiche_enc, language, updated_at) "
"VALUES(?,?,?,?,?) ON CONFLICT(serial, profile_id) DO UPDATE SET "
"fiche_enc=excluded.fiche_enc, language=excluded.language, updated_at=excluded.updated_at",
(serial, profile_id, v.seal(json.dumps(fiche)), language, now))
self._db.commit()
def set_fiche_language(self, serial: str, profile_id: str, language: str | None, *, now: int) -> None:
with self._lock:
self._db.execute(
"INSERT INTO patient_fiche(serial, profile_id, language, updated_at) VALUES(?,?,?,?) "
"ON CONFLICT(serial, profile_id) DO UPDATE SET language=excluded.language, updated_at=excluded.updated_at",
(serial, profile_id, language, now))
self._db.commit()
def fiche(self, serial: str, profile_id: str) -> dict[str, Any] | None:
v = self._require_vault()
with self._lock:
row = self._db.execute(
"SELECT * FROM patient_fiche WHERE serial=? AND profile_id=?",
(serial, profile_id)).fetchone()
if not row:
return None
fx = v.open(row["fiche_enc"])
return {"fiche": json.loads(fx) if fx else {}, "language": row["language"],
"updated_at": row["updated_at"]}
def fiches(self, serial: str) -> dict[str, dict[str, Any]]:
v = self._require_vault()
with self._lock:
rows = self._db.execute(
"SELECT * FROM patient_fiche WHERE serial=?", (serial,)).fetchall()
out = {}
for r in rows:
fx = v.open(r["fiche_enc"])
out[r["profile_id"]] = {"fiche": json.loads(fx) if fx else {}, "language": r["language"]}
return out
# ---- audit (§8) -------------------------------------------------------
def audit(self, action: str, *, actor: str | None = None, target: str | None = None,
detail: dict[str, Any] | None = None, now: int) -> None:
with self._lock:
self._db.execute(
"INSERT INTO audit(ts, actor, action, target, detail) VALUES(?,?,?,?,?)",
(now, actor, action, target, json.dumps(detail) if detail else None),
)
self._db.commit()
def audit_log(self, limit: int = 200) -> list[dict[str, Any]]:
with self._lock:
return [dict(r) for r in self._db.execute(
"SELECT * FROM audit ORDER BY id DESC LIMIT ?", (limit,)
)]