feat(backend): surveille la derive du modele de prevision
EC06 attendait une reponse a « comment savez-vous que le modele se degrade ? ». Elle n'existait nulle part : `docs/architecture/00-vue-ensemble.md` et `docs/ML-START.md` le disaient tous les deux. Le calcul vit dans le backend, et `ml/` ne gagne pas une ligne. Trois raisons : `prediction` n'est pas dans le perimetre de lecture que `ML_DATABASE_URL` vise (ADR 0003 et ML-START le bornent a `reading` et `site`) ; l'alignement prevu contre realise existe deja une fois ici, dans `AlertService._detect_anomaly`, et le dupliquer en SQL brut creerait une seconde source de verite, ce que l'ADR 0006 refuse ; et FastAPI continue de ne jamais faire tourner LightGBM. Ce qui est mesure : la jointure `prediction` x `reading` sur `(site_id, target_at)`, avec un `DISTINCT ON` des deux cotes. Les runs de scoring s'empilent volontairement, et `uq_reading_source` autorise deux lectures au meme instant quand la source differe : sans ce dedoublonnage, la meme heure pesait plusieurs fois dans la moyenne. La fenetre est fermee a droite par un delai de grace, sinon la derniere heure, dont le realise n'est pas encore ingere, ferait chuter la couverture a chaque execution. Le verdict a trois valeurs, pas deux : avec trois points on ne declare pas une derive, on dit qu'on ne sait pas. La comparaison se fait entre deux fenetres vives de meme duree, jamais contre la metrique loguee a l'entrainement : celle-ci mesure un backtest a meteo connue, le scoring prevoit une heure dont la meteo ne l'est pas. `drift_report` porte une ligne par site plus une ligne globale, que `site_id` a NULL designe. L'idempotence passe par un index a `coalesce` et non par une contrainte d'unicite, sans quoi deux lignes globales ne seraient jamais egales.
This commit is contained in:
Johan LEROY
@@ -56,6 +56,7 @@ ROLE_MINIMUM: Final[dict[Route, Role]] = {
|
||||
("GET", "/api/v1/readings"): Role.LECTEUR,
|
||||
("GET", "/api/v1/predictions"): Role.LECTEUR,
|
||||
("GET", "/api/v1/sensors/status"): Role.ADMIN,
|
||||
("GET", "/api/v1/monitoring/drift"): Role.OPERATEUR,
|
||||
("GET", "/api/v1/users"): Role.ADMIN,
|
||||
("POST", "/api/v1/users"): Role.ADMIN,
|
||||
("PATCH", "/api/v1/users/{user_id}"): Role.ADMIN,
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
from collections.abc import Iterator, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from httpx import AsyncClient
|
||||
|
||||
from app.api.deps import get_current_principal, get_drift_service
|
||||
from app.core.principal import Principal
|
||||
from app.core.roles import AccountKind, Role
|
||||
from app.models.energy import DriftReport
|
||||
|
||||
INSTANT = datetime(2026, 9, 22, 12, tzinfo=UTC)
|
||||
|
||||
|
||||
def operateur() -> Principal:
|
||||
return Principal(
|
||||
id=uuid4(),
|
||||
email="operateur@enervision.fr",
|
||||
role=Role.OPERATEUR,
|
||||
kind=AccountKind.HUMAIN,
|
||||
must_change_password=False,
|
||||
)
|
||||
|
||||
|
||||
def rapport(*, site_id: str | None) -> DriftReport:
|
||||
return DriftReport(
|
||||
drift_report_id=1,
|
||||
computed_at=INSTANT,
|
||||
site_id=site_id,
|
||||
window_start=INSTANT - timedelta(hours=168),
|
||||
window_end=INSTANT,
|
||||
reference_start=None,
|
||||
reference_end=None,
|
||||
n_observations=48,
|
||||
mae=1.5,
|
||||
mape=12.0,
|
||||
bias=0.3,
|
||||
reference_mae=1.2,
|
||||
coverage_ratio=0.95,
|
||||
insufficient_data_ratio=0.0,
|
||||
model_references=["lightgbm-aaa"],
|
||||
status="stable",
|
||||
reason=None,
|
||||
)
|
||||
|
||||
|
||||
class FauxService:
|
||||
def __init__(self, rapports: Sequence[DriftReport]) -> None:
|
||||
self.rapports = list(rapports)
|
||||
self.site_demande: str | None = None
|
||||
|
||||
async def derniers(self, *, site_id: str | None = None) -> Sequence[DriftReport]:
|
||||
self.site_demande = site_id
|
||||
return self.rapports
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def servi(app: FastAPI) -> Iterator[list[DriftReport]]:
|
||||
rapports = [rapport(site_id="SITE001"), rapport(site_id=None)]
|
||||
service = FauxService(rapports)
|
||||
app.dependency_overrides[get_current_principal] = operateur
|
||||
app.dependency_overrides[get_drift_service] = lambda: service
|
||||
yield rapports
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
async def test_drift_returns_the_latest_report_of_every_site(
|
||||
servi: list[DriftReport], client: AsyncClient
|
||||
) -> None:
|
||||
reponse = await client.get("/api/v1/monitoring/drift")
|
||||
|
||||
assert reponse.status_code == 200
|
||||
assert [ligne["site_id"] for ligne in reponse.json()] == ["SITE001", None]
|
||||
|
||||
|
||||
async def test_drift_exposes_the_metrics_of_the_stored_report(
|
||||
servi: list[DriftReport], client: AsyncClient
|
||||
) -> None:
|
||||
reponse = await client.get("/api/v1/monitoring/drift")
|
||||
|
||||
premier = reponse.json()[0]
|
||||
assert premier["status"] == "stable"
|
||||
assert premier["mae"] == 1.5
|
||||
assert premier["model_references"] == ["lightgbm-aaa"]
|
||||
|
||||
|
||||
async def test_drift_returns_an_empty_list_when_no_report_exists(
|
||||
app: FastAPI, client: AsyncClient
|
||||
) -> None:
|
||||
app.dependency_overrides[get_current_principal] = operateur
|
||||
app.dependency_overrides[get_drift_service] = lambda: FauxService([])
|
||||
|
||||
reponse = await client.get("/api/v1/monitoring/drift")
|
||||
|
||||
assert reponse.status_code == 200
|
||||
assert reponse.json() == []
|
||||
app.dependency_overrides.clear()
|
||||
@@ -1,5 +1,5 @@
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
@@ -9,7 +9,15 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.models.energy import Alert, Dataset, Prediction, Reading, Recommendation, Site
|
||||
from app.models.energy import (
|
||||
Alert,
|
||||
Dataset,
|
||||
DriftReport,
|
||||
Prediction,
|
||||
Reading,
|
||||
Recommendation,
|
||||
Site,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
MOMENT = datetime(2024, 1, 1, tzinfo=UTC)
|
||||
@@ -269,3 +277,62 @@ async def test_recommendation_is_unique_when_alert_and_rule_match(
|
||||
with pytest.raises(IntegrityError):
|
||||
async with savepoint:
|
||||
await data_connection.execute(statement)
|
||||
|
||||
|
||||
def _rapport(**remplacements: object) -> dict[str, object]:
|
||||
defauts: dict[str, object] = {
|
||||
"site_id": None,
|
||||
"window_start": MOMENT,
|
||||
"window_end": MOMENT,
|
||||
"n_observations": 12,
|
||||
"model_references": ["lightgbm-aaa"],
|
||||
"status": "stable",
|
||||
"reason": None,
|
||||
}
|
||||
return {**defauts, **remplacements}
|
||||
|
||||
|
||||
async def test_drift_report_rejects_an_unknown_status(data_connection: AsyncConnection) -> None:
|
||||
statement = insert(DriftReport).values(**_rapport(status="douteux", reason="x"))
|
||||
savepoint = data_connection.begin_nested()
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
async with savepoint:
|
||||
await data_connection.execute(statement)
|
||||
|
||||
|
||||
async def test_drift_report_rejects_a_drift_without_a_reason(
|
||||
data_connection: AsyncConnection,
|
||||
) -> None:
|
||||
statement = insert(DriftReport).values(**_rapport(status="derive"))
|
||||
savepoint = data_connection.begin_nested()
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
async with savepoint:
|
||||
await data_connection.execute(statement)
|
||||
|
||||
|
||||
async def test_drift_report_accepts_one_global_row_without_a_site(
|
||||
data_connection: AsyncConnection,
|
||||
) -> None:
|
||||
identifiant = (
|
||||
await data_connection.execute(
|
||||
insert(DriftReport).values(**_rapport()).returning(DriftReport.drift_report_id)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
assert identifiant is not None
|
||||
|
||||
|
||||
async def test_drift_report_is_unique_when_window_and_site_match(
|
||||
data_connection: AsyncConnection,
|
||||
) -> None:
|
||||
fenetre = MOMENT + timedelta(days=1)
|
||||
statement = insert(DriftReport).values(**_rapport(window_end=fenetre))
|
||||
await data_connection.execute(statement)
|
||||
|
||||
savepoint = data_connection.begin_nested()
|
||||
|
||||
with pytest.raises(IntegrityError):
|
||||
async with savepoint:
|
||||
await data_connection.execute(statement)
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.dialects import postgresql
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.sql import ClauseElement
|
||||
|
||||
from app.repositories.drift import (
|
||||
DriftRepository,
|
||||
NouveauRapportDerive,
|
||||
_lectures_retenues,
|
||||
_predictions_retenues,
|
||||
)
|
||||
from tests.repositories.test_prediction import creer_prediction
|
||||
from tests.repositories.test_reading import creer_lecture
|
||||
from tests.repositories.test_site import creer as creer_site
|
||||
|
||||
DEBUT = datetime(2026, 9, 15, tzinfo=UTC)
|
||||
FIN = datetime(2026, 9, 22, tzinfo=UTC)
|
||||
CIBLE = datetime(2026, 9, 16, 12, tzinfo=UTC)
|
||||
|
||||
|
||||
def sql(requete: ClauseElement) -> str:
|
||||
return str(requete.compile(dialect=postgresql.dialect())) # type: ignore[no-untyped-call]
|
||||
|
||||
|
||||
def rapport(**remplacements: object) -> NouveauRapportDerive:
|
||||
defauts: dict[str, object] = {
|
||||
"site_id": None,
|
||||
"window_start": DEBUT,
|
||||
"window_end": FIN,
|
||||
"reference_start": None,
|
||||
"reference_end": None,
|
||||
"n_observations": 10,
|
||||
"mae": 1.0,
|
||||
"mape": 5.0,
|
||||
"bias": 0.1,
|
||||
"reference_mae": None,
|
||||
"coverage_ratio": 1.0,
|
||||
"insufficient_data_ratio": 0.0,
|
||||
"model_references": ["lightgbm-aaa"],
|
||||
"status": "stable",
|
||||
"reason": None,
|
||||
}
|
||||
return NouveauRapportDerive(**{**defauts, **remplacements}) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_predictions_keep_one_row_per_site_and_target_in_sql() -> None:
|
||||
requete = sql(_predictions_retenues(debut=DEBUT, fin=FIN, site_id=None).element)
|
||||
|
||||
assert "DISTINCT ON (prediction.site_id, prediction.target_at)" in requete
|
||||
assert "prediction.prediction_id DESC" in requete
|
||||
|
||||
|
||||
def test_readings_keep_one_row_per_site_and_instant_in_sql() -> None:
|
||||
requete = sql(_lectures_retenues(debut=DEBUT, fin=FIN, site_id=None).element)
|
||||
|
||||
assert "DISTINCT ON (reading.site_id, reading.timestamp)" in requete
|
||||
assert "reading.reading_id DESC" in requete
|
||||
|
||||
|
||||
def test_predictions_restrict_themselves_to_the_requested_site_in_sql() -> None:
|
||||
requete = sql(_predictions_retenues(debut=DEBUT, fin=FIN, site_id="SITE001").element)
|
||||
|
||||
assert requete.count("prediction.site_id = ") == 1
|
||||
|
||||
|
||||
def test_readings_ignore_a_missing_consumption_in_sql() -> None:
|
||||
requete = sql(_lectures_retenues(debut=DEBUT, fin=FIN, site_id=None).element)
|
||||
|
||||
assert "reading.consumption_kwh IS NOT NULL" in requete
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_pairs_a_prediction_with_the_reading_of_the_same_instant(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
site = await creer_site(session)
|
||||
await creer_prediction(session, site_id=site.site_id, target_at=CIBLE, predicted_value=12.0)
|
||||
await creer_lecture(session, site_id=site.site_id, timestamp=CIBLE, consumption_kwh=10.0)
|
||||
|
||||
paires = await DriftRepository(session).paires(debut=DEBUT, fin=FIN, site_id=site.site_id)
|
||||
await session.rollback()
|
||||
|
||||
assert [(p.predicted_value, p.actual_value) for p in paires] == [(12.0, 10.0)]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_keeps_the_latest_run_when_several_predictions_share_a_target(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
site = await creer_site(session)
|
||||
await creer_prediction(session, site_id=site.site_id, target_at=CIBLE, predicted_value=12.0)
|
||||
await creer_prediction(session, site_id=site.site_id, target_at=CIBLE, predicted_value=99.0)
|
||||
await creer_lecture(session, site_id=site.site_id, timestamp=CIBLE, consumption_kwh=10.0)
|
||||
|
||||
paires = await DriftRepository(session).paires(debut=DEBUT, fin=FIN, site_id=site.site_id)
|
||||
await session.rollback()
|
||||
|
||||
assert [p.predicted_value for p in paires] == [99.0]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_keeps_one_reading_per_instant_when_two_sources_wrote_the_same_hour(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
site = await creer_site(session)
|
||||
await creer_prediction(session, site_id=site.site_id, target_at=CIBLE, predicted_value=12.0)
|
||||
await creer_lecture(
|
||||
session, site_id=site.site_id, timestamp=CIBLE, source="api_current", consumption_kwh=10.0
|
||||
)
|
||||
await creer_lecture(
|
||||
session, site_id=site.site_id, timestamp=CIBLE, source="api_history", consumption_kwh=20.0
|
||||
)
|
||||
|
||||
paires = await DriftRepository(session).paires(debut=DEBUT, fin=FIN, site_id=site.site_id)
|
||||
await session.rollback()
|
||||
|
||||
assert [p.actual_value for p in paires] == [20.0]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_excludes_an_insufficient_data_prediction_from_the_pairs(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
site = await creer_site(session)
|
||||
await creer_prediction(
|
||||
session,
|
||||
site_id=site.site_id,
|
||||
target_at=CIBLE,
|
||||
predicted_value=None,
|
||||
status="insufficient_data",
|
||||
failure_reason="historique trop court",
|
||||
)
|
||||
await creer_lecture(session, site_id=site.site_id, timestamp=CIBLE, consumption_kwh=10.0)
|
||||
|
||||
depot = DriftRepository(session)
|
||||
paires = await depot.paires(debut=DEBUT, fin=FIN, site_id=site.site_id)
|
||||
comptages = await depot.comptages(debut=DEBUT, fin=FIN, site_id=site.site_id)
|
||||
await session.rollback()
|
||||
|
||||
assert paires == []
|
||||
assert [(c.status, c.nombre) for c in comptages] == [("insufficient_data", 1)]
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_excludes_a_target_outside_the_window(session: AsyncSession) -> None:
|
||||
site = await creer_site(session)
|
||||
hors_fenetre = FIN + timedelta(hours=1)
|
||||
await creer_prediction(
|
||||
session, site_id=site.site_id, target_at=hors_fenetre, predicted_value=12.0
|
||||
)
|
||||
await creer_lecture(session, site_id=site.site_id, timestamp=hors_fenetre, consumption_kwh=10.0)
|
||||
|
||||
paires = await DriftRepository(session).paires(debut=DEBUT, fin=FIN, site_id=site.site_id)
|
||||
await session.rollback()
|
||||
|
||||
assert paires == []
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_reads_back_the_global_report_it_wrote(session: AsyncSession) -> None:
|
||||
depot = DriftRepository(session)
|
||||
fenetre = datetime(2035, 3, 1, tzinfo=UTC)
|
||||
|
||||
ecrites = await depot.enregistre([rapport(window_end=fenetre)])
|
||||
derniers = await depot.derniers()
|
||||
globaux = [r for r in derniers if r.site_id is None and r.window_end == fenetre]
|
||||
await session.rollback()
|
||||
|
||||
assert ecrites == 1
|
||||
assert len(globaux) == 1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
async def test_repository_ignores_a_second_report_for_the_same_window_and_site(
|
||||
session: AsyncSession,
|
||||
) -> None:
|
||||
depot = DriftRepository(session)
|
||||
fenetre = datetime(2035, 4, 1, tzinfo=UTC)
|
||||
|
||||
premiere = await depot.enregistre([rapport(window_end=fenetre)])
|
||||
seconde = await depot.enregistre([rapport(window_end=fenetre, status="derive", reason="x")])
|
||||
await session.rollback()
|
||||
|
||||
assert premiere == 1
|
||||
assert seconde == 0
|
||||
@@ -0,0 +1,220 @@
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from app.repositories.drift import ComptageStatut, PaireDerive
|
||||
from app.services.drift import (
|
||||
STATUT_DERIVE,
|
||||
STATUT_INDETERMINE,
|
||||
STATUT_STABLE,
|
||||
DriftService,
|
||||
Seuils,
|
||||
mesure,
|
||||
)
|
||||
|
||||
INSTANT = datetime(2026, 9, 22, 12, 0, tzinfo=UTC)
|
||||
|
||||
|
||||
def paire(
|
||||
*, site_id: str = "SITE001", prevu: float, reel: float, reference: str = "lightgbm-aaa"
|
||||
) -> PaireDerive:
|
||||
return PaireDerive(
|
||||
site_id=site_id,
|
||||
target_at=INSTANT,
|
||||
predicted_value=prevu,
|
||||
actual_value=reel,
|
||||
model_reference=reference,
|
||||
)
|
||||
|
||||
|
||||
def paires(
|
||||
*, site_id: str = "SITE001", nombre: int, prevu: float, reel: float
|
||||
) -> list[PaireDerive]:
|
||||
return [paire(site_id=site_id, prevu=prevu, reel=reel) for _ in range(nombre)]
|
||||
|
||||
|
||||
class FauxDepot:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
recentes: Sequence[PaireDerive] = (),
|
||||
anciennes: Sequence[PaireDerive] = (),
|
||||
comptages: Sequence[ComptageStatut] = (),
|
||||
) -> None:
|
||||
self.recentes = list(recentes)
|
||||
self.anciennes = list(anciennes)
|
||||
self._comptages = list(comptages)
|
||||
self.fenetres: list[tuple[datetime, datetime]] = []
|
||||
|
||||
async def paires(
|
||||
self, *, debut: datetime, fin: datetime, site_id: str | None = None
|
||||
) -> Sequence[PaireDerive]:
|
||||
self.fenetres.append((debut, fin))
|
||||
return self.recentes if len(self.fenetres) == 1 else self.anciennes
|
||||
|
||||
async def comptages(
|
||||
self, *, debut: datetime, fin: datetime, site_id: str | None = None
|
||||
) -> Sequence[ComptageStatut]:
|
||||
return self._comptages
|
||||
|
||||
|
||||
def service(depot: FauxDepot, **surcharges: object) -> DriftService:
|
||||
return DriftService(depot, seuils=Seuils(**surcharges)) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_drift_averages_the_absolute_gap_between_forecast_and_actual() -> None:
|
||||
metriques = mesure([paire(prevu=12.0, reel=10.0), paire(prevu=8.0, reel=10.0)])
|
||||
|
||||
assert metriques.mae == 2.0
|
||||
assert metriques.n_observations == 2
|
||||
|
||||
|
||||
def test_drift_computes_a_signed_bias_when_the_model_overforecasts() -> None:
|
||||
metriques = mesure([paire(prevu=12.0, reel=10.0), paire(prevu=14.0, reel=10.0)])
|
||||
|
||||
assert metriques.bias == 3.0
|
||||
|
||||
|
||||
def test_drift_computes_a_negative_bias_when_the_model_underforecasts() -> None:
|
||||
metriques = mesure([paire(prevu=8.0, reel=10.0), paire(prevu=6.0, reel=10.0)])
|
||||
|
||||
assert metriques.bias == -3.0
|
||||
|
||||
|
||||
def test_drift_excludes_a_zero_actual_from_the_mape_only() -> None:
|
||||
metriques = mesure([paire(prevu=11.0, reel=10.0), paire(prevu=5.0, reel=0.0)])
|
||||
|
||||
assert metriques.mape == 10.0
|
||||
assert metriques.n_observations == 2
|
||||
assert metriques.mae == 3.0
|
||||
|
||||
|
||||
def test_drift_reports_no_mape_when_every_actual_is_zero() -> None:
|
||||
metriques = mesure([paire(prevu=1.0, reel=0.0)])
|
||||
|
||||
assert metriques.mape is None
|
||||
|
||||
|
||||
def test_drift_lists_every_model_reference_seen_in_the_window() -> None:
|
||||
metriques = mesure(
|
||||
[paire(prevu=10.0, reel=10.0, reference="lightgbm-bbb"), paire(prevu=10.0, reel=10.0)]
|
||||
)
|
||||
|
||||
assert metriques.model_references == ["lightgbm-aaa", "lightgbm-bbb"]
|
||||
|
||||
|
||||
async def test_drift_reports_indetermine_when_the_window_holds_too_few_observations() -> None:
|
||||
depot = FauxDepot(recentes=paires(nombre=3, prevu=10.0, reel=10.0))
|
||||
|
||||
rapports = await service(depot, min_observations=24).evaluate(now=INSTANT)
|
||||
|
||||
assert {rapport.status for rapport in rapports} == {STATUT_INDETERMINE}
|
||||
assert all(rapport.reason for rapport in rapports)
|
||||
|
||||
|
||||
async def test_drift_reports_derive_when_the_recent_mae_exceeds_the_reference_ratio() -> None:
|
||||
depot = FauxDepot(
|
||||
recentes=paires(nombre=30, prevu=14.0, reel=10.0),
|
||||
anciennes=paires(nombre=30, prevu=11.0, reel=10.0),
|
||||
comptages=[ComptageStatut(site_id="SITE001", status="available", nombre=30)],
|
||||
)
|
||||
|
||||
rapports = await service(depot, min_observations=10).evaluate(now=INSTANT)
|
||||
|
||||
global_ = next(rapport for rapport in rapports if rapport.site_id is None)
|
||||
assert global_.status == STATUT_DERIVE
|
||||
assert global_.mae == 4.0
|
||||
assert global_.reference_mae == 1.0
|
||||
|
||||
|
||||
async def test_drift_reports_stable_when_the_recent_mae_stays_close_to_the_reference() -> None:
|
||||
depot = FauxDepot(
|
||||
recentes=paires(nombre=30, prevu=11.0, reel=10.0),
|
||||
anciennes=paires(nombre=30, prevu=11.0, reel=10.0),
|
||||
comptages=[ComptageStatut(site_id="SITE001", status="available", nombre=30)],
|
||||
)
|
||||
|
||||
rapports = await service(depot, min_observations=10).evaluate(now=INSTANT)
|
||||
|
||||
global_ = next(rapport for rapport in rapports if rapport.site_id is None)
|
||||
assert global_.status == STATUT_STABLE
|
||||
assert global_.reason is None
|
||||
|
||||
|
||||
async def test_drift_reports_derive_when_the_coverage_ratio_falls_under_the_threshold() -> None:
|
||||
depot = FauxDepot(
|
||||
recentes=paires(nombre=30, prevu=10.0, reel=10.0),
|
||||
anciennes=paires(nombre=30, prevu=10.0, reel=10.0),
|
||||
comptages=[ComptageStatut(site_id="SITE001", status="available", nombre=100)],
|
||||
)
|
||||
|
||||
rapports = await service(depot, min_observations=10).evaluate(now=INSTANT)
|
||||
|
||||
global_ = next(rapport for rapport in rapports if rapport.site_id is None)
|
||||
assert global_.status == STATUT_DERIVE
|
||||
assert global_.coverage_ratio == 0.3
|
||||
|
||||
|
||||
async def test_drift_reports_one_line_per_site_and_one_global_line() -> None:
|
||||
depot = FauxDepot(
|
||||
recentes=[
|
||||
*paires(site_id="SITE001", nombre=12, prevu=10.0, reel=10.0),
|
||||
*paires(site_id="SITE002", nombre=12, prevu=10.0, reel=10.0),
|
||||
],
|
||||
comptages=[
|
||||
ComptageStatut(site_id="SITE001", status="available", nombre=12),
|
||||
ComptageStatut(site_id="SITE002", status="available", nombre=12),
|
||||
],
|
||||
)
|
||||
|
||||
rapports = await service(depot, min_observations=10).evaluate(now=INSTANT)
|
||||
|
||||
assert [rapport.site_id for rapport in rapports] == ["SITE001", "SITE002", None]
|
||||
assert next(r for r in rapports if r.site_id is None).n_observations == 24
|
||||
|
||||
|
||||
async def test_drift_measures_the_share_of_sites_left_without_enough_history() -> None:
|
||||
depot = FauxDepot(
|
||||
recentes=paires(nombre=30, prevu=10.0, reel=10.0),
|
||||
comptages=[
|
||||
ComptageStatut(site_id="SITE001", status="available", nombre=30),
|
||||
ComptageStatut(site_id="SITE001", status="insufficient_data", nombre=10),
|
||||
],
|
||||
)
|
||||
|
||||
rapports = await service(depot, min_observations=10).evaluate(now=INSTANT)
|
||||
|
||||
assert next(r for r in rapports if r.site_id is None).insufficient_data_ratio == 0.25
|
||||
|
||||
|
||||
async def test_drift_closes_the_window_before_the_grace_delay() -> None:
|
||||
depot = FauxDepot()
|
||||
|
||||
await service(depot, grace=timedelta(hours=2), fenetre=timedelta(hours=168)).evaluate(
|
||||
now=INSTANT
|
||||
)
|
||||
|
||||
recente, reference = depot.fenetres
|
||||
assert recente[1] == INSTANT - timedelta(hours=2)
|
||||
assert recente[0] == INSTANT - timedelta(hours=170)
|
||||
assert reference[1] == recente[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prevu", "attendu"),
|
||||
[(10.0, STATUT_STABLE), (30.0, STATUT_DERIVE)],
|
||||
ids=["mae_stable", "mae_triplee"],
|
||||
)
|
||||
async def test_drift_compares_the_recent_window_to_the_reference_one(
|
||||
prevu: float, attendu: str
|
||||
) -> None:
|
||||
depot = FauxDepot(
|
||||
recentes=paires(nombre=30, prevu=prevu, reel=10.0),
|
||||
anciennes=paires(nombre=30, prevu=10.0, reel=10.0),
|
||||
comptages=[ComptageStatut(site_id="SITE001", status="available", nombre=30)],
|
||||
)
|
||||
|
||||
rapports = await service(depot, min_observations=10, mae_plancher=1.0).evaluate(now=INSTANT)
|
||||
|
||||
assert next(r for r in rapports if r.site_id is None).status == attendu
|
||||
@@ -0,0 +1,108 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from app.monitoring import drift as cli
|
||||
from app.repositories.drift import NouveauRapportDerive
|
||||
from app.services.drift import STATUT_DERIVE, STATUT_STABLE, Seuils
|
||||
|
||||
INSTANT = datetime(2026, 9, 22, 12, tzinfo=UTC)
|
||||
|
||||
|
||||
def rapport(*, site_id: str | None, status: str, reason: str | None = None) -> NouveauRapportDerive:
|
||||
return NouveauRapportDerive(
|
||||
site_id=site_id,
|
||||
window_start=INSTANT - timedelta(hours=168),
|
||||
window_end=INSTANT,
|
||||
reference_start=None,
|
||||
reference_end=None,
|
||||
n_observations=48,
|
||||
mae=1.5,
|
||||
mape=12.0,
|
||||
bias=0.3,
|
||||
reference_mae=1.2,
|
||||
coverage_ratio=1.0,
|
||||
insufficient_data_ratio=0.0,
|
||||
model_references=["lightgbm-aaa"],
|
||||
status=status,
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def installe(monkeypatch: pytest.MonkeyPatch, rapports: list[NouveauRapportDerive]) -> None:
|
||||
async def fausse_execution(
|
||||
*, now: datetime | None, site_id: str | None, seuils: Seuils | None
|
||||
) -> list[NouveauRapportDerive]:
|
||||
return rapports
|
||||
|
||||
monkeypatch.setattr(cli, "run_drift", fausse_execution)
|
||||
|
||||
|
||||
def test_parse_args_defaults_to_the_standard_window() -> None:
|
||||
arguments = cli.parse_args([])
|
||||
|
||||
assert arguments.window_hours == 168
|
||||
assert arguments.grace_hours == 2
|
||||
assert arguments.fail_on_drift is False
|
||||
|
||||
|
||||
def test_parse_args_reads_the_site_id() -> None:
|
||||
assert cli.parse_args(["--site-id", "SITE001"]).site_id == "SITE001"
|
||||
|
||||
|
||||
def test_parse_args_parses_the_instant_option() -> None:
|
||||
arguments = cli.parse_args(["--now", "2026-09-22T12:00:00+00:00"])
|
||||
|
||||
assert arguments.now == INSTANT
|
||||
|
||||
|
||||
def test_parse_instant_treats_a_naive_datetime_as_utc() -> None:
|
||||
assert cli._parse_instant("2026-09-22T12:00:00") == INSTANT
|
||||
|
||||
|
||||
def test_seuils_depuis_translates_the_hour_options_into_durations() -> None:
|
||||
seuils = cli.seuils_depuis(cli.parse_args(["--window-hours", "24", "--grace-hours", "1"]))
|
||||
|
||||
assert seuils.fenetre == timedelta(hours=24)
|
||||
assert seuils.grace == timedelta(hours=1)
|
||||
|
||||
|
||||
def test_main_prints_the_verdict_of_every_line(
|
||||
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
installe(
|
||||
monkeypatch,
|
||||
[
|
||||
rapport(site_id="SITE001", status=STATUT_STABLE),
|
||||
rapport(site_id=None, status=STATUT_STABLE),
|
||||
],
|
||||
)
|
||||
|
||||
code = cli.main([])
|
||||
|
||||
sortie = capsys.readouterr().out
|
||||
assert code == 0
|
||||
assert "SITE001" in sortie
|
||||
assert "TOUS SITES" in sortie
|
||||
|
||||
|
||||
def test_main_exits_non_zero_when_drift_is_detected_and_the_flag_is_set(
|
||||
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
installe(monkeypatch, [rapport(site_id=None, status=STATUT_DERIVE, reason="MAE doublée")])
|
||||
|
||||
code = cli.main(["--fail-on-drift"])
|
||||
|
||||
assert code == 1
|
||||
assert "MAE doublée" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_main_exits_zero_when_drift_is_detected_without_the_flag(
|
||||
monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
|
||||
) -> None:
|
||||
installe(monkeypatch, [rapport(site_id=None, status=STATUT_DERIVE, reason="MAE doublée")])
|
||||
|
||||
code = cli.main([])
|
||||
|
||||
assert code == 0
|
||||
assert capsys.readouterr().out != ""
|
||||
Reference in New Issue
Block a user