johanleroy/ENI-projet-piscine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(backend,airflow,ml): cloture la reconciliation entre les deux sources de lectures (#15)

Browse Source
This commit is contained in:
Dorian
2026-09-23 14:45:38 +02:00
parent c7744483b4
commit b00c39277b
9 changed files with 269 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ml/enervision_ml/data.py
+10 -4
View File
@@ -47,9 +47,15 @@ NUMERIC_COLUMNS = [
# `bool` : `astype(bool)` ferait un `True` d'une absence, et les deux chargeurs divergeraient.
FLAG_COLUMNS = ["is_working_hours"]
# `uq_reading_source` autorise deux lignes au meme (site_id, timestamp) des que `source` differe
# (cf. `app/etl/mock_api_import.py`, qui refuse desormais d'importer une fenetre deja couverte par
# le CSV, mais ne protege pas le sens inverse). `build_features` suppose une ligne par
# (site_id, timestamp) sans doublon : le `DISTINCT ON` l'impose plutot que de la supposer.
# 'csv' gagne sur 'api_history' en cas de recouvrement, l'historique etant une source verifiee
# alors que l'API Mock est traitee comme une entree hostile (cf. OWASP API10).
_READING_QUERY = text(
"""
SELECT
SELECT DISTINCT ON (r.site_id, r.timestamp)
r.site_id,
r.timestamp,
r.consumption_kwh,
@@ -61,14 +67,14 @@ _READING_QUERY = text(
s.capacity_kw
FROM reading r
JOIN site s ON s.site_id = r.site_id
ORDER BY r.site_id, r.timestamp
ORDER BY r.site_id, r.timestamp, (r.source = 'csv') DESC, r.reading_id DESC
"""
)
_RECENT_READING_QUERY = text(
"""
SELECT
SELECT DISTINCT ON (r.site_id, r.timestamp)
r.site_id,
r.timestamp,
r.consumption_kwh,
@@ -81,7 +87,7 @@ _RECENT_READING_QUERY = text(
FROM reading r
JOIN site s ON s.site_id = r.site_id
WHERE r.timestamp >= :since AND r.timestamp <= :until
ORDER BY r.site_id, r.timestamp
ORDER BY r.site_id, r.timestamp, (r.source = 'csv') DESC, r.reading_id DESC
"""
)
ml/tests/conftest.py
+38 -5
View File
@@ -16,7 +16,7 @@ from collections.abc import Iterator
from dataclasses import dataclass, field
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from typing import Any, cast
from uuid import uuid4
import lightgbm as lgb
@@ -44,20 +44,30 @@ _INSERT_SITE = text(
"""
)
# `source = 'api_history'` impose `dataset_id IS NULL` (ck_reading_dataset_source), ce qui evite
# de creer une ligne `dataset`. `raw_data` est NOT NULL, d'ou le litteral jsonb.
# `source = 'api_history'` impose `dataset_id IS NULL` (ck_reading_dataset_source) : le defaut
# `dataset_id=None` evite de creer une ligne `dataset` pour la plupart des tests. `source='csv'`
# impose l'inverse, d'ou `insere_dataset()` quand un test a besoin de cette source precise.
# `raw_data` est NOT NULL, d'ou le litteral jsonb.
_INSERT_READING = text(
"""
INSERT INTO reading (
site_id, timestamp, source, consumption_kwh, temperature_celsius,
site_id, timestamp, source, dataset_id, consumption_kwh, temperature_celsius,
humidity_percent, solar_irradiance_wm2, is_working_hours, raw_data
) VALUES (
:site_id, :timestamp, :source, :consumption_kwh, :temperature_celsius,
:site_id, :timestamp, :source, :dataset_id, :consumption_kwh, :temperature_celsius,
:humidity_percent, :solar_irradiance_wm2, :is_working_hours, '{}'::jsonb
)
"""
)
_INSERT_DATASET = text(
"""
INSERT INTO dataset (dataset_name, archive_sha256, storage_uri, source_timezone, metadata)
VALUES (:dataset_name, :archive_sha256, :storage_uri, 'UTC', '{}'::jsonb)
RETURNING dataset_id
"""
)
_SELECT_PREDICTIONS = text(
"""
SELECT target_at, predicted_value, status, failure_reason, model_reference
@@ -111,6 +121,25 @@ def insere_site(
return site_id
def insere_dataset(connexion: Connection) -> int:
"""Ligne `dataset` minimale, requise pour inserer une lecture `source='csv'`
(`ck_reading_dataset_source` impose `dataset_id IS NOT NULL` pour cette seule source).
"""
marque = uuid4().hex
return cast(
int,
connexion.execute(
_INSERT_DATASET,
{
"dataset_name": f"jeu de test {marque}",
"archive_sha256": marque.rjust(64, "0"),
"storage_uri": f"file:///test/{marque}.csv",
},
).scalar_one(),
)
def insere_lectures(
connexion: Connection,
site_id: str,
@@ -119,6 +148,7 @@ def insere_lectures(
fin: datetime,
valeur: float = 50.0,
source: str = "api_history",
dataset_id: int | None = None,
is_working_hours: bool | None = True,
) -> list[datetime]:
"""Grille horaire contigue finissant a `fin`, incluse.
@@ -134,6 +164,7 @@ def insere_lectures(
"site_id": site_id,
"timestamp": instant,
"source": source,
"dataset_id": dataset_id,
"consumption_kwh": valeur + math.sin(rang / 12.0) * 10.0,
"temperature_celsius": 15.0,
"humidity_percent": 50.0,
@@ -153,6 +184,7 @@ def insere_lecture(
instant: datetime,
consumption_kwh: float | None = 50.0,
source: str = "api_history",
dataset_id: int | None = None,
is_working_hours: bool | None = True,
) -> None:
"""Une lecture isolee, quand le test pilote sa valeur plutot que sa forme."""
@@ -162,6 +194,7 @@ def insere_lecture(
"site_id": site_id,
"timestamp": instant,
"source": source,
"dataset_id": dataset_id,
"consumption_kwh": consumption_kwh,
"temperature_celsius": 15.0,
"humidity_percent": 50.0,
ml/tests/test_data_integration.py
+53 -1
View File
@@ -11,7 +11,7 @@ from enervision_ml.data import (
load_from_database,
load_recent_from_database,
)
from tests.conftest import ANCRAGE, insere_lecture, insere_lectures, insere_site
from tests.conftest import ANCRAGE, insere_dataset, insere_lecture, insere_lectures, insere_site
pytestmark = pytest.mark.integration
@@ -39,6 +39,58 @@ def test_load_from_database_joins_the_site_attributes_to_every_reading(
assert set(mien["capacity_kw"]) == {250.0}
def test_load_from_database_deduplicates_two_sources_at_the_same_instant(
connexion_ml: Connection,
) -> None:
# `uq_reading_source` autorise deux lignes au meme (site_id, timestamp) des que `source`
# differe : le garde-fou vit dans `mock_api_import.py`, pas dans le schema. Le chargeur ML
# doit donc imposer lui-meme "une ligne par (site_id, timestamp)", pas la supposer.
site_id = insere_site(connexion_ml)
dataset_id = insere_dataset(connexion_ml)
insere_lecture(
connexion_ml, site_id, instant=ANCRAGE, consumption_kwh=10.0, source="api_history"
)
insere_lecture(
connexion_ml,
site_id,
instant=ANCRAGE,
consumption_kwh=99.0,
source="csv",
dataset_id=dataset_id,
)
frame = load_from_database(connexion_ml)
mien = frame[frame["site_id"] == site_id]
assert len(mien) == 1
assert mien["consumption_kwh"].iloc[0] == 99.0
def test_load_recent_from_database_prefers_csv_when_two_sources_share_an_instant(
connexion_ml: Connection,
) -> None:
site_id = insere_site(connexion_ml)
dataset_id = insere_dataset(connexion_ml)
insere_lecture(
connexion_ml, site_id, instant=ANCRAGE, consumption_kwh=10.0, source="api_history"
)
insere_lecture(
connexion_ml,
site_id,
instant=ANCRAGE,
consumption_kwh=99.0,
source="csv",
dataset_id=dataset_id,
)
frame = load_recent_from_database(
connexion_ml, since=ANCRAGE, until=ANCRAGE + timedelta(hours=3)
)
assert len(frame) == 1
assert frame["consumption_kwh"].iloc[0] == 99.0
def test_load_recent_from_database_excludes_readings_before_the_since_bound(
connexion_ml: Connection,
) -> None: