625 lines
16 KiB
Python
625 lines
16 KiB
Python
import json
|
|
import sys
|
|
from datetime import datetime
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
from httpx import AsyncClient, MockTransport, Request, Response
|
|
from sqlalchemy import text
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
import app.etl.mock_api_import as mock_api_import
|
|
from app.etl.mock_api_import import (
|
|
READING_INSERT,
|
|
SOURCE_HISTORY,
|
|
build_reading_batch,
|
|
build_reading_row,
|
|
fetch_readings,
|
|
fetch_sites,
|
|
upsert_sites,
|
|
)
|
|
|
|
|
|
def make_site() -> dict[str, Any]:
|
|
return {
|
|
"site_id": "SITE001",
|
|
"site_type": "office",
|
|
"site_name": "Bureau Paris La Défense",
|
|
"location": "Paris, France",
|
|
"capacity_kw": 200,
|
|
"status": "active",
|
|
}
|
|
|
|
|
|
def make_reading() -> dict[str, Any]:
|
|
return {
|
|
"timestamp": "2024-06-15T12:00:00Z",
|
|
"site_id": "SITE001",
|
|
"site_type": "office",
|
|
"consumption_kw": 87.34,
|
|
"consumption_kwh": 87.34,
|
|
"voltage_v": 401.2,
|
|
"current_a": 132.5,
|
|
"power_factor": 0.923,
|
|
"temperature_celsius": 22.1,
|
|
"humidity_percent": 58.4,
|
|
"null_reasons": [],
|
|
"data_quality": "good",
|
|
}
|
|
|
|
|
|
async def test_fetch_sites_returns_sites() -> None:
|
|
def handler(request: Request) -> Response:
|
|
assert request.url.path == "/api/v1/sites"
|
|
|
|
return Response(
|
|
status_code=200,
|
|
json=[make_site()],
|
|
)
|
|
|
|
transport = MockTransport(handler)
|
|
|
|
async with AsyncClient(
|
|
transport=transport,
|
|
base_url="https://mock.test",
|
|
) as client:
|
|
sites = await fetch_sites(client)
|
|
|
|
assert len(sites) == 1
|
|
assert sites[0]["site_id"] == "SITE001"
|
|
assert sites[0]["site_type"] == "office"
|
|
|
|
|
|
async def test_fetch_readings_sends_expected_query_parameters() -> None:
|
|
captured_params: dict[str, str] = {}
|
|
|
|
def handler(request: Request) -> Response:
|
|
nonlocal captured_params
|
|
|
|
captured_params = dict(request.url.params)
|
|
|
|
return Response(
|
|
status_code=200,
|
|
json=[make_reading()],
|
|
)
|
|
|
|
transport = MockTransport(handler)
|
|
|
|
start_time = datetime.fromisoformat("2024-06-15T12:00:00")
|
|
end_time = datetime.fromisoformat("2024-06-15T13:00:00")
|
|
|
|
async with AsyncClient(
|
|
transport=transport,
|
|
base_url="https://mock.test",
|
|
) as client:
|
|
readings = await fetch_readings(
|
|
client=client,
|
|
site_id="SITE001",
|
|
start_time=start_time,
|
|
end_time=end_time,
|
|
limit=60,
|
|
)
|
|
|
|
assert len(readings) == 1
|
|
assert captured_params["site_id"] == "SITE001"
|
|
assert captured_params["start_time"] == "2024-06-15T12:00:00"
|
|
assert captured_params["end_time"] == "2024-06-15T13:00:00"
|
|
assert captured_params["limit"] == "60"
|
|
|
|
|
|
async def test_fetch_readings_rejects_non_list_response() -> None:
|
|
def handler(request: Request) -> Response:
|
|
return Response(
|
|
status_code=200,
|
|
json={"unexpected": "payload"},
|
|
)
|
|
|
|
transport = MockTransport(handler)
|
|
|
|
async with AsyncClient(
|
|
transport=transport,
|
|
base_url="https://mock.test",
|
|
) as client:
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="La réponse /api/v1/readings doit être une liste",
|
|
):
|
|
await fetch_readings(
|
|
client=client,
|
|
site_id="SITE001",
|
|
start_time=datetime.fromisoformat("2024-06-15T12:00:00"),
|
|
end_time=datetime.fromisoformat("2024-06-15T13:00:00"),
|
|
limit=60,
|
|
)
|
|
|
|
|
|
async def test_fetch_readings_raises_on_http_error() -> None:
|
|
def handler(request: Request) -> Response:
|
|
return Response(
|
|
status_code=404,
|
|
json={"detail": "Site non trouvé"},
|
|
)
|
|
|
|
transport = MockTransport(handler)
|
|
|
|
async with AsyncClient(
|
|
transport=transport,
|
|
base_url="https://mock.test",
|
|
) as client:
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
await fetch_readings(
|
|
client=client,
|
|
site_id="SITE999",
|
|
start_time=datetime.fromisoformat("2024-06-15T12:00:00"),
|
|
end_time=datetime.fromisoformat("2024-06-15T13:00:00"),
|
|
limit=60,
|
|
)
|
|
|
|
|
|
def test_build_reading_row_respects_database_contract() -> None:
|
|
reading = make_reading()
|
|
|
|
row = build_reading_row(reading)
|
|
|
|
assert row["site_id"] == "SITE001"
|
|
assert row["source"] == SOURCE_HISTORY
|
|
assert row["source"] == "api_history"
|
|
assert row["dataset_id"] is None
|
|
|
|
assert row["timestamp"] == datetime.fromisoformat(
|
|
"2024-06-15T12:00:00+00:00"
|
|
)
|
|
|
|
assert row["consumption_kw"] == 87.34
|
|
assert row["consumption_kwh"] == 87.34
|
|
assert row["data_quality"] == "good"
|
|
assert row["null_reasons"] == []
|
|
|
|
assert row["imputed_values"] is None
|
|
assert row["imputation_method"] is None
|
|
|
|
|
|
def test_build_reading_row_keeps_null_values_and_quality() -> None:
|
|
reading = make_reading()
|
|
|
|
reading["consumption_kw"] = None
|
|
reading["consumption_kwh"] = None
|
|
reading["voltage_v"] = None
|
|
reading["current_a"] = None
|
|
reading["power_factor"] = None
|
|
reading["data_quality"] = "degraded"
|
|
reading["null_reasons"] = [
|
|
"consumption_sensor_failure",
|
|
"electrical_sensor_failure",
|
|
]
|
|
|
|
row = build_reading_row(reading)
|
|
|
|
assert row["consumption_kw"] is None
|
|
assert row["consumption_kwh"] is None
|
|
assert row["voltage_v"] is None
|
|
assert row["current_a"] is None
|
|
assert row["power_factor"] is None
|
|
|
|
assert row["data_quality"] == "degraded"
|
|
assert row["null_reasons"] == [
|
|
"consumption_sensor_failure",
|
|
"electrical_sensor_failure",
|
|
]
|
|
|
|
assert row["imputed_values"] is None
|
|
assert row["imputation_method"] is None
|
|
|
|
|
|
def test_build_reading_row_keeps_raw_source_data() -> None:
|
|
reading = make_reading()
|
|
|
|
row = build_reading_row(reading)
|
|
|
|
raw_data = json.loads(row["raw_data"])
|
|
|
|
assert raw_data == reading
|
|
|
|
|
|
def test_build_reading_batch_transforms_all_readings() -> None:
|
|
first = make_reading()
|
|
|
|
second = make_reading()
|
|
second["timestamp"] = "2024-06-15T12:01:00Z"
|
|
second["consumption_kw"] = 90.5
|
|
|
|
rows = build_reading_batch([first, second])
|
|
|
|
assert len(rows) == 2
|
|
|
|
assert rows[0]["site_id"] == "SITE001"
|
|
assert rows[0]["consumption_kw"] == 87.34
|
|
|
|
assert rows[1]["site_id"] == "SITE001"
|
|
assert rows[1]["consumption_kw"] == 90.5
|
|
|
|
|
|
def test_create_mock_api_client_requires_credentials(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
settings = SimpleNamespace(
|
|
mock_api_username=None,
|
|
mock_api_password=None,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"get_settings",
|
|
lambda: settings,
|
|
)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="Les identifiants de l'API Mock ne sont pas configurés",
|
|
):
|
|
mock_api_import.create_mock_api_client()
|
|
|
|
|
|
async def test_create_mock_api_client_uses_configuration(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
password = MagicMock()
|
|
password.get_secret_value.return_value = "test-password"
|
|
|
|
settings = SimpleNamespace(
|
|
mock_api_base_url="https://mock.test/",
|
|
mock_api_username="test-user",
|
|
mock_api_password=password,
|
|
mock_api_timeout_seconds=10.0,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"get_settings",
|
|
lambda: settings,
|
|
)
|
|
|
|
client = mock_api_import.create_mock_api_client()
|
|
|
|
try:
|
|
assert str(client.base_url) == "https://mock.test"
|
|
assert client.timeout.connect == 10.0
|
|
finally:
|
|
await client.aclose()
|
|
|
|
|
|
async def test_upsert_sites_with_empty_list_does_nothing() -> None:
|
|
connection = AsyncMock()
|
|
|
|
await upsert_sites(
|
|
connection,
|
|
[],
|
|
)
|
|
|
|
connection.execute.assert_not_awaited()
|
|
|
|
|
|
async def test_import_mock_api_history_dry_run_does_not_write(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
def handler(request: Request) -> Response:
|
|
if request.url.path == "/api/v1/sites":
|
|
return Response(
|
|
status_code=200,
|
|
json=[make_site()],
|
|
)
|
|
|
|
if request.url.path == "/api/v1/readings":
|
|
return Response(
|
|
status_code=200,
|
|
json=[make_reading()],
|
|
)
|
|
|
|
return Response(status_code=404)
|
|
|
|
transport = MockTransport(handler)
|
|
|
|
client = AsyncClient(
|
|
transport=transport,
|
|
base_url="https://mock.test",
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"create_mock_api_client",
|
|
lambda: client,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"get_settings",
|
|
lambda: SimpleNamespace(
|
|
database_url="postgresql+asyncpg://unused",
|
|
),
|
|
)
|
|
|
|
create_engine_mock = MagicMock()
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"create_async_engine",
|
|
create_engine_mock,
|
|
)
|
|
|
|
await mock_api_import.import_mock_api_history(
|
|
start_time=datetime.fromisoformat("2024-06-15T12:00:00"),
|
|
end_time=datetime.fromisoformat("2024-06-15T13:00:00"),
|
|
limit=60,
|
|
dry_run=True,
|
|
)
|
|
|
|
create_engine_mock.assert_not_called()
|
|
|
|
|
|
async def test_import_mock_api_history_loads_data(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
def handler(request: Request) -> Response:
|
|
if request.url.path == "/api/v1/sites":
|
|
return Response(
|
|
status_code=200,
|
|
json=[make_site()],
|
|
)
|
|
|
|
if request.url.path == "/api/v1/readings":
|
|
return Response(
|
|
status_code=200,
|
|
json=[make_reading()],
|
|
)
|
|
|
|
return Response(status_code=404)
|
|
|
|
transport = MockTransport(handler)
|
|
|
|
client = AsyncClient(
|
|
transport=transport,
|
|
base_url="https://mock.test",
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"create_mock_api_client",
|
|
lambda: client,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"get_settings",
|
|
lambda: SimpleNamespace(
|
|
database_url="postgresql+asyncpg://test:test@localhost/test",
|
|
),
|
|
)
|
|
|
|
connection = AsyncMock()
|
|
|
|
transaction_context = MagicMock()
|
|
transaction_context.__aenter__ = AsyncMock(
|
|
return_value=connection,
|
|
)
|
|
transaction_context.__aexit__ = AsyncMock(
|
|
return_value=None,
|
|
)
|
|
|
|
engine = MagicMock()
|
|
engine.begin.return_value = transaction_context
|
|
engine.dispose = AsyncMock()
|
|
|
|
create_engine_mock = MagicMock(
|
|
return_value=engine,
|
|
)
|
|
|
|
upsert_sites_mock = AsyncMock()
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"create_async_engine",
|
|
create_engine_mock,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"upsert_sites",
|
|
upsert_sites_mock,
|
|
)
|
|
|
|
await mock_api_import.import_mock_api_history(
|
|
start_time=datetime.fromisoformat("2024-06-15T12:00:00"),
|
|
end_time=datetime.fromisoformat("2024-06-15T13:00:00"),
|
|
limit=60,
|
|
dry_run=False,
|
|
)
|
|
|
|
create_engine_mock.assert_called_once_with(
|
|
"postgresql+asyncpg://test:test@localhost/test",
|
|
pool_pre_ping=True,
|
|
)
|
|
|
|
upsert_sites_mock.assert_awaited_once_with(
|
|
connection,
|
|
[make_site()],
|
|
)
|
|
|
|
connection.execute.assert_awaited_once()
|
|
engine.dispose.assert_awaited_once()
|
|
|
|
|
|
def test_parse_datetime_accepts_z_suffix() -> None:
|
|
result = mock_api_import.parse_datetime(
|
|
"2024-06-15T12:00:00Z",
|
|
)
|
|
|
|
assert result == datetime.fromisoformat(
|
|
"2024-06-15T12:00:00+00:00",
|
|
)
|
|
|
|
|
|
def test_parse_args_reads_cli_parameters(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"mock_api_import",
|
|
"--start-time",
|
|
"2024-06-15T12:00:00Z",
|
|
"--end-time",
|
|
"2024-06-15T13:00:00Z",
|
|
"--limit",
|
|
"60",
|
|
"--dry-run",
|
|
],
|
|
)
|
|
|
|
args = mock_api_import.parse_args()
|
|
|
|
assert args.start_time == datetime.fromisoformat(
|
|
"2024-06-15T12:00:00+00:00",
|
|
)
|
|
assert args.end_time == datetime.fromisoformat(
|
|
"2024-06-15T13:00:00+00:00",
|
|
)
|
|
assert args.limit == 60
|
|
assert args.dry_run is True
|
|
|
|
|
|
def test_main_rejects_limit_out_of_bounds(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"mock_api_import",
|
|
"--start-time",
|
|
"2024-06-15T12:00:00Z",
|
|
"--end-time",
|
|
"2024-06-15T13:00:00Z",
|
|
"--limit",
|
|
"0",
|
|
],
|
|
)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="--limit doit être compris entre 1 et 1000",
|
|
):
|
|
mock_api_import.main()
|
|
|
|
|
|
def test_main_rejects_invalid_period(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
"mock_api_import",
|
|
"--start-time",
|
|
"2024-06-15T14:00:00Z",
|
|
"--end-time",
|
|
"2024-06-15T13:00:00Z",
|
|
"--limit",
|
|
"60",
|
|
],
|
|
)
|
|
|
|
with pytest.raises(
|
|
ValueError,
|
|
match="--start-time doit être antérieur à --end-time",
|
|
):
|
|
mock_api_import.main()
|
|
|
|
|
|
def test_main_runs_import(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
start_time = datetime.fromisoformat(
|
|
"2024-06-15T12:00:00+00:00",
|
|
)
|
|
end_time = datetime.fromisoformat(
|
|
"2024-06-15T13:00:00+00:00",
|
|
)
|
|
|
|
import_mock = AsyncMock()
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"parse_args",
|
|
lambda: SimpleNamespace(
|
|
start_time=start_time,
|
|
end_time=end_time,
|
|
limit=60,
|
|
dry_run=True,
|
|
),
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
mock_api_import,
|
|
"import_mock_api_history",
|
|
import_mock,
|
|
)
|
|
|
|
mock_api_import.main()
|
|
|
|
import_mock.assert_awaited_once_with(
|
|
start_time=start_time,
|
|
end_time=end_time,
|
|
limit=60,
|
|
dry_run=True,
|
|
)
|
|
|
|
|
|
@pytest.mark.integration
|
|
async def test_reading_insert_is_idempotent(
|
|
session: AsyncSession,
|
|
) -> None:
|
|
reading = make_reading()
|
|
row = build_reading_row(reading)
|
|
|
|
connection = await session.connection()
|
|
|
|
await upsert_sites(
|
|
connection,
|
|
[make_site()],
|
|
)
|
|
|
|
await session.execute(
|
|
READING_INSERT,
|
|
[row],
|
|
)
|
|
|
|
await session.execute(
|
|
READING_INSERT,
|
|
[row],
|
|
)
|
|
|
|
result = await session.execute(
|
|
text(
|
|
"""
|
|
SELECT COUNT(*)
|
|
FROM reading
|
|
WHERE site_id = :site_id
|
|
AND timestamp = :timestamp
|
|
AND source = :source
|
|
"""
|
|
),
|
|
{
|
|
"site_id": row["site_id"],
|
|
"timestamp": row["timestamp"],
|
|
"source": row["source"],
|
|
},
|
|
)
|
|
|
|
assert result.scalar_one() == 1
|
|
|
|
await session.rollback()
|