import json import sys from datetime import datetime from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock, MagicMock import httpx import pytest from httpx import AsyncClient, MockTransport, Request, Response from sqlalchemy import text from sqlalchemy.ext.asyncio import AsyncSession import app.etl.mock_api_import as mock_api_import from app.etl.mock_api_import import ( READING_INSERT, SOURCE_HISTORY, build_reading_batch, build_reading_row, fetch_readings, fetch_sites, upsert_sites, ) def make_site() -> dict[str, Any]: return { "site_id": "SITE001", "site_type": "office", "site_name": "Bureau Paris La Défense", "location": "Paris, France", "capacity_kw": 200, "status": "active", } def make_reading() -> dict[str, Any]: return { "timestamp": "2024-06-15T12:00:00Z", "site_id": "SITE001", "site_type": "office", "consumption_kw": 87.34, "consumption_kwh": 87.34, "voltage_v": 401.2, "current_a": 132.5, "power_factor": 0.923, "temperature_celsius": 22.1, "humidity_percent": 58.4, "null_reasons": [], "data_quality": "good", } async def test_fetch_sites_returns_sites() -> None: def handler(request: Request) -> Response: assert request.url.path == "/api/v1/sites" return Response( status_code=200, json=[make_site()], ) transport = MockTransport(handler) async with AsyncClient( transport=transport, base_url="https://mock.test", ) as client: sites = await fetch_sites(client) assert len(sites) == 1 assert sites[0]["site_id"] == "SITE001" assert sites[0]["site_type"] == "office" async def test_fetch_readings_sends_expected_query_parameters() -> None: captured_params: dict[str, str] = {} def handler(request: Request) -> Response: nonlocal captured_params captured_params = dict(request.url.params) return Response( status_code=200, json=[make_reading()], ) transport = MockTransport(handler) start_time = datetime.fromisoformat("2024-06-15T12:00:00") end_time = datetime.fromisoformat("2024-06-15T13:00:00") async with AsyncClient( transport=transport, base_url="https://mock.test", ) as client: readings = await fetch_readings( client=client, site_id="SITE001", start_time=start_time, end_time=end_time, limit=60, ) assert len(readings) == 1 assert captured_params["site_id"] == "SITE001" assert captured_params["start_time"] == "2024-06-15T12:00:00" assert captured_params["end_time"] == "2024-06-15T13:00:00" assert captured_params["limit"] == "60" async def test_fetch_readings_rejects_non_list_response() -> None: def handler(request: Request) -> Response: return Response( status_code=200, json={"unexpected": "payload"}, ) transport = MockTransport(handler) async with AsyncClient( transport=transport, base_url="https://mock.test", ) as client: with pytest.raises( ValueError, match="La réponse /api/v1/readings doit être une liste", ): await fetch_readings( client=client, site_id="SITE001", start_time=datetime.fromisoformat("2024-06-15T12:00:00"), end_time=datetime.fromisoformat("2024-06-15T13:00:00"), limit=60, ) async def test_fetch_readings_raises_on_http_error() -> None: def handler(request: Request) -> Response: return Response( status_code=404, json={"detail": "Site non trouvé"}, ) transport = MockTransport(handler) async with AsyncClient( transport=transport, base_url="https://mock.test", ) as client: with pytest.raises(httpx.HTTPStatusError): await fetch_readings( client=client, site_id="SITE999", start_time=datetime.fromisoformat("2024-06-15T12:00:00"), end_time=datetime.fromisoformat("2024-06-15T13:00:00"), limit=60, ) def test_build_reading_row_respects_database_contract() -> None: reading = make_reading() row = build_reading_row(reading) assert row["site_id"] == "SITE001" assert row["source"] == SOURCE_HISTORY assert row["source"] == "api_history" assert row["dataset_id"] is None assert row["timestamp"] == datetime.fromisoformat("2024-06-15T12:00:00+00:00") assert row["consumption_kw"] == 87.34 assert row["consumption_kwh"] == 87.34 assert row["data_quality"] == "good" assert row["null_reasons"] == [] assert row["imputed_values"] is None assert row["imputation_method"] is None def test_build_reading_row_keeps_null_values_and_quality() -> None: reading = make_reading() reading["consumption_kw"] = None reading["consumption_kwh"] = None reading["voltage_v"] = None reading["current_a"] = None reading["power_factor"] = None reading["data_quality"] = "degraded" reading["null_reasons"] = [ "consumption_sensor_failure", "electrical_sensor_failure", ] row = build_reading_row(reading) assert row["consumption_kw"] is None assert row["consumption_kwh"] is None assert row["voltage_v"] is None assert row["current_a"] is None assert row["power_factor"] is None assert row["data_quality"] == "degraded" assert row["null_reasons"] == [ "consumption_sensor_failure", "electrical_sensor_failure", ] assert row["imputed_values"] is None assert row["imputation_method"] is None def test_build_reading_row_keeps_raw_source_data() -> None: reading = make_reading() row = build_reading_row(reading) raw_data = json.loads(row["raw_data"]) assert raw_data == reading def test_build_reading_batch_transforms_all_readings() -> None: first = make_reading() second = make_reading() second["timestamp"] = "2024-06-15T12:01:00Z" second["consumption_kw"] = 90.5 rows = build_reading_batch([first, second]) assert len(rows) == 2 assert rows[0]["site_id"] == "SITE001" assert rows[0]["consumption_kw"] == 87.34 assert rows[1]["site_id"] == "SITE001" assert rows[1]["consumption_kw"] == 90.5 def test_create_mock_api_client_requires_credentials( monkeypatch: pytest.MonkeyPatch, ) -> None: settings = SimpleNamespace( mock_api_username=None, mock_api_password=None, ) monkeypatch.setattr( mock_api_import, "get_settings", lambda: settings, ) with pytest.raises( ValueError, match="Les identifiants de l'API Mock ne sont pas configurés", ): mock_api_import.create_mock_api_client() async def test_create_mock_api_client_uses_configuration( monkeypatch: pytest.MonkeyPatch, ) -> None: password = MagicMock() password.get_secret_value.return_value = "test-password" settings = SimpleNamespace( mock_api_base_url="https://mock.test/", mock_api_username="test-user", mock_api_password=password, mock_api_timeout_seconds=10.0, ) monkeypatch.setattr( mock_api_import, "get_settings", lambda: settings, ) client = mock_api_import.create_mock_api_client() try: assert str(client.base_url) == "https://mock.test" assert client.timeout.connect == 10.0 finally: await client.aclose() async def test_upsert_sites_with_empty_list_does_nothing() -> None: connection = AsyncMock() await upsert_sites( connection, [], ) connection.execute.assert_not_awaited() async def test_import_mock_api_history_dry_run_does_not_write( monkeypatch: pytest.MonkeyPatch, ) -> None: def handler(request: Request) -> Response: if request.url.path == "/api/v1/sites": return Response( status_code=200, json=[make_site()], ) if request.url.path == "/api/v1/readings": return Response( status_code=200, json=[make_reading()], ) return Response(status_code=404) transport = MockTransport(handler) client = AsyncClient( transport=transport, base_url="https://mock.test", ) monkeypatch.setattr( mock_api_import, "create_mock_api_client", lambda: client, ) monkeypatch.setattr( mock_api_import, "get_settings", lambda: SimpleNamespace( database_url="postgresql+asyncpg://unused", ), ) create_engine_mock = MagicMock() monkeypatch.setattr( mock_api_import, "create_async_engine", create_engine_mock, ) await mock_api_import.import_mock_api_history( start_time=datetime.fromisoformat("2024-06-15T12:00:00"), end_time=datetime.fromisoformat("2024-06-15T13:00:00"), limit=60, dry_run=True, ) create_engine_mock.assert_not_called() async def test_import_mock_api_history_loads_data( monkeypatch: pytest.MonkeyPatch, ) -> None: def handler(request: Request) -> Response: if request.url.path == "/api/v1/sites": return Response( status_code=200, json=[make_site()], ) if request.url.path == "/api/v1/readings": return Response( status_code=200, json=[make_reading()], ) return Response(status_code=404) transport = MockTransport(handler) client = AsyncClient( transport=transport, base_url="https://mock.test", ) monkeypatch.setattr( mock_api_import, "create_mock_api_client", lambda: client, ) monkeypatch.setattr( mock_api_import, "get_settings", lambda: SimpleNamespace( database_url="postgresql+asyncpg://test:test@localhost/test", ), ) connection = AsyncMock() transaction_context = MagicMock() transaction_context.__aenter__ = AsyncMock( return_value=connection, ) transaction_context.__aexit__ = AsyncMock( return_value=None, ) engine = MagicMock() engine.begin.return_value = transaction_context engine.dispose = AsyncMock() create_engine_mock = MagicMock( return_value=engine, ) upsert_sites_mock = AsyncMock() monkeypatch.setattr( mock_api_import, "create_async_engine", create_engine_mock, ) monkeypatch.setattr( mock_api_import, "upsert_sites", upsert_sites_mock, ) await mock_api_import.import_mock_api_history( start_time=datetime.fromisoformat("2024-06-15T12:00:00"), end_time=datetime.fromisoformat("2024-06-15T13:00:00"), limit=60, dry_run=False, ) create_engine_mock.assert_called_once_with( "postgresql+asyncpg://test:test@localhost/test", pool_pre_ping=True, ) upsert_sites_mock.assert_awaited_once_with( connection, [make_site()], ) connection.execute.assert_awaited_once() engine.dispose.assert_awaited_once() def test_parse_datetime_accepts_z_suffix() -> None: result = mock_api_import.parse_datetime( "2024-06-15T12:00:00Z", ) assert result == datetime.fromisoformat( "2024-06-15T12:00:00+00:00", ) def test_parse_args_reads_cli_parameters( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( sys, "argv", [ "mock_api_import", "--start-time", "2024-06-15T12:00:00Z", "--end-time", "2024-06-15T13:00:00Z", "--limit", "60", "--dry-run", ], ) args = mock_api_import.parse_args() assert args.start_time == datetime.fromisoformat( "2024-06-15T12:00:00+00:00", ) assert args.end_time == datetime.fromisoformat( "2024-06-15T13:00:00+00:00", ) assert args.limit == 60 assert args.dry_run is True def test_main_rejects_limit_out_of_bounds( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( sys, "argv", [ "mock_api_import", "--start-time", "2024-06-15T12:00:00Z", "--end-time", "2024-06-15T13:00:00Z", "--limit", "0", ], ) with pytest.raises( ValueError, match="--limit doit être compris entre 1 et 1000", ): mock_api_import.main() def test_main_rejects_invalid_period( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( sys, "argv", [ "mock_api_import", "--start-time", "2024-06-15T14:00:00Z", "--end-time", "2024-06-15T13:00:00Z", "--limit", "60", ], ) with pytest.raises( ValueError, match="--start-time doit être antérieur à --end-time", ): mock_api_import.main() def test_main_runs_import( monkeypatch: pytest.MonkeyPatch, ) -> None: start_time = datetime.fromisoformat( "2024-06-15T12:00:00+00:00", ) end_time = datetime.fromisoformat( "2024-06-15T13:00:00+00:00", ) import_mock = AsyncMock() monkeypatch.setattr( mock_api_import, "parse_args", lambda: SimpleNamespace( start_time=start_time, end_time=end_time, limit=60, dry_run=True, ), ) monkeypatch.setattr( mock_api_import, "import_mock_api_history", import_mock, ) mock_api_import.main() import_mock.assert_awaited_once_with( start_time=start_time, end_time=end_time, limit=60, dry_run=True, ) @pytest.mark.integration async def test_reading_insert_is_idempotent( session: AsyncSession, ) -> None: reading = make_reading() row = build_reading_row(reading) connection = await session.connection() await upsert_sites( connection, [make_site()], ) await session.execute( READING_INSERT, [row], ) await session.execute( READING_INSERT, [row], ) result = await session.execute( text( """ SELECT COUNT(*) FROM reading WHERE site_id = :site_id AND timestamp = :timestamp AND source = :source """ ), { "site_id": row["site_id"], "timestamp": row["timestamp"], "source": row["source"], }, ) assert result.scalar_one() == 1 await session.rollback()