Compare commits
375
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e06b08546 | ||
|
|
931ec9e527 | ||
|
|
c8eddbc02b | ||
|
|
68fc1052fb | ||
|
|
0a7dd2e692 | ||
|
|
45cc67266d | ||
|
|
182a2f4a6c | ||
|
|
7ecc0b2e64 | ||
|
|
9ee0de9d55 | ||
|
|
9f343e9f42 | ||
|
|
5a7e2a94a7 | ||
|
|
f011fce84e | ||
|
|
7730f184e7 | ||
|
|
fa815f49b6 | ||
|
|
0462dd01ba | ||
|
|
e53c7e441c | ||
|
|
0a7a802b38 | ||
|
|
e0089537d0 | ||
|
|
6b41ec900c | ||
|
|
49175d8ff7 | ||
|
|
da7fc52299 | ||
|
|
5afe2fc88e | ||
|
|
6c09beeb3c | ||
|
|
14eed08ff5 | ||
|
|
1232646f68 | ||
|
|
cbbfaf4910 | ||
|
|
b78322bd61 | ||
|
|
101ebd404f | ||
|
|
32f1bef643 | ||
|
|
b00c39277b | ||
|
|
c2f360c591 | ||
|
|
7f4364df77 | ||
|
|
933f0a3360 | ||
|
|
dbcd5c4240 | ||
|
|
46d10209f1 | ||
|
|
db6ee6e56d | ||
|
|
84969d3375 | ||
|
|
93d5cad5af | ||
|
|
4d88604a07 | ||
|
|
22a88e193c | ||
|
|
d687d7dc58 | ||
|
|
288970df77 | ||
|
|
985c188106 | ||
|
|
b786f27a4d | ||
|
|
3e871a3e8b | ||
|
|
fe0d4222a5 | ||
|
|
30bb3b838c | ||
|
|
c3ec8b79c7 | ||
|
|
7644bf49ad | ||
|
|
3ca4839a02 | ||
|
|
10cc408b03 | ||
|
|
c7744483b4 | ||
|
|
ac05da7001 | ||
|
|
0284cc0cd8 | ||
|
|
dc952d13aa | ||
|
|
cfc194a3fb | ||
|
|
2ed9e1cee4 | ||
|
|
a197af91ff | ||
|
|
a646635b4c | ||
|
|
d54cd963f1 | ||
|
|
18307e9be3 | ||
|
|
59f050ec5e | ||
|
|
6e9c830557 | ||
|
|
5a3c526856 | ||
|
|
314e3b72c0 | ||
|
|
e118c008bf | ||
|
|
d86224a0f7 | ||
|
|
f21a843fc2 | ||
|
|
6b3908d321 | ||
|
|
b16861e211 | ||
|
|
57b9735804 | ||
|
|
c007ea01bd | ||
|
|
5472b19504 | ||
|
|
44163bfb98 | ||
|
|
ea8f9d0a3a | ||
|
|
f58fc4ba81 | ||
|
|
cb961ec2c5 | ||
|
|
68239371f6 | ||
|
|
9bf2f27127 | ||
|
|
568060a903 | ||
|
|
2c7ee2c064 | ||
|
|
aa4af62290 | ||
|
|
5b5c97532d | ||
|
|
e220f8f0c6 | ||
|
|
fb4235df3a | ||
|
|
357cf996ed | ||
|
|
4fb8b5a784 | ||
|
|
c1f9dec5fe | ||
|
|
d9103ee4ed | ||
|
|
fca36649f9 | ||
|
|
0db69fd023 | ||
|
|
be6be42947 | ||
|
|
3ddeb24207 | ||
|
|
8760ebc701 | ||
|
|
a013dfa87f | ||
|
|
b5993f994e | ||
|
|
091ee344d1 | ||
|
|
f81959c1a9 | ||
|
|
5c361c6952 | ||
|
|
a27e0c6535 | ||
|
|
7a97a4d1ac | ||
|
|
20abcfee33 | ||
|
|
3d6d2eb760 | ||
|
|
154deca511 | ||
|
|
a577356198 | ||
|
|
19ff152445 | ||
|
|
bf3584182f | ||
|
|
28e8260216 | ||
|
|
e90e563a1c | ||
|
|
d7ca9fd8b0 | ||
|
|
67204b5eb6 | ||
|
|
985466e9cc | ||
|
|
c347212c93 | ||
|
|
c67c7802c1 | ||
|
|
e824423889 | ||
|
|
436147c8cb | ||
|
|
8fb7ab8680 | ||
|
|
65baf3fddb | ||
|
|
19f4ddc62f | ||
|
|
282472bae9 | ||
|
|
8d80cdc79d | ||
|
|
b61287c8d2 | ||
|
|
a4be46c6e6 | ||
|
|
66a65991b5 | ||
|
|
095cb4559b | ||
|
|
d48b0eef4d | ||
|
|
c8e8a44390 | ||
|
|
401d5a3174 | ||
|
|
6a70443d52 | ||
|
|
ebf6476fb6 | ||
|
|
d2ca2644d2 | ||
|
|
6d741e45fb | ||
|
|
1bec2c1376 | ||
|
|
00fab49d80 | ||
|
|
91a1bfda33 | ||
|
|
de88c4f156 | ||
|
|
67dcf506e9 | ||
|
|
fe100653d0 | ||
|
|
bccf7ed774 | ||
|
|
1502eea036 | ||
|
|
bf2b66e4ad | ||
|
|
8ea388ce9a | ||
|
|
9cd4f0de1c | ||
|
|
5cc99178c2 | ||
|
|
b4da0bbefc | ||
|
|
25e0bb4ce1 | ||
|
|
b41a16364b | ||
|
|
ce3d1d1992 | ||
|
|
33aeea835b | ||
|
|
cb23895026 | ||
|
|
9312d3b60f | ||
|
|
2c86e00629 | ||
|
|
cf50d3d36c | ||
|
|
7e504bee72 | ||
|
|
32155cbfc5 | ||
|
|
cb4df846cb | ||
|
|
d16de84559 | ||
|
|
ae4082c584 | ||
|
|
f18d4f9ef9 | ||
|
|
308769b325 | ||
|
|
90639b5618 | ||
|
|
e147ea69bc | ||
|
|
f0ff953a5e | ||
|
|
7fd8d1ce30 | ||
|
|
6785d06a9a | ||
|
|
c2bd1317ed | ||
|
|
d276070ee2 | ||
|
|
e10ab44dc7 | ||
|
|
32dd0587e9 | ||
|
|
4afcfb3ced | ||
|
|
83caa9006e | ||
|
|
91752a2b0b | ||
|
|
79b157d37e | ||
|
|
851e0279f6 | ||
|
|
7f025f72ac | ||
|
|
1cd206adc8 | ||
|
|
3c378c177f | ||
|
|
2adfdf0eb0 | ||
|
|
44f3416ffe | ||
|
|
342128ccff | ||
|
|
62d81e901d | ||
|
|
31886944ab | ||
|
|
63cbeafe3b | ||
|
|
f8d08c8686 | ||
|
|
be44b97d7a | ||
|
|
306c5a52e5 | ||
|
|
801379f956 | ||
|
|
2686880185 | ||
|
|
ae58a896d9 | ||
|
|
5a29faaa16 | ||
|
|
bc75528616 | ||
|
|
268496a8c4 | ||
|
|
d7457e9fd9 | ||
|
|
3cd9a6b272 | ||
|
|
26f834485c | ||
|
|
777cd0ac64 | ||
|
|
c528ed239b | ||
|
|
a88e51c92a | ||
|
|
f3ea2785b3 | ||
|
|
901ceffd72 | ||
|
|
5545c166fd | ||
|
|
f0ad8e9990 | ||
|
|
3c01ab3ecc | ||
|
|
6f6f451eb4 | ||
|
|
cc3e38efa3 | ||
|
|
0a2ed5ad8f | ||
|
|
459ddf1792 | ||
|
|
de697b080d | ||
|
|
f238940867 | ||
|
|
b941880c22 | ||
|
|
9d2384a639 | ||
|
|
0c487fa7be | ||
|
|
b3efb98208 | ||
|
|
2d7b4bd74d | ||
|
|
56c6b79a5a | ||
|
|
19c38fe571 | ||
|
|
9a1af94d88 | ||
|
|
452cfdef85 | ||
|
|
96dd1f834c | ||
|
|
e66ef86729 | ||
|
|
0318ee6cc5 | ||
|
|
0ddfb1997d | ||
|
|
b96546cea3 | ||
|
|
c059f838bb | ||
|
|
f9c2a4610c | ||
|
|
b5fa7b0010 | ||
|
|
a9e124a97d | ||
|
|
edd5e82d29 | ||
|
|
aeb07e14db | ||
|
|
619024f547 | ||
|
|
460d6c1b3e | ||
|
|
eb4291b10a | ||
|
|
9a72bb3a8f | ||
|
|
bd1da05587 | ||
|
|
9376420868 | ||
|
|
d535959fbe | ||
|
|
33807e3038 | ||
|
|
c914d41f33 | ||
|
|
d3047f53e8 | ||
|
|
ec63798807 | ||
|
|
1e168ef03e | ||
|
|
ddf7e17788 | ||
|
|
56e8f95729 | ||
|
|
91d435748c | ||
|
|
0b41580310 | ||
|
|
e3d436ea53 | ||
|
|
c453700b13 | ||
|
|
f427f8a8f3 | ||
|
|
7be8a44e89 | ||
|
|
2095ad6bd3 | ||
|
|
fb06bf0062 | ||
|
|
e27142c7db | ||
|
|
77feabcbad | ||
|
|
18a4be6e38 | ||
|
|
feee6c3ffc | ||
|
|
173f91f26f | ||
|
|
1cd3688256 | ||
|
|
7eef960a30 | ||
|
|
db81290026 | ||
|
|
7f710c9084 | ||
|
|
12fb8860d1 | ||
|
|
d9229e5a93 | ||
|
|
7c2936f2ef | ||
|
|
9d3e402ca4 | ||
|
|
e9376a98bf | ||
|
|
7db57f162c | ||
|
|
9ccc603d88 | ||
|
|
297d85a0ca | ||
|
|
af58172742 | ||
|
|
b433e01fa8 | ||
|
|
5eb74aa64a | ||
|
|
cc0a58ac4c | ||
|
|
2400b6f05e | ||
|
|
9e33c276d6 | ||
|
|
6cb9ac00cb | ||
|
|
c1f63889c1 | ||
|
|
5875e8c239 | ||
|
|
c8383014a8 | ||
|
|
3cf9194d4c | ||
|
|
8def1e23af | ||
|
|
e26948932e | ||
|
|
41acdb54b6 | ||
|
|
5d921a9b1e | ||
|
|
56c134beb0 | ||
|
|
8fb5ab9f65 | ||
|
|
e22feac2c4 | ||
|
|
5581cb1ef3 | ||
|
|
1d8c986386 | ||
|
|
85cb7c9eeb | ||
|
|
39b1d28ead | ||
|
|
5394257855 | ||
|
|
c741ffc827 | ||
|
|
d7f775f9f7 | ||
|
|
e381e0de09 | ||
|
|
7674955637 | ||
|
|
19cfac1cff | ||
|
|
1fce577a78 | ||
|
|
063092f2c7 | ||
|
|
1afaee069f | ||
|
|
7bc9a09489 | ||
|
|
921da48eb1 | ||
|
|
4ee2109628 | ||
|
|
ec1c05ad54 | ||
|
|
334ca5982b | ||
|
|
2465021d61 | ||
|
|
278299c2b1 | ||
|
|
4f69199734 | ||
|
|
1cca4f130c | ||
|
|
016f226fdb | ||
|
|
88f4f9a601 | ||
|
|
ed7311d4ef | ||
|
|
ff68a51424 | ||
|
|
11f9b1bcd5 | ||
|
|
41c18a3bb1 | ||
|
|
00ef725249 | ||
|
|
34f35f3ca0 | ||
|
|
f5cac1c2a8 | ||
|
|
8e07168a5e | ||
|
|
916b5d246a | ||
|
|
d167b64188 | ||
|
|
7913518c4b | ||
|
|
2ad7692f1c | ||
|
|
a158d6f84c | ||
|
|
7dfd7a7e74 | ||
|
|
8d28113f03 | ||
|
|
62932e57c3 | ||
|
|
cd4fd962be | ||
|
|
517144f7e5 | ||
|
|
cc7ca2d359 | ||
|
|
bbafe7d119 | ||
|
|
9c78c6dc38 | ||
|
|
9161b74874 | ||
|
|
6798d35572 | ||
|
|
f03dce5fe3 | ||
|
|
74ac1b4577 | ||
|
|
ebb72fb399 | ||
|
|
b2d52823ba | ||
|
|
fcbfcc8eb2 | ||
|
|
3692d486c6 | ||
|
|
24bf8bf4b9 | ||
|
|
9f465538bf | ||
|
|
515a92b395 | ||
|
|
0174272bdd | ||
|
|
c740b61b24 | ||
|
|
5669cd63ec | ||
|
|
2f97e4d434 | ||
|
|
07ea8d21dc | ||
|
|
06cb60463c | ||
|
|
1c6b6105bd | ||
|
|
77440281f8 | ||
|
|
63ee79cf32 | ||
|
|
76fa90dfcb | ||
|
|
6ecec1afef | ||
|
|
e13096c62a | ||
|
|
970a4a50b8 | ||
|
|
3fb907d6f6 | ||
|
|
1654e4dd81 | ||
|
|
e50921c907 | ||
|
|
730adb69b1 | ||
|
|
f43c9f76a0 | ||
|
|
04e4913952 | ||
|
|
d1e4d8cfa0 | ||
|
|
1f0eb410eb | ||
|
|
078983a41d | ||
|
|
596cf43eda | ||
|
|
11baea7117 | ||
|
|
61b3494d12 | ||
|
|
b5cffbf56f | ||
|
|
3ef7de5baa | ||
|
|
ff6e3c288c | ||
|
|
2390e58f78 | ||
|
|
b300be5186 | ||
|
|
b8f806518f | ||
|
|
83392c7ff4 | ||
|
|
c3fd9327ea |
@@ -0,0 +1,27 @@
|
|||||||
|
# Dépendances (réinstallées dans l'image)
|
||||||
|
node_modules/
|
||||||
|
vendor/
|
||||||
|
__pycache__/
|
||||||
|
*.pyc
|
||||||
|
|
||||||
|
# Git et IDE
|
||||||
|
.git/
|
||||||
|
.gitignore
|
||||||
|
.vscode/
|
||||||
|
.idea/
|
||||||
|
*.swp
|
||||||
|
|
||||||
|
# Fichiers de build locaux
|
||||||
|
dist/
|
||||||
|
build/
|
||||||
|
*.log
|
||||||
|
|
||||||
|
# Secrets et config locale (CRITIQUE : risque d'exfiltration)
|
||||||
|
.env
|
||||||
|
.env.local
|
||||||
|
*.pem
|
||||||
|
*.key
|
||||||
|
secrets/
|
||||||
|
.npmrc
|
||||||
|
.pypirc
|
||||||
|
kubeconfig
|
||||||
+100
@@ -17,3 +17,103 @@ APP_LOG_LEVEL=INFO
|
|||||||
APP_SECRET_KEY=change_me
|
APP_SECRET_KEY=change_me
|
||||||
APP_CORS_ORIGINS=http://localhost:4200
|
APP_CORS_ORIGINS=http://localhost:4200
|
||||||
BACKEND_PORT=8000
|
BACKEND_PORT=8000
|
||||||
|
FRONTEND_PORT=3000
|
||||||
|
|
||||||
|
# Mailpit capture les courriels du backend, rien ne sort vers l'extérieur.
|
||||||
|
MAILPIT_SMTP_PORT=1025
|
||||||
|
MAILPIT_UI_PORT=8025
|
||||||
|
|
||||||
|
# API Mock EnerVision
|
||||||
|
APP_MOCK_API_BASE_URL=https://api-mock.charlieandre.fr
|
||||||
|
APP_MOCK_API_USERNAME=change_me
|
||||||
|
APP_MOCK_API_PASSWORD=change_me
|
||||||
|
APP_MOCK_API_TIMEOUT_SECONDS=10
|
||||||
|
|
||||||
|
# Airflow (api-server + scheduler + dag-processor, LocalExecutor). Base de métadonnées dédiée `airflow` dans le
|
||||||
|
# même conteneur `db` (cf. db/init/120-airflow-database.sql), pas un conteneur de plus.
|
||||||
|
AIRFLOW_PORT=8080
|
||||||
|
# Chiffre les connexions/variables stockées par Airflow. Générer la vôtre :
|
||||||
|
# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
|
||||||
|
AIRFLOW_FERNET_KEY=change_me
|
||||||
|
# Clé de session de l'api-server Airflow, distincte de la précédente. Générer la
|
||||||
|
# vôtre : python -c "import secrets; print(secrets.token_urlsafe(48))"
|
||||||
|
AIRFLOW_API_SECRET_KEY=change_me
|
||||||
|
# Secret des jetons JWT entre scheduler, tâches et api-server (conteneurs distincts, le secret
|
||||||
|
# doit être partagé). Même générateur que ci-dessus.
|
||||||
|
AIRFLOW_JWT_SECRET=change_me
|
||||||
|
AIRFLOW_ADMIN_USERNAME=admin
|
||||||
|
# Compte Airflow créé au premier démarrage (service `airflow-init`), sans rapport avec les
|
||||||
|
# comptes `app_user` d'EnerVision.
|
||||||
|
AIRFLOW_ADMIN_PASSWORD=change_me
|
||||||
|
AIRFLOW_ADMIN_EMAIL=admin@enervision.fr
|
||||||
|
# `APP_SECRET_KEY` du backend, que le DAG `alertes` lance en sous-processus. Distincte de
|
||||||
|
# celle de l'API : la détection ne signe aucun jeton, et Airflow exécute du code depuis son
|
||||||
|
# interface (cf. ADR 0008). Générer la vôtre :
|
||||||
|
# python -c "import secrets; print(secrets.token_urlsafe(48))"
|
||||||
|
AIRFLOW_APP_SECRET_KEY=change_me
|
||||||
|
|
||||||
|
# Garage, stockage objet S3 par environnement (ADR 0019) : un conteneur par projet Compose, publié
|
||||||
|
# sur 127.0.0.1 seulement. Les six secrets ci-dessous sont exigés par `make services-up` et
|
||||||
|
# `make stack-up` ; scripts/provision-host.sh les génère sur la VM.
|
||||||
|
# 32 octets en hexadécimal, rien d'autre n'est accepté : openssl rand -hex 32
|
||||||
|
GARAGE_RPC_SECRET=change_me
|
||||||
|
# Jetons de l'API d'administration et de /metrics (port 3903). Même générateur qu'APP_SECRET_KEY.
|
||||||
|
GARAGE_ADMIN_TOKEN=change_me
|
||||||
|
GARAGE_METRICS_TOKEN=change_me
|
||||||
|
# Clé S3 créée au premier démarrage (`--default-bucket`). Identifiant : echo "GK$(openssl rand -hex 12)"
|
||||||
|
# Secret : openssl rand -hex 32. Ne plus le changer ensuite, Garage refuserait de démarrer.
|
||||||
|
GARAGE_ACCESS_KEY=change_me
|
||||||
|
GARAGE_SECRET_KEY=change_me
|
||||||
|
GARAGE_BUCKET=enervision-archives
|
||||||
|
# Ports S3 et admin sur 127.0.0.1. Recette : 3910 et 3913, dev : 3920 et 3923.
|
||||||
|
GARAGE_S3_PORT=3900
|
||||||
|
GARAGE_ADMIN_PORT=3903
|
||||||
|
|
||||||
|
# Rétention des mesures (ADR 0019, 0020) : le DAG `retention` exporte chaque nuit vers Garage les
|
||||||
|
# chunks de `reading` plus vieux que cette borne, puis les supprime. L'historique de démonstration
|
||||||
|
# s'arrête fin 2024 : sous 21 mois, la démo disparaîtrait.
|
||||||
|
READING_RETENTION_DAYS=1095
|
||||||
|
# Clé SSE-C des archives, 32 octets en base64 : openssl rand -base64 32. La perdre rend les
|
||||||
|
# archives illisibles ; la sauvegarder hors de la VM.
|
||||||
|
GARAGE_SSE_KEY=change_me
|
||||||
|
|
||||||
|
# Stack complète derrière le reverse proxy (docker-compose.prod.yml).
|
||||||
|
# PUBLIC_HOST alimente l'origine CORS, le lien de réinitialisation et le certificat.
|
||||||
|
PUBLIC_HOST=enervision.local
|
||||||
|
ACME_EMAIL=
|
||||||
|
|
||||||
|
# Trois environnements sur la même machine (ADR 0009, 0017) : un dossier, un `.env` et un projet
|
||||||
|
# Compose chacun. Le nom de projet préfixe volumes, réseau et conteneurs et l'emporte sur `name:`.
|
||||||
|
# Vide sur un poste de développement : le projet reste `enervision`.
|
||||||
|
COMPOSE_PROJECT_NAME=
|
||||||
|
# Origine publique, avec le port si le proxy HTTPS n'écoute pas 443. Vide : https://PUBLIC_HOST.
|
||||||
|
# Sur la VM, provision-host.sh pose https://<nom de l'environnement>, sans port (frontal SNI).
|
||||||
|
PUBLIC_ORIGIN=
|
||||||
|
# Ports publiés par le proxy. Vides : 80 et 443. Sur la VM, provision-host.sh les pose sur 127.0.0.1,
|
||||||
|
# derrière le frontal SNI, et décale aussi base, Mailpit et Airflow par environnement.
|
||||||
|
PROXY_HTTP_PORT=
|
||||||
|
PROXY_HTTPS_PORT=
|
||||||
|
# Écouteur PROXY protocol du proxy, que seul le frontal de la VM joint (infra/front, ADR 0018).
|
||||||
|
# Vide : port aléatoire sur 127.0.0.1. VM : 127.0.0.1:10444 en prod, 8444 en recette, 9444 en dev.
|
||||||
|
PROXY_FRONT_PORT=
|
||||||
|
# Réglages mémoire de la stack déployée. Sans eux, timescaledb-tune réserve 25 % de la RAM de la
|
||||||
|
# machine à chaque base au premier démarrage. L'api-server Airflow 3 n'a rien à régler ici : son
|
||||||
|
# nombre de workers vaut 1 par défaut, contre 4 pour le webserver d'Airflow 2.
|
||||||
|
TS_TUNE_MEMORY=2GB
|
||||||
|
TS_TUNE_NUM_CPUS=2
|
||||||
|
|
||||||
|
# Supervision (ADR 0016) : `monitoring` la démarre avec `make stack-up`, réglage de la prod.
|
||||||
|
# Vide ailleurs, où `make monitoring-up` la lance à la demande.
|
||||||
|
COMPOSE_PROFILES=
|
||||||
|
# Jeton présenté par Prometheus sur `/metrics`, exigé par l'API dès qu'il est posé. Requis dès
|
||||||
|
# que la supervision tourne ; même générateur que APP_SECRET_KEY.
|
||||||
|
APP_METRICS_TOKEN=change_me
|
||||||
|
# Compte `admin` de Grafana. Sans lui, le conteneur refuse de démarrer.
|
||||||
|
GRAFANA_ADMIN_PASSWORD=change_me
|
||||||
|
# Rôle PostgreSQL `supervision`, en lecture seule, de Grafana et de postgres-exporter
|
||||||
|
# (db/roles/supervision.sql, posé par `make db-ensure-supervision`).
|
||||||
|
SUPERVISION_DB_PASSWORD=change_me
|
||||||
|
# Interfaces publiées sur 127.0.0.1 seulement, par tunnel SSH. 3000 est pris par le frontend.
|
||||||
|
GRAFANA_PORT=3001
|
||||||
|
PROMETHEUS_PORT=9090
|
||||||
|
ALERTMANAGER_PORT=9093
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
self-hosted-runner:
|
||||||
|
labels:
|
||||||
|
- eni-g3
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
# Frontend — npm
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/apps/frontend"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
groups:
|
||||||
|
frontend-dependencies:
|
||||||
|
patterns:
|
||||||
|
- "*"
|
||||||
|
ignore:
|
||||||
|
# Pourquoi : @angular/build épingle typescript (>=6.0 <6.1) et vitest (^4). Une majeure
|
||||||
|
# de l'un ou l'autre casse `npm ci` tant qu'Angular ne suit pas.
|
||||||
|
- dependency-name: "typescript"
|
||||||
|
update-types: ["version-update:semver-major"]
|
||||||
|
- dependency-name: "vitest"
|
||||||
|
update-types: ["version-update:semver-major"]
|
||||||
|
- dependency-name: "@vitest/coverage-v8"
|
||||||
|
update-types: ["version-update:semver-major"]
|
||||||
|
|
||||||
|
# Tests de bout en bout, paquet npm distinct du frontend
|
||||||
|
- package-ecosystem: "npm"
|
||||||
|
directory: "/tests/e2e"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
open-pull-requests-limit: 2
|
||||||
|
groups:
|
||||||
|
e2e-dependencies:
|
||||||
|
patterns:
|
||||||
|
- "*"
|
||||||
|
|
||||||
|
# Backend — uv (lit pyproject.toml / uv.lock)
|
||||||
|
- package-ecosystem: "uv"
|
||||||
|
directory: "/apps/backend"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
groups:
|
||||||
|
backend-dependencies:
|
||||||
|
patterns:
|
||||||
|
- "*"
|
||||||
|
|
||||||
|
# Les workflows GitHub Actions eux-mêmes ont aussi des dépendances à jour
|
||||||
|
- package-ecosystem: "github-actions"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
|
||||||
|
# Si un Dockerfile existe pour le backend
|
||||||
|
- package-ecosystem: "docker"
|
||||||
|
directory: "/apps/backend"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
|
||||||
|
- package-ecosystem: "docker"
|
||||||
|
directory: "/apps/frontend"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
|
|
||||||
|
# Images du reverse proxy et du compagnon ACME, épinglées dans les fichiers Compose
|
||||||
|
- package-ecosystem: "docker-compose"
|
||||||
|
directory: "/"
|
||||||
|
schedule:
|
||||||
|
interval: "weekly"
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
name: Airflow
|
||||||
|
|
||||||
|
# Piège : la version de Python vient de etl/airflow/.python-version. C'est 3.12 et non 3.14
|
||||||
|
# (contrairement à backend.yml et ml.yml) : celui de l'image apache/airflow retenue, et les tests
|
||||||
|
# doivent tourner sur le même interpréteur qu'elle. Le 3.14 de ml/ ne vit que dans l'image
|
||||||
|
# Docker, dans son propre environnement (cf. etl/airflow/Dockerfile).
|
||||||
|
#
|
||||||
|
# Piège : l'image COPY les fichiers de dépendances et le code de ml/ et de apps/backend/. Une
|
||||||
|
# modification de l'un ou de l'autre peut donc casser sa construction : le filtre `airflow` de
|
||||||
|
# ci.yml, qui appelle ce workflow, inclut ces chemins alors qu'il ne teste ni le modèle ni l'API.
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
verification:
|
||||||
|
name: Lint et intégrité des DAGs
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: etl/airflow
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Action tierce, épinglée sur le commit du tag (règle Sonar githubactions:S7637).
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: true
|
||||||
|
cache-dependency-glob: etl/airflow/uv.lock
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
|
- name: Installe l'interpréteur déclaré par .python-version
|
||||||
|
run: uv python install
|
||||||
|
|
||||||
|
- name: Synchronise les dépendances sur le verrou
|
||||||
|
run: uv sync --all-groups --locked
|
||||||
|
|
||||||
|
- name: Vérifie le formatage
|
||||||
|
run: uv run ruff format --check .
|
||||||
|
|
||||||
|
- name: Analyse statique
|
||||||
|
run: uv run ruff check --output-format=github .
|
||||||
|
|
||||||
|
# Aucun test ne lance de tâche ni de scheduler : DagBag charge les fichiers de dags/ et
|
||||||
|
# vérifie import, planification, plafonds d'exécution et commande de chaque tâche.
|
||||||
|
- name: Tests d'intégrité des DAGs
|
||||||
|
run: uv run pytest
|
||||||
|
|
||||||
|
image:
|
||||||
|
name: Construction de l'image
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 25
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Construit l'image (contexte à la racine, elle COPY ml/ et apps/backend/)
|
||||||
|
run: docker build -f etl/airflow/Dockerfile -t enervision-airflow:ci .
|
||||||
|
|
||||||
|
# Vérifie ce qui ne casse qu'à l'exécution, pas à la construction : libgomp1 absent
|
||||||
|
# (`OSError: libgomp.so.1` au premier import) ou environnement ml/ non figé.
|
||||||
|
- name: Vérifie que le pipeline ML s'importe sans réseau
|
||||||
|
run: >
|
||||||
|
docker run --rm --network none enervision-airflow:ci
|
||||||
|
bash -c "cd /opt/ml && env -u VIRTUAL_ENV uv run --no-sync python -m enervision_ml.train --help"
|
||||||
|
|
||||||
|
# `--help` sort par argparse avant `get_settings()` : ni base ni secret requis, et
|
||||||
|
# l'import des modules prouve que l'environnement /opt/backend est complet.
|
||||||
|
- name: Vérifie que les cinq commandes backend s'importent sans réseau
|
||||||
|
run: >
|
||||||
|
docker run --rm --network none enervision-airflow:ci
|
||||||
|
bash -c "cd /opt/backend
|
||||||
|
&& env -u VIRTUAL_ENV uv run --no-sync python -m app.detection.internal_alerts --help
|
||||||
|
&& env -u VIRTUAL_ENV uv run --no-sync python -m app.cli generate-recommendations --help
|
||||||
|
&& env -u VIRTUAL_ENV uv run --no-sync python -m app.etl.historical_import --help
|
||||||
|
&& env -u VIRTUAL_ENV uv run --no-sync python -m app.etl.mock_api_import --help
|
||||||
|
&& env -u VIRTUAL_ENV uv run --no-sync python -m app.etl.reading_retention --help"
|
||||||
+135
-17
@@ -2,47 +2,42 @@ name: Backend
|
|||||||
|
|
||||||
# Piège : la version de Python vient de apps/backend/.python-version, et elle doit rester
|
# Piège : la version de Python vient de apps/backend/.python-version, et elle doit rester
|
||||||
# en 3.14. Le code utilise le PEP 758, qu'un interpréteur 3.13 refuse de compiler.
|
# en 3.14. Le code utilise le PEP 758, qu'un interpréteur 3.13 refuse de compiler.
|
||||||
|
# Pourquoi : aucun déclencheur propre. ci.yml appelle ce workflow quand le backend change, et
|
||||||
|
# Sonar y reprend la couverture versée par le job `verification` (ADR 0014).
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
workflow_call:
|
||||||
paths:
|
|
||||||
- "apps/backend/**"
|
|
||||||
- ".github/workflows/backend.yml"
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- "apps/backend/**"
|
|
||||||
- ".github/workflows/backend.yml"
|
|
||||||
|
|
||||||
permissions:
|
permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: backend-${{ github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
verification:
|
verification:
|
||||||
name: Lint, typage et tests
|
name: Lint, typage et tests
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
defaults:
|
defaults:
|
||||||
run:
|
run:
|
||||||
working-directory: apps/backend
|
working-directory: apps/backend
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Récupère le dépôt
|
- name: Récupère le dépôt
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Action tierce, épinglée sur le commit du tag (règle Sonar githubactions:S7637).
|
||||||
- name: Installe uv
|
- name: Installe uv
|
||||||
uses: astral-sh/setup-uv@v5
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
with:
|
with:
|
||||||
enable-cache: true
|
enable-cache: true
|
||||||
cache-dependency-glob: apps/backend/uv.lock
|
cache-dependency-glob: apps/backend/uv.lock
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
- name: Installe l'interpréteur déclaré par .python-version
|
- name: Installe l'interpréteur déclaré par .python-version
|
||||||
run: uv python install
|
run: uv python install
|
||||||
|
|
||||||
- name: Synchronise les dépendances sans dévier du verrou
|
# `--locked` et non `--frozen` : un verrou qui ne suit plus pyproject.toml doit casser ici.
|
||||||
run: uv sync --all-groups --frozen
|
- name: Synchronise les dépendances sur le verrou
|
||||||
|
run: uv sync --all-groups --locked
|
||||||
|
|
||||||
- name: Vérifie le formatage
|
- name: Vérifie le formatage
|
||||||
run: uv run ruff format --check .
|
run: uv run ruff format --check .
|
||||||
@@ -55,4 +50,127 @@ jobs:
|
|||||||
|
|
||||||
# Le marqueur `integration` est exclu par défaut, donc aucune base n'est nécessaire ici.
|
# Le marqueur `integration` est exclu par défaut, donc aucune base n'est nécessaire ici.
|
||||||
- name: Tests et couverture
|
- name: Tests et couverture
|
||||||
run: uv run pytest --cov-fail-under=85
|
run: uv run pytest --cov-fail-under=85 --cov-report=xml
|
||||||
|
|
||||||
|
- name: Verse la couverture pour Sonar
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: backend-coverage
|
||||||
|
path: apps/backend/coverage.xml
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
# Piège : même image que docker-compose.yml, pas un `postgres` nu. La première migration refuse
|
||||||
|
# de s'appliquer sans TimescaleDB, et une autre image testerait une base qui n'est pas la nôtre.
|
||||||
|
integration:
|
||||||
|
name: Tests exigeant une base
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: timescale/timescaledb-ha:pg17
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: enervision
|
||||||
|
POSTGRES_PASSWORD: change_me
|
||||||
|
POSTGRES_DB: enervision_test
|
||||||
|
ports:
|
||||||
|
- "5433:5432"
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U enervision -d enervision_test"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 12
|
||||||
|
--health-start-period 40s
|
||||||
|
|
||||||
|
env:
|
||||||
|
DATABASE_URL: postgresql+asyncpg://enervision:change_me@localhost:5433/enervision_test
|
||||||
|
APP_SECRET_KEY: secret-de-test-assez-long-pour-le-validateur
|
||||||
|
PGPASSWORD: change_me
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: true
|
||||||
|
cache-dependency-glob: apps/backend/uv.lock
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
|
- name: Installe l'interpréteur déclaré par .python-version
|
||||||
|
run: uv python install
|
||||||
|
|
||||||
|
- name: Synchronise les dépendances sur le verrou
|
||||||
|
run: uv sync --all-groups --locked
|
||||||
|
|
||||||
|
# Sur le poste, c'est db/init/110-test-database.sql qui pose l'extension. Ce fichier n'est
|
||||||
|
# pas monté ici, et sans lui `alembic upgrade head` s'arrête sur la garde de la révision 1.
|
||||||
|
- name: Active TimescaleDB sur la base de test
|
||||||
|
run: psql -h localhost -p 5433 -U enervision -d enervision_test -c "CREATE EXTENSION IF NOT EXISTS timescaledb"
|
||||||
|
|
||||||
|
- name: Applique les migrations
|
||||||
|
run: uv run alembic upgrade head
|
||||||
|
|
||||||
|
# Couverture désactivée : ce job ne joue qu'une partie de la suite, son taux n'aurait
|
||||||
|
# aucun sens face au seuil de 85 %.
|
||||||
|
- name: Tests d'intégration
|
||||||
|
run: uv run pytest -m integration --no-cov
|
||||||
|
|
||||||
|
security-audit:
|
||||||
|
name: Audit des dépendances
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: true
|
||||||
|
cache-dependency-glob: apps/backend/uv.lock
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
|
# L'audit porte sur le verrou, pas sur l'environnement : sinon pip-audit auditerait
|
||||||
|
# aussi les paquets que son propre `--with` injecte, hors dépendances du projet.
|
||||||
|
- name: Audite les dépendances livrées
|
||||||
|
# Piège : sans `shell: bash`, un échec de `uv export` serait masqué par le pipe.
|
||||||
|
shell: bash
|
||||||
|
run: uv export --locked --no-dev --no-emit-project --no-hashes | uvx pip-audit --requirement /dev/stdin --no-deps
|
||||||
|
|
||||||
|
sast:
|
||||||
|
name: Analyse statique de sécurité
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Pourquoi : pas de cache ici. uvx n'installe pas le projet, le verrou n'alimente donc
|
||||||
|
# aucune clé de cache ; la seule roue téléchargée est celle de Bandit.
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: false
|
||||||
|
|
||||||
|
# Pourquoi : le périmètre est `app`, le code livré. Les tests emploient légitimement des
|
||||||
|
# secrets factices et des `assert` que Bandit signalerait sans qu'aucun n'atteigne la prod.
|
||||||
|
- name: Analyse le code livré (bloquant à partir de MEDIUM)
|
||||||
|
run: uvx bandit==1.9.4 --recursive app --severity-level medium --confidence-level medium
|
||||||
|
|
||||||
|
# Piège : sans cette seconde passe, un constat LOW disparaîtrait du journal sans trace.
|
||||||
|
- name: Rapport complet, tous niveaux
|
||||||
|
continue-on-error: true
|
||||||
|
run: uvx bandit==1.9.4 --recursive app
|
||||||
|
|||||||
@@ -0,0 +1,227 @@
|
|||||||
|
# Pourquoi : un seul point d'entrée pour toute la CI (ADR 0014) - workflow CI. Chaque composant
|
||||||
|
# ne tourne que si ses fichiers changent, Sonar reprend les couvertures déjà produites au lieu de
|
||||||
|
# tout rejouer, et le déploiement ne part que d'un commit dont la CI est verte.
|
||||||
|
# Piège : le seul check à exiger dans les règles de branche est « CI ok ». Un job sauté par son
|
||||||
|
# filtre ne publie pas les checks de son workflow, qui resteraient en attente s'ils étaient exigés.
|
||||||
|
# Piège : sur un push vers dev ou main, tous les filtres valent vrai. paths-filter comparerait
|
||||||
|
# sinon à la base de fusion avec main, et Sonar n'analyserait qu'une partie de la branche.
|
||||||
|
# Piège : pas d'annulation des runs de push. Un run coupé en plein `make stack-up` laisserait la
|
||||||
|
# stack à moitié redémarrée ; le groupe par SHA évite aussi de mettre `dev` en file derrière lui.
|
||||||
|
|
||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches: [dev, main]
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: ci-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
|
||||||
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
changes:
|
||||||
|
name: Périmètre modifié
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 5
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
pull-requests: read
|
||||||
|
outputs:
|
||||||
|
backend: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.backend == 'true' }}
|
||||||
|
frontend: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.frontend == 'true' }}
|
||||||
|
ml: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.ml == 'true' }}
|
||||||
|
airflow: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.airflow == 'true' }}
|
||||||
|
terraform: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.terraform == 'true' }}
|
||||||
|
compose: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.compose == 'true' }}
|
||||||
|
workflows: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.workflows == 'true' }}
|
||||||
|
e2e: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.e2e == 'true' }}
|
||||||
|
sonar: ${{ github.event_name != 'pull_request' || steps.filtre.outputs.ci == 'true' || steps.filtre.outputs.sonar == 'true' }}
|
||||||
|
|
||||||
|
steps:
|
||||||
|
# Sur une PR, la liste des fichiers vient de l'API : ni checkout ni historique requis.
|
||||||
|
- name: Calcule le périmètre de la PR
|
||||||
|
id: filtre
|
||||||
|
if: github.event_name == 'pull_request'
|
||||||
|
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
|
||||||
|
with:
|
||||||
|
filters: |
|
||||||
|
ci:
|
||||||
|
- ".github/workflows/ci.yml"
|
||||||
|
backend:
|
||||||
|
- "apps/backend/**"
|
||||||
|
- ".github/workflows/backend.yml"
|
||||||
|
frontend:
|
||||||
|
- "apps/frontend/**"
|
||||||
|
- ".github/workflows/frontend.yml"
|
||||||
|
ml:
|
||||||
|
- "ml/**"
|
||||||
|
- "apps/backend/alembic/**"
|
||||||
|
- "apps/backend/app/models/**"
|
||||||
|
- "apps/backend/tests/test_chaine_ml_api.py"
|
||||||
|
- "apps/backend/pyproject.toml"
|
||||||
|
- "apps/backend/uv.lock"
|
||||||
|
- ".github/workflows/ml.yml"
|
||||||
|
airflow:
|
||||||
|
- "etl/airflow/**"
|
||||||
|
- "ml/pyproject.toml"
|
||||||
|
- "ml/uv.lock"
|
||||||
|
- "ml/enervision_ml/**"
|
||||||
|
- "apps/backend/pyproject.toml"
|
||||||
|
- "apps/backend/uv.lock"
|
||||||
|
- "apps/backend/app/**"
|
||||||
|
- ".github/workflows/airflow.yml"
|
||||||
|
terraform:
|
||||||
|
- "infra/terraform/**"
|
||||||
|
- ".github/workflows/infra.yml"
|
||||||
|
compose:
|
||||||
|
- "docker-compose*.yml"
|
||||||
|
- ".env.example"
|
||||||
|
- "infra/front/**"
|
||||||
|
- "infra/garage/**"
|
||||||
|
- "tests/garage/**"
|
||||||
|
- "monitoring/**"
|
||||||
|
- ".github/workflows/infra.yml"
|
||||||
|
workflows:
|
||||||
|
- ".github/**"
|
||||||
|
e2e:
|
||||||
|
- "apps/frontend/**"
|
||||||
|
- "apps/backend/app/**"
|
||||||
|
- "apps/backend/alembic/**"
|
||||||
|
- "apps/backend/Dockerfile"
|
||||||
|
- "apps/backend/pyproject.toml"
|
||||||
|
- "apps/backend/uv.lock"
|
||||||
|
- "infra/proxy/**"
|
||||||
|
- "docker-compose*.yml"
|
||||||
|
- "db/**"
|
||||||
|
- "tests/**"
|
||||||
|
- "scripts/comptes-test.sh"
|
||||||
|
- "scripts/tls-selfsigned.sh"
|
||||||
|
- "Makefile"
|
||||||
|
- ".env.example"
|
||||||
|
- ".github/workflows/e2e.yml"
|
||||||
|
sonar:
|
||||||
|
- "apps/backend/**"
|
||||||
|
- "apps/frontend/**"
|
||||||
|
- "ml/**"
|
||||||
|
- "etl/airflow/**"
|
||||||
|
- "sonar-project.properties"
|
||||||
|
|
||||||
|
backend:
|
||||||
|
name: Backend
|
||||||
|
needs: changes
|
||||||
|
if: needs.changes.outputs.backend == 'true'
|
||||||
|
uses: ./.github/workflows/backend.yml
|
||||||
|
|
||||||
|
frontend:
|
||||||
|
name: Frontend
|
||||||
|
needs: changes
|
||||||
|
if: needs.changes.outputs.frontend == 'true'
|
||||||
|
uses: ./.github/workflows/frontend.yml
|
||||||
|
|
||||||
|
ml:
|
||||||
|
name: ML
|
||||||
|
needs: changes
|
||||||
|
if: needs.changes.outputs.ml == 'true'
|
||||||
|
uses: ./.github/workflows/ml.yml
|
||||||
|
|
||||||
|
airflow:
|
||||||
|
name: Airflow
|
||||||
|
needs: changes
|
||||||
|
if: needs.changes.outputs.airflow == 'true'
|
||||||
|
uses: ./.github/workflows/airflow.yml
|
||||||
|
|
||||||
|
infra:
|
||||||
|
name: Infra
|
||||||
|
needs: changes
|
||||||
|
if: >-
|
||||||
|
needs.changes.outputs.terraform == 'true'
|
||||||
|
|| needs.changes.outputs.compose == 'true'
|
||||||
|
|| needs.changes.outputs.workflows == 'true'
|
||||||
|
uses: ./.github/workflows/infra.yml
|
||||||
|
with:
|
||||||
|
terraform: ${{ needs.changes.outputs.terraform == 'true' }}
|
||||||
|
compose: ${{ needs.changes.outputs.compose == 'true' }}
|
||||||
|
workflows: ${{ needs.changes.outputs.workflows == 'true' }}
|
||||||
|
|
||||||
|
e2e:
|
||||||
|
name: E2E
|
||||||
|
needs: changes
|
||||||
|
if: needs.changes.outputs.e2e == 'true'
|
||||||
|
uses: ./.github/workflows/e2e.yml
|
||||||
|
|
||||||
|
# Ni dependabot[bot] ni une PR de fork ne reçoivent SONAR_TOKEN : le scan échouerait sans rien
|
||||||
|
# analyser. Tests et couverture restent joués par leurs jobs.
|
||||||
|
sonar:
|
||||||
|
name: SonarQube
|
||||||
|
needs: [changes, backend, frontend, ml]
|
||||||
|
if: >-
|
||||||
|
always() && !cancelled()
|
||||||
|
&& !contains(needs.*.result, 'failure')
|
||||||
|
&& needs.changes.outputs.sonar == 'true'
|
||||||
|
&& github.actor != 'dependabot[bot]'
|
||||||
|
&& (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
|
||||||
|
# Un téléchargement par rapport : backend et ML nomment tous deux le leur `coverage.xml`.
|
||||||
|
- name: Couverture du backend
|
||||||
|
if: needs.backend.result == 'success'
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: backend-coverage
|
||||||
|
path: apps/backend
|
||||||
|
|
||||||
|
- name: Couverture du pipeline ML
|
||||||
|
if: needs.ml.result == 'success'
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: ml-coverage
|
||||||
|
path: ml
|
||||||
|
|
||||||
|
- name: Couverture du frontend
|
||||||
|
if: needs.frontend.result == 'success'
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: frontend-coverage
|
||||||
|
path: apps/frontend/coverage/frontend
|
||||||
|
|
||||||
|
# Action tierce, épinglée sur le commit du tag (règle Sonar githubactions:S7637).
|
||||||
|
- name: Analyse SonarQube
|
||||||
|
uses: SonarSource/sonarqube-scan-action@ba9859eae8dd6bd29e412f25ddbbef3d032000f4 # v8.2.2
|
||||||
|
env:
|
||||||
|
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||||
|
|
||||||
|
ci-ok:
|
||||||
|
name: CI ok
|
||||||
|
needs: [changes, backend, frontend, ml, airflow, infra, e2e, sonar]
|
||||||
|
if: always()
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 5
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Refuse si un job a échoué ou a été annulé
|
||||||
|
env:
|
||||||
|
RESULTATS: ${{ toJSON(needs.*.result) }}
|
||||||
|
run: |
|
||||||
|
echo "$RESULTATS"
|
||||||
|
if grep -qE '"(failure|cancelled)"' <<<"$RESULTATS"; then
|
||||||
|
echo "::error::Au moins un job de la CI a échoué ou a été annulé."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
deploy:
|
||||||
|
name: Déploiement
|
||||||
|
needs: ci-ok
|
||||||
|
if: ${{ !cancelled() && needs.ci-ok.result == 'success' && github.event_name == 'push' }}
|
||||||
|
uses: ./.github/workflows/deploy.yml
|
||||||
@@ -0,0 +1,316 @@
|
|||||||
|
name: DAST
|
||||||
|
|
||||||
|
# Scan dynamique OWASP ZAP de l'API (issue #41). Il attaque une API qui tourne : le job démarre
|
||||||
|
# la base et le backend sur le runner, sème le jeu de démonstration (sans ça le scan ne frappe que
|
||||||
|
# des gestionnaires d'erreur), crée des comptes jetables (scripts/dast-token.sh), puis lance ZAP
|
||||||
|
# sur le contrat OpenAPI avec le jeton du `lecteur`.
|
||||||
|
#
|
||||||
|
# Non bloquant pour l'instant sur les alertes (`continue-on-error` sur la seule étape du scan) :
|
||||||
|
# le volume d'un premier passage trié est inconnu. Deux étapes suivantes, elles, bloquent si le
|
||||||
|
# scan n'a rien testé (import du contrat, absence de toute réponse de succès) : un job vert doit
|
||||||
|
# vouloir dire qu'un scan a eu lieu.
|
||||||
|
#
|
||||||
|
# Piège : ce scan tape la configuration par défaut du backend (`APP_ENV=local`, pas de TLS, pas
|
||||||
|
# de reverse proxy). Il ne dit rien des en-têtes ni du TLS posés par le proxy en production, et
|
||||||
|
# remontera des alertes (HSTS absent...) qui n'existent pas derrière lui.
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
schedule:
|
||||||
|
# Un scan actif est long : hebdomadaire plutôt qu'à chaque PR.
|
||||||
|
- cron: "0 3 * * 1"
|
||||||
|
pull_request:
|
||||||
|
# Ne se lance sur une PR que si le scan lui-même change.
|
||||||
|
paths:
|
||||||
|
- ".github/workflows/dast.yml"
|
||||||
|
- "scripts/dast-token.sh"
|
||||||
|
- "scripts/comptes-test.sh"
|
||||||
|
- "db/seeds/**"
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
concurrency:
|
||||||
|
group: dast-${{ github.ref }}
|
||||||
|
cancel-in-progress: true
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
zap:
|
||||||
|
name: Scan OWASP ZAP de l'API
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
# Généreux face aux ~2 minutes observées de bout en bout : le vrai plafond est
|
||||||
|
# `scanner.maxScanDurationInMins` (étape Scan ZAP), sous le TTL du jeton. Une annulation par
|
||||||
|
# ce timeout-ci n'exécute pas les étapes `always()` : mieux vaut ne jamais l'atteindre.
|
||||||
|
timeout-minutes: 30
|
||||||
|
|
||||||
|
# Même image que docker-compose.yml : la première migration refuse de s'appliquer sans
|
||||||
|
# l'extension TimescaleDB (cf. backend.yml).
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: timescale/timescaledb-ha:pg17
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: enervision
|
||||||
|
POSTGRES_PASSWORD: change_me
|
||||||
|
POSTGRES_DB: enervision_dast
|
||||||
|
ports:
|
||||||
|
- "5433:5432"
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U enervision -d enervision_dast"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 12
|
||||||
|
--health-start-period 40s
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Base jetable : ZAP y écrira et le script y crée deux comptes.
|
||||||
|
DATABASE_URL: postgresql+asyncpg://enervision:change_me@localhost:5433/enervision_dast
|
||||||
|
APP_SECRET_KEY: secret-de-scan-assez-long-pour-le-validateur
|
||||||
|
APP_ENV: local
|
||||||
|
# Le jeton du lecteur doit survivre à toute la durée du scan (15 minutes par défaut).
|
||||||
|
# 3600 est le plafond accepté par la configuration ; `scanner.maxScanDurationInMins`
|
||||||
|
# (étape Scan ZAP) reste très en dessous, marge comprise pour les étapes qui l'entourent.
|
||||||
|
APP_ACCESS_TOKEN_TTL_SECONDS: "3600"
|
||||||
|
PGPASSWORD: change_me
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Installe uv
|
||||||
|
# Épinglé sur le commit du tag v7 (règle Sonar githubactions:S7637 : dépendance tierce,
|
||||||
|
# contrairement à actions/checkout ou actions/upload-artifact, premières parties).
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: true
|
||||||
|
cache-dependency-glob: apps/backend/uv.lock
|
||||||
|
# `prune-cache` vaut `true` par défaut (encore sur ce commit) : l'étape de post-job
|
||||||
|
# « Pruning cache » est restée bloquée 5 minutes avant d'échouer (exit code 2) sur un
|
||||||
|
# run où les 16 étapes précédentes passaient, sans lien avec le scan. Le prune n'est
|
||||||
|
# qu'une optimisation de taille de cache entre deux runs, pas une garantie : le
|
||||||
|
# désactiver retire le blocage sans rien changer au comportement du job.
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
|
- name: Installe l'interpréteur déclaré par .python-version
|
||||||
|
run: uv python install
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
# `--no-build` : aucune dépendance n'est construite depuis ses sources, donc aucun script de
|
||||||
|
# build exécuté (règle Sonar S8541). Le projet lui-même n'est pas installé : il tourne depuis
|
||||||
|
# `apps/backend`, comme dans son Dockerfile. Les `uv run` suivants portent `--frozen
|
||||||
|
# --no-sync` pour ne rien résoudre ni reconstruire (règle S8544).
|
||||||
|
- name: Synchronise les dépendances sans dévier du verrou
|
||||||
|
run: uv sync --locked --no-dev --no-install-project --no-build
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
- name: Active TimescaleDB sur la base du scan
|
||||||
|
run: psql -h localhost -p 5433 -U enervision -d enervision_dast -c "CREATE EXTENSION IF NOT EXISTS timescaledb"
|
||||||
|
|
||||||
|
- name: Applique les migrations
|
||||||
|
run: uv run --frozen --no-sync --no-build alembic upgrade head
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
# Sans données, `GET /sites` rend `[]`, chaque `/{site_id}` rend 404 et le scan actif ne
|
||||||
|
# frappe que des gestionnaires d'erreur plutôt que la logique métier.
|
||||||
|
- name: Sème le jeu de démonstration
|
||||||
|
run: psql -h localhost -p 5433 -U enervision -d enervision_dast -v ON_ERROR_STOP=1 -f db/seeds/demo.sql
|
||||||
|
|
||||||
|
- name: Démarre l'API
|
||||||
|
run: |
|
||||||
|
nohup uv run --frozen --no-sync --no-build uvicorn app.main:create_app --factory \
|
||||||
|
--host 0.0.0.0 --port 8000 > "$RUNNER_TEMP/api.log" 2>&1 &
|
||||||
|
for _ in $(seq 1 30); do
|
||||||
|
curl -fsS http://localhost:8000/api/v1/health/ready >/dev/null 2>&1 && exit 0
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
echo "L'API ne répond pas sur /health/ready" >&2
|
||||||
|
cat "$RUNNER_TEMP/api.log" >&2
|
||||||
|
exit 1
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
- name: Crée le compte lecteur du scan
|
||||||
|
id: jeton
|
||||||
|
run: |
|
||||||
|
jeton="$(../../scripts/dast-token.sh)"
|
||||||
|
echo "::add-mask::$jeton"
|
||||||
|
echo "jeton=$jeton" >> "$GITHUB_OUTPUT"
|
||||||
|
working-directory: apps/backend
|
||||||
|
|
||||||
|
# Étape distincte du scan lui-même, et sans `continue-on-error` : un `curl` qui échoue ici
|
||||||
|
# (API tombée juste après la sonde de readiness, par exemple) doit rester un échec visible,
|
||||||
|
# pas se travestir en « ZAP n'a importé aucune URL » à l'étape de garde suivante.
|
||||||
|
- name: Prépare le contrat pour ZAP
|
||||||
|
run: |
|
||||||
|
mkdir -p zap-out zap-logs
|
||||||
|
curl -fsS http://localhost:8000/openapi.json -o zap-out/openapi.json
|
||||||
|
# Le dossier passe à l'uid 1000 (utilisateur du conteneur ZAP) : le runner n'y écrit
|
||||||
|
# plus après ce chown, d'où `zap-logs/` (uid du runner) pour les journaux ci-dessous.
|
||||||
|
# Pas de `chmod 777` (règle Sonar S2612).
|
||||||
|
sudo chown -R 1000:1000 zap-out
|
||||||
|
|
||||||
|
# `--network host` : ZAP atteint l'API sur le localhost du runner.
|
||||||
|
#
|
||||||
|
# Piège vécu : la clé du nom d'en-tête est `matchstr`, pas `matchstring`. ZAP accepte
|
||||||
|
# n'importe quelle clé `-config` sans erreur ; avec la mauvaise, il ajoutait à TOUTES les
|
||||||
|
# requêtes un en-tête au nom vide (`: Bearer <jeton>`), qu'uvicorn refuse par un 400
|
||||||
|
# (« Invalid HTTP request received »), y compris sur les routes publiques.
|
||||||
|
#
|
||||||
|
# Le jeton ne passe ni par `${{ }}` dans ce script (il finirait en clair dans le fichier de
|
||||||
|
# commande que GitHub écrit sur le disque du runner pour toute la durée de l'étape), ni par
|
||||||
|
# l'argv de `docker run` (visible par `ps aux` et par `docker inspect zap` tant que le
|
||||||
|
# conteneur existe) : il est écrit dans un fichier de configuration ZAP séparé, monté en
|
||||||
|
# lecture seule hors de `/zap/wrk` pour ne jamais atterrir dans l'artefact publié.
|
||||||
|
#
|
||||||
|
# Les routes d'authentification qui changent l'état du compte du scan sont exclues : un
|
||||||
|
# scan actif y déclencherait la limitation de débit du login, la réinitialisation de mots de
|
||||||
|
# passe et la fermeture des sessions, sans rien apprendre de plus.
|
||||||
|
#
|
||||||
|
# `scanner.maxScanDurationInMins`/`maxRuleDurationInMins` bornent le scan actif, que `-T` ne
|
||||||
|
# couvre pas (il ne borne que le démarrage et le scan passif) : sans ça, une règle qui
|
||||||
|
# traîne peut dépasser le TTL du jeton (401 muets en fin de scan) ou le timeout du job (qui
|
||||||
|
# annule sans exécuter les étapes `always()`, rapport et journaux perdus).
|
||||||
|
- name: Scan ZAP
|
||||||
|
id: zap
|
||||||
|
continue-on-error: true
|
||||||
|
env:
|
||||||
|
JETON: ${{ steps.jeton.outputs.jeton }}
|
||||||
|
run: |
|
||||||
|
set -o pipefail
|
||||||
|
printf 'replacer.full_list(0).description=auth\nreplacer.full_list(0).enabled=true\nreplacer.full_list(0).matchtype=REQ_HEADER\nreplacer.full_list(0).matchstr=Authorization\nreplacer.full_list(0).regex=false\nreplacer.full_list(0).replacement=Bearer %s\n' "$JETON" > "$RUNNER_TEMP/zap-auth.conf"
|
||||||
|
# Piège vécu : `chmod 600` seul rend le fichier illisible pour le conteneur, qui lit un
|
||||||
|
# montage bind avec son propre uid (1000), distinct de celui du runner qui l'a écrit.
|
||||||
|
# ZAP échoue alors dès le lancement (« File not readable: /zap/auth.conf »), et
|
||||||
|
# `zap-api-scan.py` attend `-T` minutes complètes avant d'abandonner : dix minutes qui
|
||||||
|
# ressemblent à un scan actif, pour un daemon mort depuis le début.
|
||||||
|
#
|
||||||
|
# Piège vécu (numéro deux) : une fois le fichier passé à l'uid 1000 par `sudo chown`,
|
||||||
|
# l'utilisateur du runner n'en est plus propriétaire et un `chmod` sans `sudo` échoue
|
||||||
|
# (« Operation not permitted »). Avec le `-e` implicite de bash sur les étapes GitHub
|
||||||
|
# Actions, cette erreur arrêtait toute l'étape avant même `docker run` : scan « réussi »
|
||||||
|
# en une fraction de seconde, sans le moindre journal ni rapport produit.
|
||||||
|
sudo chown 1000:1000 "$RUNNER_TEMP/zap-auth.conf"
|
||||||
|
sudo chmod 644 "$RUNNER_TEMP/zap-auth.conf"
|
||||||
|
docker run --name zap --network host \
|
||||||
|
-v "$PWD/zap-out:/zap/wrk:rw" \
|
||||||
|
-v "$RUNNER_TEMP/zap-auth.conf:/zap/auth.conf:ro" \
|
||||||
|
ghcr.io/zaproxy/zaproxy:stable zap-api-scan.py \
|
||||||
|
-t /zap/wrk/openapi.json -f openapi -O http://localhost:8000 \
|
||||||
|
-T 10 \
|
||||||
|
-r zap-report.html -J zap-report.json -w zap-report.md \
|
||||||
|
-z "-configfile /zap/auth.conf \
|
||||||
|
-config globalexcludeurl.url_list.url(0).description=auth-etat \
|
||||||
|
-config globalexcludeurl.url_list.url(0).enabled=true \
|
||||||
|
-config globalexcludeurl.url_list.url(0).regex='.*/api/v1/auth/(login|password|logout-all|forgot-password|reset-password).*' \
|
||||||
|
-config scanner.maxScanDurationInMins=15 \
|
||||||
|
-config scanner.maxRuleDurationInMins=5" \
|
||||||
|
2>&1 | tee "$RUNNER_TEMP/zap-stdout.log"
|
||||||
|
|
||||||
|
- name: Récupère les journaux de ZAP
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
mkdir -p zap-logs
|
||||||
|
# ZAP journalise la valeur de chaque `-config`/`-configfile` chargé, y compris le jeton,
|
||||||
|
# à un niveau visible sans `-d` : les copies publiées en artefact sont donc caviardées,
|
||||||
|
# même si `::add-mask::` (posé à la création du jeton) protège déjà le journal du job.
|
||||||
|
masque() { sed -E 's/(Bearer )[A-Za-z0-9._-]+/\1[MASQUE]/Ig'; }
|
||||||
|
[ -f "$RUNNER_TEMP/zap-stdout.log" ] && masque < "$RUNNER_TEMP/zap-stdout.log" > zap-logs/zap-stdout.log
|
||||||
|
docker cp zap:/home/zap/.ZAP/zap.log "$RUNNER_TEMP/zap-internal.log" 2>/dev/null || true
|
||||||
|
[ -f "$RUNNER_TEMP/zap-internal.log" ] && masque < "$RUNNER_TEMP/zap-internal.log" > zap-logs/zap.log
|
||||||
|
[ -f "$RUNNER_TEMP/api.log" ] && masque < "$RUNNER_TEMP/api.log" > zap-logs/api.log
|
||||||
|
rm -f "$RUNNER_TEMP/zap-auth.conf"
|
||||||
|
docker rm -f zap >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
# `continue-on-error` sur le scan ne doit pas faire passer pour vert un scan qui n'a rien
|
||||||
|
# testé. Constaté une première fois : 2 URL importées sur 26 opérations, ZAP n'avait envoyé
|
||||||
|
# que des requêtes vouées au 404. Le seuil est dérivé du contrat plutôt que d'un nombre fixe
|
||||||
|
# : un contrat qui grossit ne doit pas rendre la garde plus permissive qu'elle ne l'était.
|
||||||
|
- name: Vérifie que le contrat a bien été importé
|
||||||
|
run: |
|
||||||
|
attendu="$(python3 -c "
|
||||||
|
import json
|
||||||
|
d = json.load(open('zap-out/openapi.json'))
|
||||||
|
methodes = ('get', 'post', 'put', 'patch', 'delete', 'head', 'options')
|
||||||
|
print(sum(1 for chemin in d['paths'].values() for m in chemin if m in methodes))
|
||||||
|
")"
|
||||||
|
minimum=$((attendu * 80 / 100))
|
||||||
|
importees="$(sed -n 's/.*Number of Imported URLs: \([0-9]*\).*/\1/p' "$RUNNER_TEMP/zap-stdout.log" | tail -1)"
|
||||||
|
echo "URL importées depuis le contrat OpenAPI : ${importees:-aucune} (contrat : $attendu opérations, minimum accepté : $minimum)"
|
||||||
|
if [ "${importees:-0}" -lt "$minimum" ]; then
|
||||||
|
echo "::error::ZAP n'a importé que ${importees:-0} URL sur $attendu opérations du contrat OpenAPI (minimum attendu : $minimum, soit 80%). Le scan n'a pas testé l'API, voir zap-logs/zap.log dans l'artefact zap-report."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Deuxième garde-fou : le contrat peut être importé et ZAP n'obtenir que des erreurs
|
||||||
|
# (constaté : base sans données, toutes les routes de site répondaient 404).
|
||||||
|
#
|
||||||
|
# Piège de conception, trouvé en répétant ce job en local avant de l'écrire ici : borner le
|
||||||
|
# pourcentage de 4xx ne marche pas. Un scan actif fuzze délibérément un grand nombre
|
||||||
|
# d'entrées invalides (identifiants inventés, méthodes non supportées...), donc même un scan
|
||||||
|
# sain, contre l'API seedée juste au-dessus, reste à 98% de 4xx avec seulement 1% de 2xx :
|
||||||
|
# c'est la forme normale d'un scan actif, pas un signe d'échec. Le signal qui distingue
|
||||||
|
# vraiment un scan cassé (0% de 2xx, `insight.code.2xx` absent du rapport dans le premier
|
||||||
|
# incident) d'un scan sain (2xx non nul, aussi faible soit-il) est donc l'absence de succès,
|
||||||
|
# pas la part d'échecs. Dérivé de `zap-report.json` (champ structuré `insights[]`) plutôt
|
||||||
|
# que du texte libre du rapport Markdown, qui aurait le même défaut de conception en plus
|
||||||
|
# d'être fragile au format.
|
||||||
|
- name: Vérifie que le scan a obtenu au moins une réponse de succès
|
||||||
|
run: |
|
||||||
|
python3 - <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
try:
|
||||||
|
rapport = json.load(open("zap-out/zap-report.json"))
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("::error::Aucun rapport ZAP produit : le scan n'a rien testé.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
pourcentage_2xx = 0.0
|
||||||
|
for insight in rapport.get("insights", []):
|
||||||
|
if insight.get("key") == "insight.code.2xx":
|
||||||
|
pourcentage_2xx = float(insight.get("statistic", 0))
|
||||||
|
break
|
||||||
|
|
||||||
|
print(f"Pourcentage de réponses 2xx : {pourcentage_2xx}%")
|
||||||
|
if pourcentage_2xx <= 0:
|
||||||
|
print(
|
||||||
|
"::error::Aucune réponse 2xx (succès) reçue : le scan n'a atteint aucune route "
|
||||||
|
"réelle de l'API. Voir zap-logs/api.log et zap-logs/zap.log dans l'artefact "
|
||||||
|
"zap-report."
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
PY
|
||||||
|
|
||||||
|
# Uniquement la synthèse (jusqu'à « Alert Detail » exclu) : `$GITHUB_STEP_SUMMARY` est
|
||||||
|
# limité à 1 Mio, et cette étape tourne sous `always()` - son échec ferait échouer le job
|
||||||
|
# après le passage des deux garde-fous, pour une simple raison de mise en forme. Le rapport
|
||||||
|
# complet reste dans l'artefact `zap-report`.
|
||||||
|
- name: Publie le résumé
|
||||||
|
if: always()
|
||||||
|
run: |
|
||||||
|
if [ -f zap-out/zap-report.md ]; then
|
||||||
|
{
|
||||||
|
awk '/^## Alert Detail/{exit} {print}' zap-out/zap-report.md
|
||||||
|
echo ""
|
||||||
|
echo "Rapport complet (HTML/JSON/Markdown) dans l'artefact \`zap-report\`."
|
||||||
|
} >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
else
|
||||||
|
echo "Aucun rapport ZAP produit, voir le journal du job." >> "$GITHUB_STEP_SUMMARY"
|
||||||
|
fi
|
||||||
|
|
||||||
|
- name: Publie les rapports
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: zap-report
|
||||||
|
path: |
|
||||||
|
zap-out/
|
||||||
|
zap-logs/
|
||||||
|
if-no-files-found: warn
|
||||||
|
|
||||||
|
# Diagnostic de dernier recours : les journaux de l'API sont déjà dans l'artefact
|
||||||
|
# (zap-logs/api.log) via l'étape « Récupère les journaux de ZAP » (always()), mais les
|
||||||
|
# afficher directement dans le journal du job évite d'avoir à le télécharger pour un échec
|
||||||
|
# évident (l'API n'a jamais démarré, par exemple).
|
||||||
|
- name: Journal de l'API en cas d'échec
|
||||||
|
if: failure() || steps.zap.outcome == 'failure'
|
||||||
|
run: cat "$RUNNER_TEMP/api.log" || true
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
# Pourquoi : le runner tourne sur la VM ENI, adresse privée que les runners hébergés par GitHub
|
||||||
|
# ne joignent pas, et travaille dans un dossier stable par environnement plutôt que dans son
|
||||||
|
# espace de travail : `.env`, certificats et volumes y survivent d'un déploiement à l'autre.
|
||||||
|
# Pourquoi : appelé par ci.yml une fois « CI ok » vert, jamais directement par un push, et il
|
||||||
|
# déploie `GITHUB_SHA`, le commit testé, pas la pointe de branche du moment (ADR 0014).
|
||||||
|
# Pourquoi : `main` va en prod, `dev` en recette, et toute autre branche lancée à la main
|
||||||
|
# (workflow_dispatch) va dans `dev`, la vitrine d'une branche de travail (ADR 0017).
|
||||||
|
# Piège : jamais de déclencheur `pull_request` ici. Sur un dépôt public, une PR de fork
|
||||||
|
# exécuterait son code sur la machine de production (ADR 0009) - job deploy.
|
||||||
|
# Piège : les CI de deux push finissent parfois dans le désordre. Un commit qui précède celui déjà
|
||||||
|
# déployé depuis la même branche est ignoré, et le verrou est un `flock` sur le dossier de
|
||||||
|
# l'environnement plutôt qu'un groupe `concurrency` : GitHub n'y garde qu'un job en attente, et
|
||||||
|
# le suivant l'évince sans bruit.
|
||||||
|
|
||||||
|
name: Déploiement
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
deploy:
|
||||||
|
name: Déploie sur la VM
|
||||||
|
runs-on: [self-hosted, linux, eni-g3]
|
||||||
|
timeout-minutes: 30
|
||||||
|
environment:
|
||||||
|
name: ${{ github.ref_name == 'main' && 'prod' || github.ref_name == 'dev' && 'rec' || 'dev' }}
|
||||||
|
url: ${{ github.ref_name == 'main' && 'https://prod.enervision-g3.dynv6.net' || github.ref_name == 'dev' && 'https://rec.enervision-g3.dynv6.net' || 'https://dev.enervision-g3.dynv6.net' }}
|
||||||
|
env:
|
||||||
|
ENVIRONNEMENT: ${{ github.ref_name == 'main' && 'prod' || github.ref_name == 'dev' && 'rec' || 'dev' }}
|
||||||
|
PORT_HTTPS: ${{ github.ref_name == 'main' && '10443' || github.ref_name == 'dev' && '8443' || '9443' }}
|
||||||
|
steps:
|
||||||
|
# Un seul step : le verrou tombe avec le shell qui l'a posé.
|
||||||
|
- name: Déploie le commit testé, sans jamais reculer
|
||||||
|
run: |
|
||||||
|
cd "/srv/enervision/${ENVIRONNEMENT}"
|
||||||
|
exec 9>"$(git rev-parse --git-dir)/verrou-deploiement"
|
||||||
|
flock 9
|
||||||
|
|
||||||
|
echo "::group::Aligne le dossier de l'environnement sur le commit testé"
|
||||||
|
git fetch --quiet origin "${GITHUB_REF_NAME}"
|
||||||
|
deploye="$(git rev-parse HEAD)"
|
||||||
|
if [ "$(git branch --show-current)" = "$GITHUB_REF_NAME" ] && [ "$deploye" != "$GITHUB_SHA" ] \
|
||||||
|
&& git merge-base --is-ancestor "$GITHUB_SHA" "$deploye"; then
|
||||||
|
echo "::notice::${GITHUB_SHA:0:7} précède le commit déjà déployé (${deploye:0:7}) : rien à déployer."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
git checkout --quiet "${GITHUB_REF_NAME}"
|
||||||
|
git reset --quiet --hard "${GITHUB_SHA}"
|
||||||
|
git log -1 --format='%h %s'
|
||||||
|
echo "::endgroup::"
|
||||||
|
|
||||||
|
echo "::group::Reconstruit et redémarre la stack"
|
||||||
|
# Un `.env` pas encore réaligné par provision-host.sh porte encore un nom en `.local`.
|
||||||
|
if [ -r ../dns.token ] && ! grep -q '^PUBLIC_HOST=.*\.local$' .env; then make tls-dns01; fi
|
||||||
|
make stack-up
|
||||||
|
if [ "${ENVIRONNEMENT}" = prod ]; then make front-up; fi
|
||||||
|
echo "::endgroup::"
|
||||||
|
|
||||||
|
echo "::group::Attend que l'API réponde derrière le proxy"
|
||||||
|
for _ in $(seq 1 36); do
|
||||||
|
if curl --fail --silent --insecure "https://localhost:${PORT_HTTPS}/api/v1/health/ready"; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
echo "::endgroup::"
|
||||||
|
echo "L'API ne répond pas après 3 minutes" >&2
|
||||||
|
compose="docker compose -f docker-compose.yml -f docker-compose.prod.yml"
|
||||||
|
$compose ps
|
||||||
|
$compose logs --tail=50 backend proxy
|
||||||
|
exit 1
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
name: E2E
|
||||||
|
|
||||||
|
# Pourquoi : les parcours tournent contre la stack telle qu'elle est déployée, derrière le proxy
|
||||||
|
# TLS (cookie `__Secure-`, CSP, limitation de débit), pas contre `ng serve` - job parcours. Il
|
||||||
|
# construit aussi les images backend et frontend, que rien d'autre ne construit avant le
|
||||||
|
# déploiement (ADR 0015).
|
||||||
|
# Piège : pas d'Airflow ici. `up` nomme ses services : sans eux, la construction de l'image
|
||||||
|
# Airflow doublerait la durée du job sans rien tester de plus.
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
parcours:
|
||||||
|
name: Parcours Playwright et tirs k6
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 30
|
||||||
|
env:
|
||||||
|
COMPOSE_FILE: docker-compose.yml:docker-compose.prod.yml
|
||||||
|
PUBLIC_HOST: localhost
|
||||||
|
E2E_BASE_URL: https://localhost
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Prépare le .env de la stack
|
||||||
|
run: |
|
||||||
|
secret() { openssl rand -hex 32; }
|
||||||
|
sed -e "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$(secret)|" \
|
||||||
|
-e "s|^APP_SECRET_KEY=.*|APP_SECRET_KEY=$(secret)|" \
|
||||||
|
-e "s|^PUBLIC_HOST=.*|PUBLIC_HOST=localhost|" \
|
||||||
|
.env.example > .env
|
||||||
|
|
||||||
|
- name: Génère le certificat de démonstration
|
||||||
|
run: ./scripts/tls-selfsigned.sh
|
||||||
|
|
||||||
|
- name: Construit et démarre la stack derrière le proxy
|
||||||
|
run: docker compose up --detach --build --wait --wait-timeout 300 db mailpit backend frontend proxy
|
||||||
|
|
||||||
|
- name: Applique les migrations
|
||||||
|
run: docker compose exec -T backend alembic upgrade head
|
||||||
|
|
||||||
|
# Même cible que `make stack-up` en prod : les droits du rôle portent sur le schéma réel.
|
||||||
|
- name: Pose le rôle de supervision en lecture seule
|
||||||
|
run: make db-ensure-supervision
|
||||||
|
|
||||||
|
- name: Sème le jeu de démonstration
|
||||||
|
run: docker compose exec -T db psql -U enervision -d enervision -v ON_ERROR_STOP=1 < db/seeds/demo.sql
|
||||||
|
|
||||||
|
- name: Crée les comptes de test
|
||||||
|
env:
|
||||||
|
BASE_URL: https://localhost
|
||||||
|
APP_CLI: docker compose exec -T backend python -m app.cli
|
||||||
|
COMPTES_FICHIER: ${{ runner.temp }}/comptes.json
|
||||||
|
run: ./scripts/comptes-test.sh
|
||||||
|
|
||||||
|
- name: Installe Node
|
||||||
|
uses: actions/setup-node@v7
|
||||||
|
with:
|
||||||
|
node-version: 26
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: tests/e2e/package-lock.json
|
||||||
|
|
||||||
|
- name: Installe Playwright
|
||||||
|
working-directory: tests/e2e
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Restaure les navigateurs de Playwright
|
||||||
|
uses: actions/cache@v6
|
||||||
|
with:
|
||||||
|
path: ~/.cache/ms-playwright
|
||||||
|
key: playwright-${{ runner.os }}-${{ hashFiles('tests/e2e/package-lock.json') }}
|
||||||
|
|
||||||
|
# `--with-deps` tourne même quand le cache a servi : il pose aussi les bibliothèques système.
|
||||||
|
- name: Installe Chromium
|
||||||
|
working-directory: tests/e2e
|
||||||
|
run: npx playwright install --with-deps chromium
|
||||||
|
|
||||||
|
- name: Joue les parcours
|
||||||
|
working-directory: tests/e2e
|
||||||
|
env:
|
||||||
|
E2E_COMPTES: ${{ runner.temp }}/comptes.json
|
||||||
|
run: npx playwright test
|
||||||
|
|
||||||
|
# Direct sur `backend:8000` : ce tir mesure l'API, pas la limitation de nginx.
|
||||||
|
- name: Tir k6 de fumée sur l'API
|
||||||
|
env:
|
||||||
|
K6_RESUME: /results/resume-smoke.md
|
||||||
|
run: |
|
||||||
|
K6_EMAIL="$(jq -r .lecteur.email "$RUNNER_TEMP/comptes.json")"
|
||||||
|
K6_PASSWORD="$(jq -r .lecteur.password "$RUNNER_TEMP/comptes.json")"
|
||||||
|
echo "::add-mask::$K6_PASSWORD"
|
||||||
|
export K6_EMAIL K6_PASSWORD
|
||||||
|
make load-smoke
|
||||||
|
|
||||||
|
- name: Vérifie par k6 que le proxy limite le débit
|
||||||
|
env:
|
||||||
|
K6_RESUME: /results/resume-limitation.md
|
||||||
|
run: make load-limits
|
||||||
|
|
||||||
|
- name: Publie la synthèse k6
|
||||||
|
if: ${{ !cancelled() }}
|
||||||
|
run: cat tests/load/results/resume-*.md >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true
|
||||||
|
|
||||||
|
- name: Publie les rapports k6
|
||||||
|
if: ${{ !cancelled() }}
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: k6-rapports
|
||||||
|
path: tests/load/results/
|
||||||
|
if-no-files-found: ignore
|
||||||
|
retention-days: 14
|
||||||
|
|
||||||
|
- name: Publie le rapport Playwright
|
||||||
|
if: ${{ !cancelled() }}
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: playwright-report
|
||||||
|
path: |
|
||||||
|
tests/e2e/playwright-report/
|
||||||
|
tests/e2e/test-results/
|
||||||
|
if-no-files-found: ignore
|
||||||
|
retention-days: 14
|
||||||
|
|
||||||
|
- name: Journaux de la stack en cas d'échec
|
||||||
|
if: failure()
|
||||||
|
run: docker compose logs --tail=200 backend proxy frontend
|
||||||
|
|
||||||
|
- name: Arrête la stack
|
||||||
|
if: always()
|
||||||
|
run: docker compose down --volumes
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
name: Frontend
|
||||||
|
|
||||||
|
# Pourquoi : aucun déclencheur propre. ci.yml appelle ce workflow quand le frontend change, et
|
||||||
|
# Sonar y reprend la couverture versée par le job `verification` (ADR 0014).
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
# Un seul `npm ci` pour la construction et les tests : un job de plus ne ferait que le rejouer.
|
||||||
|
verification:
|
||||||
|
name: Construction et tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: apps/frontend
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Installe Node
|
||||||
|
uses: actions/setup-node@v7
|
||||||
|
with:
|
||||||
|
node-version: 26
|
||||||
|
cache: npm
|
||||||
|
cache-dependency-path: apps/frontend/package-lock.json
|
||||||
|
|
||||||
|
- name: Installe les dépendances
|
||||||
|
run: npm ci
|
||||||
|
|
||||||
|
- name: Construit l'application
|
||||||
|
run: npm run build
|
||||||
|
|
||||||
|
# Piège : `npm test --watch=false` garde l'option pour npm, `ng test` ne la reçoit jamais.
|
||||||
|
# La couverture lcov vient d'angular.json (`coverage: true`).
|
||||||
|
- name: Tests et couverture
|
||||||
|
run: npm run test:ci
|
||||||
|
|
||||||
|
- name: Verse la couverture pour Sonar
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: frontend-coverage
|
||||||
|
path: apps/frontend/coverage/frontend/lcov.info
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
security-audit:
|
||||||
|
name: Audit des dépendances
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: apps/frontend
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Installe Node
|
||||||
|
uses: actions/setup-node@v7
|
||||||
|
with:
|
||||||
|
node-version: 26
|
||||||
|
|
||||||
|
# Seuil high : une vulnérabilité moderate de devDependency ne doit pas bloquer une livraison.
|
||||||
|
- name: Audite le verrou
|
||||||
|
run: npm audit --audit-level=high --package-lock-only
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
name: Infra
|
||||||
|
|
||||||
|
# Pourquoi : rien de ce qui décrit l'infrastructure ne s'exécute avant le déploiement. Terraform est
|
||||||
|
# resté cassé sans que rien ne le dise, faute de job qui le joue : ce workflow n'applique rien, il
|
||||||
|
# vérifie le Terraform, les fichiers Compose et les workflows eux-mêmes - jobs terraform, compose,
|
||||||
|
# workflows. ci.yml choisit par ses entrées ceux qui tournent (ADR 0014).
|
||||||
|
# Piège : la boucle Terraform parcourt `environments/*`, pour qu'une racine ajoutée soit couverte
|
||||||
|
# sans toucher à ce fichier.
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
inputs:
|
||||||
|
terraform:
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
compose:
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
workflows:
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
terraform:
|
||||||
|
name: Formatage et validation Terraform
|
||||||
|
if: inputs.terraform
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Action tierce, épinglée sur le commit du tag (règle Sonar githubactions:S7637).
|
||||||
|
- name: Installe Terraform
|
||||||
|
uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
|
||||||
|
with:
|
||||||
|
terraform_version: 1.16.3
|
||||||
|
terraform_wrapper: false
|
||||||
|
|
||||||
|
- name: Vérifie le formatage
|
||||||
|
run: terraform fmt -check -recursive infra/terraform
|
||||||
|
|
||||||
|
- name: Valide chaque racine
|
||||||
|
run: |
|
||||||
|
for racine in infra/terraform/environments/*/; do
|
||||||
|
echo "::group::${racine}"
|
||||||
|
terraform -chdir="${racine}" init -backend=false -input=false
|
||||||
|
terraform -chdir="${racine}" validate
|
||||||
|
echo "::endgroup::"
|
||||||
|
done
|
||||||
|
|
||||||
|
compose:
|
||||||
|
name: Validation des fichiers Compose et de la supervision
|
||||||
|
if: inputs.compose
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Compose interpole tout le fichier : les `:?` exigent une valeur, pas un vrai secret.
|
||||||
|
- name: Prépare un .env d'exemple
|
||||||
|
run: cp .env.example .env
|
||||||
|
|
||||||
|
# Garage refuse un rpc_secret qui n'est pas 32 octets hexadécimaux : `change_me` ne suffit pas.
|
||||||
|
- name: Génère les secrets Garage du .env
|
||||||
|
run: |
|
||||||
|
sed -i -e "s|^GARAGE_RPC_SECRET=.*|GARAGE_RPC_SECRET=$(openssl rand -hex 32)|" \
|
||||||
|
-e "s|^GARAGE_ADMIN_TOKEN=.*|GARAGE_ADMIN_TOKEN=$(openssl rand -hex 32)|" \
|
||||||
|
-e "s|^GARAGE_METRICS_TOKEN=.*|GARAGE_METRICS_TOKEN=$(openssl rand -hex 32)|" \
|
||||||
|
-e "s|^GARAGE_ACCESS_KEY=.*|GARAGE_ACCESS_KEY=GK$(openssl rand -hex 12)|" \
|
||||||
|
-e "s|^GARAGE_SECRET_KEY=.*|GARAGE_SECRET_KEY=$(openssl rand -hex 32)|" \
|
||||||
|
-e "s|^GARAGE_SSE_KEY=.*|GARAGE_SSE_KEY=$(openssl rand -base64 32)|" .env
|
||||||
|
|
||||||
|
- name: Valide la stack de développement
|
||||||
|
run: docker compose config --quiet
|
||||||
|
|
||||||
|
- name: Valide la stack déployée, profils compris
|
||||||
|
run: docker compose -f docker-compose.yml -f docker-compose.prod.yml --profile acme --profile monitoring --profile load config --quiet
|
||||||
|
|
||||||
|
- name: Valide le frontal SNI de la VM
|
||||||
|
run: |
|
||||||
|
docker compose -f infra/front/compose.yml config --quiet
|
||||||
|
docker run --rm -v "$PWD/infra/front/nginx.conf:/etc/nginx/nginx.conf:ro" nginx:1.31-alpine nginx -t
|
||||||
|
|
||||||
|
# Mêmes commandes que `make monitoring-check` : images et montages viennent du fichier Compose.
|
||||||
|
- name: Valide la configuration de Prometheus et ses règles
|
||||||
|
run: docker compose --profile monitoring run --rm --no-deps --entrypoint promtool prometheus check config /etc/prometheus/prometheus.yml
|
||||||
|
|
||||||
|
- name: Joue les tests unitaires des règles d'alerte
|
||||||
|
run: docker compose --profile monitoring run --rm --no-deps --entrypoint promtool prometheus test rules /etc/prometheus/tests/enervision.test.yml
|
||||||
|
|
||||||
|
- name: Valide la configuration d'Alertmanager
|
||||||
|
run: docker compose --profile monitoring run --rm --no-deps --entrypoint amtool alertmanager check-config /etc/alertmanager/alertmanager.yml
|
||||||
|
|
||||||
|
- name: Valide les tableaux de bord Grafana
|
||||||
|
run: for tableau in monitoring/grafana/dashboards/*.json; do jq empty "$tableau"; done
|
||||||
|
|
||||||
|
# Action tierce, épinglée sur le commit du tag (règle Sonar githubactions:S7637).
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: false
|
||||||
|
|
||||||
|
# Même image et même healthcheck qu'en prod : `--wait` ne rend la main qu'une fois le S3 prêt.
|
||||||
|
- name: Démarre Garage
|
||||||
|
run: docker compose up -d --wait --wait-timeout 120 garage
|
||||||
|
|
||||||
|
- name: Fumée S3 sur Garage, SSE-C compris
|
||||||
|
run: |
|
||||||
|
set -a; . ./.env; set +a
|
||||||
|
uvx --no-build --with boto3==1.43.101 pytest==9.1.1 tests/garage -q
|
||||||
|
|
||||||
|
- name: Journaux de Garage en cas d'échec
|
||||||
|
if: failure()
|
||||||
|
run: docker compose logs --tail=100 garage
|
||||||
|
|
||||||
|
- name: Arrête Garage
|
||||||
|
if: always()
|
||||||
|
run: docker compose down --volumes
|
||||||
|
|
||||||
|
workflows:
|
||||||
|
name: Analyse des workflows
|
||||||
|
if: inputs.workflows
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Image épinglée par tag, comme les images des fichiers Compose. Elle embarque shellcheck,
|
||||||
|
# qui analyse aussi les blocs `run:`.
|
||||||
|
- name: actionlint
|
||||||
|
run: docker run --rm -v "$PWD:/repo" --workdir /repo rhysd/actionlint:1.7.12 -color
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
name: ML
|
||||||
|
|
||||||
|
# Piège : la version de Python vient de ml/.python-version, et doit rester en 3.14 (cf.
|
||||||
|
# .github/workflows/backend.yml, même contrainte).
|
||||||
|
# Pourquoi : aucun déclencheur propre. ci.yml l'appelle aussi quand les migrations ou les modèles
|
||||||
|
# du backend changent, dont dépend le job `integration` (ADR 0014).
|
||||||
|
|
||||||
|
on:
|
||||||
|
workflow_call:
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
verification:
|
||||||
|
name: Lint, typage et tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: ml
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Action tierce, épinglée sur le commit du tag (règle Sonar githubactions:S7637).
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: true
|
||||||
|
cache-dependency-glob: ml/uv.lock
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
|
- name: Installe l'interpréteur déclaré par .python-version
|
||||||
|
run: uv python install
|
||||||
|
|
||||||
|
- name: Synchronise les dépendances sur le verrou
|
||||||
|
run: uv sync --all-groups --locked
|
||||||
|
|
||||||
|
- name: Vérifie le formatage
|
||||||
|
run: uv run ruff format --check .
|
||||||
|
|
||||||
|
- name: Analyse statique
|
||||||
|
run: uv run ruff check --output-format=github .
|
||||||
|
|
||||||
|
- name: Typage
|
||||||
|
run: uv run mypy enervision_ml tests
|
||||||
|
|
||||||
|
# Les tests exigeant une base portent le marqueur `integration`, écarté par défaut et
|
||||||
|
# joué par le job `integration` ci-dessous.
|
||||||
|
- name: Tests et couverture
|
||||||
|
run: uv run pytest --cov-report=xml
|
||||||
|
|
||||||
|
- name: Verse la couverture pour Sonar
|
||||||
|
uses: actions/upload-artifact@v7
|
||||||
|
with:
|
||||||
|
name: ml-coverage
|
||||||
|
path: ml/coverage.xml
|
||||||
|
if-no-files-found: error
|
||||||
|
|
||||||
|
# Piège : le schéma de la base ML est celui du backend (apps/backend/alembic, propriétaire du
|
||||||
|
# schéma). Le reconstruire ici à la main rendrait ce job vert sur une base qui n'est pas la nôtre.
|
||||||
|
integration:
|
||||||
|
name: ML - DB et chaîne ML - DB - API
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 20
|
||||||
|
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: timescale/timescaledb-ha:pg17
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: enervision
|
||||||
|
POSTGRES_PASSWORD: change_me
|
||||||
|
POSTGRES_DB: enervision_test
|
||||||
|
ports:
|
||||||
|
- "5433:5432"
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U enervision -d enervision_test"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 12
|
||||||
|
--health-start-period 40s
|
||||||
|
|
||||||
|
env:
|
||||||
|
# Deux variables, deux dialectes : Alembic et l'API parlent asyncpg, le pipeline ML parle
|
||||||
|
# psycopg en synchrone. Cf. docs/ML-START.md, section 1.
|
||||||
|
DATABASE_URL: postgresql+asyncpg://enervision:change_me@localhost:5433/enervision_test
|
||||||
|
ML_DATABASE_URL: postgresql+psycopg://enervision:change_me@localhost:5433/enervision_test
|
||||||
|
APP_SECRET_KEY: secret-de-test-assez-long-pour-le-validateur
|
||||||
|
PGPASSWORD: change_me
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: true
|
||||||
|
cache-dependency-glob: |
|
||||||
|
ml/uv.lock
|
||||||
|
apps/backend/uv.lock
|
||||||
|
prune-cache: false
|
||||||
|
|
||||||
|
- name: Installe l'interpréteur déclaré par .python-version
|
||||||
|
working-directory: ml
|
||||||
|
run: uv python install
|
||||||
|
|
||||||
|
- name: Synchronise le pipeline ML sur le verrou
|
||||||
|
working-directory: ml
|
||||||
|
run: uv sync --all-groups --locked
|
||||||
|
|
||||||
|
# Le backend est installé ici parce qu'il porte les migrations, seule source du schéma, et
|
||||||
|
# le test de chaîne, qui interroge l'API.
|
||||||
|
- name: Synchronise le backend sur le verrou
|
||||||
|
working-directory: apps/backend
|
||||||
|
run: uv sync --all-groups --locked
|
||||||
|
|
||||||
|
# db/init/110-test-database.sql n'est pas monté ici, et sans l'extension la première
|
||||||
|
# révision Alembic refuse de s'appliquer.
|
||||||
|
- name: Active TimescaleDB sur la base de test
|
||||||
|
run: psql -h localhost -p 5433 -U enervision -d enervision_test -c "CREATE EXTENSION IF NOT EXISTS timescaledb"
|
||||||
|
|
||||||
|
- name: Applique les migrations du backend, propriétaire du schéma
|
||||||
|
working-directory: apps/backend
|
||||||
|
run: uv run alembic upgrade head
|
||||||
|
|
||||||
|
# Couverture désactivée : ce job ne joue qu'une partie de la suite, son taux n'aurait pas
|
||||||
|
# de sens (même raison que backend.yml).
|
||||||
|
- name: Tests ML exigeant une base
|
||||||
|
working-directory: ml
|
||||||
|
run: uv run pytest -m integration --no-cov
|
||||||
|
|
||||||
|
# Lance les vrais binaires enervision_ml.train et .score en sous-processus, comme les DAGs
|
||||||
|
# ml_train et ml_score, puis relit le résultat par GET /api/v1/predictions.
|
||||||
|
- name: Chaîne complète ML vers DB vers API
|
||||||
|
working-directory: apps/backend
|
||||||
|
env:
|
||||||
|
ML_PYTHON: ${{ github.workspace }}/ml/.venv/bin/python
|
||||||
|
run: uv run pytest -m chaine --no-cov
|
||||||
|
|
||||||
|
sast:
|
||||||
|
name: Analyse statique de sécurité
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 10
|
||||||
|
defaults:
|
||||||
|
run:
|
||||||
|
working-directory: ml
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Récupère le dépôt
|
||||||
|
uses: actions/checkout@v7
|
||||||
|
|
||||||
|
# Pourquoi : pas de cache ici. uvx n'installe pas le projet, le verrou n'alimente donc
|
||||||
|
# aucune clé de cache ; la seule roue téléchargée est celle de Bandit.
|
||||||
|
- name: Installe uv
|
||||||
|
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
||||||
|
with:
|
||||||
|
enable-cache: false
|
||||||
|
|
||||||
|
- name: Analyse le code livré (bloquant à partir de MEDIUM)
|
||||||
|
run: uvx bandit==1.9.4 --recursive enervision_ml --severity-level medium --confidence-level medium
|
||||||
|
|
||||||
|
- name: Rapport complet, tous niveaux
|
||||||
|
continue-on-error: true
|
||||||
|
run: uvx bandit==1.9.4 --recursive enervision_ml
|
||||||
+24
-4
@@ -22,6 +22,12 @@ apps/frontend/.angular/
|
|||||||
npm-debug.log*
|
npm-debug.log*
|
||||||
yarn-error.log*
|
yarn-error.log*
|
||||||
|
|
||||||
|
# Tests de bout en bout et de charge : rapports générés et identifiants des comptes de test
|
||||||
|
playwright-report/
|
||||||
|
blob-report/
|
||||||
|
tests/e2e/.comptes.json
|
||||||
|
tests/load/results/
|
||||||
|
|
||||||
# Terraform
|
# Terraform
|
||||||
.terraform/
|
.terraform/
|
||||||
# .terraform.lock.hcl est versionne (pas ignore) pour figer les versions de provider entre contributeurs/CI
|
# .terraform.lock.hcl est versionne (pas ignore) pour figer les versions de provider entre contributeurs/CI
|
||||||
@@ -40,7 +46,6 @@ kubeconfig
|
|||||||
# Airflow
|
# Airflow
|
||||||
etl/airflow/logs/
|
etl/airflow/logs/
|
||||||
airflow.db
|
airflow.db
|
||||||
airflow-webserver.pid
|
|
||||||
standalone_admin_password.txt
|
standalone_admin_password.txt
|
||||||
|
|
||||||
# Environnement et secrets
|
# Environnement et secrets
|
||||||
@@ -52,10 +57,25 @@ standalone_admin_password.txt
|
|||||||
secrets/
|
secrets/
|
||||||
|
|
||||||
# Donnees locales
|
# Donnees locales
|
||||||
data/
|
data/raw/*
|
||||||
|
!data/raw/.gitkeep
|
||||||
*.sqlite3
|
*.sqlite3
|
||||||
monitoring/grafana/data/
|
|
||||||
monitoring/prometheus/data/
|
# ML : jeu de donnees, modeles entraines et suivi MLflow local, tous generes/volumineux
|
||||||
|
ml/data/
|
||||||
|
ml/models/*
|
||||||
|
!ml/models/.gitkeep
|
||||||
|
ml/mlruns/
|
||||||
|
ml/mlartifacts/
|
||||||
|
ml/mlflow.db*
|
||||||
|
ml/.env
|
||||||
|
|
||||||
|
# Airflow : base sqlite locale generee par les tests d'integrite des DAGs (etl/airflow/tests)
|
||||||
|
etl/airflow/tests/.airflow_home/
|
||||||
|
|
||||||
|
# TLS : certificats du reverse proxy, générés par script ou par certbot
|
||||||
|
infra/proxy/tls/*.pem
|
||||||
|
infra/proxy/acme/
|
||||||
|
|
||||||
# IDE et OS
|
# IDE et OS
|
||||||
.idea/
|
.idea/
|
||||||
|
|||||||
@@ -1,15 +1,94 @@
|
|||||||
BACKEND := apps/backend
|
BACKEND := apps/backend
|
||||||
FRONTEND := apps/frontend
|
FRONTEND := apps/frontend
|
||||||
|
ML := ml
|
||||||
|
AIRFLOW := etl/airflow
|
||||||
|
E2E := tests/e2e
|
||||||
|
COMPOSE_PROD := docker compose -f docker-compose.yml -f docker-compose.prod.yml
|
||||||
|
|
||||||
|
# Piège : sans `export`, une valeur passée en ligne de commande n'atteindrait pas docker compose.
|
||||||
|
# PUBLIC_HOST retombe sur le `.env`, que make ne lit pas, puis sur la valeur de `.env.example`.
|
||||||
|
PUBLIC_HOST ?= $(shell sed -n 's/^PUBLIC_HOST=//p' .env 2>/dev/null | tail -1)
|
||||||
|
PUBLIC_HOST := $(or $(strip $(PUBLIC_HOST)),enervision.local)
|
||||||
|
export PUBLIC_HOST
|
||||||
|
ifdef ACME_EMAIL
|
||||||
|
export ACME_EMAIL
|
||||||
|
endif
|
||||||
|
|
||||||
|
# Piege : make ne lit pas `.env`, que seul docker compose interpole. Les cibles hors conteneur
|
||||||
|
# (ml-*, demo-data, db-wait) joignent la base par le port publie et ont besoin de ces valeurs.
|
||||||
|
env-val = $(shell sed -n 's/^$(1)=//p' .env 2>/dev/null | tail -1)
|
||||||
|
PG_USER := $(or $(strip $(call env-val,POSTGRES_USER)),enervision)
|
||||||
|
PG_PASSWORD := $(or $(strip $(call env-val,POSTGRES_PASSWORD)),change_me)
|
||||||
|
PG_DB := $(or $(strip $(call env-val,POSTGRES_DB)),enervision)
|
||||||
|
PG_PORT := $(or $(strip $(call env-val,POSTGRES_PORT)),5433)
|
||||||
|
ml-env-val = $(shell sed -n 's/^$(1)=//p' ml/.env 2>/dev/null | tail -1)
|
||||||
|
ML_ENV_DB_PASSWORD := $(call ml-env-val,MLFLOW_DB_PASSWORD)
|
||||||
|
AIRFLOW_PORT := $(or $(strip $(call env-val,AIRFLOW_PORT)),8080)
|
||||||
|
MAILPIT_UI_PORT := $(or $(strip $(call env-val,MAILPIT_UI_PORT)),8025)
|
||||||
|
ML_DATABASE_URL ?= postgresql+psycopg://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_DB)
|
||||||
|
export ML_DATABASE_URL
|
||||||
|
|
||||||
|
# Piege : la base des tests d'integration n'est pas la base de developpement. Ces tests ecrivent
|
||||||
|
# et suppriment des lignes, et leurs fixtures refusent de demarrer ailleurs que sur
|
||||||
|
# `enervision_test` (garde sur le nom, cf. ml/tests/conftest.py).
|
||||||
|
PG_TEST_DB ?= enervision_test
|
||||||
|
TEST_DATABASE_URL ?= postgresql+asyncpg://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_TEST_DB)
|
||||||
|
ML_TEST_DATABASE_URL ?= postgresql+psycopg://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_TEST_DB)
|
||||||
|
|
||||||
|
# Piege : ni make ni ces cibles ne lisent `.env` pour COMPOSE_PROFILES, que docker compose y lit
|
||||||
|
# seul. `stack-up` le relit ici pour savoir s'il doit poser le role `supervision` apres migration.
|
||||||
|
SUPERVISION := $(findstring monitoring,$(COMPOSE_PROFILES) $(call env-val,COMPOSE_PROFILES))
|
||||||
|
SERVICES_SUPERVISION := prometheus alertmanager grafana postgres-exporter node-exporter cadvisor
|
||||||
|
GRAFANA_PORT := $(or $(strip $(call env-val,GRAFANA_PORT)),3001)
|
||||||
|
PROMETHEUS_PORT := $(or $(strip $(call env-val,PROMETHEUS_PORT)),9090)
|
||||||
|
supervision-garde = for cle in APP_METRICS_TOKEN GRAFANA_ADMIN_PASSWORD SUPERVISION_DB_PASSWORD GARAGE_METRICS_TOKEN; do \
|
||||||
|
sed -n "s/^$$cle=//p" .env 2>/dev/null | tail -1 | grep -q . \
|
||||||
|
|| { echo "$$cle manquant dans .env, requis par la supervision (cf. .env.example)"; exit 1; }; \
|
||||||
|
done
|
||||||
|
MONITORING := docker compose --profile monitoring
|
||||||
|
|
||||||
|
# Piege : l'image Garage n'a pas de shell, elle ne peut pas porter sa garde comme grafana ou
|
||||||
|
# airflow-init. Un secret vide ou laisse a change_me la ferait redemarrer en boucle (ADR 0019).
|
||||||
|
CLES_GARAGE := GARAGE_RPC_SECRET GARAGE_ADMIN_TOKEN GARAGE_METRICS_TOKEN GARAGE_ACCESS_KEY GARAGE_SECRET_KEY GARAGE_SSE_KEY
|
||||||
|
garage-garde = for cle in $(CLES_GARAGE); do \
|
||||||
|
sed -n "s/^$$cle=//p" .env 2>/dev/null | tail -1 | grep -qv '^change_me$$' \
|
||||||
|
|| { echo "$$cle manquant ou laisse a change_me dans .env, requis par Garage (cf. .env.example)"; exit 1; }; \
|
||||||
|
done
|
||||||
|
PROMTOOL := $(MONITORING) run --rm --no-deps --entrypoint promtool prometheus
|
||||||
|
|
||||||
|
# Piege : `e2e-prepare` ajoute trois sites `demo-*` et des comptes `test-*` a la base visee. Elle
|
||||||
|
# vise la base de `make dev` ; ne jamais la lancer contre la recette ou la prod.
|
||||||
|
E2E_COMPTES ?= $(CURDIR)/$(E2E)/.comptes.json
|
||||||
|
E2E_API ?= http://localhost:$(or $(strip $(call env-val,BACKEND_PORT)),8000)
|
||||||
|
|
||||||
|
# Piege : `run` ne demarre que k6, la stack doit deja tourner. `--user` fait ecrire les rapports
|
||||||
|
# de tests/load/results avec l'uid du poste, pas celui de l'image (12345), qui n'y a pas acces.
|
||||||
|
k6-run = mkdir -p tests/load/results && $(COMPOSE_PROD) --profile load run --rm \
|
||||||
|
--user "$$(id -u):$$(id -g)" -e K6_WEB_DASHBOARD=true \
|
||||||
|
-e K6_WEB_DASHBOARD_EXPORT=/results/$(1)-$$(date +%Y%m%dT%H%M%S).html \
|
||||||
|
k6 run /scripts/$(1).js
|
||||||
|
|
||||||
|
# Le jeu historique s'arrete au 31/12/2024 : score et detection ancres a l'horloge reelle ne
|
||||||
|
# verraient qu'un parc muet depuis des mois. Cf. `--now` de enervision_ml.score.
|
||||||
|
DEMO_NOW ?= 2024-12-31T00:00:00Z
|
||||||
|
|
||||||
.DEFAULT_GOAL := help
|
.DEFAULT_GOAL := help
|
||||||
.PHONY: help install install-backend install-frontend dev dev-backend dev-frontend \
|
.PHONY: help install install-backend install-frontend install-ml install-airflow \
|
||||||
lint format typecheck test test-cov test-integration check \
|
dev dev-backend dev-frontend \
|
||||||
openapi docker-build db-up db-down db-reset db-logs db-psql migrate bootstrap-admin
|
lint format typecheck test test-cov test-integration ml-test-integration \
|
||||||
|
test-chaine check \
|
||||||
|
openapi docker-build db-up db-down db-reset db-logs db-psql db-wait db-ensure-airflow \
|
||||||
|
migrate migrate-test bootstrap-admin services-up demo-data demo-data-force \
|
||||||
|
ml-lint ml-typecheck ml-test ml-check ml-train ml-score mlflow-up detect-alerts recommendations \
|
||||||
|
airflow-lint airflow-test airflow-check airflow-up airflow-down airflow-logs \
|
||||||
|
tls-selfsigned tls-acme tls-renew tls-dns01 front-up stack-up stack-down stack-logs \
|
||||||
|
e2e-install e2e-prepare e2e load-smoke load-test load-stress load-limits \
|
||||||
|
db-ensure-supervision monitoring-up monitoring-down monitoring-logs monitoring-check
|
||||||
|
|
||||||
help: ## Liste les cibles disponibles
|
help: ## Liste les cibles disponibles
|
||||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}'
|
@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}'
|
||||||
|
|
||||||
install: install-backend install-frontend ## Installe les dépendances backend et frontend
|
install: install-backend install-frontend install-ml install-airflow ## Installe les dépendances backend, frontend, ML et Airflow
|
||||||
|
|
||||||
install-backend: ## Installe les dépendances du backend
|
install-backend: ## Installe les dépendances du backend
|
||||||
cd $(BACKEND) && uv sync --all-groups
|
cd $(BACKEND) && uv sync --all-groups
|
||||||
@@ -17,12 +96,26 @@ install-backend: ## Installe les dépendances du backend
|
|||||||
install-frontend: ## Installe les dépendances du frontend
|
install-frontend: ## Installe les dépendances du frontend
|
||||||
cd $(FRONTEND) && npm ci
|
cd $(FRONTEND) && npm ci
|
||||||
|
|
||||||
dev: ## Lance toute la stack (backend + frontend) en rechargement à chaud
|
install-ml: ## Installe les dépendances du pipeline ML
|
||||||
|
cd $(ML) && uv sync --all-groups
|
||||||
|
|
||||||
|
install-airflow: ## Installe les dépendances de lint/test des DAGs Airflow
|
||||||
|
cd $(AIRFLOW) && uv sync --all-groups
|
||||||
|
|
||||||
|
dev: services-up migrate demo-data ## Lance toute la stack : base, Mailpit, Airflow, puis backend et frontend
|
||||||
|
@echo "airflow -> http://localhost:$(AIRFLOW_PORT) mailpit -> http://localhost:$(MAILPIT_UI_PORT)"
|
||||||
@trap 'kill 0' EXIT INT TERM; \
|
@trap 'kill 0' EXIT INT TERM; \
|
||||||
$(MAKE) --no-print-directory dev-backend & \
|
$(MAKE) --no-print-directory dev-backend & \
|
||||||
$(MAKE) --no-print-directory dev-frontend & \
|
$(MAKE) --no-print-directory dev-frontend & \
|
||||||
wait
|
wait
|
||||||
|
|
||||||
|
services-up: ## Démarre les services conteneurisés dont `make dev` dépend (base, Mailpit, Garage, Airflow)
|
||||||
|
@$(garage-garde)
|
||||||
|
docker compose up -d db mailpit garage
|
||||||
|
@$(MAKE) --no-print-directory db-wait
|
||||||
|
@$(MAKE) --no-print-directory db-ensure-airflow
|
||||||
|
docker compose up -d airflow-init airflow-apiserver airflow-scheduler airflow-dag-processor
|
||||||
|
|
||||||
dev-backend: ## Lance l'API seule en rechargement à chaud
|
dev-backend: ## Lance l'API seule en rechargement à chaud
|
||||||
@echo "backend -> http://localhost:8000 (docs sur /docs)"
|
@echo "backend -> http://localhost:8000 (docs sur /docs)"
|
||||||
cd $(BACKEND) && uv run uvicorn app.main:create_app --factory --reload --host 0.0.0.0 --port 8000
|
cd $(BACKEND) && uv run uvicorn app.main:create_app --factory --reload --host 0.0.0.0 --port 8000
|
||||||
@@ -55,9 +148,179 @@ check: lint typecheck test ## Chaîne de vérification complète
|
|||||||
openapi: ## Régénère apps/backend/openapi.json depuis les routes déclarées
|
openapi: ## Régénère apps/backend/openapi.json depuis les routes déclarées
|
||||||
cd $(BACKEND) && uv run python -m app.cli export-openapi
|
cd $(BACKEND) && uv run python -m app.cli export-openapi
|
||||||
|
|
||||||
|
ml-lint: ## Analyse statique du pipeline ML
|
||||||
|
cd $(ML) && uv run ruff check .
|
||||||
|
|
||||||
|
ml-typecheck: ## Vérifie le typage du pipeline ML
|
||||||
|
cd $(ML) && uv run mypy enervision_ml tests
|
||||||
|
|
||||||
|
ml-test: ## Exécute les tests du pipeline ML (donnees synthetiques, sans base ni serveur MLflow)
|
||||||
|
cd $(ML) && uv run pytest
|
||||||
|
|
||||||
|
ml-check: ml-lint ml-typecheck ml-test ## Chaîne de vérification complète du pipeline ML
|
||||||
|
|
||||||
|
# La cible surcharge ML_DATABASE_URL, que ce Makefile exporte vers la base de développement : la
|
||||||
|
# garde du conftest ferait échouer la cible sans cette surcharge.
|
||||||
|
ml-test-integration: ML_DATABASE_URL := $(ML_TEST_DATABASE_URL)
|
||||||
|
ml-test-integration: ## Tests ML exigeant une base migrée. Faire `make db-up migrate-test` avant
|
||||||
|
cd $(ML) && uv run pytest -m integration --no-cov
|
||||||
|
|
||||||
|
test-chaine: ## Chaîne ML -> DB -> API, vrais binaires. Exige les deux environnements uv
|
||||||
|
cd $(BACKEND) && DATABASE_URL=$(TEST_DATABASE_URL) ML_PYTHON=$(CURDIR)/$(ML)/.venv/bin/python \
|
||||||
|
uv run pytest -m chaine --no-cov
|
||||||
|
|
||||||
|
ml-train: ## Entraine le modele LightGBM. CSV=chemin optionnel, sinon lit ML_DATABASE_URL
|
||||||
|
cd $(ML) && uv run python -m enervision_ml.train $(if $(CSV),--csv $(CSV),)
|
||||||
|
|
||||||
|
ml-score: ## Score le prochain pas horaire et l'ecrit dans `prediction`. CSV= et NOW= optionnels
|
||||||
|
cd $(ML) && uv run python -m enervision_ml.score $(if $(CSV),--csv $(CSV),) $(if $(NOW),--now $(NOW),)
|
||||||
|
|
||||||
|
mlflow-up: ## Démarre le serveur MLflow (tracking + registry) en conteneur. ml/.env requis
|
||||||
|
@test -n "$(strip $(ML_ENV_DB_PASSWORD))" \
|
||||||
|
|| { echo "MLFLOW_DB_PASSWORD absente de ml/.env (copier ml/.env.example)"; exit 1; }
|
||||||
|
@echo "$(ML_ENV_DB_PASSWORD)" | grep -qE '^[A-Za-z0-9]+$$' \
|
||||||
|
|| { echo "MLFLOW_DB_PASSWORD doit contenir uniquement lettres et chiffres (interpolee dans l'URI postgresql://)"; exit 1; }
|
||||||
|
cd $(ML) && docker compose -f docker-compose.mlflow.yml up -d --build
|
||||||
|
@echo "mlflow -> http://localhost:5000"
|
||||||
|
|
||||||
|
detect-alerts: ## Détecte les alertes internes depuis les lectures en base. SITE= et NOW= optionnels
|
||||||
|
cd $(BACKEND) && uv run python -m app.detection.internal_alerts $(if $(SITE),--site-id $(SITE),) $(if $(NOW),--now $(NOW),)
|
||||||
|
|
||||||
|
recommendations: ## Genere les recommandations depuis les alertes en base. SITE=identifiant optionnel
|
||||||
|
cd $(BACKEND) && uv run python -m app.cli generate-recommendations $(if $(SITE),--site-id $(SITE),)
|
||||||
|
|
||||||
|
airflow-lint: ## Analyse statique des DAGs Airflow
|
||||||
|
cd $(AIRFLOW) && uv run ruff check .
|
||||||
|
|
||||||
|
airflow-test: ## Verifie que les DAGs s'importent sans erreur et ont la structure attendue
|
||||||
|
cd $(AIRFLOW) && uv run pytest
|
||||||
|
|
||||||
|
airflow-check: airflow-lint airflow-test ## Chaîne de vérification complète des DAGs Airflow
|
||||||
|
|
||||||
|
airflow-up: db-ensure-airflow ## Démarre Airflow (api-server + scheduler + dag-processor, LocalExecutor). db-up requis avant.
|
||||||
|
docker compose up -d airflow-init airflow-apiserver airflow-scheduler airflow-dag-processor
|
||||||
|
@echo "airflow -> http://localhost:$${AIRFLOW_PORT:-8080}"
|
||||||
|
|
||||||
|
airflow-down: ## Arrête l'api-server, le scheduler et le dag-processor Airflow
|
||||||
|
docker compose stop airflow-apiserver airflow-scheduler airflow-dag-processor
|
||||||
|
|
||||||
|
airflow-logs: ## Suit les journaux du scheduler Airflow (où tournent les tâches, LocalExecutor)
|
||||||
|
docker compose logs -f airflow-scheduler
|
||||||
|
|
||||||
docker-build: ## Construit l'image du backend
|
docker-build: ## Construit l'image du backend
|
||||||
docker build -t enervision-backend:local $(BACKEND)
|
docker build -t enervision-backend:local $(BACKEND)
|
||||||
|
|
||||||
|
tls-selfsigned: ## Génère le certificat de démonstration. PUBLIC_HOST=..., FORCE=1 pour écraser
|
||||||
|
./scripts/tls-selfsigned.sh $(if $(FORCE),--force,)
|
||||||
|
|
||||||
|
# Piège : l'image backend ne migre pas au démarrage, et `/health/ready` ne teste que la connexion
|
||||||
|
# et l'extension. Sans `alembic upgrade head`, la stack démarre verte sur une base sans schéma.
|
||||||
|
stack-up: ## Démarre la stack derrière le reverse proxy, puis migre la base. PUBLIC_HOST=... au besoin
|
||||||
|
@test -f infra/proxy/tls/fullchain.pem \
|
||||||
|
|| { echo "Aucun certificat dans infra/proxy/tls. Lancer d'abord make tls-selfsigned"; exit 1; }
|
||||||
|
@openssl x509 -in infra/proxy/tls/fullchain.pem -noout -checkhost "$(PUBLIC_HOST)" >/dev/null \
|
||||||
|
|| { echo "Le certificat ne couvre pas $(PUBLIC_HOST). Relancer make tls-selfsigned PUBLIC_HOST=$(PUBLIC_HOST) FORCE=1"; exit 1; }
|
||||||
|
@$(if $(SUPERVISION),$(supervision-garde),true)
|
||||||
|
@$(garage-garde)
|
||||||
|
$(COMPOSE_PROD) up -d --build
|
||||||
|
$(COMPOSE_PROD) exec -T backend alembic upgrade head
|
||||||
|
@$(if $(SUPERVISION),$(MAKE) --no-print-directory db-ensure-supervision,true)
|
||||||
|
|
||||||
|
stack-down: ## Arrête la stack complète en conservant les données
|
||||||
|
$(COMPOSE_PROD) stop
|
||||||
|
|
||||||
|
stack-logs: ## Suit les journaux du reverse proxy
|
||||||
|
$(COMPOSE_PROD) logs -f proxy
|
||||||
|
|
||||||
|
tls-acme: ## Demande un certificat Let's Encrypt. PUBLIC_HOST public et ACME_EMAIL requis
|
||||||
|
@test "$(PUBLIC_HOST)" != enervision.local \
|
||||||
|
|| { echo "PUBLIC_HOST doit être un domaine public résolvable, pas le nom de démonstration"; exit 1; }
|
||||||
|
$(COMPOSE_PROD) --profile acme run --rm certbot certonly --webroot -w /var/www/certbot \
|
||||||
|
-d $(PUBLIC_HOST) \
|
||||||
|
--email $${ACME_EMAIL:?ACME_EMAIL=... requis} \
|
||||||
|
--agree-tos --no-eff-email --deploy-hook /deploy-hook.sh
|
||||||
|
$(COMPOSE_PROD) exec proxy nginx -s reload
|
||||||
|
|
||||||
|
tls-renew: ## Renouvelle les certificats Let's Encrypt et recharge le proxy
|
||||||
|
$(COMPOSE_PROD) --profile acme run --rm certbot renew --deploy-hook /deploy-hook.sh
|
||||||
|
$(COMPOSE_PROD) exec proxy nginx -s reload
|
||||||
|
|
||||||
|
# Pourquoi : la VM n'a qu'une IP privée, que Let's Encrypt ne joint pas ; le défi DNS-01 passe
|
||||||
|
# par l'API du fournisseur DNS, dynv6 par défaut (ADR 0018). Le jeton ne passe jamais par `argv`.
|
||||||
|
ACME_SH := neilpang/acme.sh:3.1.6
|
||||||
|
DNS01_API ?= dns_dynv6
|
||||||
|
DNS01_JETON_VAR ?= DYNV6_TOKEN
|
||||||
|
DNS01_JETON_FICHIER ?= $(abspath $(CURDIR)/../dns.token)
|
||||||
|
acme-sh = docker run --rm --user "$$(id -u):$$(id -g)" -e $(DNS01_JETON_VAR) -e AUTO_UPGRADE=0 \
|
||||||
|
-v "$(CURDIR)/infra/proxy/acme:/acme.sh" -v "$(CURDIR)/infra/proxy/tls:/tls" $(ACME_SH)
|
||||||
|
|
||||||
|
# acme.sh sort en 2 quand le certificat n'est pas à renouveler, et recopie le jeton dans
|
||||||
|
# acme/account.conf, d'où le chmod. `--dnssleep` : Let's Encrypt valide depuis plusieurs réseaux.
|
||||||
|
tls-dns01: ## Certificat Let's Encrypt par DNS-01, renouvelé seulement à échéance. Jeton : ../dns.token
|
||||||
|
@case "$(PUBLIC_HOST)" in *.local | localhost) echo "PUBLIC_HOST=$(PUBLIC_HOST) n'est pas un nom public"; exit 1 ;; esac
|
||||||
|
@test -r "$(DNS01_JETON_FICHIER)" || { echo "Jeton DNS illisible : $(DNS01_JETON_FICHIER)"; exit 1; }
|
||||||
|
@mkdir -p infra/proxy/acme && chmod 700 infra/proxy/acme
|
||||||
|
@$(DNS01_JETON_VAR)="$$(tr -d '[:space:]' < "$(DNS01_JETON_FICHIER)")"; export $(DNS01_JETON_VAR); \
|
||||||
|
$(acme-sh) --issue --server letsencrypt --dns $(DNS01_API) --dnssleep 90 -d "$(PUBLIC_HOST)"; \
|
||||||
|
code=$$?; chmod -R go-rwx infra/proxy/acme; [ $$code -eq 0 ] || [ $$code -eq 2 ] || exit $$code
|
||||||
|
@$(acme-sh) --install-cert --ecc -d "$(PUBLIC_HOST)" \
|
||||||
|
--fullchain-file /tls/fullchain.pem --key-file /tls/privkey.pem
|
||||||
|
@$(COMPOSE_PROD) exec -T proxy nginx -s reload 2>/dev/null \
|
||||||
|
|| echo "Proxy arrêté : il lira le certificat à son démarrage"
|
||||||
|
|
||||||
|
front-up: ## Démarre ou recharge le frontal SNI de la VM, sur les ports 80 et 443 de l'hôte
|
||||||
|
docker compose -f infra/front/compose.yml up -d
|
||||||
|
docker compose -f infra/front/compose.yml exec -T front nginx -s reload
|
||||||
|
|
||||||
|
e2e-install: ## Installe Playwright et Chromium pour les tests de bout en bout
|
||||||
|
cd $(E2E) && npm ci && npx playwright install chromium
|
||||||
|
|
||||||
|
e2e-prepare: ## Sème le jeu de démonstration et crée les comptes de test sur la base de `make dev`
|
||||||
|
docker compose exec -T db psql -U $(PG_USER) -d $(PG_DB) -v ON_ERROR_STOP=1 < db/seeds/demo.sql
|
||||||
|
cd $(BACKEND) && BASE_URL=$(E2E_API) COMPTES_FICHIER=$(E2E_COMPTES) ADMIN_SUPPLEMENTAIRE=1 \
|
||||||
|
../../scripts/comptes-test.sh
|
||||||
|
|
||||||
|
e2e: ## Joue les parcours Playwright. E2E_BASE_URL= optionnel (défaut http://localhost:4200)
|
||||||
|
cd $(E2E) && E2E_COMPTES=$(E2E_COMPTES) npx playwright test
|
||||||
|
|
||||||
|
load-smoke: ## Tir k6 d'une minute. K6_EMAIL= et K6_PASSWORD= d'un lecteur, K6_BASE_URL= optionnel
|
||||||
|
$(call k6-run,smoke)
|
||||||
|
|
||||||
|
load-test: ## Charge nominale k6, 50 utilisateurs pendant 8 minutes. Rapport HTML dans tests/load/results
|
||||||
|
$(call k6-run,charge)
|
||||||
|
|
||||||
|
load-stress: ## Monte le débit jusqu'à la rupture de l'API. Sur la VM, la prod partage la machine
|
||||||
|
$(call k6-run,stress)
|
||||||
|
|
||||||
|
load-limits: ## Vérifie par le proxy que nginx limite le débit d'une même adresse (429)
|
||||||
|
$(call k6-run,limitation-debit)
|
||||||
|
|
||||||
|
# Piege : le mot de passe est lu dans `.env` par le shell et passe a psql sur son entree
|
||||||
|
# standard. Developpe par make, il apparaitrait en clair dans la ligne de commande (`ps`).
|
||||||
|
db-ensure-supervision: ## Crée ou réaligne le rôle `supervision`, en lecture seule, de Grafana et de l'exportateur
|
||||||
|
@mdp="$$(sed -n 's/^SUPERVISION_DB_PASSWORD=//p' .env 2>/dev/null | tail -1)"; \
|
||||||
|
[ -n "$$mdp" ] || { echo "SUPERVISION_DB_PASSWORD manquant dans .env"; exit 1; }; \
|
||||||
|
{ printf '\\set mot_de_passe %s\n' "$$mdp"; cat db/roles/supervision.sql; } \
|
||||||
|
| docker compose exec -T db psql -U $(PG_USER) -d $(PG_DB) -v ON_ERROR_STOP=1 -v base=$(PG_DB) -q
|
||||||
|
|
||||||
|
monitoring-up: ## Démarre la supervision sur la stack en cours : Prometheus, Alertmanager, Grafana, exporteurs
|
||||||
|
@$(supervision-garde)
|
||||||
|
$(MONITORING) up -d --no-deps $(SERVICES_SUPERVISION)
|
||||||
|
@$(MAKE) --no-print-directory db-ensure-supervision
|
||||||
|
@echo "grafana -> http://localhost:$(GRAFANA_PORT) prometheus -> http://localhost:$(PROMETHEUS_PORT)"
|
||||||
|
|
||||||
|
monitoring-down: ## Arrête la supervision en conservant ses données
|
||||||
|
$(MONITORING) stop $(SERVICES_SUPERVISION)
|
||||||
|
|
||||||
|
monitoring-logs: ## Suit les journaux de Prometheus, Alertmanager et Grafana
|
||||||
|
$(MONITORING) logs -f prometheus alertmanager grafana
|
||||||
|
|
||||||
|
monitoring-check: ## Valide la configuration de supervision et joue les tests des règles d'alerte, comme la CI
|
||||||
|
$(PROMTOOL) check config /etc/prometheus/prometheus.yml
|
||||||
|
$(PROMTOOL) test rules /etc/prometheus/tests/enervision.test.yml
|
||||||
|
$(MONITORING) run --rm --no-deps --entrypoint amtool alertmanager check-config /etc/alertmanager/alertmanager.yml
|
||||||
|
@for tableau in monitoring/grafana/dashboards/*.json; do jq empty "$$tableau" || exit 1; done
|
||||||
|
|
||||||
db-up: ## Démarre la base PostgreSQL TimescaleDB
|
db-up: ## Démarre la base PostgreSQL TimescaleDB
|
||||||
docker compose up -d db
|
docker compose up -d db
|
||||||
|
|
||||||
@@ -73,8 +336,39 @@ db-logs: ## Suit les journaux de la base
|
|||||||
db-psql: ## Ouvre une session psql sur la base applicative
|
db-psql: ## Ouvre une session psql sur la base applicative
|
||||||
docker compose exec db psql -U $${POSTGRES_USER:-enervision} -d $${POSTGRES_DB:-enervision}
|
docker compose exec db psql -U $${POSTGRES_USER:-enervision} -d $${POSTGRES_DB:-enervision}
|
||||||
|
|
||||||
|
db-wait: ## Attend que la base accepte les connexions
|
||||||
|
@for _ in $$(seq 1 60); do \
|
||||||
|
docker compose exec -T db pg_isready -U $(PG_USER) -d $(PG_DB) >/dev/null 2>&1 && exit 0; \
|
||||||
|
sleep 1; \
|
||||||
|
done; \
|
||||||
|
echo "La base n'accepte toujours pas de connexion apres 60s"; exit 1
|
||||||
|
|
||||||
|
# Piege : db/init ne rejoue qu'a la premiere initialisation du volume. Un `pgdata` cree avant
|
||||||
|
# db/init/120-airflow-database.sql n'a pas de base `airflow`, et airflow-init boucle dessus.
|
||||||
|
db-ensure-airflow: ## Crée la base de métadonnées Airflow si le volume pgdata est antérieur à db/init/120
|
||||||
|
@docker compose exec -T db psql -U $(PG_USER) -d postgres -tAc \
|
||||||
|
"SELECT 1 FROM pg_database WHERE datname = 'airflow'" | grep -q 1 \
|
||||||
|
|| docker compose exec -T db psql -U $(PG_USER) -d postgres -c "CREATE DATABASE airflow"
|
||||||
|
|
||||||
migrate: ## Applique les migrations Alembic
|
migrate: ## Applique les migrations Alembic
|
||||||
cd $(BACKEND) && uv run alembic upgrade head
|
cd $(BACKEND) && uv run alembic upgrade head
|
||||||
|
|
||||||
|
migrate-test: ## Applique les migrations sur enervision_test, la base des tests d'intégration
|
||||||
|
cd $(BACKEND) && DATABASE_URL=$(TEST_DATABASE_URL) uv run alembic upgrade head
|
||||||
|
|
||||||
bootstrap-admin: ## Crée le premier administrateur, mot de passe saisi au clavier
|
bootstrap-admin: ## Crée le premier administrateur, mot de passe saisi au clavier
|
||||||
cd $(BACKEND) && uv run python -m app.cli create-admin --email $${EMAIL:?EMAIL=... requis}
|
cd $(BACKEND) && uv run python -m app.cli create-admin --email $${EMAIL:?EMAIL=... requis}
|
||||||
|
|
||||||
|
demo-data: ## Renseigne prédictions, alertes et recommandations si elles manquent. NOW= optionnel
|
||||||
|
@nombre=$$(docker compose exec -T db psql -U $(PG_USER) -d $(PG_DB) -tAc 'SELECT count(*) FROM alert') \
|
||||||
|
|| { echo "demo-data : base injoignable ou migrations non appliquees"; exit 1; }; \
|
||||||
|
if [ "$$nombre" = 0 ]; then \
|
||||||
|
$(MAKE) --no-print-directory demo-data-force; \
|
||||||
|
else \
|
||||||
|
echo "demo-data : $$nombre alerte(s) deja en base (make demo-data-force pour rejouer)"; \
|
||||||
|
fi
|
||||||
|
|
||||||
|
demo-data-force: ## Rejoue le peuplement sans regarder l'existant. Les trois etapes sont idempotentes
|
||||||
|
$(MAKE) --no-print-directory ml-score NOW=$(DEMO_NOW)
|
||||||
|
$(MAKE) --no-print-directory detect-alerts NOW=$(DEMO_NOW)
|
||||||
|
$(MAKE) --no-print-directory recommendations
|
||||||
|
|||||||
@@ -9,27 +9,34 @@ series temporelles energetiques, deployee sur une machine on-premise.
|
|||||||
|-------|----------------------------------------------------------|
|
|-------|----------------------------------------------------------|
|
||||||
| J1 | Valider la préparation de l'environnement et du repo |
|
| J1 | Valider la préparation de l'environnement et du repo |
|
||||||
| J2 | Valider le périmètre retenu et les choix technologiques |
|
| J2 | Valider le périmètre retenu et les choix technologiques |
|
||||||
| J3 | Valider l'architecture et la gestion de la sécurité |
|
| J3 | Ingestion & backend |
|
||||||
| J4 | Valider la robustesse et assurer les livrables |
|
| J4 | Architecture, sécurité & frontend |
|
||||||
|
| J5 | Valider la robustesse et assurer les livrables |
|
||||||
|
| J6 | Amélioration possible |
|
||||||
|
|
||||||
Ce que la documentation apporte à chacun : [docs/architecture/00-vue-ensemble.md](docs/architecture/00-vue-ensemble.md).
|
Ce que la documentation apporte à chacun : [docs/architecture/00-vue-ensemble.md](docs/architecture/00-vue-ensemble.md).
|
||||||
|
|
||||||
## Stack cible
|
## Stack
|
||||||
|
|
||||||
| Domaine | Technologie | Emplacement | Etat |
|
| Domaine | Technologie | Emplacement | Etat |
|
||||||
|------------|-------------------------------------|---------------------|---------------|
|
|------------|-------------------------------------|---------------------|---------------|
|
||||||
| Backend | FastAPI, Python 3.14 | `apps/backend` | Initialise |
|
| Backend | FastAPI, Python 3.14 | `apps/backend` | En place |
|
||||||
| Frontend | Angular 22, Node 24 LTS | `apps/frontend` | Tableau de bord |
|
| Frontend | Angular 22, Node 26 | `apps/frontend` | En place |
|
||||||
| Base | PostgreSQL 17 + TimescaleDB | `db` | Initialise |
|
| Base | PostgreSQL 17 + TimescaleDB | `db` | En place |
|
||||||
| ETL | Apache Airflow | `etl/airflow` | A initialiser |
|
| ETL | Apache Airflow | `etl/airflow` | Sept DAGs |
|
||||||
| Infra | Terraform (k3s single-node) | `infra/terraform` | Initialise |
|
| Infra | Terraform (VM ENI ; module k3s) | `infra/terraform` | VM appliquée, k3s écrit non appliqué |
|
||||||
| CI/CD | GitHub Actions | `.github/workflows` | Backend en place |
|
| Reverse proxy | Nginx, TLS, frontal SNI | `infra/proxy`, `infra/front` | En place, certificats Let's Encrypt |
|
||||||
| Monitoring | Prometheus, Grafana, Alertmanager | `monitoring` | A initialiser |
|
| CI/CD | GitHub Actions | `.github/workflows` | En place |
|
||||||
|
| Monitoring | Prometheus, Grafana, Alertmanager | `monitoring` | En place, profil Compose |
|
||||||
|
| Stockage objet | Garage (S3), un par environnement | `infra/garage` | En place, archives de `reading` |
|
||||||
|
| Tests e2e et de charge | Playwright, k6 | `tests` | En place |
|
||||||
|
| ML | LightGBM, MLflow | `ml` | En place |
|
||||||
|
|
||||||
Le backend, la base et l'infrastructure (Terraform/k3s) sont initialises a ce stade. Le frontend
|
Toutes ces briques tournent sur la machine du groupe, en trois environnements (production,
|
||||||
sert un tableau de bord sur `/dashboard`, dont les données proviennent de fixtures : les endpoints
|
recette, dev). Le frontend sert le tableau de bord, les vues sites, recommandations et
|
||||||
correspondants restent à écrire côté API. Les autres dossiers portent l'arborescence et un README
|
supervision des capteurs, toutes branchées sur l'API réelle : les fixtures sont coupées
|
||||||
de cadrage, leur contenu fait l'objet d'un ticket dedie.
|
(`useMockFixtures: false`). Le module Terraform k3s reste une cible, écrite et validée, jamais
|
||||||
|
appliquée.
|
||||||
|
|
||||||
L'etat detaille de chaque brique et les vues d'architecture sont dans
|
L'etat detaille de chaque brique et les vues d'architecture sont dans
|
||||||
[docs/architecture](docs/architecture/README.md).
|
[docs/architecture](docs/architecture/README.md).
|
||||||
@@ -44,51 +51,91 @@ L'etat detaille de chaque brique et les vues d'architecture sont dans
|
|||||||
├── db/
|
├── db/
|
||||||
│ ├── init/ Bootstrap PostgreSQL + TimescaleDB
|
│ ├── init/ Bootstrap PostgreSQL + TimescaleDB
|
||||||
│ ├── migrations/ Migrations SQL versionnees
|
│ ├── migrations/ Migrations SQL versionnees
|
||||||
│ └── seeds/ Jeux de donnees de reference
|
│ ├── roles/ Roles PostgreSQL hors schema (supervision)
|
||||||
|
│ └── seeds/ Jeu de demonstration des tests
|
||||||
├── etl/airflow/
|
├── etl/airflow/
|
||||||
│ ├── dags/ DAGs d'ingestion et d'agregation
|
│ ├── dags/ DAGs d'orchestration (pipeline ML, alertes, imports, dérive, rétention)
|
||||||
│ ├── plugins/ Operateurs et hooks maison
|
│ ├── plugins/ Operateurs et hooks maison
|
||||||
│ ├── include/ Requetes SQL et ressources des DAGs
|
│ ├── include/ Requetes SQL et ressources des DAGs
|
||||||
│ └── tests/ Tests d'integrite des DAGs
|
│ └── tests/ Tests d'integrite des DAGs
|
||||||
├── infra/terraform/
|
├── infra/
|
||||||
|
│ ├── front/ Frontal SNI de la machine : ports 80 et 443, aiguillage par nom
|
||||||
|
│ ├── garage/ Stockage objet S3 : configuration sans secret
|
||||||
|
│ ├── proxy/ Reverse proxy Nginx : terminaison TLS et routage
|
||||||
|
│ └── terraform/
|
||||||
│ ├── modules/ Modules reutilisables
|
│ ├── modules/ Modules reutilisables
|
||||||
│ └── environments/ Racines Terraform, une par environnement
|
│ └── environments/ Racines Terraform, une par environnement
|
||||||
|
├── ml/ Pipeline d'entrainement LightGBM, suivi MLflow
|
||||||
├── monitoring/
|
├── monitoring/
|
||||||
│ ├── prometheus/ Collecte et regles d'alerte
|
│ ├── prometheus/ Collecte et regles d'alerte
|
||||||
│ ├── grafana/ Provisioning et dashboards
|
│ ├── grafana/ Provisioning et dashboards
|
||||||
│ └── alertmanager/ Routage des alertes
|
│ └── alertmanager/ Routage des alertes
|
||||||
├── docs/ ADR et vues d'architecture
|
├── tests/
|
||||||
|
│ ├── e2e/ Parcours Playwright contre la stack
|
||||||
|
│ ├── garage/ Tests de fumée S3 joués par la CI contre Garage
|
||||||
|
│ └── load/ Scenarios de charge k6
|
||||||
|
├── docs/ ADR, vues d'architecture, runbook de pilotage, livrables de rendu
|
||||||
└── scripts/ Outillage local
|
└── scripts/ Outillage local
|
||||||
```
|
```
|
||||||
|
|
||||||
## Demarrage
|
## Demarrage
|
||||||
|
|
||||||
Prerequis : uv, Docker, Node 24 LTS (npm fourni). Le poste doit disposer de Python 3.14, que
|
Prerequis : uv, Docker, Node 26 (version de la CI et de l'image frontend, npm fourni). Le poste doit disposer de Python 3.14, que
|
||||||
`uv` installe seul.
|
`uv` installe seul.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
cp .env.example .env # variables de docker-compose
|
cp .env.example .env # variables de docker-compose
|
||||||
cp apps/backend/.env.example apps/backend/.env # variables du backend hors conteneur
|
cp apps/backend/.env.example apps/backend/.env # variables du backend hors conteneur
|
||||||
|
|
||||||
make db-up # PostgreSQL + TimescaleDB, publie sur le port 5433
|
make install # dependances du backend, du frontend, du ML et des DAGs
|
||||||
make install # dependances du backend et du frontend
|
make dev # toute la stack, voir ci-dessous
|
||||||
make migrate # applique les migrations Alembic
|
|
||||||
make dev # backend sur http://localhost:8000 (docs sur /docs), frontend sur http://localhost:4200
|
|
||||||
make check # lint + typage + tests
|
make check # lint + typage + tests
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`make dev` enchaine tout : demarrage des services conteneurises (base sur le port 5433, Mailpit,
|
||||||
|
Airflow), migrations Alembic, peuplement de demonstration si les alertes manquent, puis backend
|
||||||
|
et frontend en rechargement a chaud sur le poste.
|
||||||
|
|
||||||
|
| Service | Adresse |
|
||||||
|
|---|---|
|
||||||
|
| Backend | <http://localhost:8000> (documentation sur `/docs`) |
|
||||||
|
| Frontend | <http://localhost:4200> |
|
||||||
|
| Airflow | <http://localhost:8080> (`AIRFLOW_ADMIN_USERNAME` / `AIRFLOW_ADMIN_PASSWORD` du `.env`) |
|
||||||
|
| Mailpit | <http://localhost:8025> |
|
||||||
|
|
||||||
|
Le `.env` doit porter les cles Airflow avant le premier `make dev` : `AIRFLOW_FERNET_KEY`,
|
||||||
|
`AIRFLOW_API_SECRET_KEY`, `AIRFLOW_JWT_SECRET`, `AIRFLOW_APP_SECRET_KEY` et
|
||||||
|
`AIRFLOW_ADMIN_PASSWORD`. Sans elles `airflow-init` refuse de demarrer, et `airflow-apiserver`,
|
||||||
|
`airflow-scheduler` et `airflow-dag-processor` avec lui. Il doit aussi porter les six clés
|
||||||
|
`GARAGE_*` (rpc, jetons, clé S3, clé SSE-C) : `make services-up` refuse sinon de démarrer Garage,
|
||||||
|
où le DAG `retention` archive les mesures anciennes ([ADR 0019](docs/adr/0019-stockage-objet-garage-et-cycle-de-vie-des-mesures.md)).
|
||||||
|
|
||||||
|
Les cibles d'origine restent disponibles pour ne demarrer qu'une partie : `make db-up`,
|
||||||
|
`make airflow-up`, `make dev-backend`, `make dev-frontend`.
|
||||||
|
|
||||||
`make help` liste les cibles disponibles.
|
`make help` liste les cibles disponibles.
|
||||||
|
|
||||||
Deux fichiers d'environnement, deux usages : `.env` a la racine alimente `docker-compose.yml`,
|
Deux fichiers d'environnement, deux usages : `.env` a la racine alimente `docker-compose.yml`,
|
||||||
`apps/backend/.env` alimente le backend lance sur le poste. Le port 5433 est publie plutot que
|
`apps/backend/.env` alimente le backend lance sur le poste. Le port 5433 est publie plutot que
|
||||||
5432, souvent deja pris par une autre base.
|
5432, souvent deja pris par une autre base.
|
||||||
|
|
||||||
La boucle de developpement est `make db-up` puis `make dev` : seule la base tourne en
|
Le backend et le frontend tournent sur le poste, lances ensemble par `make dev` (logs
|
||||||
conteneur, le backend et le frontend tournent tous les deux sur le poste, lances ensemble par
|
entrelaces dans le meme terminal, Ctrl+C arrete les deux) ; la base, Mailpit et Airflow tournent
|
||||||
`make dev` (logs entrelaces dans le meme terminal, Ctrl+C arrete les deux). `make dev-backend`
|
en conteneur. Le service `backend` du `docker-compose.yml` sert la stack complete et la recette,
|
||||||
et `make dev-frontend` restent disponibles pour lancer un seul des deux. Le service `backend`
|
et n'embarque pas le source, donc toute modification y demande un
|
||||||
du `docker-compose.yml` sert la stack complete et la recette, et n'embarque pas le source, donc
|
`docker compose up -d --build backend`.
|
||||||
toute modification y demande un `docker compose up -d --build backend`.
|
|
||||||
|
### Donnees de demonstration
|
||||||
|
|
||||||
|
Le jeu historique s'arrete au 31/12/2024. `make demo-data` renseigne les tables que les vues
|
||||||
|
alertes, recommandations et previsions lisent, en ancrant le scoring et la detection a cette
|
||||||
|
date (`DEMO_NOW`) plutot qu'a l'horloge reelle, qui ne verrait qu'un parc muet depuis des mois.
|
||||||
|
La cible ne fait rien si des alertes existent deja ; `make demo-data-force` rejoue les trois
|
||||||
|
etapes, toutes idempotentes en base.
|
||||||
|
|
||||||
|
Un volume `pgdata` cree avant `db/init/120-airflow-database.sql` n'a pas de base `airflow` :
|
||||||
|
`db/init` ne rejoue qu'a la premiere initialisation. `make db-ensure-airflow`, appelee par
|
||||||
|
`make dev` et `make airflow-up`, la cree au besoin, sans detruire les donnees applicatives.
|
||||||
|
|
||||||
Verifier que la base repond et que l'extension est chargee :
|
Verifier que la base repond et que l'extension est chargee :
|
||||||
|
|
||||||
@@ -96,6 +143,45 @@ Verifier que la base repond et que l'extension est chargee :
|
|||||||
curl -s localhost:8000/api/v1/health/ready
|
curl -s localhost:8000/api/v1/health/ready
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Stack complète derrière le reverse proxy
|
||||||
|
|
||||||
|
Pour servir l'application comme sur la machine cible, en HTTPS et sous une seule origine.
|
||||||
|
L'overlay emploie `!override` et `!reset`, donc **Docker Compose 2.24.4 ou plus récent** :
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make tls-selfsigned PUBLIC_HOST=enervision.local # certificat de démonstration
|
||||||
|
make stack-up PUBLIC_HOST=enervision.local # nginx en 80/443, le reste sur 127.0.0.1
|
||||||
|
```
|
||||||
|
|
||||||
|
Le navigateur avertit d'un émetteur inconnu : sur le poste, le certificat est auto-signé. Sur la
|
||||||
|
machine, les certificats viennent de Let's Encrypt par défi DNS-01
|
||||||
|
([ADR 0018](docs/adr/0018-noms-publics-certificats-dns01-et-frontal-sni.md)). Routage, mode ACME et renouvellement dans
|
||||||
|
[`infra/proxy/README.md`](infra/proxy/README.md) ; la décision et ses motifs dans
|
||||||
|
[l'ADR 0007](docs/adr/0007-terminaison-tls-et-reverse-proxy-nginx.md).
|
||||||
|
|
||||||
|
Sur la VM ENI, trois environnements cohabitent, production sur `main`, recette sur `dev`, et
|
||||||
|
`dev` pour toute autre branche lancée à la main
|
||||||
|
([ADR 0017](docs/adr/0017-environnement-dev-a-la-demande.md)), chacun dans son dossier et son
|
||||||
|
projet Compose, derrière un frontal SNI commun : `scripts/provision-host.sh` les prépare, le
|
||||||
|
workflow `deploy.yml` les redéploie par un runner auto-hébergé, une fois la CI du commit poussé
|
||||||
|
verte ([ADR 0014](docs/adr/0014-pipeline-ci-unique-et-deploiement-conditionne.md)). Ports, noms
|
||||||
|
d'hôte et garde-fous dans [`docs/architecture/10-infra.md`](docs/architecture/10-infra.md) et
|
||||||
|
[l'ADR 0009](docs/adr/0009-deux-environnements-compose-sur-la-vm-eni.md).
|
||||||
|
|
||||||
|
## Tests de bout en bout, charge et supervision
|
||||||
|
|
||||||
|
| Besoin | Commandes | Détail |
|
||||||
|
|---|---|---|
|
||||||
|
| Parcours utilisateur (Playwright) | `make e2e-install`, puis `make e2e-prepare e2e` contre `make dev` | [`tests/e2e/README.md`](tests/e2e/README.md) |
|
||||||
|
| Tir de charge (k6) | `make load-smoke`, `load-test`, `load-stress`, `load-limits` | [`tests/load/README.md`](tests/load/README.md) |
|
||||||
|
| Supervision | `make monitoring-up`, Grafana sur <http://localhost:3001> | [`monitoring/README.md`](monitoring/README.md) |
|
||||||
|
|
||||||
|
La CI joue les parcours, un tir de fumée et le contrôle de la limitation de débit à chaque PR
|
||||||
|
qui touche l'application, contre la stack de prod derrière le proxy
|
||||||
|
([ADR 0015](docs/adr/0015-tests-e2e-et-de-charge-contre-la-stack-compose.md)). La supervision
|
||||||
|
est active en prod, à la demande ailleurs
|
||||||
|
([ADR 0016](docs/adr/0016-supervision-en-profil-compose.md)).
|
||||||
|
|
||||||
## Conventions
|
## Conventions
|
||||||
|
|
||||||
- Branches : `feat/`, `fix/`, `chore/`, `docs/`, `test/` suivi d'un libelle court.
|
- Branches : `feat/`, `fix/`, `chore/`, `docs/`, `test/` suivi d'un libelle court.
|
||||||
|
|||||||
@@ -8,3 +8,17 @@ APP_SECRET_KEY=change_me
|
|||||||
|
|
||||||
APP_CORS_ORIGINS=http://localhost:4200
|
APP_CORS_ORIGINS=http://localhost:4200
|
||||||
DATABASE_URL=postgresql+asyncpg://enervision:change_me@localhost:5433/enervision
|
DATABASE_URL=postgresql+asyncpg://enervision:change_me@localhost:5433/enervision
|
||||||
|
|
||||||
|
# Mot de passe oublié : lien à usage unique valable 15 minutes par défaut.
|
||||||
|
APP_FRONTEND_RESET_PASSWORD_URL=http://localhost:4200/reset-password
|
||||||
|
|
||||||
|
# SMTP local de dev (Mailpit, cf. docker-compose.yml) : aucune authentification, aucun TLS.
|
||||||
|
# À remplacer par un vrai relais en staging/prod.
|
||||||
|
APP_SMTP_HOST=localhost
|
||||||
|
APP_SMTP_PORT=1025
|
||||||
|
APP_SMTP_USE_TLS=false
|
||||||
|
APP_SMTP_FROM_ADDRESS=no-reply@enervision.fr
|
||||||
|
APP_MOCK_API_BASE_URL=https://api-mock.charlieandre.fr
|
||||||
|
APP_MOCK_API_USERNAME=change_me
|
||||||
|
APP_MOCK_API_PASSWORD=change_me
|
||||||
|
APP_MOCK_API_TIMEOUT_SECONDS=10
|
||||||
|
|||||||
@@ -11,13 +11,14 @@ WORKDIR /app
|
|||||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||||
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
||||||
uv sync --locked --no-install-project --no-dev
|
uv sync --locked --no-install-project --no-dev --no-build
|
||||||
|
|
||||||
|
# Le projet lui-meme n'est pas installe (pas de second `uv sync`) : il tourne depuis /app, le
|
||||||
|
# repertoire de travail, et rien ne lit ses metadonnees. L'installer imposerait de le construire
|
||||||
|
# (backend hatchling), donc de retirer `--no-build` de l'etape ci-dessus, qui garantit que
|
||||||
|
# l'installation des dependances n'execute aucun script de build (regle Sonar docker:S8541).
|
||||||
COPY . /app
|
COPY . /app
|
||||||
|
|
||||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
||||||
uv sync --locked --no-dev
|
|
||||||
|
|
||||||
|
|
||||||
FROM python:3.14-slim AS runtime
|
FROM python:3.14-slim AS runtime
|
||||||
|
|
||||||
|
|||||||
@@ -103,6 +103,8 @@ Le sens de dependance est unique : `endpoints` vers `services` vers `repositorie
|
|||||||
| `/api/v1/auth/logout` | Ferme la session courante | cookie, idempotente |
|
| `/api/v1/auth/logout` | Ferme la session courante | cookie, idempotente |
|
||||||
| `/api/v1/auth/logout-all` | Ferme toutes les sessions du compte | jeton |
|
| `/api/v1/auth/logout-all` | Ferme toutes les sessions du compte | jeton |
|
||||||
| `/api/v1/auth/password` | Change son propre mot de passe | jeton |
|
| `/api/v1/auth/password` | Change son propre mot de passe | jeton |
|
||||||
|
| `/api/v1/auth/forgot-password` | Demande un lien de réinitialisation par email | public |
|
||||||
|
| `/api/v1/auth/reset-password` | Choisit un nouveau mot de passe depuis ce lien | public |
|
||||||
| `/api/v1/auth/me` | Décrit le compte connecté | jeton |
|
| `/api/v1/auth/me` | Décrit le compte connecté | jeton |
|
||||||
| `/api/v1/users` | Liste et crée des comptes | `admin` |
|
| `/api/v1/users` | Liste et crée des comptes | `admin` |
|
||||||
| `/api/v1/users/{id}` | Change le rôle ou l'activation | `admin` |
|
| `/api/v1/users/{id}` | Change le rôle ou l'activation | `admin` |
|
||||||
@@ -111,6 +113,7 @@ Le sens de dependance est unique : `endpoints` vers `services` vers `repositorie
|
|||||||
| `/api/v1/sites/{site_id}` | Décrit un site | `lecteur` |
|
| `/api/v1/sites/{site_id}` | Décrit un site | `lecteur` |
|
||||||
| `/api/v1/recommendations` | Liste les recommandations | `lecteur` |
|
| `/api/v1/recommendations` | Liste les recommandations | `lecteur` |
|
||||||
| `/api/v1/recommendations/{recommendation_id}` | Décrit une recommandation | `lecteur` |
|
| `/api/v1/recommendations/{recommendation_id}` | Décrit une recommandation | `lecteur` |
|
||||||
|
| `/api/v1/recommendations/generate` | Génère les recommandations depuis les alertes (POST) | `admin` |
|
||||||
| `/metrics` | Métriques au format Prometheus | jeton si `APP_METRICS_TOKEN` |
|
| `/metrics` | Métriques au format Prometheus | jeton si `APP_METRICS_TOKEN` |
|
||||||
| `/docs`, `/openapi.json` | Documentation, fermée en `staging` et `prod` | public sinon |
|
| `/docs`, `/openapi.json` | Documentation, fermée en `staging` et `prod` | public sinon |
|
||||||
|
|
||||||
|
|||||||
+21
-3
@@ -123,6 +123,11 @@ async def test_repository_reads_back_what_it_wrote(session: AsyncSession) -> Non
|
|||||||
defaut, ce qui garde `make check` jouable sans Docker. Tout autre marqueur doit etre
|
defaut, ce qui garde `make check` jouable sans Docker. Tout autre marqueur doit etre
|
||||||
declare dans `pyproject.toml` : `--strict-markers` refuse les marqueurs inconnus.
|
declare dans `pyproject.toml` : `--strict-markers` refuse les marqueurs inconnus.
|
||||||
|
|
||||||
|
Ces tests ne sont pas pour autant facultatifs : le job `integration` de
|
||||||
|
`.github/workflows/backend.yml` monte un service TimescaleDB, applique les migrations et
|
||||||
|
les joue a chaque poussee. Un test `integration` casse donc la CI comme un autre. En local,
|
||||||
|
`make db-up` puis `make test-integration`.
|
||||||
|
|
||||||
## Couverture
|
## Couverture
|
||||||
|
|
||||||
Les branches sont mesurees, pas seulement les lignes. Le seuil de 85 % ne s'applique
|
Les branches sont mesurees, pas seulement les lignes. Le seuil de 85 % ne s'applique
|
||||||
@@ -142,14 +147,27 @@ uv run pytest tests/api/test_health.py # un seul fichier
|
|||||||
uv run pytest -k readiness # par motif de nom
|
uv run pytest -k readiness # par motif de nom
|
||||||
```
|
```
|
||||||
|
|
||||||
## Trois fichiers à connaître avant de toucher à l'authentification
|
## Quatre fichiers à connaître avant de toucher à l'authentification
|
||||||
|
|
||||||
|
`tests/api/acces.py` porte la classification des routes du contrat, en quatre ensembles :
|
||||||
|
`ROUTES_PUBLIQUES`, `ROUTE_COOKIE`, `ROUTES_SANS_ROLE` et la table `ROLE_MINIMUM`. Ce n'est pas
|
||||||
|
un fichier de test, c'est la référence que les trois autres confrontent au comportement observé.
|
||||||
|
**Toute route ajoutée doit y être classée** : `test_every_declared_route_is_classified` échoue
|
||||||
|
sinon, et échoue aussi sur une entrée qui ne correspond plus à aucune route.
|
||||||
|
|
||||||
`tests/api/test_route_protection.py` interroge réellement chaque route sans identifiant et
|
`tests/api/test_route_protection.py` interroge réellement chaque route sans identifiant et
|
||||||
échoue si l'une d'elles répond autre chose qu'un 401 ou un 403. Il n'inspecte pas l'arbre de
|
échoue si l'une d'elles répond autre chose qu'un 401 ou un 403. Il n'inspecte pas l'arbre de
|
||||||
dépendances : celui-ci n'est accessible que par l'API privée de FastAPI, et surtout une route
|
dépendances : celui-ci n'est accessible que par l'API privée de FastAPI, et surtout une route
|
||||||
peut porter la bonne dépendance tout en répondant quand même. **Rendre une route publique impose
|
peut porter la bonne dépendance tout en répondant quand même. **Rendre une route publique impose
|
||||||
donc de modifier la liste `ROUTES_PUBLIQUES` de ce fichier**, ce qui apparaît en clair dans la
|
donc de modifier `ROUTES_PUBLIQUES` dans `acces.py`**, ce qui apparaît en clair dans la diff
|
||||||
diff d'une pull request.
|
d'une pull request.
|
||||||
|
|
||||||
|
`tests/api/test_matrice_acces.py` croise chaque route gardée avec chacun des trois rôles, dans
|
||||||
|
les deux sens : un rôle insuffisant reçoit un 403 `Droits insuffisants`, un rôle suffisant ne le
|
||||||
|
reçoit jamais. Le second sens est ce qui rend visible une garde posée trop haut, par exemple
|
||||||
|
`AdminDep` sur une route de lecture. La même matrice est rejouée sous `integration` avec de vrais
|
||||||
|
jetons, donc en traversant le décodage du JWT et la relecture du compte en base, que
|
||||||
|
`dependency_overrides` court-circuite.
|
||||||
|
|
||||||
`tests/services/test_auth.py` donne au faux hacheur un **compteur d'appels**. C'est ce qui rend
|
`tests/services/test_auth.py` donne au faux hacheur un **compteur d'appels**. C'est ce qui rend
|
||||||
possibles les deux assertions qui prouvent la conception, et qu'aucune autre forme de test
|
possibles les deux assertions qui prouvent la conception, et qu'aucune autre forme de test
|
||||||
|
|||||||
@@ -0,0 +1,96 @@
|
|||||||
|
"""jetons et tentatives de reinitialisation de mot de passe
|
||||||
|
|
||||||
|
Revision ID: c0adab96238c
|
||||||
|
Revises: e6d2026091501
|
||||||
|
Create Date: 2026-09-17 10:37:12.571314
|
||||||
|
|
||||||
|
Meme schema que `refresh_token` pour `password_reset_token` : seule l'empreinte SHA-256 du
|
||||||
|
jeton est stockee, jamais le jeton lui-meme, pour la meme raison (revocation en cascade,
|
||||||
|
aucune session utilisable dans un pg_dump qui fuiterait).
|
||||||
|
|
||||||
|
`password_reset_attempt` vit hors de `audit_log`, comme `login_attempt`, car son volume est
|
||||||
|
pilote par l'attaquant : une campagne de demandes y ecrirait des lignes que l'audit, en ajout
|
||||||
|
seul, ne devrait jamais purger.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "c0adab96238c"
|
||||||
|
down_revision: str | Sequence[str] | None = "e6d2026091501"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
JETONS_VIVANTS = "consumed_at is null"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"password_reset_attempt",
|
||||||
|
sa.Column("id", sa.BigInteger(), sa.Identity(always=True), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"occurred_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("email_tried", sa.String(length=320), nullable=False),
|
||||||
|
sa.Column("client_ip", postgresql.INET(), nullable=True),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_password_reset_attempt"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_password_reset_attempt_email_date",
|
||||||
|
"password_reset_attempt",
|
||||||
|
["email_tried", "occurred_at"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_password_reset_attempt_ip_date", "password_reset_attempt", ["client_ip", "occurred_at"]
|
||||||
|
)
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"password_reset_token",
|
||||||
|
sa.Column("id", sa.UUID(), server_default=sa.text("gen_random_uuid()"), nullable=False),
|
||||||
|
sa.Column("user_id", sa.UUID(), nullable=False),
|
||||||
|
sa.Column("token_hash", sa.LargeBinary(), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"issued_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("consumed_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("client_ip", postgresql.INET(), nullable=True),
|
||||||
|
sa.Column("user_agent", sa.Text(), nullable=True),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["user_id"],
|
||||||
|
["app_user.id"],
|
||||||
|
name="fk_password_reset_token_user",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name="pk_password_reset_token"),
|
||||||
|
sa.UniqueConstraint("token_hash", name="uq_password_reset_token_hash"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_password_reset_token_user", "password_reset_token", ["user_id"])
|
||||||
|
op.create_index(
|
||||||
|
"ix_password_reset_token_vivants",
|
||||||
|
"password_reset_token",
|
||||||
|
["user_id"],
|
||||||
|
postgresql_where=JETONS_VIVANTS,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index(
|
||||||
|
"ix_password_reset_token_vivants",
|
||||||
|
table_name="password_reset_token",
|
||||||
|
postgresql_where=JETONS_VIVANTS,
|
||||||
|
)
|
||||||
|
op.drop_index("ix_password_reset_token_user", table_name="password_reset_token")
|
||||||
|
op.drop_table("password_reset_token")
|
||||||
|
op.drop_index("ix_password_reset_attempt_ip_date", table_name="password_reset_attempt")
|
||||||
|
op.drop_index("ix_password_reset_attempt_email_date", table_name="password_reset_attempt")
|
||||||
|
op.drop_table("password_reset_attempt")
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""rapports de derive du modele de prevision
|
||||||
|
|
||||||
|
Revision ID: d3f1a2b7c904
|
||||||
|
Revises: c0adab96238c
|
||||||
|
Create Date: 2026-09-22 14:40:00.000000
|
||||||
|
|
||||||
|
`site_id` est nullable, et c'est le coeur du schema : une ligne par site, plus une ligne
|
||||||
|
globale tous sites confondus, que `NULL` designe. Un seul site qui derive est invisible dans
|
||||||
|
une moyenne d'ensemble, et une derive d'ensemble sans rupture par site signale un changement
|
||||||
|
de modele ou de saison, pas une panne.
|
||||||
|
|
||||||
|
L'unicite passe par un index a `coalesce` et non par une `UniqueConstraint` : deux lignes
|
||||||
|
globales successives ont toutes deux `site_id` a NULL, et NULL n'est egal a aucune valeur, pas
|
||||||
|
meme a lui-meme. Meme forme que `uq_reading_source`.
|
||||||
|
|
||||||
|
Les trois `CHECK` sont portees par la base, comme `ck_prediction_status` : un verdict sans
|
||||||
|
motif, ou un statut inconnu, ne doit pas dependre de la vigilance de l'appelant.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "d3f1a2b7c904"
|
||||||
|
down_revision: str | Sequence[str] | None = "c0adab96238c"
|
||||||
|
branch_labels: str | Sequence[str] | None = None
|
||||||
|
depends_on: str | Sequence[str] | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"drift_report",
|
||||||
|
sa.Column("drift_report_id", sa.BigInteger(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"computed_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("site_id", sa.Text(), nullable=True),
|
||||||
|
sa.Column("window_start", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("window_end", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("reference_start", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("reference_end", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("n_observations", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("mae", sa.Double(), nullable=True),
|
||||||
|
sa.Column("mape", sa.Double(), nullable=True),
|
||||||
|
sa.Column("bias", sa.Double(), nullable=True),
|
||||||
|
sa.Column("reference_mae", sa.Double(), nullable=True),
|
||||||
|
sa.Column("coverage_ratio", sa.Double(), nullable=True),
|
||||||
|
sa.Column("insufficient_data_ratio", sa.Double(), nullable=True),
|
||||||
|
sa.Column("model_references", postgresql.ARRAY(sa.Text()), nullable=False),
|
||||||
|
sa.Column("status", sa.Text(), nullable=False),
|
||||||
|
sa.Column("reason", sa.Text(), nullable=True),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status IN ('stable', 'derive', 'indetermine')", name="ck_drift_report_status"
|
||||||
|
),
|
||||||
|
sa.CheckConstraint(
|
||||||
|
"status = 'stable' OR reason IS NOT NULL", name="ck_drift_report_reason"
|
||||||
|
),
|
||||||
|
sa.CheckConstraint("n_observations >= 0", name="ck_drift_report_observations"),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["site_id"], ["site.site_id"], name="fk_drift_report_site", ondelete="RESTRICT"
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("drift_report_id"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_drift_report_site_computed", "drift_report", ["site_id", "computed_at"], unique=False
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"uq_drift_report_window",
|
||||||
|
"drift_report",
|
||||||
|
["window_end", sa.literal_column("coalesce(site_id, '')")],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("drift_report")
|
||||||
@@ -16,20 +16,31 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.hashing import Argon2Hasher, build_hasher
|
from app.core.hashing import Argon2Hasher, build_hasher
|
||||||
|
from app.core.mailer import Mailer, SmtpConfig
|
||||||
from app.core.principal import Principal
|
from app.core.principal import Principal
|
||||||
from app.core.roles import AccountKind, Role, has_at_least
|
from app.core.roles import AccountKind, Role, has_at_least
|
||||||
from app.core.security import TokenExpiredError, TokenInvalidError, TokenPolicy
|
from app.core.security import TokenExpiredError, TokenInvalidError, TokenPolicy
|
||||||
from app.core.security import decode_access_token as decode_token
|
from app.core.security import decode_access_token as decode_token
|
||||||
from app.db.session import get_session
|
from app.db.session import get_session
|
||||||
|
from app.repositories.alert import AlertRepository
|
||||||
from app.repositories.audit_log import AuditLogRepository
|
from app.repositories.audit_log import AuditLogRepository
|
||||||
|
from app.repositories.drift import DriftRepository
|
||||||
from app.repositories.login_attempt import LoginAttemptRepository
|
from app.repositories.login_attempt import LoginAttemptRepository
|
||||||
|
from app.repositories.password_reset_attempt import PasswordResetAttemptRepository
|
||||||
|
from app.repositories.password_reset_token import PasswordResetTokenRepository
|
||||||
|
from app.repositories.prediction import PredictionRepository
|
||||||
from app.repositories.reading import ReadingRepository
|
from app.repositories.reading import ReadingRepository
|
||||||
from app.repositories.recommendation import RecommendationRepository
|
from app.repositories.recommendation import RecommendationRepository
|
||||||
from app.repositories.refresh_token import RefreshTokenRepository
|
from app.repositories.refresh_token import RefreshTokenRepository
|
||||||
from app.repositories.site import SiteRepository
|
from app.repositories.site import SiteRepository
|
||||||
from app.repositories.user import UserRepository
|
from app.repositories.user import UserRepository
|
||||||
from app.services.auth import AuthService, LoginPolicy
|
from app.services.alert import AlertService
|
||||||
|
from app.services.auth import AuthService, LoginPolicy, PasswordResetPolicy
|
||||||
|
from app.services.drift import DriftService
|
||||||
|
from app.services.prediction import PredictionService
|
||||||
|
from app.services.reading import ReadingService
|
||||||
from app.services.recommendation import RecommendationService
|
from app.services.recommendation import RecommendationService
|
||||||
|
from app.services.sensor import SensorService
|
||||||
from app.services.site import SiteService
|
from app.services.site import SiteService
|
||||||
from app.services.stats import StatsService
|
from app.services.stats import StatsService
|
||||||
from app.services.user import UserService
|
from app.services.user import UserService
|
||||||
@@ -39,7 +50,9 @@ SettingsDep = Annotated[Settings, Depends(get_settings)]
|
|||||||
|
|
||||||
CODE_CHANGEMENT_REQUIS = "password_change_required"
|
CODE_CHANGEMENT_REQUIS = "password_change_required"
|
||||||
|
|
||||||
_porteur = HTTPBearer(auto_error=False, scheme_name="Jeton d'accès")
|
# Nom ASCII : un outillage tiers (ZAP, cf. .github/workflows/dast.yml) peut mal analyser un nom
|
||||||
|
# de schéma accentué dans le contrat OpenAPI. Piège vécu, pas anticipé.
|
||||||
|
_porteur = HTTPBearer(auto_error=False, scheme_name="JetonAcces")
|
||||||
CredentialsDep = Annotated[HTTPAuthorizationCredentials | None, Depends(_porteur)]
|
CredentialsDep = Annotated[HTTPAuthorizationCredentials | None, Depends(_porteur)]
|
||||||
|
|
||||||
|
|
||||||
@@ -94,11 +107,27 @@ def get_client_ip(request: Request, settings: SettingsDep) -> str | None:
|
|||||||
return request.client.host if request.client else None
|
return request.client.host if request.client else None
|
||||||
|
|
||||||
|
|
||||||
|
def get_mailer(settings: SettingsDep) -> Mailer:
|
||||||
|
return Mailer(
|
||||||
|
SmtpConfig(
|
||||||
|
host=settings.smtp_host,
|
||||||
|
port=settings.smtp_port,
|
||||||
|
username=settings.smtp_username,
|
||||||
|
password=(
|
||||||
|
settings.smtp_password.get_secret_value() if settings.smtp_password else None
|
||||||
|
),
|
||||||
|
use_tls=settings.smtp_use_tls,
|
||||||
|
from_address=settings.smtp_from_address,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_auth_service(
|
def get_auth_service(
|
||||||
session: SessionDep,
|
session: SessionDep,
|
||||||
settings: SettingsDep,
|
settings: SettingsDep,
|
||||||
hasher: Annotated[Argon2Hasher, Depends(get_hasher)],
|
hasher: Annotated[Argon2Hasher, Depends(get_hasher)],
|
||||||
token_policy: Annotated[TokenPolicy, Depends(get_token_policy)],
|
token_policy: Annotated[TokenPolicy, Depends(get_token_policy)],
|
||||||
|
mailer: Annotated[Mailer, Depends(get_mailer)],
|
||||||
) -> AuthService:
|
) -> AuthService:
|
||||||
return AuthService(
|
return AuthService(
|
||||||
users=UserRepository(session),
|
users=UserRepository(session),
|
||||||
@@ -115,6 +144,16 @@ def get_auth_service(
|
|||||||
max_failures_per_identifier=settings.login_max_failures_per_identifier,
|
max_failures_per_identifier=settings.login_max_failures_per_identifier,
|
||||||
),
|
),
|
||||||
refresh_ttl=timedelta(seconds=settings.refresh_token_ttl_seconds),
|
refresh_ttl=timedelta(seconds=settings.refresh_token_ttl_seconds),
|
||||||
|
reset_tokens=PasswordResetTokenRepository(session),
|
||||||
|
reset_attempts=PasswordResetAttemptRepository(session),
|
||||||
|
reset_policy=PasswordResetPolicy(
|
||||||
|
window_seconds=settings.password_reset_window_seconds,
|
||||||
|
max_requests_per_identifier=settings.password_reset_max_requests_per_identifier,
|
||||||
|
max_requests_per_ip=settings.password_reset_max_requests_per_ip,
|
||||||
|
token_ttl=timedelta(seconds=settings.password_reset_ttl_seconds),
|
||||||
|
frontend_reset_url=settings.frontend_reset_password_url,
|
||||||
|
),
|
||||||
|
mailer=mailer,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -138,14 +177,30 @@ UserServiceDep = Annotated[UserService, Depends(get_user_service)]
|
|||||||
|
|
||||||
|
|
||||||
def get_site_service(session: SessionDep) -> SiteService:
|
def get_site_service(session: SessionDep) -> SiteService:
|
||||||
return SiteService(sites=SiteRepository(session))
|
return SiteService(sites=SiteRepository(session), readings=ReadingRepository(session))
|
||||||
|
|
||||||
|
|
||||||
SiteServiceDep = Annotated[SiteService, Depends(get_site_service)]
|
SiteServiceDep = Annotated[SiteService, Depends(get_site_service)]
|
||||||
|
|
||||||
|
|
||||||
|
def get_alert_service(session: SessionDep) -> AlertService:
|
||||||
|
return AlertService(
|
||||||
|
alerts=AlertRepository(session),
|
||||||
|
readings=ReadingRepository(session),
|
||||||
|
predictions=PredictionRepository(session),
|
||||||
|
sites=SiteRepository(session),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
AlertServiceDep = Annotated[AlertService, Depends(get_alert_service)]
|
||||||
|
|
||||||
|
|
||||||
def get_recommendation_service(session: SessionDep) -> RecommendationService:
|
def get_recommendation_service(session: SessionDep) -> RecommendationService:
|
||||||
return RecommendationService(recommendations=RecommendationRepository(session))
|
return RecommendationService(
|
||||||
|
recommendations=RecommendationRepository(session),
|
||||||
|
alerts=AlertRepository(session),
|
||||||
|
transaction=session,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
RecommendationServiceDep = Annotated[RecommendationService, Depends(get_recommendation_service)]
|
RecommendationServiceDep = Annotated[RecommendationService, Depends(get_recommendation_service)]
|
||||||
@@ -158,6 +213,36 @@ def get_stats_service(session: SessionDep) -> StatsService:
|
|||||||
StatsServiceDep = Annotated[StatsService, Depends(get_stats_service)]
|
StatsServiceDep = Annotated[StatsService, Depends(get_stats_service)]
|
||||||
|
|
||||||
|
|
||||||
|
def get_reading_service(session: SessionDep) -> ReadingService:
|
||||||
|
return ReadingService(readings=ReadingRepository(session))
|
||||||
|
|
||||||
|
|
||||||
|
ReadingServiceDep = Annotated[ReadingService, Depends(get_reading_service)]
|
||||||
|
|
||||||
|
|
||||||
|
def get_sensor_service(session: SessionDep) -> SensorService:
|
||||||
|
return SensorService(sites=SiteRepository(session), readings=ReadingRepository(session))
|
||||||
|
|
||||||
|
|
||||||
|
SensorServiceDep = Annotated[SensorService, Depends(get_sensor_service)]
|
||||||
|
|
||||||
|
|
||||||
|
def get_prediction_service(session: SessionDep) -> PredictionService:
|
||||||
|
return PredictionService(
|
||||||
|
sites=SiteRepository(session), predictions=PredictionRepository(session)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
PredictionServiceDep = Annotated[PredictionService, Depends(get_prediction_service)]
|
||||||
|
|
||||||
|
|
||||||
|
def get_drift_service(session: SessionDep) -> DriftService:
|
||||||
|
return DriftService(DriftRepository(session))
|
||||||
|
|
||||||
|
|
||||||
|
DriftServiceDep = Annotated[DriftService, Depends(get_drift_service)]
|
||||||
|
|
||||||
|
|
||||||
async def get_current_principal(
|
async def get_current_principal(
|
||||||
credentials: CredentialsDep,
|
credentials: CredentialsDep,
|
||||||
session: SessionDep,
|
session: SessionDep,
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ EN_TETES: Final[dict[str, str]] = {
|
|||||||
"X-Content-Type-Options": "nosniff",
|
"X-Content-Type-Options": "nosniff",
|
||||||
"X-Frame-Options": "DENY",
|
"X-Frame-Options": "DENY",
|
||||||
"Referrer-Policy": "no-referrer",
|
"Referrer-Policy": "no-referrer",
|
||||||
|
# same-origin : aucun client ne charge l'API en no-cors depuis une autre origine
|
||||||
|
# (proxy.conf.json en dev, reverse proxy nginx ensuite, cf. docs/architecture/20-backend.md).
|
||||||
|
"Cross-Origin-Resource-Policy": "same-origin",
|
||||||
}
|
}
|
||||||
|
|
||||||
PREFIXE_AUTHENTIFICATION: Final = "/auth"
|
PREFIXE_AUTHENTIFICATION: Final = "/auth"
|
||||||
|
|||||||
@@ -54,11 +54,16 @@ TAGS: Final[list[dict[str, Any]]] = [
|
|||||||
"name": "sites",
|
"name": "sites",
|
||||||
"description": "Consultation du parc de sites. Accessible à partir du rôle `lecteur`.",
|
"description": "Consultation du parc de sites. Accessible à partir du rôle `lecteur`.",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "alerts",
|
||||||
|
"description": "Consultation des alertes de consommation. Accessible à partir du rôle "
|
||||||
|
"`lecteur`.",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "recommendations",
|
"name": "recommendations",
|
||||||
"description": (
|
"description": (
|
||||||
"Consultation des recommandations issues des alertes. Accessible à partir du rôle "
|
"Consultation des recommandations issues des alertes. Accessible à partir du rôle "
|
||||||
"`lecteur`."
|
"`lecteur`. Leur génération par le moteur de règles est réservée au rôle `admin`."
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -66,11 +71,40 @@ TAGS: Final[list[dict[str, Any]]] = [
|
|||||||
"description": "Statistiques agrégées de consommation. Accessible à partir du rôle "
|
"description": "Statistiques agrégées de consommation. Accessible à partir du rôle "
|
||||||
"`lecteur`.",
|
"`lecteur`.",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "readings",
|
||||||
|
"description": (
|
||||||
|
"Historique des lectures de consommation. Fenêtre temporelle plafonnée à 90 jours, "
|
||||||
|
"24 dernières heures par défaut si `start`/`end` sont omis. Accessible à partir du "
|
||||||
|
"rôle `lecteur`."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sensors",
|
||||||
|
"description": "État de santé des capteurs par site. Réservé au rôle `admin`.",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "predictions",
|
||||||
|
"description": (
|
||||||
|
"Dernière prévision de consommation par site, calculée hors ligne par le pipeline "
|
||||||
|
"de scoring (`ml/`) et simplement lue ici. Accessible à partir du rôle `lecteur`."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "monitoring",
|
||||||
|
"description": (
|
||||||
|
"Surveillance de la dérive du modèle : écart entre les prévisions déjà écrites et "
|
||||||
|
"les lectures réellement arrivées, par site et tous sites confondus. Réservé à "
|
||||||
|
"partir du rôle `operateur`, qui agit sur un pipeline dégradé."
|
||||||
|
),
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
cookie_de_rafraichissement = APIKeyCookie(
|
cookie_de_rafraichissement = APIKeyCookie(
|
||||||
name=REFRESH_COOKIE_DEFAUT,
|
name=REFRESH_COOKIE_DEFAUT,
|
||||||
scheme_name="Cookie de rafraîchissement",
|
# Nom ASCII : un outillage tiers (ZAP, cf. .github/workflows/dast.yml) peut mal analyser un
|
||||||
|
# nom de schéma accentué dans le contrat OpenAPI. Piège vécu, pas anticipé.
|
||||||
|
scheme_name="CookieRafraichissement",
|
||||||
description=(
|
description=(
|
||||||
"Cookie `HttpOnly` posé par `/auth/login` et tourné par `/auth/refresh`. Il prend le "
|
"Cookie `HttpOnly` posé par `/auth/login` et tourné par `/auth/refresh`. Il prend le "
|
||||||
"préfixe `__Secure-` dès que l'API tourne derrière TLS, et n'est émis que vers "
|
"préfixe `__Secure-` dès que l'API tourne derrière TLS, et n'est émis que vers "
|
||||||
@@ -129,6 +163,17 @@ REPONSES_ADMIN: Final[Reponses] = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
REPONSES_OPERATEUR: Final[Reponses] = {
|
||||||
|
**REPONSES_AUTHENTIFIEES,
|
||||||
|
403: {
|
||||||
|
"model": ErrorResponse,
|
||||||
|
"description": (
|
||||||
|
"Droits insuffisants, ou mot de passe provisoire à changer quand `detail` vaut "
|
||||||
|
"`password_change_required`."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
# `lecteur` est le rôle minimum : `require_role` n'y refuse jamais un 403 pour droits
|
# `lecteur` est le rôle minimum : `require_role` n'y refuse jamais un 403 pour droits
|
||||||
# insuffisants, seulement pour le mot de passe provisoire.
|
# insuffisants, seulement pour le mot de passe provisoire.
|
||||||
REPONSES_LECTEUR: Final[Reponses] = {
|
REPONSES_LECTEUR: Final[Reponses] = {
|
||||||
@@ -147,3 +192,16 @@ REPONSE_ORIGINE_REFUSEE: Final[Reponses] = {
|
|||||||
"description": "Origine non autorisée (protection CSRF de `require_trusted_origin`).",
|
"description": "Origine non autorisée (protection CSRF de `require_trusted_origin`).",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
REPONSE_LIMITE: Final[Reponses] = {
|
||||||
|
429: {
|
||||||
|
"model": ErrorResponse,
|
||||||
|
"description": "Trop de demandes sur cette fenêtre glissante.",
|
||||||
|
"headers": {
|
||||||
|
"Retry-After": {
|
||||||
|
"description": "Secondes à attendre avant une nouvelle tentative.",
|
||||||
|
"schema": {"type": "integer"},
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,23 @@
|
|||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from app.api.deps import AlertServiceDep, LecteurDep
|
||||||
|
from app.api.openapi import REPONSE_VALIDATION
|
||||||
|
from app.schemas.alert import AlertResponse, AlertSeverity
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"",
|
||||||
|
response_model=list[AlertResponse],
|
||||||
|
summary="Liste les alertes",
|
||||||
|
responses=REPONSE_VALIDATION,
|
||||||
|
)
|
||||||
|
async def list_alerts(
|
||||||
|
_: LecteurDep,
|
||||||
|
service: AlertServiceDep,
|
||||||
|
site_id: str | None = None,
|
||||||
|
severity: AlertSeverity | None = None,
|
||||||
|
) -> list[AlertResponse]:
|
||||||
|
alertes = await service.list_all(site_id=site_id, severity=severity)
|
||||||
|
return [AlertResponse.model_validate(alerte) for alerte in alertes]
|
||||||
@@ -2,7 +2,7 @@
|
|||||||
# d'accès ne va jamais dans un cookie. C'est ce qui réduit la surface CSRF aux trois routes de
|
# d'accès ne va jamais dans un cookie. C'est ce qui réduit la surface CSRF aux trois routes de
|
||||||
# ce module : partout ailleurs, le navigateur n'attache rien de lui-même.
|
# ce module : partout ailleurs, le navigateur n'attache rien de lui-même.
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request, Response, status
|
||||||
|
|
||||||
from app.api.deps import (
|
from app.api.deps import (
|
||||||
AuthServiceDep,
|
AuthServiceDep,
|
||||||
@@ -12,6 +12,7 @@ from app.api.deps import (
|
|||||||
require_trusted_origin,
|
require_trusted_origin,
|
||||||
)
|
)
|
||||||
from app.api.openapi import (
|
from app.api.openapi import (
|
||||||
|
REPONSE_LIMITE,
|
||||||
REPONSE_ORIGINE_REFUSEE,
|
REPONSE_ORIGINE_REFUSEE,
|
||||||
REPONSE_VALIDATION,
|
REPONSE_VALIDATION,
|
||||||
REPONSES_AUTHENTIFIEES,
|
REPONSES_AUTHENTIFIEES,
|
||||||
@@ -21,15 +22,19 @@ from app.api.openapi import (
|
|||||||
from app.core.cookies import RefreshCookie, cookie_name
|
from app.core.cookies import RefreshCookie, cookie_name
|
||||||
from app.core.logging import get_logger
|
from app.core.logging import get_logger
|
||||||
from app.schemas.auth import (
|
from app.schemas.auth import (
|
||||||
|
ForgotPasswordRequest,
|
||||||
LoginRequest,
|
LoginRequest,
|
||||||
PasswordChangeRequest,
|
PasswordChangeRequest,
|
||||||
PrincipalResponse,
|
PrincipalResponse,
|
||||||
|
ResetPasswordRequest,
|
||||||
|
ResetTokenValidationResponse,
|
||||||
TokenResponse,
|
TokenResponse,
|
||||||
)
|
)
|
||||||
from app.schemas.errors import ErrorResponse
|
from app.schemas.errors import ErrorResponse
|
||||||
from app.services.auth import (
|
from app.services.auth import (
|
||||||
AuthenticatedSession,
|
AuthenticatedSession,
|
||||||
InvalidCredentialsError,
|
InvalidCredentialsError,
|
||||||
|
InvalidOrExpiredResetTokenError,
|
||||||
RateLimitedError,
|
RateLimitedError,
|
||||||
SessionRejectedError,
|
SessionRejectedError,
|
||||||
)
|
)
|
||||||
@@ -39,6 +44,7 @@ logger = get_logger(__name__)
|
|||||||
|
|
||||||
DETAIL_IDENTIFIANTS = "Identifiants invalides"
|
DETAIL_IDENTIFIANTS = "Identifiants invalides"
|
||||||
DETAIL_SESSION = "Session invalide"
|
DETAIL_SESSION = "Session invalide"
|
||||||
|
DETAIL_LIEN_RESET = "Lien invalide ou expiré"
|
||||||
|
|
||||||
REPONSES_LOGIN: Reponses = {
|
REPONSES_LOGIN: Reponses = {
|
||||||
**REPONSE_VALIDATION,
|
**REPONSE_VALIDATION,
|
||||||
@@ -85,6 +91,20 @@ REPONSES_MOT_DE_PASSE: Reponses = {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
REPONSES_FORGOT_PASSWORD: Reponses = {
|
||||||
|
**REPONSE_VALIDATION,
|
||||||
|
**REPONSE_LIMITE,
|
||||||
|
}
|
||||||
|
|
||||||
|
REPONSES_RESET_PASSWORD: Reponses = {
|
||||||
|
**REPONSE_VALIDATION,
|
||||||
|
**REPONSE_ORIGINE_REFUSEE,
|
||||||
|
400: {
|
||||||
|
"model": ErrorResponse,
|
||||||
|
"description": "Lien invalide, déjà utilisé, ou expiré (durée de vie : 15 minutes).",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def repond(
|
def repond(
|
||||||
response: Response, settings: SettingsDep, session: AuthenticatedSession
|
response: Response, settings: SettingsDep, session: AuthenticatedSession
|
||||||
@@ -267,3 +287,79 @@ async def change_password(
|
|||||||
|
|
||||||
logger.info("auth.password_changed user_id=%s", principal.id)
|
logger.info("auth.password_changed user_id=%s", principal.id)
|
||||||
return repond(response, settings, session)
|
return repond(response, settings, session)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/forgot-password",
|
||||||
|
status_code=status.HTTP_202_ACCEPTED,
|
||||||
|
summary="Demande un lien de réinitialisation par email",
|
||||||
|
responses=REPONSES_FORGOT_PASSWORD,
|
||||||
|
)
|
||||||
|
async def forgot_password(
|
||||||
|
payload: ForgotPasswordRequest,
|
||||||
|
request: Request,
|
||||||
|
response: Response,
|
||||||
|
service: AuthServiceDep,
|
||||||
|
background_tasks: BackgroundTasks,
|
||||||
|
client_ip: str | None = Depends(get_client_ip),
|
||||||
|
) -> None:
|
||||||
|
response.headers["Cache-Control"] = "no-store"
|
||||||
|
|
||||||
|
try:
|
||||||
|
await service.request_password_reset(
|
||||||
|
email=payload.email,
|
||||||
|
client_ip=client_ip,
|
||||||
|
user_agent=request.headers.get("user-agent"),
|
||||||
|
background_tasks=background_tasks,
|
||||||
|
)
|
||||||
|
except RateLimitedError as erreur:
|
||||||
|
logger.warning("auth.password_reset.rate_limited ip=%s", client_ip)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_429_TOO_MANY_REQUESTS,
|
||||||
|
detail="Trop de demandes, réessayez plus tard",
|
||||||
|
headers={"Retry-After": str(erreur.retry_after)},
|
||||||
|
) from erreur
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/reset-password/validate",
|
||||||
|
response_model=ResetTokenValidationResponse,
|
||||||
|
summary="Vérifie sans le consommer si un lien de réinitialisation est encore valide",
|
||||||
|
responses=REPONSE_VALIDATION,
|
||||||
|
)
|
||||||
|
async def validate_reset_token(token: str, service: AuthServiceDep) -> ResetTokenValidationResponse:
|
||||||
|
return ResetTokenValidationResponse(valid=await service.is_reset_token_valid(token=token))
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/reset-password",
|
||||||
|
response_model=TokenResponse,
|
||||||
|
summary="Choisit un nouveau mot de passe depuis un lien reçu par email",
|
||||||
|
dependencies=[Depends(require_trusted_origin)],
|
||||||
|
responses=REPONSES_RESET_PASSWORD,
|
||||||
|
)
|
||||||
|
async def reset_password(
|
||||||
|
payload: ResetPasswordRequest,
|
||||||
|
request: Request,
|
||||||
|
response: Response,
|
||||||
|
settings: SettingsDep,
|
||||||
|
service: AuthServiceDep,
|
||||||
|
client_ip: str | None = Depends(get_client_ip),
|
||||||
|
) -> TokenResponse:
|
||||||
|
response.headers["Cache-Control"] = "no-store"
|
||||||
|
|
||||||
|
try:
|
||||||
|
session = await service.confirm_password_reset(
|
||||||
|
token=payload.token,
|
||||||
|
new_password=payload.new_password,
|
||||||
|
client_ip=client_ip,
|
||||||
|
user_agent=request.headers.get("user-agent"),
|
||||||
|
)
|
||||||
|
except InvalidOrExpiredResetTokenError as erreur:
|
||||||
|
logger.warning("auth.password_reset.invalid_token ip=%s", client_ip)
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST, detail=DETAIL_LIEN_RESET
|
||||||
|
) from erreur
|
||||||
|
|
||||||
|
logger.info("auth.password_reset.success user_id=%s", session.principal.id)
|
||||||
|
return repond(response, settings, session)
|
||||||
|
|||||||
@@ -0,0 +1,20 @@
|
|||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from app.api.deps import DriftServiceDep, OperateurDep
|
||||||
|
from app.api.openapi import REPONSE_VALIDATION
|
||||||
|
from app.schemas.drift import DriftReportResponse
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/drift",
|
||||||
|
response_model=list[DriftReportResponse],
|
||||||
|
summary="Dernier rapport de dérive par site, plus la ligne globale",
|
||||||
|
responses=REPONSE_VALIDATION,
|
||||||
|
)
|
||||||
|
async def get_drift(
|
||||||
|
_: OperateurDep, service: DriftServiceDep, site_id: str | None = None
|
||||||
|
) -> list[DriftReportResponse]:
|
||||||
|
rapports = await service.derniers(site_id=site_id)
|
||||||
|
return [DriftReportResponse.model_validate(rapport) for rapport in rapports]
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from app.api.deps import LecteurDep, PredictionServiceDep
|
||||||
|
from app.schemas.prediction import PredictionSummaryResponse
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"",
|
||||||
|
response_model=PredictionSummaryResponse,
|
||||||
|
summary="Dernière prédiction de consommation par site",
|
||||||
|
)
|
||||||
|
async def get_predictions(
|
||||||
|
_: LecteurDep, service: PredictionServiceDep
|
||||||
|
) -> PredictionSummaryResponse:
|
||||||
|
resume = await service.summary()
|
||||||
|
return PredictionSummaryResponse.model_validate(resume)
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from fastapi import APIRouter, HTTPException, Query, status
|
||||||
|
|
||||||
|
from app.api.deps import LecteurDep, ReadingServiceDep
|
||||||
|
from app.api.openapi import REPONSE_VALIDATION, Reponses
|
||||||
|
from app.schemas.errors import ErrorResponse
|
||||||
|
from app.schemas.reading import ReadingResponse
|
||||||
|
from app.services.reading import FenetreInverseeError, FenetreTropLargeError
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
REPONSES_FENETRE: Reponses = {
|
||||||
|
**REPONSE_VALIDATION,
|
||||||
|
400: {
|
||||||
|
"model": ErrorResponse,
|
||||||
|
"description": (
|
||||||
|
"Fenêtre temporelle invalide : `start` postérieur ou égal à `end`, ou écart entre "
|
||||||
|
"les deux supérieur à 90 jours."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"",
|
||||||
|
response_model=list[ReadingResponse],
|
||||||
|
summary="Liste l'historique des lectures",
|
||||||
|
responses=REPONSES_FENETRE,
|
||||||
|
)
|
||||||
|
async def list_readings(
|
||||||
|
_: LecteurDep,
|
||||||
|
service: ReadingServiceDep,
|
||||||
|
site_id: str | None = None,
|
||||||
|
start: datetime | None = None,
|
||||||
|
end: datetime | None = None,
|
||||||
|
limit: int = Query(500, ge=1, le=2000),
|
||||||
|
offset: int = Query(0, ge=0),
|
||||||
|
) -> list[ReadingResponse]:
|
||||||
|
try:
|
||||||
|
lectures = await service.list_history(
|
||||||
|
site_id=site_id, start=start, end=end, limit=limit, offset=offset
|
||||||
|
)
|
||||||
|
except FenetreInverseeError as erreur:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="`start` doit être strictement antérieur à `end`",
|
||||||
|
) from erreur
|
||||||
|
except FenetreTropLargeError as erreur:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_400_BAD_REQUEST,
|
||||||
|
detail="L'écart entre `start` et `end` ne peut pas dépasser 90 jours",
|
||||||
|
) from erreur
|
||||||
|
return [ReadingResponse.model_validate(lecture) for lecture in lectures]
|
||||||
@@ -1,13 +1,18 @@
|
|||||||
from fastapi import APIRouter, HTTPException, status
|
from fastapi import APIRouter, HTTPException, status
|
||||||
|
|
||||||
from app.api.deps import LecteurDep, RecommendationServiceDep
|
from app.api.deps import AdminDep, LecteurDep, RecommendationServiceDep
|
||||||
from app.api.openapi import REPONSE_VALIDATION, Reponses
|
from app.api.openapi import REPONSE_VALIDATION, REPONSES_ADMIN, Reponses
|
||||||
from app.schemas.errors import ErrorResponse
|
from app.schemas.errors import ErrorResponse
|
||||||
from app.schemas.recommendation import RecommendationResponse
|
from app.schemas.recommendation import (
|
||||||
|
RecommendationGenerationResponse,
|
||||||
|
RecommendationResponse,
|
||||||
|
)
|
||||||
from app.services.recommendation import RecommendationNotFoundError
|
from app.services.recommendation import RecommendationNotFoundError
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
REPONSES_GENERATION: Reponses = {**REPONSES_ADMIN, **REPONSE_VALIDATION}
|
||||||
|
|
||||||
REPONSES_INTROUVABLE: Reponses = {
|
REPONSES_INTROUVABLE: Reponses = {
|
||||||
**REPONSE_VALIDATION,
|
**REPONSE_VALIDATION,
|
||||||
404: {"model": ErrorResponse, "description": "Aucune recommandation ne porte cet identifiant."},
|
404: {"model": ErrorResponse, "description": "Aucune recommandation ne porte cet identifiant."},
|
||||||
@@ -38,3 +43,22 @@ async def get_recommendation(
|
|||||||
status_code=status.HTTP_404_NOT_FOUND, detail="Recommandation introuvable"
|
status_code=status.HTTP_404_NOT_FOUND, detail="Recommandation introuvable"
|
||||||
) from erreur
|
) from erreur
|
||||||
return RecommendationResponse.model_validate(recommendation)
|
return RecommendationResponse.model_validate(recommendation)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/generate",
|
||||||
|
response_model=RecommendationGenerationResponse,
|
||||||
|
summary="Génère les recommandations à partir des alertes",
|
||||||
|
responses=REPONSES_GENERATION,
|
||||||
|
)
|
||||||
|
async def generate_recommendations(
|
||||||
|
_: AdminDep,
|
||||||
|
service: RecommendationServiceDep,
|
||||||
|
site_id: str | None = None,
|
||||||
|
) -> RecommendationGenerationResponse:
|
||||||
|
rapport = await service.generate(site_id=site_id)
|
||||||
|
return RecommendationGenerationResponse(
|
||||||
|
alerts_examined=rapport.alertes_examinees,
|
||||||
|
recommendations_created=rapport.recommandations_creees,
|
||||||
|
already_present=rapport.deja_presentes,
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
from fastapi import APIRouter
|
||||||
|
|
||||||
|
from app.api.deps import AdminDep, SensorServiceDep
|
||||||
|
from app.schemas.sensor import SensorStatusResponse
|
||||||
|
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/status",
|
||||||
|
response_model=SensorStatusResponse,
|
||||||
|
summary="État de santé des capteurs par site",
|
||||||
|
)
|
||||||
|
async def get_status(_: AdminDep, service: SensorServiceDep) -> SensorStatusResponse:
|
||||||
|
etat = await service.status()
|
||||||
|
return SensorStatusResponse.model_validate(etat)
|
||||||
@@ -3,7 +3,7 @@ from fastapi import APIRouter, HTTPException, status
|
|||||||
from app.api.deps import LecteurDep, SiteServiceDep
|
from app.api.deps import LecteurDep, SiteServiceDep
|
||||||
from app.api.openapi import REPONSE_VALIDATION, Reponses
|
from app.api.openapi import REPONSE_VALIDATION, Reponses
|
||||||
from app.schemas.errors import ErrorResponse
|
from app.schemas.errors import ErrorResponse
|
||||||
from app.schemas.site import SiteResponse
|
from app.schemas.site import SiteCurrentResponse, SiteResponse
|
||||||
from app.services.site import SiteNotFoundError
|
from app.services.site import SiteNotFoundError
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
@@ -34,3 +34,19 @@ async def get_site(site_id: str, _: LecteurDep, service: SiteServiceDep) -> Site
|
|||||||
status_code=status.HTTP_404_NOT_FOUND, detail="Site introuvable"
|
status_code=status.HTTP_404_NOT_FOUND, detail="Site introuvable"
|
||||||
) from erreur
|
) from erreur
|
||||||
return SiteResponse.model_validate(site)
|
return SiteResponse.model_validate(site)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/{site_id}/current",
|
||||||
|
response_model=SiteCurrentResponse,
|
||||||
|
summary="Dernière mesure d'un site",
|
||||||
|
responses=REPONSES_INTROUVABLE,
|
||||||
|
)
|
||||||
|
async def get_current(site_id: str, _: LecteurDep, service: SiteServiceDep) -> SiteCurrentResponse:
|
||||||
|
try:
|
||||||
|
actuel = await service.current(site_id)
|
||||||
|
except SiteNotFoundError as erreur:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_404_NOT_FOUND, detail="Site introuvable"
|
||||||
|
) from erreur
|
||||||
|
return SiteCurrentResponse.model_validate(actuel)
|
||||||
|
|||||||
@@ -1,13 +1,33 @@
|
|||||||
from fastapi import APIRouter
|
from fastapi import APIRouter
|
||||||
|
|
||||||
from app.api.openapi import REPONSE_SERVEUR, REPONSES_ADMIN, REPONSES_LECTEUR
|
from app.api.openapi import (
|
||||||
from app.api.v1.endpoints import auth, health, recommendations, sites, stats, users
|
REPONSE_SERVEUR,
|
||||||
|
REPONSES_ADMIN,
|
||||||
|
REPONSES_LECTEUR,
|
||||||
|
REPONSES_OPERATEUR,
|
||||||
|
)
|
||||||
|
from app.api.v1.endpoints import (
|
||||||
|
alerts,
|
||||||
|
auth,
|
||||||
|
health,
|
||||||
|
monitoring,
|
||||||
|
predictions,
|
||||||
|
readings,
|
||||||
|
recommendations,
|
||||||
|
sensors,
|
||||||
|
sites,
|
||||||
|
stats,
|
||||||
|
users,
|
||||||
|
)
|
||||||
|
|
||||||
api_router = APIRouter(responses=REPONSE_SERVEUR)
|
api_router = APIRouter(responses=REPONSE_SERVEUR)
|
||||||
api_router.include_router(health.router, prefix="/health", tags=["health"])
|
api_router.include_router(health.router, prefix="/health", tags=["health"])
|
||||||
api_router.include_router(auth.router, prefix="/auth", tags=["auth"])
|
api_router.include_router(auth.router, prefix="/auth", tags=["auth"])
|
||||||
api_router.include_router(users.router, prefix="/users", tags=["users"], responses=REPONSES_ADMIN)
|
api_router.include_router(users.router, prefix="/users", tags=["users"], responses=REPONSES_ADMIN)
|
||||||
api_router.include_router(sites.router, prefix="/sites", tags=["sites"], responses=REPONSES_LECTEUR)
|
api_router.include_router(sites.router, prefix="/sites", tags=["sites"], responses=REPONSES_LECTEUR)
|
||||||
|
api_router.include_router(
|
||||||
|
alerts.router, prefix="/alerts", tags=["alerts"], responses=REPONSES_LECTEUR
|
||||||
|
)
|
||||||
api_router.include_router(
|
api_router.include_router(
|
||||||
recommendations.router,
|
recommendations.router,
|
||||||
prefix="/recommendations",
|
prefix="/recommendations",
|
||||||
@@ -15,3 +35,15 @@ api_router.include_router(
|
|||||||
responses=REPONSES_LECTEUR,
|
responses=REPONSES_LECTEUR,
|
||||||
)
|
)
|
||||||
api_router.include_router(stats.router, prefix="/stats", tags=["stats"], responses=REPONSES_LECTEUR)
|
api_router.include_router(stats.router, prefix="/stats", tags=["stats"], responses=REPONSES_LECTEUR)
|
||||||
|
api_router.include_router(
|
||||||
|
readings.router, prefix="/readings", tags=["readings"], responses=REPONSES_LECTEUR
|
||||||
|
)
|
||||||
|
api_router.include_router(
|
||||||
|
sensors.router, prefix="/sensors", tags=["sensors"], responses=REPONSES_ADMIN
|
||||||
|
)
|
||||||
|
api_router.include_router(
|
||||||
|
predictions.router, prefix="/predictions", tags=["predictions"], responses=REPONSES_LECTEUR
|
||||||
|
)
|
||||||
|
api_router.include_router(
|
||||||
|
monitoring.router, prefix="/monitoring", tags=["monitoring"], responses=REPONSES_OPERATEUR
|
||||||
|
)
|
||||||
|
|||||||
+55
-4
@@ -9,6 +9,7 @@ import argparse
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import secrets
|
import secrets
|
||||||
|
import string
|
||||||
import sys
|
import sys
|
||||||
from getpass import getpass
|
from getpass import getpass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -21,10 +22,13 @@ from app.core.hashing import build_hasher
|
|||||||
from app.core.roles import Role
|
from app.core.roles import Role
|
||||||
from app.db.session import get_session_factory
|
from app.db.session import get_session_factory
|
||||||
from app.main import create_app
|
from app.main import create_app
|
||||||
|
from app.repositories.alert import AlertRepository
|
||||||
|
from app.repositories.recommendation import RecommendationRepository
|
||||||
from app.repositories.user import UserRepository
|
from app.repositories.user import UserRepository
|
||||||
|
from app.schemas.auth import PASSWORD_MIN_LENGTH, SPECIAL_CHARACTERS, valide_complexite
|
||||||
|
from app.services.recommendation import RecommendationService
|
||||||
|
|
||||||
LONGUEUR_MOT_DE_PASSE_GENERE = 24
|
LONGUEUR_MOT_DE_PASSE_GENERE = 24
|
||||||
LONGUEUR_MINIMALE = 12
|
|
||||||
CHEMIN_CONTRAT = Path(__file__).resolve().parent.parent / "openapi.json"
|
CHEMIN_CONTRAT = Path(__file__).resolve().parent.parent / "openapi.json"
|
||||||
|
|
||||||
|
|
||||||
@@ -62,6 +66,22 @@ async def create_admin(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def generate_recommendations(*, site_id: str | None) -> str:
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
service = RecommendationService(
|
||||||
|
recommendations=RecommendationRepository(session),
|
||||||
|
alerts=AlertRepository(session),
|
||||||
|
transaction=session,
|
||||||
|
)
|
||||||
|
rapport = await service.generate(site_id=site_id)
|
||||||
|
|
||||||
|
return (
|
||||||
|
f"{rapport.alertes_examinees} alerte(s) examinée(s), "
|
||||||
|
f"{rapport.recommandations_creees} recommandation(s) créée(s), "
|
||||||
|
f"{rapport.deja_presentes} déjà présente(s)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# Piège : le schéma ne doit dépendre ni du `.env` du poste ni des variables `APP_*`, sinon le
|
# Piège : le schéma ne doit dépendre ni du `.env` du poste ni des variables `APP_*`, sinon le
|
||||||
# fichier versionné changerait de machine en machine et le test de dérive deviendrait un oracle
|
# fichier versionné changerait de machine en machine et le test de dérive deviendrait un oracle
|
||||||
# de configuration locale. Tout ce qui atteint le schéma est donc posé ici, `_env_file` compris.
|
# de configuration locale. Tout ce qui atteint le schéma est donc posé ici, `_env_file` compris.
|
||||||
@@ -108,18 +128,45 @@ def build_parser() -> argparse.ArgumentParser:
|
|||||||
"export-openapi", help="Écrit le contrat OpenAPI sur disque"
|
"export-openapi", help="Écrit le contrat OpenAPI sur disque"
|
||||||
)
|
)
|
||||||
contrat.add_argument("--output", default=str(CHEMIN_CONTRAT))
|
contrat.add_argument("--output", default=str(CHEMIN_CONTRAT))
|
||||||
|
|
||||||
|
recommandations = sous_commandes.add_parser(
|
||||||
|
"generate-recommendations",
|
||||||
|
help="Applique le moteur de règles aux alertes en base",
|
||||||
|
)
|
||||||
|
recommandations.add_argument(
|
||||||
|
"--site-id", default=None, help="Limite le traitement aux alertes d'un site"
|
||||||
|
)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def genere_mot_de_passe() -> str:
|
||||||
|
tirage = secrets.SystemRandom()
|
||||||
|
classes = [
|
||||||
|
string.ascii_uppercase,
|
||||||
|
string.ascii_lowercase,
|
||||||
|
string.digits,
|
||||||
|
SPECIAL_CHARACTERS,
|
||||||
|
]
|
||||||
|
reste = LONGUEUR_MOT_DE_PASSE_GENERE - len(classes)
|
||||||
|
caracteres = [tirage.choice(classe) for classe in classes]
|
||||||
|
caracteres += [tirage.choice("".join(classes)) for _ in range(reste)]
|
||||||
|
tirage.shuffle(caracteres)
|
||||||
|
return "".join(caracteres)
|
||||||
|
|
||||||
|
|
||||||
def read_password(*, generate: bool) -> str:
|
def read_password(*, generate: bool) -> str:
|
||||||
if generate:
|
if generate:
|
||||||
mot_de_passe = secrets.token_urlsafe(LONGUEUR_MOT_DE_PASSE_GENERE)
|
mot_de_passe = genere_mot_de_passe()
|
||||||
print(f"Mot de passe généré, il ne sera plus affiché : {mot_de_passe}")
|
print(f"Mot de passe généré, il ne sera plus affiché : {mot_de_passe}")
|
||||||
return mot_de_passe
|
return mot_de_passe
|
||||||
|
|
||||||
mot_de_passe = getpass("Mot de passe : ")
|
mot_de_passe = getpass("Mot de passe : ")
|
||||||
if len(mot_de_passe) < LONGUEUR_MINIMALE:
|
if len(mot_de_passe) < PASSWORD_MIN_LENGTH:
|
||||||
raise SystemExit(f"Le mot de passe doit faire au moins {LONGUEUR_MINIMALE} caractères")
|
raise SystemExit(f"Le mot de passe doit faire au moins {PASSWORD_MIN_LENGTH} caractères")
|
||||||
|
try:
|
||||||
|
valide_complexite(mot_de_passe)
|
||||||
|
except ValueError as erreur:
|
||||||
|
raise SystemExit(str(erreur)) from erreur
|
||||||
if mot_de_passe != getpass("Confirmation : "):
|
if mot_de_passe != getpass("Confirmation : "):
|
||||||
raise SystemExit("Les deux saisies diffèrent")
|
raise SystemExit("Les deux saisies diffèrent")
|
||||||
return mot_de_passe
|
return mot_de_passe
|
||||||
@@ -132,6 +179,10 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
print(export_openapi(Path(arguments.output)))
|
print(export_openapi(Path(arguments.output)))
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
if arguments.commande == "generate-recommendations":
|
||||||
|
print(asyncio.run(generate_recommendations(site_id=arguments.site_id)))
|
||||||
|
return 0
|
||||||
|
|
||||||
mot_de_passe = read_password(generate=arguments.generate)
|
mot_de_passe = read_password(generate=arguments.generate)
|
||||||
|
|
||||||
succes, message = asyncio.run(
|
succes, message = asyncio.run(
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from typing import Literal, Self
|
from typing import Literal, Self
|
||||||
|
|
||||||
from pydantic import Field, SecretStr, model_validator
|
from pydantic import Field, SecretStr, field_validator, model_validator
|
||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
|
|
||||||
Environment = Literal["local", "dev", "staging", "prod"]
|
Environment = Literal["local", "dev", "staging", "prod"]
|
||||||
@@ -34,6 +34,11 @@ class Settings(BaseSettings):
|
|||||||
database_pool_size: int = 5
|
database_pool_size: int = 5
|
||||||
database_max_overflow: int = 10
|
database_max_overflow: int = 10
|
||||||
|
|
||||||
|
mock_api_base_url: str = "https://api-mock.charlieandre.fr"
|
||||||
|
mock_api_username: str | None = None
|
||||||
|
mock_api_password: SecretStr | None = None
|
||||||
|
mock_api_timeout_seconds: float = Field(default=10.0, gt=0)
|
||||||
|
|
||||||
jwt_issuer: str = "enervision-api"
|
jwt_issuer: str = "enervision-api"
|
||||||
jwt_audience: str = "enervision-web"
|
jwt_audience: str = "enervision-web"
|
||||||
access_token_ttl_seconds: int = Field(default=900, ge=60, le=3600)
|
access_token_ttl_seconds: int = Field(default=900, ge=60, le=3600)
|
||||||
@@ -54,10 +59,46 @@ class Settings(BaseSettings):
|
|||||||
login_max_failures_per_ip: int = Field(default=20, ge=1)
|
login_max_failures_per_ip: int = Field(default=20, ge=1)
|
||||||
login_max_failures_per_identifier: int = Field(default=50, ge=1)
|
login_max_failures_per_identifier: int = Field(default=50, ge=1)
|
||||||
|
|
||||||
|
password_reset_ttl_seconds: int = Field(default=900, ge=60, le=3600)
|
||||||
|
password_reset_window_seconds: int = Field(default=900, ge=60)
|
||||||
|
password_reset_max_requests_per_identifier: int = Field(default=3, ge=1)
|
||||||
|
password_reset_max_requests_per_ip: int = Field(default=10, ge=1)
|
||||||
|
|
||||||
|
smtp_host: str = "localhost"
|
||||||
|
smtp_port: int = Field(default=587, ge=1, le=65535)
|
||||||
|
smtp_username: str | None = None
|
||||||
|
smtp_password: SecretStr | None = None
|
||||||
|
smtp_use_tls: bool = False
|
||||||
|
smtp_from_address: str = "no-reply@enervision.fr"
|
||||||
|
frontend_reset_password_url: str = "http://localhost:4200/reset-password" # noqa: S105
|
||||||
|
|
||||||
trust_proxy_headers: bool = False
|
trust_proxy_headers: bool = False
|
||||||
expose_api_docs: bool | None = None
|
expose_api_docs: bool | None = None
|
||||||
metrics_token: SecretStr | None = None
|
metrics_token: SecretStr | None = None
|
||||||
|
|
||||||
|
s3_endpoint_url: str | None = None
|
||||||
|
s3_region: str = "garage"
|
||||||
|
s3_access_key: str | None = None
|
||||||
|
s3_secret_key: SecretStr | None = None
|
||||||
|
s3_bucket: str | None = None
|
||||||
|
s3_sse_key: SecretStr | None = None
|
||||||
|
reading_retention_days: int = Field(default=1095, ge=30)
|
||||||
|
|
||||||
|
# Compose passe `APP_METRICS_TOKEN` et les réglages S3 vides quand rien n'est posé : vide vaut
|
||||||
|
# absent, sinon `/metrics` exigerait un `Bearer` sans valeur et l'archivage un endpoint vide.
|
||||||
|
@field_validator(
|
||||||
|
"metrics_token",
|
||||||
|
"s3_endpoint_url",
|
||||||
|
"s3_access_key",
|
||||||
|
"s3_secret_key",
|
||||||
|
"s3_bucket",
|
||||||
|
"s3_sse_key",
|
||||||
|
mode="before",
|
||||||
|
)
|
||||||
|
@classmethod
|
||||||
|
def _jeton_vide_vaut_absent(cls, valeur: object) -> object:
|
||||||
|
return None if valeur == "" else valeur
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def allowed_origins(self) -> list[str]:
|
def allowed_origins(self) -> list[str]:
|
||||||
return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()]
|
return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()]
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# Piège : l'URL de réinitialisation porte le jeton en clair. Ne jamais la journaliser :
|
||||||
|
# `send_password_reset_email()` ne logue que le destinataire, jamais `reset_url`.
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from email.message import EmailMessage
|
||||||
|
|
||||||
|
import aiosmtplib
|
||||||
|
|
||||||
|
from app.core.logging import get_logger
|
||||||
|
|
||||||
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SmtpConfig:
|
||||||
|
host: str
|
||||||
|
port: int
|
||||||
|
username: str | None
|
||||||
|
password: str | None
|
||||||
|
use_tls: bool
|
||||||
|
from_address: str
|
||||||
|
|
||||||
|
|
||||||
|
class Mailer:
|
||||||
|
def __init__(self, config: SmtpConfig) -> None:
|
||||||
|
self._config = config
|
||||||
|
|
||||||
|
async def send_password_reset_email(self, *, to: str, reset_url: str) -> None:
|
||||||
|
message = EmailMessage()
|
||||||
|
message["From"] = self._config.from_address
|
||||||
|
message["To"] = to
|
||||||
|
message["Subject"] = "Réinitialisation de votre mot de passe EnerVision"
|
||||||
|
message.set_content(
|
||||||
|
"Une réinitialisation de mot de passe a été demandée pour ce compte.\n\n"
|
||||||
|
f"Ouvrez ce lien dans les 15 minutes pour choisir un nouveau mot de passe : "
|
||||||
|
f"{reset_url}\n\n"
|
||||||
|
"Si vous n'êtes pas à l'origine de cette demande, ignorez cet email."
|
||||||
|
)
|
||||||
|
|
||||||
|
_, message_recu = await aiosmtplib.send(
|
||||||
|
message,
|
||||||
|
hostname=self._config.host,
|
||||||
|
port=self._config.port,
|
||||||
|
username=self._config.username,
|
||||||
|
password=self._config.password,
|
||||||
|
use_tls=self._config.use_tls,
|
||||||
|
)
|
||||||
|
logger.info("mailer.password_reset_sent to=%s smtp_response=%s", to, message_recu)
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
# Détection d'alertes internes EnerVision (issue #104) : script lancé à la main pour l'instant,
|
||||||
|
# comme `enervision_ml.score` côté ML, sans automatisation Airflow pour l'ordonnancer.
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from app.repositories.alert import AlertRepository
|
||||||
|
from app.repositories.prediction import PredictionRepository
|
||||||
|
from app.repositories.reading import ReadingRepository
|
||||||
|
from app.repositories.site import SiteRepository
|
||||||
|
from app.services.alert import AlertService
|
||||||
|
|
||||||
|
|
||||||
|
async def run_detection(*, now: datetime | None = None, site_id: str | None = None) -> int:
|
||||||
|
"""Exécute les cinq règles de détection et enregistre les nouvelles alertes. Rend le nombre de
|
||||||
|
lignes effectivement insérées (les doublons de `source_alert_id` sont silencieusement
|
||||||
|
ignorés)."""
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
service = AlertService(
|
||||||
|
alerts=AlertRepository(session),
|
||||||
|
readings=ReadingRepository(session),
|
||||||
|
predictions=PredictionRepository(session),
|
||||||
|
sites=SiteRepository(session),
|
||||||
|
)
|
||||||
|
nouvelles = await service.detect(now=now, site_id=site_id)
|
||||||
|
await session.commit()
|
||||||
|
return len(nouvelles)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_instant(valeur: str) -> datetime:
|
||||||
|
instant = datetime.fromisoformat(valeur)
|
||||||
|
return instant if instant.tzinfo is not None else instant.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="python -m app.detection.internal_alerts",
|
||||||
|
description="Détection d'alertes internes EnerVision",
|
||||||
|
)
|
||||||
|
parser.add_argument("--site-id", default=None, help="Limite la détection à un seul site.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--now",
|
||||||
|
type=_parse_instant,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Instant de référence (ISO 8601, UTC si le fuseau est omis). Défaut : l'heure courante."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
args = parse_args(argv)
|
||||||
|
# Échoue tôt si `APP_SECRET_KEY`/`DATABASE_URL` manquent, avant toute requête à la base.
|
||||||
|
get_settings()
|
||||||
|
nombre = asyncio.run(run_detection(now=args.now, site_id=args.site_id))
|
||||||
|
print(f"{nombre} nouvelle(s) alerte(s) enregistrée(s).")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,621 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, cast
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
|
||||||
|
REQUIRED_COLUMNS = {
|
||||||
|
"timestamp",
|
||||||
|
"site_id",
|
||||||
|
"site_type",
|
||||||
|
"site_name",
|
||||||
|
"consumption_kwh",
|
||||||
|
"consumption_euros",
|
||||||
|
"temperature_celsius",
|
||||||
|
"humidity_percent",
|
||||||
|
"solar_irradiance_wm2",
|
||||||
|
"hour",
|
||||||
|
"day_of_week",
|
||||||
|
"day_name",
|
||||||
|
"month",
|
||||||
|
"is_weekend",
|
||||||
|
"is_working_hours",
|
||||||
|
}
|
||||||
|
|
||||||
|
MEASURE_COLUMNS = [
|
||||||
|
"consumption_kwh",
|
||||||
|
"consumption_euros",
|
||||||
|
"temperature_celsius",
|
||||||
|
"humidity_percent",
|
||||||
|
"solar_irradiance_wm2",
|
||||||
|
]
|
||||||
|
|
||||||
|
SOURCE_NAME = "csv"
|
||||||
|
|
||||||
|
|
||||||
|
def compute_sha256(path: Path) -> str:
|
||||||
|
"""Calcule l'empreinte SHA-256 du fichier source."""
|
||||||
|
sha256 = hashlib.sha256()
|
||||||
|
|
||||||
|
with path.open("rb") as source:
|
||||||
|
for block in iter(lambda: source.read(1024 * 1024), b""):
|
||||||
|
sha256.update(block)
|
||||||
|
|
||||||
|
return sha256.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def load_metadata(path: Path) -> dict[str, Any]:
|
||||||
|
"""Charge les métadonnées fournies avec le dataset."""
|
||||||
|
with path.open("r", encoding="utf-8") as source:
|
||||||
|
metadata = json.load(source)
|
||||||
|
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
raise ValueError("Le fichier de métadonnées doit contenir un objet JSON.")
|
||||||
|
|
||||||
|
return cast(dict[str, Any], metadata)
|
||||||
|
|
||||||
|
|
||||||
|
def classify_quality(
|
||||||
|
row: dict[str, Any],
|
||||||
|
) -> tuple[str, list[str]]:
|
||||||
|
"""
|
||||||
|
Déduit une qualité technique à partir des champs manquants.
|
||||||
|
|
||||||
|
Les valeurs NULL sont conservées. On ne cherche pas ici à
|
||||||
|
déterminer la cause physique exacte de leur absence.
|
||||||
|
"""
|
||||||
|
missing = [column for column in MEASURE_COLUMNS if pd.isna(row.get(column))]
|
||||||
|
|
||||||
|
if not missing:
|
||||||
|
quality = "good"
|
||||||
|
elif len(missing) == len(MEASURE_COLUMNS):
|
||||||
|
quality = "critical"
|
||||||
|
elif "consumption_kwh" in missing:
|
||||||
|
quality = "degraded"
|
||||||
|
else:
|
||||||
|
quality = "partial"
|
||||||
|
|
||||||
|
reasons = [f"missing:{column}" for column in missing]
|
||||||
|
|
||||||
|
return quality, reasons
|
||||||
|
|
||||||
|
|
||||||
|
def validate_source(
|
||||||
|
frame: pd.DataFrame,
|
||||||
|
metadata: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
"""Valide le dataset avant tout chargement en base."""
|
||||||
|
missing_columns = REQUIRED_COLUMNS.difference(frame.columns)
|
||||||
|
|
||||||
|
if missing_columns:
|
||||||
|
raise ValueError(f"Colonnes obligatoires absentes : {sorted(missing_columns)}")
|
||||||
|
|
||||||
|
expected_records = int(metadata["total_records"])
|
||||||
|
|
||||||
|
if len(frame) != expected_records:
|
||||||
|
raise ValueError(f"Nombre de lignes inattendu : {len(frame)} au lieu de {expected_records}")
|
||||||
|
|
||||||
|
expected_sites = set(metadata["sites"].keys())
|
||||||
|
actual_sites = set(frame["site_id"].unique())
|
||||||
|
|
||||||
|
if actual_sites != expected_sites:
|
||||||
|
raise ValueError(
|
||||||
|
f"Sites incohérents. Attendus={sorted(expected_sites)}, trouvés={sorted(actual_sites)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
duplicated = frame.duplicated(subset=["site_id", "timestamp"]).sum()
|
||||||
|
|
||||||
|
if duplicated:
|
||||||
|
raise ValueError(f"{duplicated} doublons (site_id, timestamp) détectés")
|
||||||
|
|
||||||
|
static_variants = frame.groupby("site_id")[["site_type", "site_name"]].nunique()
|
||||||
|
|
||||||
|
if (static_variants > 1).any().any():
|
||||||
|
raise ValueError("Un site possède plusieurs valeurs de site_type ou site_name.")
|
||||||
|
|
||||||
|
# Vérifie également que tous les timestamps
|
||||||
|
# peuvent être interprétés correctement.
|
||||||
|
pd.to_datetime(
|
||||||
|
frame["timestamp"],
|
||||||
|
errors="raise",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_timestamps(
|
||||||
|
frame: pd.DataFrame,
|
||||||
|
source_timezone: str,
|
||||||
|
) -> pd.DataFrame:
|
||||||
|
"""
|
||||||
|
Normalise les timestamps et leur associe une timezone.
|
||||||
|
|
||||||
|
Les timestamps originaux sont conservés dans une colonne
|
||||||
|
temporaire afin de pouvoir les stocker dans raw_data.
|
||||||
|
"""
|
||||||
|
normalized = frame.copy()
|
||||||
|
|
||||||
|
normalized["_source_timestamp"] = normalized["timestamp"]
|
||||||
|
|
||||||
|
timestamps = pd.to_datetime(
|
||||||
|
normalized["timestamp"],
|
||||||
|
errors="raise",
|
||||||
|
)
|
||||||
|
|
||||||
|
if timestamps.dt.tz is None:
|
||||||
|
timestamps = timestamps.dt.tz_localize(source_timezone)
|
||||||
|
else:
|
||||||
|
timestamps = timestamps.dt.tz_convert(source_timezone)
|
||||||
|
|
||||||
|
normalized["timestamp"] = timestamps
|
||||||
|
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def to_json_value(value: Any) -> Any:
|
||||||
|
"""
|
||||||
|
Convertit une valeur Pandas/Numpy en valeur
|
||||||
|
compatible JSON.
|
||||||
|
"""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
if pd.isna(value):
|
||||||
|
return None
|
||||||
|
except TypeError, ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if isinstance(value, pd.Timestamp):
|
||||||
|
return value.isoformat()
|
||||||
|
|
||||||
|
if hasattr(value, "item"):
|
||||||
|
return value.item()
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
async def ensure_dataset(
|
||||||
|
connection: AsyncConnection,
|
||||||
|
metadata: dict[str, Any],
|
||||||
|
sha256: str,
|
||||||
|
source_timezone: str,
|
||||||
|
storage_uri: str,
|
||||||
|
) -> int:
|
||||||
|
"""
|
||||||
|
Crée l'entrée dataset si elle n'existe pas.
|
||||||
|
|
||||||
|
Le SHA-256 permet de reconnaître un fichier déjà importé
|
||||||
|
et participe à l'idempotence et à la traçabilité.
|
||||||
|
"""
|
||||||
|
result = await connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT dataset_id
|
||||||
|
FROM dataset
|
||||||
|
WHERE archive_sha256 = :sha256
|
||||||
|
LIMIT 1
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"sha256": sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
existing = result.scalar_one_or_none()
|
||||||
|
|
||||||
|
if existing is not None:
|
||||||
|
return int(existing)
|
||||||
|
|
||||||
|
metadata_summary = {
|
||||||
|
"generator_version": metadata.get("generator_version"),
|
||||||
|
"total_sites": metadata.get("total_sites"),
|
||||||
|
"total_records": metadata.get("total_records"),
|
||||||
|
"date_range": metadata.get("date_range"),
|
||||||
|
"frequency": metadata.get("frequency"),
|
||||||
|
"null_injection_enabled": metadata.get("null_injection_enabled"),
|
||||||
|
"null_strategies": metadata.get("null_strategies"),
|
||||||
|
"importer": "historical_import_v1",
|
||||||
|
}
|
||||||
|
|
||||||
|
result = await connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset (
|
||||||
|
dataset_name,
|
||||||
|
archive_sha256,
|
||||||
|
storage_uri,
|
||||||
|
source_timezone,
|
||||||
|
"metadata"
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:dataset_name,
|
||||||
|
:archive_sha256,
|
||||||
|
:storage_uri,
|
||||||
|
:source_timezone,
|
||||||
|
CAST(:metadata AS jsonb)
|
||||||
|
)
|
||||||
|
RETURNING dataset_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"dataset_name": ("EnerVision historical dataset 2023-2024"),
|
||||||
|
"archive_sha256": sha256,
|
||||||
|
"storage_uri": storage_uri,
|
||||||
|
"source_timezone": source_timezone,
|
||||||
|
"metadata": json.dumps(
|
||||||
|
metadata_summary,
|
||||||
|
ensure_ascii=False,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return int(result.scalar_one())
|
||||||
|
|
||||||
|
|
||||||
|
async def upsert_sites(
|
||||||
|
connection: AsyncConnection,
|
||||||
|
frame: pd.DataFrame,
|
||||||
|
) -> None:
|
||||||
|
"""Insère ou met à jour les sites du dataset."""
|
||||||
|
sites = cast(
|
||||||
|
list[dict[str, Any]],
|
||||||
|
frame[
|
||||||
|
[
|
||||||
|
"site_id",
|
||||||
|
"site_type",
|
||||||
|
"site_name",
|
||||||
|
]
|
||||||
|
]
|
||||||
|
.drop_duplicates(subset=["site_id"])
|
||||||
|
.to_dict(orient="records"),
|
||||||
|
)
|
||||||
|
|
||||||
|
await connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO site (
|
||||||
|
site_id,
|
||||||
|
site_type,
|
||||||
|
site_name
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:site_id,
|
||||||
|
:site_type,
|
||||||
|
:site_name
|
||||||
|
)
|
||||||
|
ON CONFLICT (site_id)
|
||||||
|
DO UPDATE SET
|
||||||
|
site_type = EXCLUDED.site_type,
|
||||||
|
site_name = EXCLUDED.site_name
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
sites,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_reading_batch(
|
||||||
|
chunk: pd.DataFrame,
|
||||||
|
dataset_id: int,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Transforme un chunk Pandas en lignes prêtes
|
||||||
|
à être chargées dans la table reading.
|
||||||
|
"""
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
records = cast(
|
||||||
|
list[dict[str, Any]],
|
||||||
|
chunk.to_dict(orient="records"),
|
||||||
|
)
|
||||||
|
|
||||||
|
for record in records:
|
||||||
|
quality, reasons = classify_quality(record)
|
||||||
|
|
||||||
|
raw_data = {
|
||||||
|
column: to_json_value(value)
|
||||||
|
for column, value in record.items()
|
||||||
|
if column != "_source_timestamp"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Dans raw_data, on conserve le timestamp
|
||||||
|
# exactement tel qu'il était dans le CSV.
|
||||||
|
raw_data["timestamp"] = to_json_value(record["_source_timestamp"])
|
||||||
|
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"site_id": record["site_id"],
|
||||||
|
"timestamp": record["timestamp"],
|
||||||
|
"source": SOURCE_NAME,
|
||||||
|
"dataset_id": dataset_id,
|
||||||
|
# Non fourni par le dataset historique.
|
||||||
|
"consumption_kw": None,
|
||||||
|
"consumption_kwh": to_json_value(record["consumption_kwh"]),
|
||||||
|
"consumption_euros": to_json_value(record["consumption_euros"]),
|
||||||
|
# Non fournis par le CSV historique.
|
||||||
|
"voltage_v": None,
|
||||||
|
"current_a": None,
|
||||||
|
"power_factor": None,
|
||||||
|
"temperature_celsius": (to_json_value(record["temperature_celsius"])),
|
||||||
|
"humidity_percent": (to_json_value(record["humidity_percent"])),
|
||||||
|
"solar_irradiance_wm2": (to_json_value(record["solar_irradiance_wm2"])),
|
||||||
|
"is_working_hours": bool(record["is_working_hours"]),
|
||||||
|
"data_quality": quality,
|
||||||
|
"null_reasons": reasons,
|
||||||
|
# Aucune imputation pendant l'ingestion RAW.
|
||||||
|
# Les valeurs manquantes sont conservées telles quelles
|
||||||
|
# afin de préserver la donnée source.
|
||||||
|
"imputed_values": None,
|
||||||
|
"imputation_method": None,
|
||||||
|
# Conservation de la donnée source
|
||||||
|
# pour la traçabilité.
|
||||||
|
"raw_data": json.dumps(
|
||||||
|
raw_data,
|
||||||
|
ensure_ascii=False,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
READING_INSERT = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO reading (
|
||||||
|
site_id,
|
||||||
|
timestamp,
|
||||||
|
source,
|
||||||
|
dataset_id,
|
||||||
|
consumption_kw,
|
||||||
|
consumption_kwh,
|
||||||
|
consumption_euros,
|
||||||
|
voltage_v,
|
||||||
|
current_a,
|
||||||
|
power_factor,
|
||||||
|
temperature_celsius,
|
||||||
|
humidity_percent,
|
||||||
|
solar_irradiance_wm2,
|
||||||
|
is_working_hours,
|
||||||
|
data_quality,
|
||||||
|
null_reasons,
|
||||||
|
imputed_values,
|
||||||
|
imputation_method,
|
||||||
|
raw_data
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:site_id,
|
||||||
|
:timestamp,
|
||||||
|
:source,
|
||||||
|
:dataset_id,
|
||||||
|
:consumption_kw,
|
||||||
|
:consumption_kwh,
|
||||||
|
:consumption_euros,
|
||||||
|
:voltage_v,
|
||||||
|
:current_a,
|
||||||
|
:power_factor,
|
||||||
|
:temperature_celsius,
|
||||||
|
:humidity_percent,
|
||||||
|
:solar_irradiance_wm2,
|
||||||
|
:is_working_hours,
|
||||||
|
:data_quality,
|
||||||
|
:null_reasons,
|
||||||
|
CAST(:imputed_values AS jsonb),
|
||||||
|
:imputation_method,
|
||||||
|
CAST(:raw_data AS jsonb)
|
||||||
|
)
|
||||||
|
ON CONFLICT DO NOTHING
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def import_historical(
|
||||||
|
csv_path: Path,
|
||||||
|
metadata_path: Path,
|
||||||
|
source_timezone: str,
|
||||||
|
batch_size: int,
|
||||||
|
dry_run: bool,
|
||||||
|
storage_uri: str,
|
||||||
|
) -> None:
|
||||||
|
"""
|
||||||
|
Exécute le pipeline ETL historique EnerVision.
|
||||||
|
|
||||||
|
Étapes :
|
||||||
|
1. Extract
|
||||||
|
2. Validate
|
||||||
|
3. Transform
|
||||||
|
4. Load
|
||||||
|
"""
|
||||||
|
metadata = load_metadata(metadata_path)
|
||||||
|
|
||||||
|
frame = pd.read_csv(csv_path)
|
||||||
|
|
||||||
|
validate_source(
|
||||||
|
frame,
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Lignes : {len(frame)}")
|
||||||
|
print(f"Sites : {frame['site_id'].nunique()}")
|
||||||
|
print(f"Période : {frame['timestamp'].min()} -> {frame['timestamp'].max()}")
|
||||||
|
print(f"Doublons : {frame.duplicated(['site_id', 'timestamp']).sum()}")
|
||||||
|
|
||||||
|
print("\nValeurs NULL :")
|
||||||
|
print(frame[MEASURE_COLUMNS].isna().sum())
|
||||||
|
|
||||||
|
sha256 = compute_sha256(csv_path)
|
||||||
|
|
||||||
|
print(f"\nSHA-256 : {sha256}")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print("\nDry-run terminé : aucune donnée écrite.")
|
||||||
|
return
|
||||||
|
|
||||||
|
normalized = normalize_timestamps(
|
||||||
|
frame,
|
||||||
|
source_timezone,
|
||||||
|
)
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
engine = create_async_engine(
|
||||||
|
str(settings.database_url),
|
||||||
|
pool_pre_ping=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with engine.begin() as connection:
|
||||||
|
dataset_id = await ensure_dataset(
|
||||||
|
connection=connection,
|
||||||
|
metadata=metadata,
|
||||||
|
sha256=sha256,
|
||||||
|
source_timezone=source_timezone,
|
||||||
|
storage_uri=storage_uri,
|
||||||
|
)
|
||||||
|
|
||||||
|
await upsert_sites(
|
||||||
|
connection,
|
||||||
|
normalized,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM reading
|
||||||
|
WHERE dataset_id = :dataset_id
|
||||||
|
AND source = :source
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"dataset_id": dataset_id,
|
||||||
|
"source": SOURCE_NAME,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
before = int(result.scalar_one())
|
||||||
|
|
||||||
|
for start in range(
|
||||||
|
0,
|
||||||
|
len(normalized),
|
||||||
|
batch_size,
|
||||||
|
):
|
||||||
|
chunk = normalized.iloc[start : start + batch_size]
|
||||||
|
|
||||||
|
rows = build_reading_batch(
|
||||||
|
chunk,
|
||||||
|
dataset_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
await connection.execute(
|
||||||
|
READING_INSERT,
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
loaded = min(
|
||||||
|
start + batch_size,
|
||||||
|
len(normalized),
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"Chargement : {loaded}/{len(normalized)}")
|
||||||
|
|
||||||
|
result = await connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT COUNT(*)
|
||||||
|
FROM reading
|
||||||
|
WHERE dataset_id = :dataset_id
|
||||||
|
AND source = :source
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"dataset_id": dataset_id,
|
||||||
|
"source": SOURCE_NAME,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
after = int(result.scalar_one())
|
||||||
|
|
||||||
|
print("\nImport terminé.")
|
||||||
|
print(f"dataset_id : {dataset_id}")
|
||||||
|
print(f"lectures avant : {before}")
|
||||||
|
print(f"lectures après : {after}")
|
||||||
|
print(f"nouvelles lectures : {after - before}")
|
||||||
|
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
"""Définit les arguments CLI de l'import."""
|
||||||
|
parser = argparse.ArgumentParser(description=("Import historique EnerVision"))
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--csv",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Chemin vers le CSV historique.",
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--metadata",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help=("Chemin vers le fichier dataset_metadata.json."),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--source-timezone",
|
||||||
|
default="UTC",
|
||||||
|
help=("Timezone associée aux timestamps du dataset. Défaut : UTC."),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--batch-size",
|
||||||
|
type=int,
|
||||||
|
default=1000,
|
||||||
|
help=("Nombre de lignes insérées par batch. Défaut : 1000."),
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--dry-run",
|
||||||
|
action="store_true",
|
||||||
|
help=("Valide les données sans écrire en base."),
|
||||||
|
)
|
||||||
|
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
"""Point d'entrée CLI du pipeline."""
|
||||||
|
args = parse_args()
|
||||||
|
|
||||||
|
if args.batch_size <= 0:
|
||||||
|
raise ValueError("--batch-size doit être strictement supérieur à 0.")
|
||||||
|
|
||||||
|
# resolve() est volontairement exécuté ici,
|
||||||
|
# dans la partie synchrone du programme.
|
||||||
|
# Cela évite une opération filesystem bloquante
|
||||||
|
# à l'intérieur d'une fonction async.
|
||||||
|
storage_uri = args.csv.resolve().as_uri()
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
import_historical(
|
||||||
|
csv_path=args.csv,
|
||||||
|
metadata_path=args.metadata,
|
||||||
|
source_timezone=(args.source_timezone),
|
||||||
|
batch_size=args.batch_size,
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
storage_uri=storage_uri,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,527 @@
|
|||||||
|
# Contrainte : la réponse de l'API Mock est une entrée hostile, pas une source de confiance.
|
||||||
|
# Voir OWASP API10 dans docs/architecture/owasp-traceabilite.md. Rien de ce qu'elle renvoie
|
||||||
|
# n'atteint la base sans passer par build_site_row() ou build_reading_row() : seuls les champs
|
||||||
|
# attendus sont recopiés, les grandeurs physiques sont bornées par PHYSICAL_BOUNDS, la taille des
|
||||||
|
# tableaux est plafonnée par MAX_SITES et par limit_for_window() (dérivé de la fenêtre, jamais
|
||||||
|
# fourni par l'appelant), et les lectures dont le timestamp déborde de la fenêtre demandée sont
|
||||||
|
# écartées (fetch_readings). Une valeur hors bornes devient NULL et laisse sa trace dans
|
||||||
|
# null_reasons plutôt que de lever : le mock émet des anomalies par construction, et raw_data
|
||||||
|
# conserve de toute façon la réponse d'origine intacte.
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.etl.historical_import import SOURCE_NAME as SOURCE_CSV
|
||||||
|
|
||||||
|
SOURCE_HISTORY = "api_history"
|
||||||
|
|
||||||
|
MAX_SITES = 100
|
||||||
|
|
||||||
|
MAX_LIMIT = 1000
|
||||||
|
|
||||||
|
# Les quatre seules valeurs que la contrainte ck_reading_quality accepte.
|
||||||
|
ACCEPTED_QUALITIES = frozenset({"good", "partial", "degraded", "critical"})
|
||||||
|
|
||||||
|
PHYSICAL_BOUNDS: dict[str, tuple[float, float]] = {
|
||||||
|
"consumption_kw": (0.0, 100_000.0),
|
||||||
|
"consumption_kwh": (0.0, 100_000.0),
|
||||||
|
"voltage_v": (0.0, 1_000.0),
|
||||||
|
"current_a": (0.0, 10_000.0),
|
||||||
|
"power_factor": (0.0, 1.0),
|
||||||
|
"temperature_celsius": (-90.0, 60.0),
|
||||||
|
"humidity_percent": (0.0, 100.0),
|
||||||
|
}
|
||||||
|
|
||||||
|
CAPACITY_BOUNDS = (0.0, 100_000.0)
|
||||||
|
|
||||||
|
|
||||||
|
def create_mock_api_client() -> httpx.AsyncClient:
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
username = settings.mock_api_username
|
||||||
|
password = (
|
||||||
|
settings.mock_api_password.get_secret_value()
|
||||||
|
if settings.mock_api_password is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
|
if not username or not username.strip() or not password or not password.strip():
|
||||||
|
raise ValueError("Les identifiants de l'API Mock ne sont pas configurés.")
|
||||||
|
|
||||||
|
return httpx.AsyncClient(
|
||||||
|
base_url=settings.mock_api_base_url.rstrip("/"),
|
||||||
|
auth=(username, password),
|
||||||
|
timeout=settings.mock_api_timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def read_text(payload: dict[str, Any], key: str) -> str:
|
||||||
|
value = payload.get(key)
|
||||||
|
|
||||||
|
if not isinstance(value, str) or not value:
|
||||||
|
raise ValueError(f"Champ {key} absent ou invalide dans la réponse de l'API Mock.")
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def optional_text(value: Any) -> str | None:
|
||||||
|
return value if isinstance(value, str) else None
|
||||||
|
|
||||||
|
|
||||||
|
def coerce_measure(
|
||||||
|
value: Any,
|
||||||
|
bounds: tuple[float, float],
|
||||||
|
) -> float | None:
|
||||||
|
if isinstance(value, bool) or not isinstance(value, int | float):
|
||||||
|
return None
|
||||||
|
|
||||||
|
lower, upper = bounds
|
||||||
|
|
||||||
|
# Écarte aussi NaN et les infinis, qu'aucune comparaison de bornes ne retient.
|
||||||
|
return float(value) if lower <= value <= upper else None
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_quality(
|
||||||
|
value: Any,
|
||||||
|
rejected: list[str],
|
||||||
|
) -> str | None:
|
||||||
|
quality = value if isinstance(value, str) and value in ACCEPTED_QUALITIES else None
|
||||||
|
|
||||||
|
if rejected:
|
||||||
|
return "critical" if quality == "critical" else "degraded"
|
||||||
|
|
||||||
|
return quality
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_null_reasons(
|
||||||
|
value: Any,
|
||||||
|
rejected: list[str],
|
||||||
|
) -> list[str]:
|
||||||
|
reported = [str(reason) for reason in value] if isinstance(value, list) else []
|
||||||
|
|
||||||
|
return reported + rejected
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_sites(
|
||||||
|
client: httpx.AsyncClient,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
response = await client.get("/api/v1/sites")
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
payload = response.json()
|
||||||
|
|
||||||
|
if not isinstance(payload, list):
|
||||||
|
raise ValueError("La réponse /api/v1/sites doit être une liste.")
|
||||||
|
|
||||||
|
if len(payload) > MAX_SITES:
|
||||||
|
raise ValueError(f"La réponse /api/v1/sites dépasse le plafond de {MAX_SITES} sites.")
|
||||||
|
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def build_site_row(
|
||||||
|
site: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"site_id": read_text(site, "site_id"),
|
||||||
|
"site_type": read_text(site, "site_type"),
|
||||||
|
"site_name": read_text(site, "site_name"),
|
||||||
|
"location": optional_text(site.get("location")),
|
||||||
|
"capacity_kw": coerce_measure(site.get("capacity_kw"), CAPACITY_BOUNDS),
|
||||||
|
"status": optional_text(site.get("status")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def upsert_sites(
|
||||||
|
connection: AsyncConnection,
|
||||||
|
sites: list[dict[str, Any]],
|
||||||
|
) -> None:
|
||||||
|
rows = [build_site_row(site) for site in sites]
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return
|
||||||
|
|
||||||
|
await connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO site (
|
||||||
|
site_id,
|
||||||
|
site_type,
|
||||||
|
site_name,
|
||||||
|
location,
|
||||||
|
capacity_kw,
|
||||||
|
status
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:site_id,
|
||||||
|
:site_type,
|
||||||
|
:site_name,
|
||||||
|
:location,
|
||||||
|
:capacity_kw,
|
||||||
|
:status
|
||||||
|
)
|
||||||
|
ON CONFLICT (site_id)
|
||||||
|
DO UPDATE SET
|
||||||
|
site_type = EXCLUDED.site_type,
|
||||||
|
site_name = EXCLUDED.site_name,
|
||||||
|
location = EXCLUDED.location,
|
||||||
|
capacity_kw = EXCLUDED.capacity_kw,
|
||||||
|
status = EXCLUDED.status
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _timestamp_in_window(reading: dict[str, Any], start_time: datetime, end_time: datetime) -> bool:
|
||||||
|
valeur = reading.get("timestamp")
|
||||||
|
if not isinstance(valeur, str):
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
instant = parse_datetime(valeur)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
return start_time <= instant < end_time
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_readings(
|
||||||
|
client: httpx.AsyncClient,
|
||||||
|
site_id: str,
|
||||||
|
start_time: datetime,
|
||||||
|
end_time: datetime,
|
||||||
|
limit: int = MAX_LIMIT,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
response = await client.get(
|
||||||
|
"/api/v1/readings",
|
||||||
|
params={
|
||||||
|
"site_id": site_id,
|
||||||
|
"start_time": start_time.isoformat(),
|
||||||
|
"end_time": end_time.isoformat(),
|
||||||
|
"limit": limit,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
payload = response.json()
|
||||||
|
|
||||||
|
if not isinstance(payload, list):
|
||||||
|
raise ValueError("La réponse /api/v1/readings doit être une liste.")
|
||||||
|
|
||||||
|
if len(payload) > limit:
|
||||||
|
raise ValueError(f"La réponse /api/v1/readings dépasse la limite demandée de {limit}.")
|
||||||
|
|
||||||
|
# Le garde-fou `refuse_if_overlaps_historical_dataset` ne vérifie que la fenêtre demandée :
|
||||||
|
# une réponse (bug du mock, ou hostile) dont les `timestamp` débordent de
|
||||||
|
# `[start_time, end_time)` contournerait ce contrôle et écrirait exactement le doublon
|
||||||
|
# inter-source qu'il doit empêcher. Écarter ces lectures ici rend le contrôle par fenêtre
|
||||||
|
# suffisant.
|
||||||
|
dans_la_fenetre = [
|
||||||
|
lecture
|
||||||
|
for lecture in payload
|
||||||
|
if isinstance(lecture, dict) and _timestamp_in_window(lecture, start_time, end_time)
|
||||||
|
]
|
||||||
|
|
||||||
|
if len(dans_la_fenetre) != len(payload):
|
||||||
|
ecartees = len(payload) - len(dans_la_fenetre)
|
||||||
|
print(f"{site_id}: {ecartees} lecture(s) hors fenêtre écartée(s).")
|
||||||
|
|
||||||
|
return dans_la_fenetre
|
||||||
|
|
||||||
|
|
||||||
|
def build_reading_row(
|
||||||
|
reading: dict[str, Any],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
measures: dict[str, float | None] = {}
|
||||||
|
rejected: list[str] = []
|
||||||
|
|
||||||
|
for name, bounds in PHYSICAL_BOUNDS.items():
|
||||||
|
received = reading.get(name)
|
||||||
|
measures[name] = coerce_measure(received, bounds)
|
||||||
|
|
||||||
|
if received is not None and measures[name] is None:
|
||||||
|
rejected.append(f"out_of_physical_bounds:{name}")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"site_id": read_text(reading, "site_id"),
|
||||||
|
"timestamp": parse_datetime(read_text(reading, "timestamp")),
|
||||||
|
"source": SOURCE_HISTORY,
|
||||||
|
"dataset_id": None,
|
||||||
|
**measures,
|
||||||
|
"consumption_euros": None,
|
||||||
|
"solar_irradiance_wm2": None,
|
||||||
|
"is_working_hours": None,
|
||||||
|
"data_quality": resolve_quality(reading.get("data_quality"), rejected),
|
||||||
|
"null_reasons": resolve_null_reasons(reading.get("null_reasons"), rejected),
|
||||||
|
"imputed_values": None,
|
||||||
|
"imputation_method": None,
|
||||||
|
"raw_data": json.dumps(
|
||||||
|
reading,
|
||||||
|
ensure_ascii=False,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# `uq_reading_source` autorise deux lignes au même (site_id, timestamp) dès que `source` diffère :
|
||||||
|
# sans ce garde-fou, importer une fenêtre déjà couverte par le dataset historique (source='csv')
|
||||||
|
# dupliquerait silencieusement chaque point plutôt que de lever une erreur. Ce garde-fou protège
|
||||||
|
# l'ingestion ; il ne dit rien de la lecture (`GET /readings` renvoie les deux lignes en cas de
|
||||||
|
# doublon malgré tout, cf. la section réconciliation de 40-data.md).
|
||||||
|
OVERLAP_CHECK = text(
|
||||||
|
"SELECT count(*) FROM reading WHERE source = :source_csv "
|
||||||
|
"AND timestamp >= :start_time AND timestamp < :end_time"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def refuse_if_overlaps_historical_dataset(
|
||||||
|
connection: AsyncConnection,
|
||||||
|
start_time: datetime,
|
||||||
|
end_time: datetime,
|
||||||
|
) -> None:
|
||||||
|
resultat = await connection.execute(
|
||||||
|
OVERLAP_CHECK,
|
||||||
|
{"source_csv": SOURCE_CSV, "start_time": start_time, "end_time": end_time},
|
||||||
|
)
|
||||||
|
nombre = resultat.scalar_one()
|
||||||
|
|
||||||
|
if nombre > 0:
|
||||||
|
raise ValueError(
|
||||||
|
f"La fenêtre [{start_time.isoformat()}, {end_time.isoformat()}) recouvre "
|
||||||
|
f"{nombre} lecture(s) déjà importée(s) du dataset historique (source='{SOURCE_CSV}') : "
|
||||||
|
"import refusé pour éviter un doublon inter-source."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Le conflit vise l'index unique uq_reading_source plutôt que la table entière : sans cible
|
||||||
|
# nommée, DO NOTHING avalerait aussi une violation de clé primaire.
|
||||||
|
READING_INSERT = text(
|
||||||
|
"""
|
||||||
|
INSERT INTO reading (
|
||||||
|
site_id,
|
||||||
|
timestamp,
|
||||||
|
source,
|
||||||
|
dataset_id,
|
||||||
|
consumption_kw,
|
||||||
|
consumption_kwh,
|
||||||
|
consumption_euros,
|
||||||
|
voltage_v,
|
||||||
|
current_a,
|
||||||
|
power_factor,
|
||||||
|
temperature_celsius,
|
||||||
|
humidity_percent,
|
||||||
|
solar_irradiance_wm2,
|
||||||
|
is_working_hours,
|
||||||
|
data_quality,
|
||||||
|
null_reasons,
|
||||||
|
imputed_values,
|
||||||
|
imputation_method,
|
||||||
|
raw_data
|
||||||
|
)
|
||||||
|
VALUES (
|
||||||
|
:site_id,
|
||||||
|
:timestamp,
|
||||||
|
:source,
|
||||||
|
:dataset_id,
|
||||||
|
:consumption_kw,
|
||||||
|
:consumption_kwh,
|
||||||
|
:consumption_euros,
|
||||||
|
:voltage_v,
|
||||||
|
:current_a,
|
||||||
|
:power_factor,
|
||||||
|
:temperature_celsius,
|
||||||
|
:humidity_percent,
|
||||||
|
:solar_irradiance_wm2,
|
||||||
|
:is_working_hours,
|
||||||
|
:data_quality,
|
||||||
|
:null_reasons,
|
||||||
|
CAST(:imputed_values AS jsonb),
|
||||||
|
:imputation_method,
|
||||||
|
CAST(:raw_data AS jsonb)
|
||||||
|
)
|
||||||
|
ON CONFLICT (site_id, timestamp, source, (coalesce(dataset_id, 0)))
|
||||||
|
DO NOTHING
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_reading_batch(
|
||||||
|
readings: list[dict[str, Any]],
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
return [build_reading_row(reading) for reading in readings]
|
||||||
|
|
||||||
|
|
||||||
|
def limit_for_window(start_time: datetime, end_time: datetime) -> int:
|
||||||
|
"""Nombre de lectures à demander pour que l'API Mock en rende une par heure, alignée.
|
||||||
|
|
||||||
|
L'API ne renvoie pas un flux à un rythme naturel : elle répartit exactement `limit` lectures,
|
||||||
|
espacées uniformément, sur toute la fenêtre `[start_time, end_time)` demandée, la première
|
||||||
|
au tout début de la fenêtre (vérifié empiriquement). Deux façons d'obtenir une lecture
|
||||||
|
alignée sur l'heure :
|
||||||
|
|
||||||
|
- une fenêtre d'exactement N heures (`start_time` sur l'heure) donne, avec `limit=N`, N
|
||||||
|
lectures espacées d'1h pile, la première à `start_time` : c'est le chemin du backfill
|
||||||
|
manuel (plusieurs jours d'historique en un seul appel).
|
||||||
|
- une fenêtre plus courte qu'une heure, ou qui n'est pas un multiple entier d'heure, ne peut
|
||||||
|
espacer plusieurs lectures d'1h pile (l'espacement de l'API vaut toujours
|
||||||
|
`durée / limit`) : seule `limit=1` reste alignée, la lecture unique atterrissant à
|
||||||
|
`start_time`. C'est le chemin du DAG horaire, dont la fenêtre part de l'heure pile qui
|
||||||
|
précède son déclenchement jusqu'à l'instant du déclenchement lui-même (`:45`), donc plus
|
||||||
|
courte qu'une heure.
|
||||||
|
|
||||||
|
Dans les deux cas, `start_time` doit tomber pile sur l'heure : c'est elle qui ancre
|
||||||
|
l'alignement, jamais `end_time`. Un `limit` plus grand que celui rendu ici fabriquerait des
|
||||||
|
lectures infra-horaires, incompatibles avec les lags positionnels de `build_features`.
|
||||||
|
"""
|
||||||
|
if start_time.minute or start_time.second or start_time.microsecond:
|
||||||
|
raise ValueError(
|
||||||
|
f"La fenêtre doit démarrer pile sur l'heure : {start_time.isoformat()} ne l'est pas."
|
||||||
|
)
|
||||||
|
|
||||||
|
duree = end_time - start_time
|
||||||
|
heures, reste = divmod(duree.total_seconds(), 3600)
|
||||||
|
|
||||||
|
# Fenêtre plus courte qu'une heure, ou pas un multiple entier : aucun `limit` supérieur à 1
|
||||||
|
# n'espacerait ses lectures d'1h pile (l'espacement vaut toujours durée / limit). Seule la
|
||||||
|
# lecture unique, ancrée sur `start_time`, reste alignée.
|
||||||
|
limit = int(heures) if reste == 0 and heures >= 1 else 1
|
||||||
|
|
||||||
|
if limit > MAX_LIMIT:
|
||||||
|
raise ValueError(
|
||||||
|
f"La fenêtre demandée couvre {limit}h, au-delà du plafond de {MAX_LIMIT} "
|
||||||
|
"lectures accepté par l'API Mock."
|
||||||
|
)
|
||||||
|
|
||||||
|
return limit
|
||||||
|
|
||||||
|
|
||||||
|
async def import_mock_api_history(
|
||||||
|
start_time: datetime,
|
||||||
|
end_time: datetime,
|
||||||
|
dry_run: bool,
|
||||||
|
) -> None:
|
||||||
|
settings = get_settings()
|
||||||
|
limit = limit_for_window(start_time, end_time)
|
||||||
|
|
||||||
|
engine = create_async_engine(
|
||||||
|
str(settings.database_url),
|
||||||
|
pool_pre_ping=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Garde-fou d'abord, y compris en dry-run : il est en lecture seule, et annoncer un
|
||||||
|
# succès pour une fenêtre que l'import réel refusera serait trompeur.
|
||||||
|
async with engine.connect() as connection:
|
||||||
|
await refuse_if_overlaps_historical_dataset(connection, start_time, end_time)
|
||||||
|
|
||||||
|
async with create_mock_api_client() as client:
|
||||||
|
sites = await fetch_sites(client)
|
||||||
|
|
||||||
|
print(f"Sites récupérés : {len(sites)}")
|
||||||
|
|
||||||
|
all_readings: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
for site in sites:
|
||||||
|
site_id = read_text(site, "site_id")
|
||||||
|
|
||||||
|
readings = await fetch_readings(
|
||||||
|
client=client,
|
||||||
|
site_id=site_id,
|
||||||
|
start_time=start_time,
|
||||||
|
end_time=end_time,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
print(f"{site_id}: {len(readings)} lectures")
|
||||||
|
|
||||||
|
all_readings.extend(readings)
|
||||||
|
|
||||||
|
print(f"Lectures récupérées : {len(all_readings)}")
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print("Dry-run terminé : aucune donnée écrite.")
|
||||||
|
return
|
||||||
|
|
||||||
|
async with engine.begin() as connection:
|
||||||
|
await upsert_sites(
|
||||||
|
connection,
|
||||||
|
sites,
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = build_reading_batch(all_readings)
|
||||||
|
|
||||||
|
if rows:
|
||||||
|
await connection.execute(
|
||||||
|
READING_INSERT,
|
||||||
|
rows,
|
||||||
|
)
|
||||||
|
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
print("Import API Mock terminé.")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_datetime(value: str) -> datetime:
|
||||||
|
# Sans fuseau, l'API le traite comme reçu, telle quelle, mais l'encodeur `timestamptz`
|
||||||
|
# d'asyncpg lirait un datetime naif dans le fuseau *local du processus* (correct dans le
|
||||||
|
# conteneur Airflow en UTC, décalé de 1-2h pour un import manuel lancé depuis un poste en
|
||||||
|
# Europe/Paris). Poser `tzinfo=UTC` explicitement, même pattern que `_vers_utc()` dans
|
||||||
|
# `app/services/reading.py`, garantit que la borne envoyée à l'API et celle comparée en SQL
|
||||||
|
# (refuse_if_overlaps_historical_dataset) désignent le même instant.
|
||||||
|
instant = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
return instant if instant.tzinfo is not None else instant.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description=("Import historique depuis l'API Mock EnerVision"))
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--start-time",
|
||||||
|
required=True,
|
||||||
|
type=parse_datetime,
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--end-time",
|
||||||
|
required=True,
|
||||||
|
type=parse_datetime,
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"--dry-run",
|
||||||
|
action="store_true",
|
||||||
|
)
|
||||||
|
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
args = parse_args()
|
||||||
|
|
||||||
|
if args.start_time >= args.end_time:
|
||||||
|
raise ValueError("--start-time doit être antérieur à --end-time.")
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
import_mock_api_history(
|
||||||
|
start_time=args.start_time,
|
||||||
|
end_time=args.end_time,
|
||||||
|
dry_run=args.dry_run,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,349 @@
|
|||||||
|
# Pourquoi : la suppression n'est pas confiée à add_retention_policy, qui ignorerait l'export.
|
||||||
|
# archive_reading_chunks() exporte chaque chunk vers Garage, le relit, puis le supprime seul.
|
||||||
|
# Piège : drop_chunks pose un verrou exclusif sur reading, site et dataset jusqu'au COMMIT. La
|
||||||
|
# suppression tient donc dans une transaction dédiée et courte, séparée de la lecture du chunk.
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import base64
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
|
import anyio.to_thread
|
||||||
|
import boto3
|
||||||
|
import pandas as pd
|
||||||
|
from botocore.exceptions import ClientError
|
||||||
|
from pydantic import SecretStr
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
|
||||||
|
|
||||||
|
from app.core.config import Settings, get_settings
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from types_boto3_s3.client import S3Client
|
||||||
|
|
||||||
|
SSE_KEY_LENGTH = 32
|
||||||
|
FORMAT_BORNE = "%Y%m%dT%H%M%SZ"
|
||||||
|
|
||||||
|
ELIGIBLE_CHUNKS = text(
|
||||||
|
"SELECT chunk_schema, chunk_name, range_start, range_end "
|
||||||
|
"FROM timescaledb_information.chunks "
|
||||||
|
"WHERE hypertable_name = 'reading' AND range_end <= :older_than "
|
||||||
|
"ORDER BY range_start"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Lecture via l'hypertable, jamais la table interne : l'exclusion de partition vise le seul chunk.
|
||||||
|
CHUNK_ROWS = text(
|
||||||
|
"SELECT * FROM reading WHERE timestamp >= :start AND timestamp < :end "
|
||||||
|
"ORDER BY timestamp, reading_id"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Les deux bornes sont inclusives pour drop_chunks : celles du chunk le désignent, et lui seul.
|
||||||
|
DROP_CHUNK = text(
|
||||||
|
"SELECT drop_chunks('reading', "
|
||||||
|
"older_than => CAST(:end AS timestamptz), newer_than => CAST(:start AS timestamptz))"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Chunk:
|
||||||
|
schema: str
|
||||||
|
name: str
|
||||||
|
range_start: datetime
|
||||||
|
range_end: datetime
|
||||||
|
|
||||||
|
@property
|
||||||
|
def qualified_name(self) -> str:
|
||||||
|
return f"{self.schema}.{self.name}"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class Rapport:
|
||||||
|
chunks_vus: int = 0
|
||||||
|
exportes: int = 0
|
||||||
|
deja_presents: int = 0
|
||||||
|
supprimes: int = 0
|
||||||
|
lignes: int = 0
|
||||||
|
|
||||||
|
|
||||||
|
def object_key(chunk: Chunk) -> str:
|
||||||
|
start = chunk.range_start.astimezone(UTC)
|
||||||
|
end = chunk.range_end.astimezone(UTC)
|
||||||
|
return (
|
||||||
|
f"reading/{start.year}/reading_{start.strftime(FORMAT_BORNE)}_"
|
||||||
|
f"{end.strftime(FORMAT_BORNE)}.csv.gz"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def eligible_chunks(conn: AsyncConnection, older_than: datetime) -> list[Chunk]:
|
||||||
|
result = await conn.execute(ELIGIBLE_CHUNKS, {"older_than": older_than})
|
||||||
|
return [
|
||||||
|
Chunk(
|
||||||
|
schema=row["chunk_schema"],
|
||||||
|
name=row["chunk_name"],
|
||||||
|
range_start=row["range_start"],
|
||||||
|
range_end=row["range_end"],
|
||||||
|
)
|
||||||
|
for row in result.mappings().all()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def read_chunk_rows(conn: AsyncConnection, chunk: Chunk) -> list[dict[str, Any]]:
|
||||||
|
result = await conn.execute(CHUNK_ROWS, {"start": chunk.range_start, "end": chunk.range_end})
|
||||||
|
return [dict(row) for row in result.mappings().all()]
|
||||||
|
|
||||||
|
|
||||||
|
def _csv_cell(value: object) -> object:
|
||||||
|
if isinstance(value, dict | list):
|
||||||
|
return json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_csv_gzip(rows: list[dict[str, Any]]) -> bytes:
|
||||||
|
if not rows:
|
||||||
|
raise ValueError("Aucune ligne à sérialiser : un CSV sans colonne ne se relit pas.")
|
||||||
|
|
||||||
|
frame = pd.DataFrame([{name: _csv_cell(value) for name, value in row.items()} for row in rows])
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
frame.to_csv(buffer, mode="wb", index=False, compression={"method": "gzip", "mtime": 0})
|
||||||
|
return buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def sha256_of(data: bytes) -> str:
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_missing_object(erreur: ClientError) -> bool:
|
||||||
|
error = erreur.response.get("Error")
|
||||||
|
metadata = erreur.response.get("ResponseMetadata")
|
||||||
|
code = error.get("Code") if error is not None else None
|
||||||
|
status = metadata.get("HTTPStatusCode") if metadata is not None else None
|
||||||
|
return code == "NoSuchKey" or status == 404
|
||||||
|
|
||||||
|
|
||||||
|
class ArchiveStore:
|
||||||
|
def __init__(self, client: S3Client, bucket: str, sse_key: bytes | None) -> None:
|
||||||
|
self._client = client
|
||||||
|
self._bucket = bucket
|
||||||
|
self._sse_key = sse_key
|
||||||
|
|
||||||
|
# boto3 encode lui-même la clé en base64 et calcule son MD5 : la fournir brute, sans MD5.
|
||||||
|
def _sse_headers(self) -> dict[str, Any]:
|
||||||
|
if self._sse_key is None:
|
||||||
|
return {}
|
||||||
|
return {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": self._sse_key}
|
||||||
|
|
||||||
|
def put(self, key: str, body: bytes, metadata: dict[str, str]) -> None:
|
||||||
|
self._client.put_object(
|
||||||
|
Bucket=self._bucket,
|
||||||
|
Key=key,
|
||||||
|
Body=body,
|
||||||
|
ContentType="text/csv",
|
||||||
|
ContentEncoding="gzip",
|
||||||
|
Metadata=metadata,
|
||||||
|
**self._sse_headers(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def fetch_sha256(self, key: str) -> str | None:
|
||||||
|
try:
|
||||||
|
response = self._client.get_object(Bucket=self._bucket, Key=key, **self._sse_headers())
|
||||||
|
except ClientError as erreur:
|
||||||
|
if _is_missing_object(erreur):
|
||||||
|
return None
|
||||||
|
raise
|
||||||
|
return sha256_of(response["Body"].read())
|
||||||
|
|
||||||
|
|
||||||
|
def decode_sse_key(encoded: SecretStr | None) -> bytes | None:
|
||||||
|
if encoded is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
key = base64.b64decode(encoded.get_secret_value(), validate=True)
|
||||||
|
if len(key) != SSE_KEY_LENGTH:
|
||||||
|
raise ValueError(
|
||||||
|
f"APP_S3_SSE_KEY doit encoder exactement {SSE_KEY_LENGTH} octets en base64, "
|
||||||
|
f"pas {len(key)}."
|
||||||
|
)
|
||||||
|
return key
|
||||||
|
|
||||||
|
|
||||||
|
def build_archive_store(settings: Settings) -> ArchiveStore:
|
||||||
|
endpoint = settings.s3_endpoint_url
|
||||||
|
access_key = settings.s3_access_key
|
||||||
|
secret_key = settings.s3_secret_key
|
||||||
|
bucket = settings.s3_bucket
|
||||||
|
|
||||||
|
if endpoint is None or access_key is None or secret_key is None or bucket is None:
|
||||||
|
raise ValueError(
|
||||||
|
"L'archivage vers Garage exige APP_S3_ENDPOINT_URL, APP_S3_ACCESS_KEY, "
|
||||||
|
"APP_S3_SECRET_KEY et APP_S3_BUCKET."
|
||||||
|
)
|
||||||
|
|
||||||
|
client = boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=endpoint,
|
||||||
|
aws_access_key_id=access_key,
|
||||||
|
aws_secret_access_key=secret_key.get_secret_value(),
|
||||||
|
region_name=settings.s3_region,
|
||||||
|
)
|
||||||
|
return ArchiveStore(client, bucket=bucket, sse_key=decode_sse_key(settings.s3_sse_key))
|
||||||
|
|
||||||
|
|
||||||
|
async def drop_chunk(conn: AsyncConnection, chunk: Chunk) -> None:
|
||||||
|
result = await conn.execute(DROP_CHUNK, {"start": chunk.range_start, "end": chunk.range_end})
|
||||||
|
supprimes = list(result.scalars().all())
|
||||||
|
|
||||||
|
if supprimes != [chunk.qualified_name]:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"drop_chunks devait supprimer exactement {chunk.qualified_name}, "
|
||||||
|
f"il a rendu {supprimes}."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _export(
|
||||||
|
store: ArchiveStore,
|
||||||
|
key: str,
|
||||||
|
rows: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
dry_run: bool,
|
||||||
|
rapport: Rapport,
|
||||||
|
) -> str:
|
||||||
|
body = serialize_csv_gzip(rows)
|
||||||
|
sha = sha256_of(body)
|
||||||
|
|
||||||
|
if await anyio.to_thread.run_sync(store.fetch_sha256, key) == sha:
|
||||||
|
rapport.deja_presents += 1
|
||||||
|
return f"{len(body)} octets déjà présents"
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
return f"{len(body)} octets à exporter"
|
||||||
|
|
||||||
|
metadata = {"sha256": sha, "rows": str(len(rows))}
|
||||||
|
await anyio.to_thread.run_sync(store.put, key, body, metadata)
|
||||||
|
relu = await anyio.to_thread.run_sync(store.fetch_sha256, key)
|
||||||
|
|
||||||
|
if relu != sha:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Relecture de {key} : sha256 {relu} au lieu de {sha}, le chunk est conservé."
|
||||||
|
)
|
||||||
|
|
||||||
|
rapport.exportes += 1
|
||||||
|
return f"{len(body)} octets exportés et relus"
|
||||||
|
|
||||||
|
|
||||||
|
async def _archive_chunk(
|
||||||
|
engine: AsyncEngine,
|
||||||
|
store: ArchiveStore,
|
||||||
|
chunk: Chunk,
|
||||||
|
*,
|
||||||
|
dry_run: bool,
|
||||||
|
rapport: Rapport,
|
||||||
|
) -> None:
|
||||||
|
async with engine.connect() as conn:
|
||||||
|
rows = await read_chunk_rows(conn, chunk)
|
||||||
|
|
||||||
|
key = object_key(chunk)
|
||||||
|
rapport.lignes += len(rows)
|
||||||
|
|
||||||
|
if rows:
|
||||||
|
action = await _export(store, key, rows, dry_run=dry_run, rapport=rapport)
|
||||||
|
else:
|
||||||
|
action = "vide, rien à exporter"
|
||||||
|
|
||||||
|
if dry_run:
|
||||||
|
print(f"{key} : {len(rows)} ligne(s), {action}, suppression simulée.")
|
||||||
|
return
|
||||||
|
|
||||||
|
async with engine.begin() as conn:
|
||||||
|
await drop_chunk(conn, chunk)
|
||||||
|
|
||||||
|
rapport.supprimes += 1
|
||||||
|
print(f"{key} : {len(rows)} ligne(s), {action}, chunk {chunk.qualified_name} supprimé.")
|
||||||
|
|
||||||
|
|
||||||
|
async def archive_reading_chunks(
|
||||||
|
engine: AsyncEngine,
|
||||||
|
store: ArchiveStore,
|
||||||
|
*,
|
||||||
|
older_than: datetime,
|
||||||
|
dry_run: bool,
|
||||||
|
) -> Rapport:
|
||||||
|
rapport = Rapport()
|
||||||
|
|
||||||
|
async with engine.connect() as conn:
|
||||||
|
chunks = await eligible_chunks(conn, older_than)
|
||||||
|
|
||||||
|
rapport.chunks_vus = len(chunks)
|
||||||
|
print(
|
||||||
|
f"{len(chunks)} chunk(s) de reading entièrement antérieur(s) au {older_than.isoformat()}."
|
||||||
|
)
|
||||||
|
|
||||||
|
for chunk in chunks:
|
||||||
|
await _archive_chunk(engine, store, chunk, dry_run=dry_run, rapport=rapport)
|
||||||
|
|
||||||
|
bilan = "Dry-run terminé : rien n'a été écrit ni supprimé." if dry_run else "Archivage terminé."
|
||||||
|
print(
|
||||||
|
f"{bilan} Chunks vus : {rapport.chunks_vus}, exportés : {rapport.exportes}, "
|
||||||
|
f"déjà présents : {rapport.deja_presents}, supprimés : {rapport.supprimes}, "
|
||||||
|
f"lignes : {rapport.lignes}."
|
||||||
|
)
|
||||||
|
return rapport
|
||||||
|
|
||||||
|
|
||||||
|
async def _run(
|
||||||
|
settings: Settings,
|
||||||
|
store: ArchiveStore,
|
||||||
|
*,
|
||||||
|
older_than: datetime,
|
||||||
|
dry_run: bool,
|
||||||
|
) -> Rapport:
|
||||||
|
engine = create_async_engine(str(settings.database_url), pool_pre_ping=True)
|
||||||
|
try:
|
||||||
|
return await archive_reading_chunks(engine, store, older_than=older_than, dry_run=dry_run)
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="python -m app.etl.reading_retention",
|
||||||
|
description=(
|
||||||
|
"Exporte vers Garage puis supprime les chunks de reading entièrement plus vieux "
|
||||||
|
"que la borne de rétention."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--older-than-days",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Borne en jours, par défaut APP_READING_RETENTION_DAYS.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--dry-run",
|
||||||
|
action="store_true",
|
||||||
|
help="Liste et mesure les chunks éligibles sans rien écrire ni supprimer.",
|
||||||
|
)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> None:
|
||||||
|
args = build_parser().parse_args(argv)
|
||||||
|
settings = get_settings()
|
||||||
|
|
||||||
|
jours = (
|
||||||
|
settings.reading_retention_days if args.older_than_days is None else args.older_than_days
|
||||||
|
)
|
||||||
|
older_than = datetime.now(UTC) - timedelta(days=jours)
|
||||||
|
store = build_archive_store(settings)
|
||||||
|
|
||||||
|
asyncio.run(_run(settings, store, older_than=older_than, dry_run=args.dry_run))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -1,9 +1,15 @@
|
|||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from fastapi import Depends, FastAPI
|
from fastapi import Depends, FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from prometheus_fastapi_instrumentator import Instrumentator
|
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
|
||||||
|
from fastapi.staticfiles import StaticFiles
|
||||||
|
from prometheus_client import CollectorRegistry, GCCollector, PlatformCollector, ProcessCollector
|
||||||
|
from prometheus_fastapi_instrumentator import Instrumentator, metrics
|
||||||
|
from starlette.requests import Request
|
||||||
|
from starlette.responses import HTMLResponse
|
||||||
|
|
||||||
from app.api.errors import register_error_handlers
|
from app.api.errors import register_error_handlers
|
||||||
from app.api.middleware import SecurityHeadersMiddleware
|
from app.api.middleware import SecurityHeadersMiddleware
|
||||||
@@ -18,6 +24,8 @@ logger = get_logger(__name__)
|
|||||||
|
|
||||||
METHODES_AUTORISEES = ["GET", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"]
|
METHODES_AUTORISEES = ["GET", "POST", "PATCH", "PUT", "DELETE", "OPTIONS"]
|
||||||
EN_TETES_AUTORISES = ["Authorization", "Content-Type"]
|
EN_TETES_AUTORISES = ["Authorization", "Content-Type"]
|
||||||
|
STATIC_DIR = Path(__file__).parent / "static"
|
||||||
|
LOGO_URL = "/static/logo-icon.png"
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
@@ -30,6 +38,16 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
|||||||
await get_engine().dispose()
|
await get_engine().dispose()
|
||||||
|
|
||||||
|
|
||||||
|
# Pourquoi : le registre global n'accepte chaque métrique qu'une fois. Toute application créée
|
||||||
|
# après la première, dans les tests notamment, n'aurait rien mesuré.
|
||||||
|
def _registre_de_metriques() -> CollectorRegistry:
|
||||||
|
registre = CollectorRegistry()
|
||||||
|
ProcessCollector(registry=registre)
|
||||||
|
PlatformCollector(registry=registre)
|
||||||
|
GCCollector(registry=registre)
|
||||||
|
return registre
|
||||||
|
|
||||||
|
|
||||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||||
resolved = settings or get_settings()
|
resolved = settings or get_settings()
|
||||||
configure_logging(resolved)
|
configure_logging(resolved)
|
||||||
@@ -43,11 +61,41 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
openapi_tags=TAGS,
|
openapi_tags=TAGS,
|
||||||
debug=resolved.debug,
|
debug=resolved.debug,
|
||||||
lifespan=lifespan,
|
lifespan=lifespan,
|
||||||
docs_url="/docs" if documentee else None,
|
docs_url=None,
|
||||||
redoc_url="/redoc" if documentee else None,
|
redoc_url=None,
|
||||||
openapi_url="/openapi.json" if documentee else None,
|
openapi_url="/openapi.json" if documentee else None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if documentee:
|
||||||
|
application.mount("/static", StaticFiles(directory=STATIC_DIR), name="static")
|
||||||
|
|
||||||
|
# ReDoc supporte nativement `info.x-logo` (extension Redocly) pour afficher un logo
|
||||||
|
# en en-tête ; Swagger UI n'a pas d'equivalent, il ne reprend que le favicon.
|
||||||
|
openapi_original = application.openapi
|
||||||
|
|
||||||
|
def openapi_avec_logo() -> dict[str, object]:
|
||||||
|
schema = openapi_original()
|
||||||
|
schema["info"]["x-logo"] = {"url": LOGO_URL, "altText": "EnerVision"}
|
||||||
|
return schema
|
||||||
|
|
||||||
|
application.openapi = openapi_avec_logo # type: ignore[method-assign]
|
||||||
|
|
||||||
|
@application.get("/docs", include_in_schema=False)
|
||||||
|
async def docs_swagger(_: Request) -> HTMLResponse:
|
||||||
|
return get_swagger_ui_html(
|
||||||
|
openapi_url="/openapi.json",
|
||||||
|
title=f"{application.title} · Swagger UI",
|
||||||
|
swagger_favicon_url=LOGO_URL,
|
||||||
|
)
|
||||||
|
|
||||||
|
@application.get("/redoc", include_in_schema=False)
|
||||||
|
async def docs_redoc(_: Request) -> HTMLResponse:
|
||||||
|
return get_redoc_html(
|
||||||
|
openapi_url="/openapi.json",
|
||||||
|
title=f"{application.title} · ReDoc",
|
||||||
|
redoc_favicon_url=LOGO_URL,
|
||||||
|
)
|
||||||
|
|
||||||
application.add_middleware(SecurityHeadersMiddleware)
|
application.add_middleware(SecurityHeadersMiddleware)
|
||||||
|
|
||||||
if resolved.allowed_origins:
|
if resolved.allowed_origins:
|
||||||
@@ -65,7 +113,14 @@ def create_app(settings: Settings | None = None) -> FastAPI:
|
|||||||
|
|
||||||
register_error_handlers(application)
|
register_error_handlers(application)
|
||||||
|
|
||||||
Instrumentator().instrument(application).expose(
|
# Les sondes de santé tombent toutes les 30 s : comptées, elles fausseraient latences et débit.
|
||||||
|
# Seaux fins autour du seuil de charge (p95 < 500 ms, ADR 0015), route par route.
|
||||||
|
registre = _registre_de_metriques()
|
||||||
|
Instrumentator(
|
||||||
|
excluded_handlers=["/metrics", f"{resolved.api_prefix}/health/.*"], registry=registre
|
||||||
|
).add(
|
||||||
|
metrics.default(latency_lowr_buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5), registry=registre)
|
||||||
|
).instrument(application).expose(
|
||||||
application,
|
application,
|
||||||
endpoint="/metrics",
|
endpoint="/metrics",
|
||||||
include_in_schema=False,
|
include_in_schema=False,
|
||||||
|
|||||||
@@ -2,8 +2,18 @@
|
|||||||
# --autogenerate`, qui générerait alors un drop de sa table.
|
# --autogenerate`, qui générerait alors un drop de sa table.
|
||||||
|
|
||||||
from app.models.audit_log import AuditLog
|
from app.models.audit_log import AuditLog
|
||||||
from app.models.energy import Alert, Dataset, Prediction, Reading, Recommendation, Site
|
from app.models.energy import (
|
||||||
|
Alert,
|
||||||
|
Dataset,
|
||||||
|
DriftReport,
|
||||||
|
Prediction,
|
||||||
|
Reading,
|
||||||
|
Recommendation,
|
||||||
|
Site,
|
||||||
|
)
|
||||||
from app.models.login_attempt import LoginAttempt
|
from app.models.login_attempt import LoginAttempt
|
||||||
|
from app.models.password_reset_attempt import PasswordResetAttempt
|
||||||
|
from app.models.password_reset_token import PasswordResetToken
|
||||||
from app.models.refresh_token import RefreshToken
|
from app.models.refresh_token import RefreshToken
|
||||||
from app.models.user import AppUser
|
from app.models.user import AppUser
|
||||||
|
|
||||||
@@ -12,7 +22,10 @@ __all__ = [
|
|||||||
"AppUser",
|
"AppUser",
|
||||||
"AuditLog",
|
"AuditLog",
|
||||||
"Dataset",
|
"Dataset",
|
||||||
|
"DriftReport",
|
||||||
"LoginAttempt",
|
"LoginAttempt",
|
||||||
|
"PasswordResetAttempt",
|
||||||
|
"PasswordResetToken",
|
||||||
"Prediction",
|
"Prediction",
|
||||||
"Reading",
|
"Reading",
|
||||||
"Recommendation",
|
"Recommendation",
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ class AuditAction(StrEnum):
|
|||||||
COMPTE_ACTIVE = "user.enabled"
|
COMPTE_ACTIVE = "user.enabled"
|
||||||
COMPTE_MOT_DE_PASSE_REINITIALISE = "user.password_reset_by_admin"
|
COMPTE_MOT_DE_PASSE_REINITIALISE = "user.password_reset_by_admin"
|
||||||
COMPTE_MOT_DE_PASSE_CHANGE = "user.password_changed"
|
COMPTE_MOT_DE_PASSE_CHANGE = "user.password_changed"
|
||||||
|
MOT_DE_PASSE_OUBLIE_DEMANDE = "auth.password_reset_requested"
|
||||||
|
MOT_DE_PASSE_REINITIALISE_PAR_SOI = "auth.password_reset_self_service"
|
||||||
REFRESH_REUTILISE = "auth.refresh_reuse_detected"
|
REFRESH_REUTILISE = "auth.refresh_reuse_detected"
|
||||||
SESSIONS_REVOQUEES = "auth.all_sessions_revoked"
|
SESSIONS_REVOQUEES = "auth.all_sessions_revoked"
|
||||||
LIMITE_PAR_IDENTIFIANT = "auth.identifier_throttled"
|
LIMITE_PAR_IDENTIFIANT = "auth.identifier_throttled"
|
||||||
|
|||||||
@@ -208,3 +208,49 @@ class Recommendation(Base):
|
|||||||
explanation: Mapped[str] = mapped_column(Text)
|
explanation: Mapped[str] = mapped_column(Text)
|
||||||
rule_reference: Mapped[str] = mapped_column(Text)
|
rule_reference: Mapped[str] = mapped_column(Text)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
|
||||||
|
class DriftReport(Base):
|
||||||
|
__tablename__ = "drift_report"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('stable', 'derive', 'indetermine')", name="ck_drift_report_status"
|
||||||
|
),
|
||||||
|
CheckConstraint("status = 'stable' OR reason IS NOT NULL", name="ck_drift_report_reason"),
|
||||||
|
CheckConstraint("n_observations >= 0", name="ck_drift_report_observations"),
|
||||||
|
Index("ix_drift_report_site_computed", "site_id", "computed_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
drift_report_id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
|
||||||
|
computed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), server_default=func.now()
|
||||||
|
)
|
||||||
|
# `NULL` porte la ligne globale, tous sites confondus : une derive d'ensemble et la derive
|
||||||
|
# d'un seul site ne se lisent pas dans le meme chiffre.
|
||||||
|
site_id: Mapped[str | None] = mapped_column(
|
||||||
|
Text, ForeignKey("site.site_id", name="fk_drift_report_site", ondelete="RESTRICT")
|
||||||
|
)
|
||||||
|
window_start: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
window_end: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
|
reference_start: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
reference_end: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
n_observations: Mapped[int] = mapped_column(Integer)
|
||||||
|
mae: Mapped[float | None] = mapped_column(Double)
|
||||||
|
mape: Mapped[float | None] = mapped_column(Double)
|
||||||
|
bias: Mapped[float | None] = mapped_column(Double)
|
||||||
|
reference_mae: Mapped[float | None] = mapped_column(Double)
|
||||||
|
coverage_ratio: Mapped[float | None] = mapped_column(Double)
|
||||||
|
insufficient_data_ratio: Mapped[float | None] = mapped_column(Double)
|
||||||
|
model_references: Mapped[list[str]] = mapped_column(ARRAY(Text))
|
||||||
|
status: Mapped[str] = mapped_column(Text)
|
||||||
|
reason: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|
||||||
|
|
||||||
|
# Piège : une `UniqueConstraint` ne dédoublonnerait pas les lignes globales, dont `site_id` est
|
||||||
|
# NULL et qu'aucune n'est égale à une autre. Même forme que `uq_reading_source`.
|
||||||
|
Index(
|
||||||
|
"uq_drift_report_window",
|
||||||
|
DriftReport.window_end,
|
||||||
|
func.coalesce(DriftReport.site_id, text("''")),
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Pourquoi : même séparation que `login_attempt` par rapport à `audit_log` : ce compteur est
|
||||||
|
# piloté par l'attaquant (une campagne de demandes) et se purge, l'audit log est en ajout seul.
|
||||||
|
# Piège : la tentative est enregistrée même quand l'email est inconnu, sinon le 429 apprendrait
|
||||||
|
# qu'un compte existe.
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import BigInteger, DateTime, Identity, Index, String, func
|
||||||
|
from sqlalchemy.dialects.postgresql import INET
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class PasswordResetAttempt(Base):
|
||||||
|
__tablename__ = "password_reset_attempt"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_password_reset_attempt_email_date", "email_tried", "occurred_at"),
|
||||||
|
Index("ix_password_reset_attempt_ip_date", "client_ip", "occurred_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(BigInteger, Identity(always=True), primary_key=True)
|
||||||
|
occurred_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
email_tried: Mapped[str] = mapped_column(String(320), nullable=False)
|
||||||
|
client_ip: Mapped[str | None] = mapped_column(INET, nullable=True)
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Pourquoi : même schéma que `refresh_token` (chaîne opaque, jamais un JWT) pour la même
|
||||||
|
# raison : un jeton de réinitialisation doit être révocable d'un coup, et un JWT ne figure
|
||||||
|
# dans aucune ligne à invalider.
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import DateTime, ForeignKey, Index, LargeBinary, Text, func
|
||||||
|
from sqlalchemy.dialects.postgresql import INET
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID as PG_UUID
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from app.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class PasswordResetToken(Base):
|
||||||
|
__tablename__ = "password_reset_token"
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_password_reset_token_user", "user_id"),
|
||||||
|
Index(
|
||||||
|
"ix_password_reset_token_vivants",
|
||||||
|
"user_id",
|
||||||
|
postgresql_where="consumed_at is null",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
PG_UUID(as_uuid=True), primary_key=True, server_default=func.gen_random_uuid()
|
||||||
|
)
|
||||||
|
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
PG_UUID(as_uuid=True), ForeignKey("app_user.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
token_hash: Mapped[bytes] = mapped_column(LargeBinary, nullable=False, unique=True)
|
||||||
|
issued_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||||
|
consumed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
client_ip: Mapped[str | None] = mapped_column(INET, nullable=True)
|
||||||
|
user_agent: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
@@ -0,0 +1,115 @@
|
|||||||
|
# Surveillance de dérive du modèle de prévision (EC06, issue #45) : même gabarit que
|
||||||
|
# `app.detection.internal_alerts`, ordonnancé par le DAG `derive`.
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from app.core.config import get_settings
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from app.repositories.drift import DriftRepository, NouveauRapportDerive
|
||||||
|
from app.services.drift import STATUT_DERIVE, DriftService, Seuils
|
||||||
|
|
||||||
|
|
||||||
|
async def run_drift(
|
||||||
|
*, now: datetime | None = None, site_id: str | None = None, seuils: Seuils | None = None
|
||||||
|
) -> list[NouveauRapportDerive]:
|
||||||
|
"""Calcule les rapports de la fenêtre et les enregistre. Rend ce qui a été calculé, que la
|
||||||
|
ligne ait été écrite ou ignorée par l'index d'idempotence."""
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
depot = DriftRepository(session)
|
||||||
|
rapports = await DriftService(depot, seuils=seuils).evaluate(now=now, site_id=site_id)
|
||||||
|
await depot.enregistre(rapports)
|
||||||
|
await session.commit()
|
||||||
|
return rapports
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_instant(valeur: str) -> datetime:
|
||||||
|
instant = datetime.fromisoformat(valeur)
|
||||||
|
return instant if instant.tzinfo is not None else instant.replace(tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||||
|
defauts = Seuils()
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
prog="python -m app.monitoring.drift",
|
||||||
|
description="Surveillance de dérive du modèle de prévision EnerVision",
|
||||||
|
)
|
||||||
|
parser.add_argument("--site-id", default=None, help="Limite le calcul à un seul site.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--now",
|
||||||
|
type=_parse_instant,
|
||||||
|
default=None,
|
||||||
|
help=(
|
||||||
|
"Instant de référence (ISO 8601, UTC si le fuseau est omis). Défaut : l'heure courante."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--window-hours",
|
||||||
|
type=int,
|
||||||
|
default=int(defauts.fenetre.total_seconds() // 3600),
|
||||||
|
help="Durée de la fenêtre récente, et de la fenêtre de référence qui la précède.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--grace-hours",
|
||||||
|
type=int,
|
||||||
|
default=int(defauts.grace.total_seconds() // 3600),
|
||||||
|
help="Délai laissé à l'ingestion avant qu'une prévision soit jugée vérifiable.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--min-observations",
|
||||||
|
type=int,
|
||||||
|
default=defauts.min_observations,
|
||||||
|
help="En deçà, le verdict est `indetermine` plutôt qu'un chiffre trompeur.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--bias-threshold",
|
||||||
|
type=float,
|
||||||
|
default=defauts.seuil_biais,
|
||||||
|
help=(
|
||||||
|
"Biais absolu en kWh au-delà duquel le verdict bascule en dérive. "
|
||||||
|
"Zéro, le défaut, laisse le biais informatif : voir l'ADR 0013."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fail-on-drift",
|
||||||
|
action="store_true",
|
||||||
|
help="Sort en code non nul si une dérive est constatée, pour que la tâche rougisse.",
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def seuils_depuis(args: argparse.Namespace) -> Seuils:
|
||||||
|
return Seuils(
|
||||||
|
fenetre=timedelta(hours=args.window_hours),
|
||||||
|
grace=timedelta(hours=args.grace_hours),
|
||||||
|
min_observations=args.min_observations,
|
||||||
|
seuil_biais=args.bias_threshold,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
args = parse_args(argv)
|
||||||
|
# Échoue tôt si `APP_SECRET_KEY`/`DATABASE_URL` manquent, avant toute requête à la base.
|
||||||
|
get_settings()
|
||||||
|
rapports = asyncio.run(
|
||||||
|
run_drift(now=args.now, site_id=args.site_id, seuils=seuils_depuis(args))
|
||||||
|
)
|
||||||
|
|
||||||
|
for rapport in rapports:
|
||||||
|
cible = rapport.site_id or "TOUS SITES"
|
||||||
|
mae = f"{rapport.mae:.2f}" if rapport.mae is not None else "-"
|
||||||
|
print(
|
||||||
|
f"{cible} : {rapport.status}, MAE {mae} kWh sur {rapport.n_observations} prévision(s)"
|
||||||
|
f"{' : ' + rapport.reason if rapport.reason else ''}"
|
||||||
|
)
|
||||||
|
|
||||||
|
derive = any(rapport.status == STATUT_DERIVE for rapport in rapports)
|
||||||
|
return 1 if derive and args.fail_on_drift else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__": # pragma: no cover
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.energy import Alert
|
||||||
|
|
||||||
|
# Douze colonnes par alerte, contre quatre pour une recommandation : le plafond asyncpg de
|
||||||
|
# 32 767 parametres tombe a 2 730 lignes, d'ou un lot plus petit que `recommendation.py`.
|
||||||
|
TAILLE_DE_LOT = 1000
|
||||||
|
|
||||||
|
|
||||||
|
class AlertRepository:
|
||||||
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
|
self._session = session
|
||||||
|
|
||||||
|
async def list_all(
|
||||||
|
self, *, site_id: str | None = None, severity: str | None = None
|
||||||
|
) -> Sequence[Alert]:
|
||||||
|
requete = select(Alert).order_by(Alert.timestamp.desc(), Alert.alert_id.desc())
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(Alert.site_id == site_id)
|
||||||
|
if severity is not None:
|
||||||
|
requete = requete.where(Alert.severity == severity)
|
||||||
|
return (await self._session.scalars(requete)).all()
|
||||||
|
|
||||||
|
async def create_many(self, alerts: Sequence[Alert]) -> Sequence[Alert]:
|
||||||
|
# `ON CONFLICT DO NOTHING` sur `uq_alert_source_reference` : rejouer la détection sur une
|
||||||
|
# fenêtre qui recouvre une exécution précédente ne doit pas dupliquer une alerte déjà
|
||||||
|
# enregistrée. `RETURNING` ne renvoie donc que les lignes effectivement insérées.
|
||||||
|
if not alerts:
|
||||||
|
return []
|
||||||
|
valeurs = [
|
||||||
|
{
|
||||||
|
"source_alert_id": alerte.source_alert_id,
|
||||||
|
"site_id": alerte.site_id,
|
||||||
|
"source": alerte.source,
|
||||||
|
"timestamp": alerte.timestamp,
|
||||||
|
"type": alerte.type,
|
||||||
|
"severity": alerte.severity,
|
||||||
|
"message": alerte.message,
|
||||||
|
"value": alerte.value,
|
||||||
|
"threshold": alerte.threshold,
|
||||||
|
"metric": alerte.metric,
|
||||||
|
"prediction_id": alerte.prediction_id,
|
||||||
|
"raw_data": alerte.raw_data,
|
||||||
|
}
|
||||||
|
for alerte in alerts
|
||||||
|
]
|
||||||
|
creees: list[Alert] = []
|
||||||
|
# Piège : asyncpg plafonne une requête à 32 767 paramètres. Une détection sur une fenêtre
|
||||||
|
# chargée dépasse ce seuil, et l'`INSERT` d'un seul tenant échouerait.
|
||||||
|
for debut in range(0, len(valeurs), TAILLE_DE_LOT):
|
||||||
|
requete = (
|
||||||
|
insert(Alert)
|
||||||
|
.values(valeurs[debut : debut + TAILLE_DE_LOT])
|
||||||
|
.on_conflict_do_nothing(constraint="uq_alert_source_reference")
|
||||||
|
.returning(Alert)
|
||||||
|
)
|
||||||
|
resultat = await self._session.execute(requete)
|
||||||
|
creees.extend(resultat.scalars().all())
|
||||||
|
await self._session.flush()
|
||||||
|
return creees
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
"""Piège : deux dédoublonnages, pas un - DriftRepository.paires()
|
||||||
|
|
||||||
|
`prediction` n'a pas d'unicité sur `(site_id, target_at)` : chaque run de scoring empile une
|
||||||
|
ligne de plus. `uq_reading_source` autorise de son côté deux lectures au même instant quand la
|
||||||
|
`source` diffère. Joindre les deux tables sans `DISTINCT ON` des deux côtés compterait donc la
|
||||||
|
même heure plusieurs fois, et la moyenne d'erreur pèserait ces sites en double.
|
||||||
|
|
||||||
|
On retient la prédiction du run le plus récent, celle que sert `GET /api/v1/predictions`, avec
|
||||||
|
`prediction_id` en départage : `created_at` vaut l'heure de début de transaction et ne
|
||||||
|
distingue pas deux lignes du même run.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import Subquery, func, select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.energy import DriftReport, Prediction, Reading
|
||||||
|
|
||||||
|
TARGET_METRIC = "consumption_kwh"
|
||||||
|
STATUT_DISPONIBLE = "available"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PaireDerive:
|
||||||
|
site_id: str
|
||||||
|
target_at: datetime
|
||||||
|
predicted_value: float
|
||||||
|
actual_value: float
|
||||||
|
model_reference: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class NouveauRapportDerive:
|
||||||
|
site_id: str | None
|
||||||
|
window_start: datetime
|
||||||
|
window_end: datetime
|
||||||
|
reference_start: datetime | None
|
||||||
|
reference_end: datetime | None
|
||||||
|
n_observations: int
|
||||||
|
mae: float | None
|
||||||
|
mape: float | None
|
||||||
|
bias: float | None
|
||||||
|
reference_mae: float | None
|
||||||
|
coverage_ratio: float | None
|
||||||
|
insufficient_data_ratio: float | None
|
||||||
|
model_references: list[str]
|
||||||
|
status: str
|
||||||
|
reason: str | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ComptageStatut:
|
||||||
|
site_id: str
|
||||||
|
status: str
|
||||||
|
nombre: int
|
||||||
|
|
||||||
|
|
||||||
|
def _predictions_retenues(*, debut: datetime, fin: datetime, site_id: str | None) -> Subquery:
|
||||||
|
requete = (
|
||||||
|
select(
|
||||||
|
Prediction.site_id,
|
||||||
|
Prediction.target_at,
|
||||||
|
Prediction.predicted_value,
|
||||||
|
Prediction.model_reference,
|
||||||
|
Prediction.status,
|
||||||
|
)
|
||||||
|
.distinct(Prediction.site_id, Prediction.target_at)
|
||||||
|
.where(
|
||||||
|
Prediction.target_metric == TARGET_METRIC,
|
||||||
|
Prediction.target_at >= debut,
|
||||||
|
Prediction.target_at < fin,
|
||||||
|
)
|
||||||
|
.order_by(Prediction.site_id, Prediction.target_at, Prediction.prediction_id.desc())
|
||||||
|
)
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(Prediction.site_id == site_id)
|
||||||
|
return requete.subquery()
|
||||||
|
|
||||||
|
|
||||||
|
def _lectures_retenues(*, debut: datetime, fin: datetime, site_id: str | None) -> Subquery:
|
||||||
|
requete = (
|
||||||
|
select(Reading.site_id, Reading.timestamp, Reading.consumption_kwh)
|
||||||
|
.distinct(Reading.site_id, Reading.timestamp)
|
||||||
|
.where(
|
||||||
|
Reading.timestamp >= debut,
|
||||||
|
Reading.timestamp < fin,
|
||||||
|
Reading.consumption_kwh.is_not(None),
|
||||||
|
)
|
||||||
|
.order_by(Reading.site_id, Reading.timestamp, Reading.reading_id.desc())
|
||||||
|
)
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(Reading.site_id == site_id)
|
||||||
|
return requete.subquery()
|
||||||
|
|
||||||
|
|
||||||
|
class DriftRepository:
|
||||||
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
|
self._session = session
|
||||||
|
|
||||||
|
async def paires(
|
||||||
|
self, *, debut: datetime, fin: datetime, site_id: str | None = None
|
||||||
|
) -> Sequence[PaireDerive]:
|
||||||
|
predictions = _predictions_retenues(debut=debut, fin=fin, site_id=site_id)
|
||||||
|
lectures = _lectures_retenues(debut=debut, fin=fin, site_id=site_id)
|
||||||
|
requete = (
|
||||||
|
select(
|
||||||
|
predictions.c.site_id,
|
||||||
|
predictions.c.target_at,
|
||||||
|
predictions.c.predicted_value,
|
||||||
|
lectures.c.consumption_kwh,
|
||||||
|
predictions.c.model_reference,
|
||||||
|
)
|
||||||
|
.select_from(predictions)
|
||||||
|
.join(
|
||||||
|
lectures,
|
||||||
|
(lectures.c.site_id == predictions.c.site_id)
|
||||||
|
& (lectures.c.timestamp == predictions.c.target_at),
|
||||||
|
)
|
||||||
|
.where(predictions.c.status == STATUT_DISPONIBLE)
|
||||||
|
.order_by(predictions.c.site_id, predictions.c.target_at)
|
||||||
|
)
|
||||||
|
|
||||||
|
lignes = await self._session.execute(requete)
|
||||||
|
return [
|
||||||
|
PaireDerive(
|
||||||
|
site_id=ligne[0],
|
||||||
|
target_at=ligne[1],
|
||||||
|
predicted_value=ligne[2],
|
||||||
|
actual_value=ligne[3],
|
||||||
|
model_reference=ligne[4],
|
||||||
|
)
|
||||||
|
for ligne in lignes
|
||||||
|
]
|
||||||
|
|
||||||
|
async def comptages(
|
||||||
|
self, *, debut: datetime, fin: datetime, site_id: str | None = None
|
||||||
|
) -> Sequence[ComptageStatut]:
|
||||||
|
predictions = _predictions_retenues(debut=debut, fin=fin, site_id=site_id)
|
||||||
|
requete = (
|
||||||
|
select(predictions.c.site_id, predictions.c.status, func.count())
|
||||||
|
.select_from(predictions)
|
||||||
|
.group_by(predictions.c.site_id, predictions.c.status)
|
||||||
|
)
|
||||||
|
|
||||||
|
lignes = await self._session.execute(requete)
|
||||||
|
return [
|
||||||
|
ComptageStatut(site_id=ligne[0], status=ligne[1], nombre=ligne[2]) for ligne in lignes
|
||||||
|
]
|
||||||
|
|
||||||
|
# Pourquoi : l'idempotence est déléguée à `uq_drift_report_window` plutôt qu'à une lecture
|
||||||
|
# préalable, comme pour les recommandations. Rejouer la commande sur la même fenêtre ne
|
||||||
|
# duplique donc rien.
|
||||||
|
async def enregistre(self, rapports: Sequence[NouveauRapportDerive]) -> int:
|
||||||
|
if not rapports:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
valeurs = [asdict(rapport) for rapport in rapports]
|
||||||
|
requete = (
|
||||||
|
insert(DriftReport)
|
||||||
|
.values(valeurs)
|
||||||
|
.on_conflict_do_nothing(
|
||||||
|
index_elements=[DriftReport.window_end, func.coalesce(DriftReport.site_id, "")]
|
||||||
|
)
|
||||||
|
.returning(DriftReport.drift_report_id)
|
||||||
|
)
|
||||||
|
return len((await self._session.scalars(requete)).all())
|
||||||
|
|
||||||
|
async def derniers(self, *, site_id: str | None = None) -> Sequence[DriftReport]:
|
||||||
|
requete = (
|
||||||
|
select(DriftReport)
|
||||||
|
.distinct(DriftReport.site_id)
|
||||||
|
.order_by(
|
||||||
|
DriftReport.site_id,
|
||||||
|
DriftReport.computed_at.desc(),
|
||||||
|
DriftReport.drift_report_id.desc(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(DriftReport.site_id == site_id)
|
||||||
|
return (await self._session.scalars(requete)).all()
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.password_reset_attempt import PasswordResetAttempt
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ResetRequestCounts:
|
||||||
|
per_identifier: int
|
||||||
|
per_ip: int
|
||||||
|
|
||||||
|
|
||||||
|
class PasswordResetAttemptRepository:
|
||||||
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
|
self._session = session
|
||||||
|
|
||||||
|
async def record(self, *, email: str, client_ip: str | None) -> None:
|
||||||
|
self._session.add(
|
||||||
|
PasswordResetAttempt(email_tried=email.strip().lower(), client_ip=client_ip)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def count_recent(
|
||||||
|
self, *, email: str, client_ip: str | None, window_seconds: int
|
||||||
|
) -> ResetRequestCounts:
|
||||||
|
identifiant = email.strip().lower()
|
||||||
|
meme_email = PasswordResetAttempt.email_tried == identifiant
|
||||||
|
meme_ip = PasswordResetAttempt.client_ip == client_ip
|
||||||
|
|
||||||
|
requete = select(
|
||||||
|
func.count().filter(meme_email),
|
||||||
|
func.count().filter(meme_ip),
|
||||||
|
).where(
|
||||||
|
PasswordResetAttempt.occurred_at
|
||||||
|
> datetime.now(UTC) - timedelta(seconds=window_seconds),
|
||||||
|
meme_email | meme_ip,
|
||||||
|
)
|
||||||
|
|
||||||
|
par_identifiant, par_ip = (await self._session.execute(requete)).one()
|
||||||
|
return ResetRequestCounts(per_identifier=par_identifiant, per_ip=par_ip)
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# Piège : `consume()` est une seule instruction, sur le modèle de `claim_for_rotation()` du
|
||||||
|
# jeton de rafraîchissement. Un SELECT puis un UPDATE laisseraient une fenêtre où deux
|
||||||
|
# soumissions concurrentes du même lien réussiraient toutes les deux.
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from sqlalchemy import func, select, update
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.password_reset_token import PasswordResetToken
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class ConsumedResetToken:
|
||||||
|
id: UUID
|
||||||
|
user_id: UUID
|
||||||
|
|
||||||
|
|
||||||
|
class PasswordResetTokenRepository:
|
||||||
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
|
self._session = session
|
||||||
|
|
||||||
|
async def create(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
user_id: UUID,
|
||||||
|
token_hash: bytes,
|
||||||
|
expires_at: datetime,
|
||||||
|
client_ip: str | None,
|
||||||
|
user_agent: str | None,
|
||||||
|
) -> PasswordResetToken:
|
||||||
|
jeton = PasswordResetToken(
|
||||||
|
user_id=user_id,
|
||||||
|
token_hash=token_hash,
|
||||||
|
expires_at=expires_at,
|
||||||
|
client_ip=client_ip,
|
||||||
|
user_agent=user_agent,
|
||||||
|
)
|
||||||
|
self._session.add(jeton)
|
||||||
|
await self._session.flush()
|
||||||
|
return jeton
|
||||||
|
|
||||||
|
async def consume(self, token_hash: bytes) -> ConsumedResetToken | None:
|
||||||
|
requete = (
|
||||||
|
update(PasswordResetToken)
|
||||||
|
.where(
|
||||||
|
PasswordResetToken.token_hash == token_hash,
|
||||||
|
PasswordResetToken.consumed_at.is_(None),
|
||||||
|
PasswordResetToken.expires_at > func.clock_timestamp(),
|
||||||
|
)
|
||||||
|
.values(consumed_at=func.clock_timestamp())
|
||||||
|
.returning(PasswordResetToken.id, PasswordResetToken.user_id)
|
||||||
|
)
|
||||||
|
ligne = (await self._session.execute(requete)).one_or_none()
|
||||||
|
if ligne is None:
|
||||||
|
return None
|
||||||
|
return ConsumedResetToken(id=ligne.id, user_id=ligne.user_id)
|
||||||
|
|
||||||
|
# Piège : simple SELECT, volontairement pas atomique avec la consommation. Sert seulement
|
||||||
|
# au feedback UX (jeton encore valide ?) ; `consume()` reste la seule source de vérité.
|
||||||
|
async def exists_valid(self, token_hash: bytes) -> bool:
|
||||||
|
requete = select(PasswordResetToken.id).where(
|
||||||
|
PasswordResetToken.token_hash == token_hash,
|
||||||
|
PasswordResetToken.consumed_at.is_(None),
|
||||||
|
PasswordResetToken.expires_at > func.clock_timestamp(),
|
||||||
|
)
|
||||||
|
return (await self._session.execute(requete)).first() is not None
|
||||||
|
|
||||||
|
async def invalidate_all_for_user(self, user_id: UUID) -> int:
|
||||||
|
resultat = await self._session.execute(
|
||||||
|
update(PasswordResetToken)
|
||||||
|
.where(PasswordResetToken.user_id == user_id, PasswordResetToken.consumed_at.is_(None))
|
||||||
|
.values(consumed_at=func.clock_timestamp())
|
||||||
|
.returning(PasswordResetToken.id)
|
||||||
|
)
|
||||||
|
return len(resultat.all())
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.energy import Prediction
|
||||||
|
|
||||||
|
|
||||||
|
class PredictionRepository:
|
||||||
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
|
self._session = session
|
||||||
|
|
||||||
|
async def list_since(
|
||||||
|
self, *, since: datetime, site_id: str | None = None
|
||||||
|
) -> Sequence[Prediction]:
|
||||||
|
# Restreint à `available` : une prévision `insufficient_data`/`error` n'a pas de
|
||||||
|
# `predicted_value` à comparer à une lecture réelle (détection d'anomalie).
|
||||||
|
# Piège : `prediction` n'a pas d'unicité sur `(site_id, target_at)` (cf.
|
||||||
|
# `enervision_ml.score`, qui insère toujours une nouvelle ligne plutôt que d'écraser la
|
||||||
|
# précédente). `prediction_id` en dernier départage donc les égalités de `target_at` par
|
||||||
|
# ordre croissant : `_detect_anomaly` construit un dict qui garde le dernier rencontré,
|
||||||
|
# c'est-à-dire le run le plus récent plutôt qu'une ligne choisie au hasard par le plan
|
||||||
|
# d'exécution.
|
||||||
|
requete = (
|
||||||
|
select(Prediction)
|
||||||
|
.where(Prediction.target_at >= since, Prediction.status == "available")
|
||||||
|
.order_by(Prediction.site_id, Prediction.target_at, Prediction.prediction_id)
|
||||||
|
)
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(Prediction.site_id == site_id)
|
||||||
|
return (await self._session.scalars(requete)).all()
|
||||||
|
|
||||||
|
async def latest_by_site(self) -> Sequence[Prediction]:
|
||||||
|
# `.distinct(site_id)` compile en `DISTINCT ON (site_id)` sous PostgreSQL : une seule
|
||||||
|
# ligne par site, la plus récente grâce à l'ordre composite qui suit. Même mécanisme que
|
||||||
|
# `ReadingRepository.latest_by_site`. Trié sur `target_at` (couvert par
|
||||||
|
# `ix_prediction_site_target`) plutôt que `created_at` : c'est la prévision la plus
|
||||||
|
# récente qui compte pour un tableau de bord, pas forcément le dernier run de scoring.
|
||||||
|
requete = (
|
||||||
|
select(Prediction)
|
||||||
|
.distinct(Prediction.site_id)
|
||||||
|
.order_by(
|
||||||
|
Prediction.site_id,
|
||||||
|
Prediction.target_at.desc(),
|
||||||
|
Prediction.prediction_id.desc(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return (await self._session.scalars(requete)).all()
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
@@ -12,10 +13,58 @@ class ReadingRepository:
|
|||||||
|
|
||||||
async def latest_by_site(self) -> Sequence[Reading]:
|
async def latest_by_site(self) -> Sequence[Reading]:
|
||||||
# `.distinct(site_id)` compile en `DISTINCT ON (site_id)` sous PostgreSQL : une seule
|
# `.distinct(site_id)` compile en `DISTINCT ON (site_id)` sous PostgreSQL : une seule
|
||||||
# ligne par site, la plus récente grâce à l'ordre composite qui suit.
|
# ligne par site, la plus récente grâce à l'ordre composite qui suit. `reading_id` départage
|
||||||
|
# les égalités de timestamp, que `uq_reading_source` autorise à `source` différente.
|
||||||
requete = (
|
requete = (
|
||||||
select(Reading)
|
select(Reading)
|
||||||
.distinct(Reading.site_id)
|
.distinct(Reading.site_id)
|
||||||
.order_by(Reading.site_id, Reading.timestamp.desc())
|
.order_by(Reading.site_id, Reading.timestamp.desc(), Reading.reading_id.desc())
|
||||||
)
|
)
|
||||||
return (await self._session.execute(requete)).scalars().all()
|
return (await self._session.execute(requete)).scalars().all()
|
||||||
|
|
||||||
|
async def latest_for_site(self, site_id: str) -> Reading | None:
|
||||||
|
# Piège : `uq_reading_source` autorise deux lignes au même `site_id`+`timestamp` quand la
|
||||||
|
# `source` diffère. Sans `reading_id` en départage, le `LIMIT 1` renverrait au hasard.
|
||||||
|
requete = (
|
||||||
|
select(Reading)
|
||||||
|
.where(Reading.site_id == site_id)
|
||||||
|
.order_by(Reading.timestamp.desc(), Reading.reading_id.desc())
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
lecture: Reading | None = await self._session.scalar(requete)
|
||||||
|
return lecture
|
||||||
|
|
||||||
|
async def list_since(self, *, since: datetime, site_id: str | None = None) -> Sequence[Reading]:
|
||||||
|
# Trié par site puis par heure croissante : la détection d'alertes (spike) a besoin de
|
||||||
|
# comparer chaque lecture à celle qui la précède immédiatement pour le même site.
|
||||||
|
# `reading_id` en dernier départage : `uq_reading_source` autorise deux lignes au même
|
||||||
|
# `site_id`+`timestamp` quand la `source` diffère (même piège que `latest_for_site`), sans
|
||||||
|
# quoi l'ordre entre elles ne serait pas garanti d'un appel à l'autre.
|
||||||
|
requete = (
|
||||||
|
select(Reading)
|
||||||
|
.where(Reading.timestamp >= since)
|
||||||
|
.order_by(Reading.site_id, Reading.timestamp, Reading.reading_id)
|
||||||
|
)
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(Reading.site_id == site_id)
|
||||||
|
return (await self._session.scalars(requete)).all()
|
||||||
|
|
||||||
|
async def list_history(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
start: datetime,
|
||||||
|
end: datetime,
|
||||||
|
site_id: str | None = None,
|
||||||
|
limit: int,
|
||||||
|
offset: int,
|
||||||
|
) -> Sequence[Reading]:
|
||||||
|
requete = (
|
||||||
|
select(Reading)
|
||||||
|
.where(Reading.timestamp >= start, Reading.timestamp < end)
|
||||||
|
.order_by(Reading.timestamp.desc(), Reading.reading_id.desc())
|
||||||
|
.limit(limit)
|
||||||
|
.offset(offset)
|
||||||
|
)
|
||||||
|
if site_id is not None:
|
||||||
|
requete = requete.where(Reading.site_id == site_id)
|
||||||
|
return (await self._session.scalars(requete)).all()
|
||||||
|
|||||||
@@ -1,11 +1,24 @@
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import asdict, dataclass
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.models.energy import Recommendation
|
from app.models.energy import Recommendation
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class NouvelleRecommandation:
|
||||||
|
alert_id: int
|
||||||
|
action: str
|
||||||
|
explanation: str
|
||||||
|
rule_reference: str
|
||||||
|
|
||||||
|
|
||||||
|
TAILLE_DE_LOT = 1000
|
||||||
|
|
||||||
|
|
||||||
class RecommendationRepository:
|
class RecommendationRepository:
|
||||||
def __init__(self, session: AsyncSession) -> None:
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
self._session = session
|
self._session = session
|
||||||
@@ -20,3 +33,19 @@ class RecommendationRepository:
|
|||||||
)
|
)
|
||||||
recommendation: Recommendation | None = await self._session.scalar(requete)
|
recommendation: Recommendation | None = await self._session.scalar(requete)
|
||||||
return recommendation
|
return recommendation
|
||||||
|
|
||||||
|
# Pourquoi : l'idempotence est déléguée à `uq_recommendation_alert_rule` plutôt qu'à une
|
||||||
|
# lecture préalable, qui laisserait une fenêtre entre le contrôle et l'insertion.
|
||||||
|
async def create_missing(self, nouvelles: Sequence[NouvelleRecommandation]) -> int:
|
||||||
|
creees = 0
|
||||||
|
# Piège : asyncpg plafonne une requête à 32 767 paramètres, soit 8 191 lignes de quatre
|
||||||
|
# colonnes. Au-delà de ce seuil un `INSERT` d'un seul tenant échouerait.
|
||||||
|
for debut in range(0, len(nouvelles), TAILLE_DE_LOT):
|
||||||
|
requete = (
|
||||||
|
insert(Recommendation)
|
||||||
|
.values([asdict(nouvelle) for nouvelle in nouvelles[debut : debut + TAILLE_DE_LOT]])
|
||||||
|
.on_conflict_do_nothing(constraint="uq_recommendation_alert_rule")
|
||||||
|
.returning(Recommendation.recommendation_id)
|
||||||
|
)
|
||||||
|
creees += len((await self._session.scalars(requete)).all())
|
||||||
|
return creees
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict
|
||||||
|
|
||||||
|
|
||||||
|
class AlertType(StrEnum):
|
||||||
|
SPIKE = "spike"
|
||||||
|
THRESHOLD = "threshold"
|
||||||
|
ANOMALY = "anomaly"
|
||||||
|
OUTAGE = "outage"
|
||||||
|
SENSOR = "sensor"
|
||||||
|
|
||||||
|
|
||||||
|
class AlertSeverity(StrEnum):
|
||||||
|
LOW = "low"
|
||||||
|
MEDIUM = "medium"
|
||||||
|
HIGH = "high"
|
||||||
|
CRITICAL = "critical"
|
||||||
|
|
||||||
|
|
||||||
|
class AlertResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
alert_id: int
|
||||||
|
site_id: str
|
||||||
|
timestamp: datetime
|
||||||
|
type: AlertType
|
||||||
|
severity: AlertSeverity
|
||||||
|
message: str
|
||||||
|
value: float | None
|
||||||
|
threshold: float | None
|
||||||
|
metric: str | None
|
||||||
|
prediction_id: int | None
|
||||||
@@ -1,17 +1,45 @@
|
|||||||
# Contrainte : le mot de passe est borné à 128 caractères. Sans plafond, une chaîne de dix
|
# Contrainte : le mot de passe est borné à 128 caractères. Sans plafond, une chaîne de dix
|
||||||
# mégaoctets ferait travailler Argon2 gratuitement, à la charge du serveur.
|
# mégaoctets ferait travailler Argon2 gratuitement, à la charge du serveur.
|
||||||
|
# Contrainte : `SPECIAL_CHARACTERS` doit rester identique à `password.validator.ts` côté
|
||||||
|
# frontend. `\w`/`\d` divergent entre Python (Unicode) et JavaScript (ASCII) : une classe
|
||||||
|
# explicite, plutôt qu'une négation, évite qu'un mot de passe soit accepté d'un côté et
|
||||||
|
# rejeté de l'autre (ex. "Sécurité1", où "é" comptait comme "spécial" pour Python seul).
|
||||||
|
|
||||||
|
import re
|
||||||
from typing import Literal, Self
|
from typing import Literal, Self
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict, EmailStr, Field
|
from pydantic import BaseModel, ConfigDict, EmailStr, Field, field_validator
|
||||||
|
|
||||||
from app.core.principal import Principal
|
from app.core.principal import Principal
|
||||||
from app.core.roles import AccountKind, Role
|
from app.core.roles import AccountKind, Role
|
||||||
|
|
||||||
PASSWORD_MIN_LENGTH = 12
|
PASSWORD_MIN_LENGTH = 8
|
||||||
PASSWORD_MAX_LENGTH = 128
|
PASSWORD_MAX_LENGTH = 128
|
||||||
|
|
||||||
|
SPECIAL_CHARACTERS = "!@#$%^&*()-_=+[]{};:,.?"
|
||||||
|
|
||||||
|
_MAJUSCULE = re.compile(r"[A-ZÀ-ÖØ-Þ]")
|
||||||
|
_MINUSCULE = re.compile(r"[a-zà-öø-þ]")
|
||||||
|
_CHIFFRE = re.compile(r"[0-9]")
|
||||||
|
_SPECIAL = re.compile(r"[" + re.escape(SPECIAL_CHARACTERS) + r"]")
|
||||||
|
|
||||||
|
|
||||||
|
def valide_complexite(mot_de_passe: str) -> str:
|
||||||
|
manquants = [
|
||||||
|
nom
|
||||||
|
for nom, motif in (
|
||||||
|
("une majuscule", _MAJUSCULE),
|
||||||
|
("une minuscule", _MINUSCULE),
|
||||||
|
("un chiffre", _CHIFFRE),
|
||||||
|
("un caractère spécial", _SPECIAL),
|
||||||
|
)
|
||||||
|
if not motif.search(mot_de_passe)
|
||||||
|
]
|
||||||
|
if manquants:
|
||||||
|
raise ValueError(f"Le mot de passe doit contenir au moins {', '.join(manquants)}")
|
||||||
|
return mot_de_passe
|
||||||
|
|
||||||
|
|
||||||
class LoginRequest(BaseModel):
|
class LoginRequest(BaseModel):
|
||||||
email: EmailStr
|
email: EmailStr
|
||||||
@@ -22,6 +50,25 @@ class PasswordChangeRequest(BaseModel):
|
|||||||
current_password: str = Field(min_length=1, max_length=PASSWORD_MAX_LENGTH)
|
current_password: str = Field(min_length=1, max_length=PASSWORD_MAX_LENGTH)
|
||||||
new_password: str = Field(min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH)
|
new_password: str = Field(min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH)
|
||||||
|
|
||||||
|
@field_validator("new_password")
|
||||||
|
@classmethod
|
||||||
|
def _new_password_est_complexe(cls, valeur: str) -> str:
|
||||||
|
return valide_complexite(valeur)
|
||||||
|
|
||||||
|
|
||||||
|
class ForgotPasswordRequest(BaseModel):
|
||||||
|
email: EmailStr
|
||||||
|
|
||||||
|
|
||||||
|
class ResetPasswordRequest(BaseModel):
|
||||||
|
token: str = Field(min_length=1)
|
||||||
|
new_password: str = Field(min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH)
|
||||||
|
|
||||||
|
@field_validator("new_password")
|
||||||
|
@classmethod
|
||||||
|
def _new_password_est_complexe(cls, valeur: str) -> str:
|
||||||
|
return valide_complexite(valeur)
|
||||||
|
|
||||||
|
|
||||||
class PrincipalResponse(BaseModel):
|
class PrincipalResponse(BaseModel):
|
||||||
model_config = ConfigDict(from_attributes=True)
|
model_config = ConfigDict(from_attributes=True)
|
||||||
@@ -37,6 +84,10 @@ class PrincipalResponse(BaseModel):
|
|||||||
return cls.model_validate(principal)
|
return cls.model_validate(principal)
|
||||||
|
|
||||||
|
|
||||||
|
class ResetTokenValidationResponse(BaseModel):
|
||||||
|
valid: bool
|
||||||
|
|
||||||
|
|
||||||
class TokenResponse(BaseModel):
|
class TokenResponse(BaseModel):
|
||||||
access_token: str
|
access_token: str
|
||||||
token_type: Literal["bearer"] = "bearer" # noqa: S105
|
token_type: Literal["bearer"] = "bearer" # noqa: S105
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict
|
||||||
|
|
||||||
|
|
||||||
|
class DriftStatus(StrEnum):
|
||||||
|
STABLE = "stable"
|
||||||
|
DERIVE = "derive"
|
||||||
|
INDETERMINE = "indetermine"
|
||||||
|
|
||||||
|
|
||||||
|
class DriftReportResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
site_id: str | None
|
||||||
|
computed_at: datetime
|
||||||
|
window_start: datetime
|
||||||
|
window_end: datetime
|
||||||
|
reference_start: datetime | None
|
||||||
|
reference_end: datetime | None
|
||||||
|
n_observations: int
|
||||||
|
mae: float | None
|
||||||
|
mape: float | None
|
||||||
|
bias: float | None
|
||||||
|
reference_mae: float | None
|
||||||
|
coverage_ratio: float | None
|
||||||
|
insufficient_data_ratio: float | None
|
||||||
|
model_references: list[str]
|
||||||
|
status: DriftStatus
|
||||||
|
reason: str | None
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from enum import StrEnum
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict
|
||||||
|
|
||||||
|
|
||||||
|
class PredictionTargetMetric(StrEnum):
|
||||||
|
CONSUMPTION_KWH = "consumption_kwh"
|
||||||
|
CONSUMPTION_KW = "consumption_kw"
|
||||||
|
|
||||||
|
|
||||||
|
class PredictionStatus(StrEnum):
|
||||||
|
AVAILABLE = "available"
|
||||||
|
INSUFFICIENT_DATA = "insufficient_data"
|
||||||
|
ERROR = "error"
|
||||||
|
|
||||||
|
|
||||||
|
class SitePredictionResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
target_at: datetime
|
||||||
|
target_metric: PredictionTargetMetric
|
||||||
|
period_minutes: int | None
|
||||||
|
predicted_value: float | None
|
||||||
|
status: PredictionStatus
|
||||||
|
failure_reason: str | None
|
||||||
|
model_reference: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class SitePredictionSummaryResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
site_id: str
|
||||||
|
site_name: str
|
||||||
|
prediction: SitePredictionResponse | None
|
||||||
|
|
||||||
|
|
||||||
|
class PredictionSummaryResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
timestamp: datetime
|
||||||
|
sites: list[SitePredictionSummaryResponse]
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from decimal import Decimal
|
||||||
|
from enum import StrEnum
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict
|
||||||
|
|
||||||
|
|
||||||
|
class ReadingSource(StrEnum):
|
||||||
|
CSV = "csv"
|
||||||
|
API_CURRENT = "api_current"
|
||||||
|
API_HISTORY = "api_history"
|
||||||
|
|
||||||
|
|
||||||
|
class ReadingDataQuality(StrEnum):
|
||||||
|
GOOD = "good"
|
||||||
|
PARTIAL = "partial"
|
||||||
|
DEGRADED = "degraded"
|
||||||
|
CRITICAL = "critical"
|
||||||
|
|
||||||
|
|
||||||
|
class ReadingResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
reading_id: int
|
||||||
|
site_id: str
|
||||||
|
timestamp: datetime
|
||||||
|
source: ReadingSource
|
||||||
|
consumption_kw: float | None
|
||||||
|
consumption_kwh: float | None
|
||||||
|
# Piège : `Decimal` (miroir de `Numeric(14, 2)` en base, pour ne pas arrondir un montant)
|
||||||
|
# sérialise en chaîne dans le JSON, pas en nombre — un consommateur qui ferait un `parseFloat`
|
||||||
|
# naïf perdrait la précision que ce choix visait à garder.
|
||||||
|
consumption_euros: Decimal | None
|
||||||
|
voltage_v: float | None
|
||||||
|
current_a: float | None
|
||||||
|
power_factor: float | None
|
||||||
|
temperature_celsius: float | None
|
||||||
|
humidity_percent: float | None
|
||||||
|
solar_irradiance_wm2: float | None
|
||||||
|
is_working_hours: bool | None
|
||||||
|
data_quality: ReadingDataQuality | None
|
||||||
|
null_reasons: list[str] | None
|
||||||
|
imputed_values: dict[str, Any] | None
|
||||||
|
imputation_method: str | None
|
||||||
@@ -12,3 +12,9 @@ class RecommendationResponse(BaseModel):
|
|||||||
explanation: str
|
explanation: str
|
||||||
rule_reference: str
|
rule_reference: str
|
||||||
created_at: datetime
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class RecommendationGenerationResponse(BaseModel):
|
||||||
|
alerts_examined: int
|
||||||
|
recommendations_created: int
|
||||||
|
already_present: int
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from pydantic import BaseModel, ConfigDict, Field
|
||||||
|
|
||||||
|
|
||||||
|
class SensorDiagnosticResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
status: Literal["ok", "failing"]
|
||||||
|
since: datetime | None = Field(
|
||||||
|
description=(
|
||||||
|
"Horodatage de la dernière lecture reçue pour ce site. Ce n'est pas le début de la "
|
||||||
|
"panne : l'historique ne permet pas de le dater sans requête supplémentaire."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class SiteSensorsResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
consumption: SensorDiagnosticResponse
|
||||||
|
electrical: SensorDiagnosticResponse
|
||||||
|
temperature: SensorDiagnosticResponse
|
||||||
|
humidity: SensorDiagnosticResponse
|
||||||
|
network: SensorDiagnosticResponse
|
||||||
|
|
||||||
|
|
||||||
|
class SiteSensorStatusResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
site_id: str
|
||||||
|
site_name: str
|
||||||
|
sensors: SiteSensorsResponse
|
||||||
|
overall: Literal["ok", "degraded", "critical"]
|
||||||
|
|
||||||
|
|
||||||
|
class SensorStatusResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
timestamp: datetime
|
||||||
|
sites: list[SiteSensorStatusResponse]
|
||||||
@@ -1,3 +1,6 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
from pydantic import BaseModel, ConfigDict
|
from pydantic import BaseModel, ConfigDict
|
||||||
|
|
||||||
|
|
||||||
@@ -10,3 +13,20 @@ class SiteResponse(BaseModel):
|
|||||||
location: str | None
|
location: str | None
|
||||||
capacity_kw: float | None
|
capacity_kw: float | None
|
||||||
status: str | None
|
status: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class SiteCurrentResponse(BaseModel):
|
||||||
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
|
timestamp: datetime | None
|
||||||
|
site_id: str
|
||||||
|
site_type: str
|
||||||
|
consumption_kw: float | None
|
||||||
|
consumption_kwh: float | None
|
||||||
|
voltage_v: float | None
|
||||||
|
current_a: float | None
|
||||||
|
power_factor: float | None
|
||||||
|
temperature_celsius: float | None
|
||||||
|
humidity_percent: float | None
|
||||||
|
null_reasons: list[str]
|
||||||
|
data_quality: Literal["good", "partial", "degraded", "critical"]
|
||||||
|
|||||||
@@ -0,0 +1,323 @@
|
|||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from app.models.energy import Alert, Prediction, Reading, Site
|
||||||
|
from app.repositories.alert import AlertRepository
|
||||||
|
from app.repositories.prediction import PredictionRepository
|
||||||
|
from app.repositories.reading import ReadingRepository
|
||||||
|
from app.repositories.site import SiteRepository
|
||||||
|
|
||||||
|
# Fenêtre de lectures/prédictions analysée à chaque exécution : assez large pour couvrir une paire
|
||||||
|
# de lectures consécutives (spike) et une coupure prolongée (outage), sans réanalyser tout
|
||||||
|
# l'historique à chaque lancement manuel du script de détection.
|
||||||
|
LOOKBACK = timedelta(hours=48)
|
||||||
|
|
||||||
|
# Cadence nominale d'une lecture : le CSV historique comme l'API Mock livrent un pas horaire.
|
||||||
|
EXPECTED_INTERVAL = timedelta(hours=1)
|
||||||
|
# Au-delà de trois pas manqués, on parle de coupure plutôt que d'un simple retard d'ingestion.
|
||||||
|
OUTAGE_THRESHOLD = EXPECTED_INTERVAL * 3
|
||||||
|
|
||||||
|
# +/-50% entre deux lectures consécutives du même site.
|
||||||
|
SPIKE_RELATIVE_THRESHOLD = 0.5
|
||||||
|
# 30% d'écart entre la consommation réelle et la prévision du même site/instant.
|
||||||
|
ANOMALY_RELATIVE_THRESHOLD = 0.3
|
||||||
|
# Une prévision quasi nulle rend l'écart relatif ininterprétable ; on l'ignore plutôt.
|
||||||
|
ANOMALY_MINIMUM_PREDICTED_VALUE = 1e-6
|
||||||
|
|
||||||
|
THRESHOLD_METRIC = "consumption_kw"
|
||||||
|
ANOMALY_METRIC = "consumption_kwh"
|
||||||
|
# `data_quality` -> sévérité du capteur défaillant. `good` est volontairement absent : il ne
|
||||||
|
# déclenche jamais d'alerte.
|
||||||
|
QUALITE_VERS_SEVERITE: dict[str, str] = {
|
||||||
|
"partial": "low",
|
||||||
|
"degraded": "medium",
|
||||||
|
"critical": "critical",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class AlertService:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
alerts: AlertRepository,
|
||||||
|
readings: ReadingRepository,
|
||||||
|
predictions: PredictionRepository,
|
||||||
|
sites: SiteRepository,
|
||||||
|
) -> None:
|
||||||
|
self._alerts = alerts
|
||||||
|
self._readings = readings
|
||||||
|
self._predictions = predictions
|
||||||
|
self._sites = sites
|
||||||
|
|
||||||
|
async def list_all(
|
||||||
|
self, *, site_id: str | None = None, severity: str | None = None
|
||||||
|
) -> Sequence[Alert]:
|
||||||
|
return await self._alerts.list_all(site_id=site_id, severity=severity)
|
||||||
|
|
||||||
|
async def detect(
|
||||||
|
self, *, now: datetime | None = None, site_id: str | None = None
|
||||||
|
) -> Sequence[Alert]:
|
||||||
|
"""Compare les lectures/prévisions récentes aux cinq règles internes et enregistre les
|
||||||
|
alertes déclenchées (`source='enervision'`). Idempotent grâce à `source_alert_id` :
|
||||||
|
rejouer sur une fenêtre déjà analysée ne recrée pas les mêmes lignes."""
|
||||||
|
instant = now or datetime.now(UTC)
|
||||||
|
depuis = instant - LOOKBACK
|
||||||
|
|
||||||
|
sites = await self._sites.list_all()
|
||||||
|
if site_id is not None:
|
||||||
|
sites = [site for site in sites if site.site_id == site_id]
|
||||||
|
sites_par_id = {site.site_id: site for site in sites}
|
||||||
|
if not sites_par_id:
|
||||||
|
return []
|
||||||
|
|
||||||
|
lectures = [
|
||||||
|
lecture
|
||||||
|
for lecture in await self._readings.list_since(since=depuis, site_id=site_id)
|
||||||
|
if lecture.site_id in sites_par_id
|
||||||
|
]
|
||||||
|
predictions = [
|
||||||
|
prediction
|
||||||
|
for prediction in await self._predictions.list_since(since=depuis, site_id=site_id)
|
||||||
|
if prediction.site_id in sites_par_id
|
||||||
|
]
|
||||||
|
dernieres_lectures = {
|
||||||
|
lecture.site_id: lecture
|
||||||
|
for lecture in await self._readings.latest_by_site()
|
||||||
|
if lecture.site_id in sites_par_id
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates = [
|
||||||
|
*_detect_threshold(lectures, sites_par_id),
|
||||||
|
*_detect_spike(lectures),
|
||||||
|
*_detect_anomaly(lectures, predictions),
|
||||||
|
*_detect_outage(sites, dernieres_lectures, instant),
|
||||||
|
*_detect_sensor(lectures),
|
||||||
|
]
|
||||||
|
if not candidates:
|
||||||
|
return []
|
||||||
|
return await self._alerts.create_many(candidates)
|
||||||
|
|
||||||
|
|
||||||
|
def _severity_from_ratio(ratio: float) -> str:
|
||||||
|
if ratio >= 2.0:
|
||||||
|
return "critical"
|
||||||
|
if ratio >= 1.5:
|
||||||
|
return "high"
|
||||||
|
if ratio >= 1.2:
|
||||||
|
return "medium"
|
||||||
|
return "low"
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_threshold(lectures: Sequence[Reading], sites_par_id: dict[str, Site]) -> list[Alert]:
|
||||||
|
# Seuil fixe = la capacité déclarée du site : dépasser `capacity_kw` est un dépassement
|
||||||
|
# matériel, pas une simple variation, et évite un seuil arbitraire non fourni par le domaine.
|
||||||
|
alertes = []
|
||||||
|
for lecture in lectures:
|
||||||
|
site = sites_par_id[lecture.site_id]
|
||||||
|
valeur = lecture.consumption_kw
|
||||||
|
if site.capacity_kw is None or site.capacity_kw <= 0 or valeur is None:
|
||||||
|
continue
|
||||||
|
if valeur <= site.capacity_kw:
|
||||||
|
continue
|
||||||
|
alertes.append(
|
||||||
|
Alert(
|
||||||
|
source_alert_id=f"threshold:{THRESHOLD_METRIC}:{lecture.timestamp.isoformat()}",
|
||||||
|
site_id=lecture.site_id,
|
||||||
|
source="enervision",
|
||||||
|
timestamp=lecture.timestamp,
|
||||||
|
type="threshold",
|
||||||
|
severity=_severity_from_ratio(valeur / site.capacity_kw),
|
||||||
|
message=(
|
||||||
|
f"Puissance appelée {valeur:.1f} kW au-dessus de la capacité du site "
|
||||||
|
f"({site.capacity_kw:.1f} kW)"
|
||||||
|
),
|
||||||
|
value=valeur,
|
||||||
|
threshold=site.capacity_kw,
|
||||||
|
metric=THRESHOLD_METRIC,
|
||||||
|
prediction_id=None,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return alertes
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_spike(lectures: Sequence[Reading]) -> list[Alert]:
|
||||||
|
# `lectures` est triée par site, heure puis `reading_id` (cf. `ReadingRepository.list_since`) :
|
||||||
|
# deux lignes consécutives du même site sont donc deux mesures consécutives dans le temps,
|
||||||
|
# sauf lorsqu'elles partagent le même horodatage (deux `source` différentes pour le même
|
||||||
|
# instant, permises par `uq_reading_source`) : ce n'est alors pas une variation réelle, on
|
||||||
|
# l'ignore plutôt que de générer une fausse alerte figée par son `source_alert_id`.
|
||||||
|
alertes = []
|
||||||
|
precedente: Reading | None = None
|
||||||
|
for lecture in lectures:
|
||||||
|
if (
|
||||||
|
precedente is None
|
||||||
|
or precedente.site_id != lecture.site_id
|
||||||
|
or precedente.timestamp == lecture.timestamp
|
||||||
|
):
|
||||||
|
precedente = lecture
|
||||||
|
continue
|
||||||
|
avant, apres = precedente.consumption_kw, lecture.consumption_kw
|
||||||
|
precedente = lecture
|
||||||
|
if avant is None or apres is None:
|
||||||
|
continue
|
||||||
|
if avant == 0:
|
||||||
|
# Une variation relative n'a pas de sens depuis zéro, mais un redémarrage direct à
|
||||||
|
# une consommation positive reste le signal le plus alarmant du lot : `critical`
|
||||||
|
# plutôt qu'un ratio indéfini.
|
||||||
|
if apres > 0:
|
||||||
|
alertes.append(_spike_alert(lecture, avant, apres, severity="critical"))
|
||||||
|
continue
|
||||||
|
variation = abs(apres - avant) / abs(avant)
|
||||||
|
if variation < SPIKE_RELATIVE_THRESHOLD:
|
||||||
|
continue
|
||||||
|
alertes.append(
|
||||||
|
_spike_alert(
|
||||||
|
lecture,
|
||||||
|
avant,
|
||||||
|
apres,
|
||||||
|
severity=_severity_from_ratio(variation / SPIKE_RELATIVE_THRESHOLD),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return alertes
|
||||||
|
|
||||||
|
|
||||||
|
def _spike_alert(lecture: Reading, avant: float, apres: float, *, severity: str) -> Alert:
|
||||||
|
return Alert(
|
||||||
|
source_alert_id=f"spike:{THRESHOLD_METRIC}:{lecture.timestamp.isoformat()}",
|
||||||
|
site_id=lecture.site_id,
|
||||||
|
source="enervision",
|
||||||
|
timestamp=lecture.timestamp,
|
||||||
|
type="spike",
|
||||||
|
severity=severity,
|
||||||
|
message=(
|
||||||
|
f"Variation brutale entre deux lectures consécutives ({avant:.1f} kW -> {apres:.1f} kW)"
|
||||||
|
),
|
||||||
|
value=apres,
|
||||||
|
threshold=avant,
|
||||||
|
metric=THRESHOLD_METRIC,
|
||||||
|
prediction_id=None,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_anomaly(lectures: Sequence[Reading], predictions: Sequence[Prediction]) -> list[Alert]:
|
||||||
|
# Alignement strict (site_id, target_at == timestamp) : `enervision_ml.score` produit une
|
||||||
|
# cible à l'heure pile suivant la dernière lecture, sur la même grille horaire que `reading`.
|
||||||
|
predictions_par_cle = {
|
||||||
|
(prediction.site_id, prediction.target_at): prediction
|
||||||
|
for prediction in predictions
|
||||||
|
if prediction.target_metric == ANOMALY_METRIC
|
||||||
|
}
|
||||||
|
alertes = []
|
||||||
|
for lecture in lectures:
|
||||||
|
prediction = predictions_par_cle.get((lecture.site_id, lecture.timestamp))
|
||||||
|
reel = lecture.consumption_kwh
|
||||||
|
if prediction is None or reel is None or prediction.predicted_value is None:
|
||||||
|
continue
|
||||||
|
predite = prediction.predicted_value
|
||||||
|
if abs(predite) < ANOMALY_MINIMUM_PREDICTED_VALUE:
|
||||||
|
continue
|
||||||
|
ecart = abs(reel - predite) / abs(predite)
|
||||||
|
if ecart < ANOMALY_RELATIVE_THRESHOLD:
|
||||||
|
continue
|
||||||
|
alertes.append(
|
||||||
|
Alert(
|
||||||
|
source_alert_id=f"anomaly:{ANOMALY_METRIC}:{lecture.timestamp.isoformat()}",
|
||||||
|
site_id=lecture.site_id,
|
||||||
|
source="enervision",
|
||||||
|
timestamp=lecture.timestamp,
|
||||||
|
type="anomaly",
|
||||||
|
severity=_severity_from_ratio(ecart / ANOMALY_RELATIVE_THRESHOLD),
|
||||||
|
message=(
|
||||||
|
f"Écart de {ecart * 100:.0f}% entre la consommation mesurée ({reel:.1f} kWh) "
|
||||||
|
f"et la prévision ({predite:.1f} kWh)"
|
||||||
|
),
|
||||||
|
value=reel,
|
||||||
|
threshold=predite,
|
||||||
|
metric=ANOMALY_METRIC,
|
||||||
|
prediction_id=prediction.prediction_id,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return alertes
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_outage(
|
||||||
|
sites: Sequence[Site], dernieres_lectures: dict[str, Reading], now: datetime
|
||||||
|
) -> list[Alert]:
|
||||||
|
alertes = []
|
||||||
|
for site in sites:
|
||||||
|
derniere = dernieres_lectures.get(site.site_id)
|
||||||
|
if derniere is None:
|
||||||
|
alertes.append(
|
||||||
|
_outage_alert(
|
||||||
|
site.site_id,
|
||||||
|
now,
|
||||||
|
reference=None,
|
||||||
|
message="Aucune lecture n'a jamais été reçue pour ce site",
|
||||||
|
severity="critical",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
absence = now - derniere.timestamp
|
||||||
|
if absence < OUTAGE_THRESHOLD:
|
||||||
|
continue
|
||||||
|
alertes.append(
|
||||||
|
_outage_alert(
|
||||||
|
site.site_id,
|
||||||
|
now,
|
||||||
|
reference=derniere.timestamp,
|
||||||
|
message=(
|
||||||
|
f"Aucune lecture depuis {absence} (dernière lecture : "
|
||||||
|
f"{derniere.timestamp.isoformat()})"
|
||||||
|
),
|
||||||
|
severity=_severity_from_ratio(absence / OUTAGE_THRESHOLD),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return alertes
|
||||||
|
|
||||||
|
|
||||||
|
def _outage_alert(
|
||||||
|
site_id: str, now: datetime, *, reference: datetime | None, message: str, severity: str
|
||||||
|
) -> Alert:
|
||||||
|
return Alert(
|
||||||
|
source_alert_id=f"outage:{reference.isoformat() if reference is not None else 'jamais'}",
|
||||||
|
site_id=site_id,
|
||||||
|
source="enervision",
|
||||||
|
timestamp=now,
|
||||||
|
type="outage",
|
||||||
|
severity=severity,
|
||||||
|
message=message,
|
||||||
|
value=None,
|
||||||
|
threshold=None,
|
||||||
|
metric=None,
|
||||||
|
prediction_id=None,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _detect_sensor(lectures: Sequence[Reading]) -> list[Alert]:
|
||||||
|
alertes = []
|
||||||
|
for lecture in lectures:
|
||||||
|
severite = QUALITE_VERS_SEVERITE.get(lecture.data_quality or "")
|
||||||
|
if severite is None:
|
||||||
|
continue
|
||||||
|
raisons = ", ".join(lecture.null_reasons or []) or "raison non précisée"
|
||||||
|
alertes.append(
|
||||||
|
Alert(
|
||||||
|
source_alert_id=f"sensor:{lecture.timestamp.isoformat()}",
|
||||||
|
site_id=lecture.site_id,
|
||||||
|
source="enervision",
|
||||||
|
timestamp=lecture.timestamp,
|
||||||
|
type="sensor",
|
||||||
|
severity=severite,
|
||||||
|
message=f"Qualité de mesure {lecture.data_quality} ({raisons})",
|
||||||
|
value=None,
|
||||||
|
threshold=None,
|
||||||
|
metric=None,
|
||||||
|
prediction_id=None,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return alertes
|
||||||
@@ -14,7 +14,11 @@ from datetime import UTC, datetime, timedelta
|
|||||||
from typing import NoReturn, Protocol
|
from typing import NoReturn, Protocol
|
||||||
from uuid import UUID, uuid4
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from fastapi import BackgroundTasks
|
||||||
|
|
||||||
from app.core.hashing import Argon2Hasher
|
from app.core.hashing import Argon2Hasher
|
||||||
|
from app.core.logging import get_logger
|
||||||
|
from app.core.mailer import Mailer
|
||||||
from app.core.principal import Principal
|
from app.core.principal import Principal
|
||||||
from app.core.roles import AccountKind, Role
|
from app.core.roles import AccountKind, Role
|
||||||
from app.core.security import (
|
from app.core.security import (
|
||||||
@@ -28,9 +32,13 @@ from app.models.login_attempt import LoginOutcome
|
|||||||
from app.models.refresh_token import RevocationReason
|
from app.models.refresh_token import RevocationReason
|
||||||
from app.repositories.audit_log import AuditLogRepository
|
from app.repositories.audit_log import AuditLogRepository
|
||||||
from app.repositories.login_attempt import LoginAttemptRepository
|
from app.repositories.login_attempt import LoginAttemptRepository
|
||||||
|
from app.repositories.password_reset_attempt import PasswordResetAttemptRepository
|
||||||
|
from app.repositories.password_reset_token import PasswordResetTokenRepository
|
||||||
from app.repositories.refresh_token import RefreshTokenRepository
|
from app.repositories.refresh_token import RefreshTokenRepository
|
||||||
from app.repositories.user import UserRepository
|
from app.repositories.user import UserRepository
|
||||||
|
|
||||||
|
logger = get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class Transaction(Protocol):
|
class Transaction(Protocol):
|
||||||
async def commit(self) -> None: ...
|
async def commit(self) -> None: ...
|
||||||
@@ -54,6 +62,10 @@ class RateLimitedError(AuthError):
|
|||||||
self.retry_after = retry_after
|
self.retry_after = retry_after
|
||||||
|
|
||||||
|
|
||||||
|
class InvalidOrExpiredResetTokenError(AuthError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class LoginPolicy:
|
class LoginPolicy:
|
||||||
window_seconds: int
|
window_seconds: int
|
||||||
@@ -62,6 +74,15 @@ class LoginPolicy:
|
|||||||
max_failures_per_identifier: int
|
max_failures_per_identifier: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PasswordResetPolicy:
|
||||||
|
window_seconds: int
|
||||||
|
max_requests_per_identifier: int
|
||||||
|
max_requests_per_ip: int
|
||||||
|
token_ttl: timedelta
|
||||||
|
frontend_reset_url: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class AuthenticatedSession:
|
class AuthenticatedSession:
|
||||||
principal: Principal
|
principal: Principal
|
||||||
@@ -83,6 +104,10 @@ class AuthService:
|
|||||||
token_policy: TokenPolicy,
|
token_policy: TokenPolicy,
|
||||||
login_policy: LoginPolicy,
|
login_policy: LoginPolicy,
|
||||||
refresh_ttl: timedelta,
|
refresh_ttl: timedelta,
|
||||||
|
reset_tokens: PasswordResetTokenRepository,
|
||||||
|
reset_attempts: PasswordResetAttemptRepository,
|
||||||
|
reset_policy: PasswordResetPolicy,
|
||||||
|
mailer: Mailer,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._users = users
|
self._users = users
|
||||||
self._attempts = attempts
|
self._attempts = attempts
|
||||||
@@ -93,6 +118,10 @@ class AuthService:
|
|||||||
self._token_policy = token_policy
|
self._token_policy = token_policy
|
||||||
self._login_policy = login_policy
|
self._login_policy = login_policy
|
||||||
self._refresh_ttl = refresh_ttl
|
self._refresh_ttl = refresh_ttl
|
||||||
|
self._reset_tokens = reset_tokens
|
||||||
|
self._reset_attempts = reset_attempts
|
||||||
|
self._reset_policy = reset_policy
|
||||||
|
self._mailer = mailer
|
||||||
|
|
||||||
async def authenticate(
|
async def authenticate(
|
||||||
self, *, email: str, password: str, client_ip: str | None, user_agent: str | None
|
self, *, email: str, password: str, client_ip: str | None, user_agent: str | None
|
||||||
@@ -200,6 +229,102 @@ class AuthService:
|
|||||||
rafraichi = await self._users.get_by_id(principal.id)
|
rafraichi = await self._users.get_by_id(principal.id)
|
||||||
return self._session(self._en_principal(rafraichi or compte), secret)
|
return self._session(self._en_principal(rafraichi or compte), secret)
|
||||||
|
|
||||||
|
async def request_password_reset(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
email: str,
|
||||||
|
client_ip: str | None,
|
||||||
|
user_agent: str | None,
|
||||||
|
background_tasks: BackgroundTasks,
|
||||||
|
) -> None:
|
||||||
|
await self._refuse_si_limite_reset(email=email, client_ip=client_ip)
|
||||||
|
|
||||||
|
compte = await self._users.get_by_email(email)
|
||||||
|
# Piège : le hachage factice équilibre le temps de réponse sur un compte inconnu, comme
|
||||||
|
# `authenticate()`. La réponse et sa forme restent identiques dans tous les cas : compte
|
||||||
|
# inconnu, compte inactif, ou email envoyé avec succès. L'envoi SMTP lui-même est différé
|
||||||
|
# en tâche de fond : le laisser dans le chemin de réponse rouvrirait le même oracle par le
|
||||||
|
# temps (aller-retour réseau) et par la forme (500 si le relais SMTP échoue, contre 202).
|
||||||
|
if compte is None or not compte.is_active or compte.kind != AccountKind.HUMAIN.value:
|
||||||
|
await self._hasher.verify_dummy()
|
||||||
|
await self._reset_attempts.record(email=email, client_ip=client_ip)
|
||||||
|
await self._transaction.commit()
|
||||||
|
return
|
||||||
|
|
||||||
|
await self._reset_tokens.invalidate_all_for_user(compte.id)
|
||||||
|
secret = generate_refresh_secret()
|
||||||
|
await self._reset_tokens.create(
|
||||||
|
user_id=compte.id,
|
||||||
|
token_hash=fingerprint_refresh(secret),
|
||||||
|
expires_at=datetime.now(UTC) + self._reset_policy.token_ttl,
|
||||||
|
client_ip=client_ip,
|
||||||
|
user_agent=user_agent,
|
||||||
|
)
|
||||||
|
await self._reset_attempts.record(email=email, client_ip=client_ip)
|
||||||
|
await self._audit.record(
|
||||||
|
action=AuditAction.MOT_DE_PASSE_OUBLIE_DEMANDE,
|
||||||
|
actor_label=compte.email,
|
||||||
|
target_type="app_user",
|
||||||
|
target_id=str(compte.id),
|
||||||
|
client_ip=client_ip,
|
||||||
|
user_agent=user_agent,
|
||||||
|
)
|
||||||
|
await self._transaction.commit()
|
||||||
|
|
||||||
|
lien = f"{self._reset_policy.frontend_reset_url}?token={secret}"
|
||||||
|
background_tasks.add_task(self._envoie_email_reset, compte.email, lien)
|
||||||
|
|
||||||
|
async def _envoie_email_reset(self, email: str, reset_url: str) -> None:
|
||||||
|
try:
|
||||||
|
await self._mailer.send_password_reset_email(to=email, reset_url=reset_url)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("auth.password_reset.mail_failed")
|
||||||
|
|
||||||
|
# Piège : lecture seule, pas d'appel à `consume()`. Aucune limitation de débit n'est
|
||||||
|
# nécessaire ici : le jeton est un secret de 256 bits (`generate_refresh_secret`), donc
|
||||||
|
# non brute-forçable, et cette route n'apprend rien sur l'existence d'un compte ou d'un
|
||||||
|
# email, seulement si le lien déjà en main du visiteur est encore valide.
|
||||||
|
async def is_reset_token_valid(self, token: str) -> bool:
|
||||||
|
return await self._reset_tokens.exists_valid(fingerprint_refresh(token))
|
||||||
|
|
||||||
|
async def confirm_password_reset(
|
||||||
|
self, *, token: str, new_password: str, client_ip: str | None, user_agent: str | None
|
||||||
|
) -> AuthenticatedSession:
|
||||||
|
revendique = await self._reset_tokens.consume(fingerprint_refresh(token))
|
||||||
|
if revendique is None:
|
||||||
|
raise InvalidOrExpiredResetTokenError("Lien invalide ou expiré")
|
||||||
|
|
||||||
|
# Piège : le jeton peut avoir été émis avant une désactivation du compte. Sans cette
|
||||||
|
# relecture, un lien encore valide (15 min) changerait quand même le mot de passe d'un
|
||||||
|
# compte désactivé, réutilisable dès sa réactivation.
|
||||||
|
compte = await self._users.get_by_id(revendique.user_id)
|
||||||
|
if compte is None or not compte.is_active or compte.kind != AccountKind.HUMAIN.value:
|
||||||
|
raise InvalidOrExpiredResetTokenError("Lien invalide ou expiré")
|
||||||
|
|
||||||
|
await self._users.update_password(
|
||||||
|
revendique.user_id, await self._hasher.hash(new_password), must_change_password=False
|
||||||
|
)
|
||||||
|
revoquees = await self._refresh.revoke_all_for_user(
|
||||||
|
revendique.user_id, RevocationReason.CHANGEMENT_MOT_DE_PASSE
|
||||||
|
)
|
||||||
|
secret = await self._ouvre_une_famille(
|
||||||
|
user_id=revendique.user_id, client_ip=client_ip, user_agent=user_agent
|
||||||
|
)
|
||||||
|
await self._audit.record(
|
||||||
|
action=AuditAction.MOT_DE_PASSE_REINITIALISE_PAR_SOI,
|
||||||
|
target_type="app_user",
|
||||||
|
target_id=str(revendique.user_id),
|
||||||
|
client_ip=client_ip,
|
||||||
|
user_agent=user_agent,
|
||||||
|
detail={"sessions_revoquees": revoquees},
|
||||||
|
)
|
||||||
|
await self._transaction.commit()
|
||||||
|
|
||||||
|
compte = await self._users.get_by_id(revendique.user_id)
|
||||||
|
if compte is None:
|
||||||
|
raise SessionRejectedError("Compte introuvable")
|
||||||
|
return self._session(self._en_principal(compte), secret)
|
||||||
|
|
||||||
async def logout_all(self, principal: Principal) -> int:
|
async def logout_all(self, principal: Principal) -> int:
|
||||||
revoquees = await self._refresh.revoke_all_for_user(
|
revoquees = await self._refresh.revoke_all_for_user(
|
||||||
principal.id, RevocationReason.DECONNEXION
|
principal.id, RevocationReason.DECONNEXION
|
||||||
@@ -307,6 +432,23 @@ class AuthService:
|
|||||||
await self._transaction.commit()
|
await self._transaction.commit()
|
||||||
raise RateLimitedError(politique.window_seconds)
|
raise RateLimitedError(politique.window_seconds)
|
||||||
|
|
||||||
|
async def _refuse_si_limite_reset(self, *, email: str, client_ip: str | None) -> None:
|
||||||
|
politique = self._reset_policy
|
||||||
|
compteurs = await self._reset_attempts.count_recent(
|
||||||
|
email=email, client_ip=client_ip, window_seconds=politique.window_seconds
|
||||||
|
)
|
||||||
|
|
||||||
|
depasse = (
|
||||||
|
compteurs.per_identifier >= politique.max_requests_per_identifier
|
||||||
|
or compteurs.per_ip >= politique.max_requests_per_ip
|
||||||
|
)
|
||||||
|
if not depasse:
|
||||||
|
return
|
||||||
|
|
||||||
|
await self._reset_attempts.record(email=email, client_ip=client_ip)
|
||||||
|
await self._transaction.commit()
|
||||||
|
raise RateLimitedError(politique.window_seconds)
|
||||||
|
|
||||||
async def _echoue(
|
async def _echoue(
|
||||||
self,
|
self,
|
||||||
email: str,
|
email: str,
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
# Contrainte : `ck_reading_quality` accepte NULL et quatre valeurs seulement, alors que le contrat
|
||||||
|
# frontend n'a aucune valeur pour l'absence de qualité. `qualite_ou_critique()` replie donc sur
|
||||||
|
# `critical`, la seule des quatre qui n'induise pas une confiance qu'on n'a pas. `QUALITES_CONNUES`
|
||||||
|
# reste exposé pour les appelants qui doivent distinguer un `critical` stocké d'un repli.
|
||||||
|
|
||||||
|
from typing import Literal, get_args
|
||||||
|
|
||||||
|
DataQuality = Literal["good", "partial", "degraded", "critical"]
|
||||||
|
|
||||||
|
QUALITES_CONNUES: frozenset[str] = frozenset(get_args(DataQuality))
|
||||||
|
|
||||||
|
_PAR_VALEUR: dict[str, DataQuality] = {valeur: valeur for valeur in get_args(DataQuality)}
|
||||||
|
|
||||||
|
|
||||||
|
def qualite_ou_critique(valeur: str | None) -> DataQuality:
|
||||||
|
if valeur is None:
|
||||||
|
return "critical"
|
||||||
|
return _PAR_VALEUR.get(valeur, "critical")
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
"""Contrainte : la dérive se mesure sur ce qui a déjà eu lieu - DriftService.evaluate()
|
||||||
|
|
||||||
|
Une prévision ne devient vérifiable que quand la lecture de son instant cible est ingérée. La
|
||||||
|
fenêtre est donc fermée à droite par un délai de grâce : sans lui, la dernière heure ferait
|
||||||
|
chuter le taux de couverture à chaque exécution, et le verdict dirait « dérive » alors que
|
||||||
|
seule l'ingestion n'avait pas fini son tour.
|
||||||
|
|
||||||
|
La comparaison se fait entre deux fenêtres vives de même durée, pas contre la métrique de
|
||||||
|
référence du modèle journalisée à l'entraînement. Ce ne sont pas les mêmes grandeurs :
|
||||||
|
l'entraînement mesure un backtest où la météo de l'heure cible est connue, le scoring prévoit
|
||||||
|
une heure future dont la météo ne l'est pas. Les comparer classerait le modèle « en dérive »
|
||||||
|
dès le premier jour, ce qui ne prouverait rien.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass, replace
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from app.models.energy import DriftReport
|
||||||
|
from app.repositories.drift import (
|
||||||
|
ComptageStatut,
|
||||||
|
DriftRepository,
|
||||||
|
NouveauRapportDerive,
|
||||||
|
PaireDerive,
|
||||||
|
)
|
||||||
|
|
||||||
|
STATUT_STABLE = "stable"
|
||||||
|
STATUT_DERIVE = "derive"
|
||||||
|
STATUT_INDETERMINE = "indetermine"
|
||||||
|
|
||||||
|
STATUT_INSUFFISANT = "insufficient_data"
|
||||||
|
STATUT_DISPONIBLE = "available"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Seuils:
|
||||||
|
# 168 h, la saisonnalité hebdomadaire que le modèle apprend par son lag principal : une
|
||||||
|
# fenêtre plus courte comparerait un week-end à une semaine ouvrée.
|
||||||
|
fenetre: timedelta = timedelta(hours=168)
|
||||||
|
grace: timedelta = timedelta(hours=2)
|
||||||
|
min_observations: int = 24
|
||||||
|
ratio_derive: float = 1.25
|
||||||
|
mae_plancher: float = 0.0
|
||||||
|
# Un biais se compte en kWh, donc ne se transpose pas d'un site à l'autre : zéro le désactive,
|
||||||
|
# sans cesser de le mesurer. Réglé par `--bias-threshold`, arbitrage dans l'ADR 0013.
|
||||||
|
seuil_biais: float = 0.0
|
||||||
|
seuil_couverture: float = 0.8
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Metriques:
|
||||||
|
n_observations: int
|
||||||
|
mae: float | None
|
||||||
|
mape: float | None
|
||||||
|
bias: float | None
|
||||||
|
model_references: list[str]
|
||||||
|
|
||||||
|
|
||||||
|
def mesure(paires: Sequence[PaireDerive]) -> Metriques:
|
||||||
|
if not paires:
|
||||||
|
return Metriques(n_observations=0, mae=None, mape=None, bias=None, model_references=[])
|
||||||
|
|
||||||
|
ecarts = [paire.predicted_value - paire.actual_value for paire in paires]
|
||||||
|
# Le MAPE diverge sur une consommation nulle : les sites à l'arrêt sortent de ce seul
|
||||||
|
# rapport, jamais des autres métriques.
|
||||||
|
ratios = [
|
||||||
|
abs(ecart / paire.actual_value)
|
||||||
|
for ecart, paire in zip(ecarts, paires, strict=True)
|
||||||
|
if paire.actual_value != 0
|
||||||
|
]
|
||||||
|
|
||||||
|
return Metriques(
|
||||||
|
n_observations=len(paires),
|
||||||
|
mae=sum(abs(ecart) for ecart in ecarts) / len(ecarts),
|
||||||
|
mape=(sum(ratios) / len(ratios) * 100) if ratios else None,
|
||||||
|
bias=sum(ecarts) / len(ecarts),
|
||||||
|
model_references=sorted({paire.model_reference for paire in paires}),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Verdict:
|
||||||
|
status: str
|
||||||
|
reason: str | None
|
||||||
|
|
||||||
|
|
||||||
|
class DriftService:
|
||||||
|
def __init__(self, depot: DriftRepository, *, seuils: Seuils | None = None) -> None:
|
||||||
|
self._depot = depot
|
||||||
|
self._seuils = seuils or Seuils()
|
||||||
|
|
||||||
|
async def derniers(self, *, site_id: str | None = None) -> Sequence[DriftReport]:
|
||||||
|
"""Ce que sert l'API : le dernier rapport de chaque site, plus la ligne globale."""
|
||||||
|
return await self._depot.derniers(site_id=site_id)
|
||||||
|
|
||||||
|
async def evaluate(
|
||||||
|
self, *, now: datetime | None = None, site_id: str | None = None
|
||||||
|
) -> list[NouveauRapportDerive]:
|
||||||
|
"""Une ligne par site, plus une ligne globale dont le `site_id` est nul."""
|
||||||
|
fin = (now or datetime.now(UTC)) - self._seuils.grace
|
||||||
|
debut = fin - self._seuils.fenetre
|
||||||
|
reference_fin = debut
|
||||||
|
reference_debut = reference_fin - self._seuils.fenetre
|
||||||
|
|
||||||
|
recentes = await self._depot.paires(debut=debut, fin=fin, site_id=site_id)
|
||||||
|
anciennes = await self._depot.paires(
|
||||||
|
debut=reference_debut, fin=reference_fin, site_id=site_id
|
||||||
|
)
|
||||||
|
comptages = await self._depot.comptages(debut=debut, fin=fin, site_id=site_id)
|
||||||
|
|
||||||
|
gabarit = NouveauRapportDerive(
|
||||||
|
site_id=None,
|
||||||
|
window_start=debut,
|
||||||
|
window_end=fin,
|
||||||
|
reference_start=reference_debut,
|
||||||
|
reference_end=reference_fin,
|
||||||
|
n_observations=0,
|
||||||
|
mae=None,
|
||||||
|
mape=None,
|
||||||
|
bias=None,
|
||||||
|
reference_mae=None,
|
||||||
|
coverage_ratio=None,
|
||||||
|
insufficient_data_ratio=None,
|
||||||
|
model_references=[],
|
||||||
|
status=STATUT_INDETERMINE,
|
||||||
|
reason=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
rapports = [
|
||||||
|
self._rapport(
|
||||||
|
gabarit,
|
||||||
|
site=site,
|
||||||
|
recentes=[p for p in recentes if p.site_id == site],
|
||||||
|
anciennes=[p for p in anciennes if p.site_id == site],
|
||||||
|
comptages=[c for c in comptages if c.site_id == site],
|
||||||
|
)
|
||||||
|
for site in sorted(
|
||||||
|
{paire.site_id for paire in recentes} | {c.site_id for c in comptages}
|
||||||
|
)
|
||||||
|
]
|
||||||
|
rapports.append(
|
||||||
|
self._rapport(
|
||||||
|
gabarit, site=None, recentes=recentes, anciennes=anciennes, comptages=comptages
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return rapports
|
||||||
|
|
||||||
|
def _rapport(
|
||||||
|
self,
|
||||||
|
gabarit: NouveauRapportDerive,
|
||||||
|
*,
|
||||||
|
site: str | None,
|
||||||
|
recentes: Sequence[PaireDerive],
|
||||||
|
anciennes: Sequence[PaireDerive],
|
||||||
|
comptages: Sequence[ComptageStatut],
|
||||||
|
) -> NouveauRapportDerive:
|
||||||
|
metriques = mesure(recentes)
|
||||||
|
reference = mesure(anciennes)
|
||||||
|
couverture = _couverture(len(recentes), comptages)
|
||||||
|
verdict = self._verdict(metriques, reference_mae=reference.mae, couverture=couverture)
|
||||||
|
|
||||||
|
return replace(
|
||||||
|
gabarit,
|
||||||
|
site_id=site,
|
||||||
|
n_observations=metriques.n_observations,
|
||||||
|
mae=metriques.mae,
|
||||||
|
mape=metriques.mape,
|
||||||
|
bias=metriques.bias,
|
||||||
|
reference_mae=reference.mae,
|
||||||
|
coverage_ratio=couverture,
|
||||||
|
insufficient_data_ratio=_part_insuffisante(comptages),
|
||||||
|
model_references=metriques.model_references,
|
||||||
|
status=verdict.status,
|
||||||
|
reason=verdict.reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _verdict(
|
||||||
|
self, metriques: Metriques, *, reference_mae: float | None, couverture: float | None
|
||||||
|
) -> Verdict:
|
||||||
|
seuils = self._seuils
|
||||||
|
if metriques.n_observations < seuils.min_observations:
|
||||||
|
return Verdict(
|
||||||
|
STATUT_INDETERMINE,
|
||||||
|
f"{metriques.n_observations} prévision(s) vérifiée(s) sur la fenêtre, "
|
||||||
|
f"minimum {seuils.min_observations}.",
|
||||||
|
)
|
||||||
|
|
||||||
|
if couverture is not None and couverture < seuils.seuil_couverture:
|
||||||
|
return Verdict(
|
||||||
|
STATUT_DERIVE,
|
||||||
|
f"Couverture de {couverture:.0%}, sous le seuil de {seuils.seuil_couverture:.0%} : "
|
||||||
|
"le pipeline, pas le modèle.",
|
||||||
|
)
|
||||||
|
|
||||||
|
plafond = _plafond(reference_mae, ratio=seuils.ratio_derive, plancher=seuils.mae_plancher)
|
||||||
|
if metriques.mae is not None and plafond is not None and metriques.mae > plafond:
|
||||||
|
return Verdict(
|
||||||
|
STATUT_DERIVE,
|
||||||
|
f"MAE de {metriques.mae:.2f} kWh au-delà de {plafond:.2f} kWh, "
|
||||||
|
"seuil dérivé de la fenêtre de référence.",
|
||||||
|
)
|
||||||
|
|
||||||
|
if (
|
||||||
|
seuils.seuil_biais > 0
|
||||||
|
and metriques.bias is not None
|
||||||
|
and abs(metriques.bias) > seuils.seuil_biais
|
||||||
|
):
|
||||||
|
return Verdict(
|
||||||
|
STATUT_DERIVE,
|
||||||
|
f"Biais de {metriques.bias:+.2f} kWh : le modèle se trompe toujours du même côté.",
|
||||||
|
)
|
||||||
|
|
||||||
|
return Verdict(STATUT_STABLE, None)
|
||||||
|
|
||||||
|
|
||||||
|
def _plafond(reference_mae: float | None, *, ratio: float, plancher: float) -> float | None:
|
||||||
|
if reference_mae is None:
|
||||||
|
return plancher or None
|
||||||
|
return max(plancher, reference_mae * ratio)
|
||||||
|
|
||||||
|
|
||||||
|
def _couverture(apparie: int, comptages: Sequence[ComptageStatut]) -> float | None:
|
||||||
|
"""Part des prévisions disponibles qui ont trouvé leur réalisé. Mesure l'ingestion et
|
||||||
|
l'ordonnancement, pas la qualité du modèle."""
|
||||||
|
disponibles = sum(c.nombre for c in comptages if c.status == STATUT_DISPONIBLE)
|
||||||
|
return apparie / disponibles if disponibles else None
|
||||||
|
|
||||||
|
|
||||||
|
def _part_insuffisante(comptages: Sequence[ComptageStatut]) -> float | None:
|
||||||
|
total = sum(c.nombre for c in comptages)
|
||||||
|
if not total:
|
||||||
|
return None
|
||||||
|
return sum(c.nombre for c in comptages if c.status == STATUT_INSUFFISANT) / total
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from app.models.energy import Prediction, Site
|
||||||
|
from app.repositories.prediction import PredictionRepository
|
||||||
|
from app.repositories.site import SiteRepository
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SitePrediction:
|
||||||
|
target_at: datetime
|
||||||
|
target_metric: str
|
||||||
|
period_minutes: int | None
|
||||||
|
predicted_value: float | None
|
||||||
|
status: str
|
||||||
|
failure_reason: str | None
|
||||||
|
model_reference: str
|
||||||
|
created_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SitePredictionSummary:
|
||||||
|
site_id: str
|
||||||
|
site_name: str
|
||||||
|
prediction: SitePrediction | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class PredictionSummary:
|
||||||
|
timestamp: datetime
|
||||||
|
sites: list[SitePredictionSummary]
|
||||||
|
|
||||||
|
|
||||||
|
class PredictionService:
|
||||||
|
def __init__(self, sites: SiteRepository, predictions: PredictionRepository) -> None:
|
||||||
|
self._sites = sites
|
||||||
|
self._predictions = predictions
|
||||||
|
|
||||||
|
async def summary(self) -> PredictionSummary:
|
||||||
|
sites = await self._sites.list_all()
|
||||||
|
dernieres = {p.site_id: p for p in await self._predictions.latest_by_site()}
|
||||||
|
|
||||||
|
return PredictionSummary(
|
||||||
|
timestamp=datetime.now(UTC),
|
||||||
|
sites=[_resume_site(site, dernieres.get(site.site_id)) for site in sites],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resume_site(site: Site, derniere: Prediction | None) -> SitePredictionSummary:
|
||||||
|
# Piège : l'absence de ligne signifie « jamais scoré », pas une valeur pseudo-statut, qui
|
||||||
|
# n'existe pas dans la contrainte de la table. `prediction` reste `None` plutôt que de
|
||||||
|
# fabriquer un statut absent du domaine `available`/`insufficient_data`/`error`.
|
||||||
|
prediction = None
|
||||||
|
if derniere is not None:
|
||||||
|
prediction = SitePrediction(
|
||||||
|
target_at=derniere.target_at,
|
||||||
|
target_metric=derniere.target_metric,
|
||||||
|
period_minutes=derniere.period_minutes,
|
||||||
|
predicted_value=derniere.predicted_value,
|
||||||
|
status=derniere.status,
|
||||||
|
failure_reason=derniere.failure_reason,
|
||||||
|
model_reference=derniere.model_reference,
|
||||||
|
created_at=derniere.created_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
return SitePredictionSummary(
|
||||||
|
site_id=site.site_id, site_name=site.site_name, prediction=prediction
|
||||||
|
)
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
from collections.abc import Sequence
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from app.models.energy import Reading
|
||||||
|
from app.repositories.reading import ReadingRepository
|
||||||
|
|
||||||
|
FENETRE_PAR_DEFAUT = timedelta(hours=24)
|
||||||
|
FENETRE_MAXIMALE = timedelta(days=90)
|
||||||
|
|
||||||
|
|
||||||
|
class FenetreInverseeError(Exception):
|
||||||
|
"""`start` est postérieur ou égal à `end`."""
|
||||||
|
|
||||||
|
|
||||||
|
class FenetreTropLargeError(Exception):
|
||||||
|
"""L'écart entre `start` et `end` dépasse `FENETRE_MAXIMALE`."""
|
||||||
|
|
||||||
|
|
||||||
|
class ReadingService:
|
||||||
|
def __init__(self, *, readings: ReadingRepository) -> None:
|
||||||
|
self._readings = readings
|
||||||
|
|
||||||
|
async def list_history(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
site_id: str | None = None,
|
||||||
|
start: datetime | None = None,
|
||||||
|
end: datetime | None = None,
|
||||||
|
limit: int,
|
||||||
|
offset: int,
|
||||||
|
) -> Sequence[Reading]:
|
||||||
|
debut, fin = self._resoudre_fenetre(start, end)
|
||||||
|
return await self._readings.list_history(
|
||||||
|
site_id=site_id, start=debut, end=fin, limit=limit, offset=offset
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resoudre_fenetre(
|
||||||
|
start: datetime | None, end: datetime | None
|
||||||
|
) -> tuple[datetime, datetime]:
|
||||||
|
# Piège : un datetime naïf (sans fuseau dans la chaîne ISO reçue) fait échouer la
|
||||||
|
# comparaison à `reading.timestamp` (`timestamptz`) au niveau du pilote, en 500 plutôt
|
||||||
|
# qu'un refus propre. On le traite comme de l'UTC plutôt que de le rejeter.
|
||||||
|
debut = _vers_utc(start)
|
||||||
|
fin = _vers_utc(end) or datetime.now(UTC)
|
||||||
|
if debut is None:
|
||||||
|
debut = fin - FENETRE_PAR_DEFAUT
|
||||||
|
|
||||||
|
if debut >= fin:
|
||||||
|
raise FenetreInverseeError
|
||||||
|
if fin - debut > FENETRE_MAXIMALE:
|
||||||
|
raise FenetreTropLargeError
|
||||||
|
return debut, fin
|
||||||
|
|
||||||
|
|
||||||
|
def _vers_utc(instant: datetime | None) -> datetime | None:
|
||||||
|
if instant is None:
|
||||||
|
return None
|
||||||
|
return instant if instant.tzinfo is not None else instant.replace(tzinfo=UTC)
|
||||||
@@ -1,7 +1,15 @@
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Protocol
|
||||||
|
|
||||||
from app.models.energy import Recommendation
|
from app.models.energy import Recommendation
|
||||||
|
from app.repositories.alert import AlertRepository
|
||||||
from app.repositories.recommendation import RecommendationRepository
|
from app.repositories.recommendation import RecommendationRepository
|
||||||
|
from app.services.recommendation_rules import applique_les_regles
|
||||||
|
|
||||||
|
|
||||||
|
class Transaction(Protocol):
|
||||||
|
async def commit(self) -> None: ...
|
||||||
|
|
||||||
|
|
||||||
class RecommendationError(Exception):
|
class RecommendationError(Exception):
|
||||||
@@ -12,9 +20,24 @@ class RecommendationNotFoundError(RecommendationError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RapportGeneration:
|
||||||
|
alertes_examinees: int
|
||||||
|
recommandations_creees: int
|
||||||
|
deja_presentes: int
|
||||||
|
|
||||||
|
|
||||||
class RecommendationService:
|
class RecommendationService:
|
||||||
def __init__(self, *, recommendations: RecommendationRepository) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
recommendations: RecommendationRepository,
|
||||||
|
alerts: AlertRepository,
|
||||||
|
transaction: Transaction,
|
||||||
|
) -> None:
|
||||||
self._recommendations = recommendations
|
self._recommendations = recommendations
|
||||||
|
self._alerts = alerts
|
||||||
|
self._transaction = transaction
|
||||||
|
|
||||||
async def list_all(self) -> Sequence[Recommendation]:
|
async def list_all(self) -> Sequence[Recommendation]:
|
||||||
return await self._recommendations.list_all()
|
return await self._recommendations.list_all()
|
||||||
@@ -24,3 +47,16 @@ class RecommendationService:
|
|||||||
if recommendation is None:
|
if recommendation is None:
|
||||||
raise RecommendationNotFoundError(recommendation_id)
|
raise RecommendationNotFoundError(recommendation_id)
|
||||||
return recommendation
|
return recommendation
|
||||||
|
|
||||||
|
async def generate(self, *, site_id: str | None = None) -> RapportGeneration:
|
||||||
|
alertes = await self._alerts.list_all(site_id=site_id)
|
||||||
|
nouvelles = [nouvelle for alerte in alertes for nouvelle in applique_les_regles(alerte)]
|
||||||
|
|
||||||
|
creees = await self._recommendations.create_missing(nouvelles)
|
||||||
|
await self._transaction.commit()
|
||||||
|
|
||||||
|
return RapportGeneration(
|
||||||
|
alertes_examinees=len(alertes),
|
||||||
|
recommandations_creees=creees,
|
||||||
|
deja_presentes=len(nouvelles) - creees,
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,117 @@
|
|||||||
|
# Piège : `rule_reference` est la clé d'idempotence en base, portée par la contrainte
|
||||||
|
# `uq_recommendation_alert_rule`. Renommer une référence déjà livrée ne remplace pas les
|
||||||
|
# recommandations existantes, il en crée de nouvelles à côté. Une règle qui change de sens
|
||||||
|
# prend donc une référence suffixée `-v2` - REGLES.
|
||||||
|
|
||||||
|
from collections.abc import Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
from app.models.energy import Alert
|
||||||
|
from app.repositories.recommendation import NouvelleRecommandation
|
||||||
|
from app.schemas.alert import AlertSeverity, AlertType
|
||||||
|
|
||||||
|
FACTEUR_DEPASSEMENT_MAJEUR: Final = 1.2
|
||||||
|
POURCENTAGE_DEPASSEMENT_MAJEUR: Final = round((FACTEUR_DEPASSEMENT_MAJEUR - 1) * 100)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class Regle:
|
||||||
|
reference: str
|
||||||
|
action: str
|
||||||
|
declencheur: Callable[[Alert], bool]
|
||||||
|
motif: Callable[[Alert], str]
|
||||||
|
|
||||||
|
|
||||||
|
def _du_type(attendu: AlertType) -> Callable[[Alert], bool]:
|
||||||
|
return lambda alerte: alerte.type == attendu
|
||||||
|
|
||||||
|
|
||||||
|
def _de_severite(attendue: AlertSeverity) -> Callable[[Alert], bool]:
|
||||||
|
return lambda alerte: alerte.severity == attendue
|
||||||
|
|
||||||
|
|
||||||
|
# Un seuil nul ou négatif rendrait le rapport `value / threshold` arbitraire : l'alerte ne
|
||||||
|
# renseigne alors aucun dépassement exploitable, et la règle ne se déclenche pas.
|
||||||
|
def _depasse_largement_le_seuil(alerte: Alert) -> bool:
|
||||||
|
if alerte.value is None or alerte.threshold is None or alerte.threshold <= 0:
|
||||||
|
return False
|
||||||
|
return alerte.value >= alerte.threshold * FACTEUR_DEPASSEMENT_MAJEUR
|
||||||
|
|
||||||
|
|
||||||
|
REGLES: Final[tuple[Regle, ...]] = (
|
||||||
|
Regle(
|
||||||
|
reference="spike-delestage-v1",
|
||||||
|
action="Délester les équipements non prioritaires sur le créneau du pic",
|
||||||
|
declencheur=_du_type(AlertType.SPIKE),
|
||||||
|
motif=lambda alerte: f"Pic de consommation signalé sur le site {alerte.site_id}",
|
||||||
|
),
|
||||||
|
Regle(
|
||||||
|
reference="threshold-reduction-v1",
|
||||||
|
action="Ramener la puissance appelée sous le seuil contractuel",
|
||||||
|
declencheur=_du_type(AlertType.THRESHOLD),
|
||||||
|
motif=lambda alerte: f"Seuil de consommation dépassé sur le site {alerte.site_id}",
|
||||||
|
),
|
||||||
|
Regle(
|
||||||
|
reference="outage-secours-v1",
|
||||||
|
action="Basculer sur l'alimentation de secours et prévenir l'exploitant",
|
||||||
|
declencheur=_du_type(AlertType.OUTAGE),
|
||||||
|
motif=lambda alerte: (
|
||||||
|
f"Risque de surcharge ou de coupure imminente sur le site {alerte.site_id}"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Regle(
|
||||||
|
reference="sensor-maintenance-v1",
|
||||||
|
action="Planifier une intervention de maintenance sur le capteur",
|
||||||
|
declencheur=_du_type(AlertType.SENSOR),
|
||||||
|
motif=lambda alerte: (
|
||||||
|
f"Capteur défaillant sur le site {alerte.site_id}, les mesures ne sont plus fiables"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Regle(
|
||||||
|
reference="anomaly-verification-v1",
|
||||||
|
action="Confronter la mesure à la prévision et vérifier le paramétrage du site",
|
||||||
|
declencheur=_du_type(AlertType.ANOMALY),
|
||||||
|
motif=lambda alerte: (
|
||||||
|
f"Écart anormal entre la mesure et le comportement attendu du site {alerte.site_id}"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Regle(
|
||||||
|
reference="escalade-astreinte-v1",
|
||||||
|
action="Escalader à l'astreinte sous une heure",
|
||||||
|
declencheur=_de_severite(AlertSeverity.CRITICAL),
|
||||||
|
motif=lambda alerte: f"Alerte de sévérité critique sur le site {alerte.site_id}",
|
||||||
|
),
|
||||||
|
Regle(
|
||||||
|
reference="contrat-puissance-v1",
|
||||||
|
action="Réévaluer la puissance souscrite au contrat",
|
||||||
|
declencheur=_depasse_largement_le_seuil,
|
||||||
|
motif=lambda alerte: (
|
||||||
|
f"Dépassement d'au moins {POURCENTAGE_DEPASSEMENT_MAJEUR} % du seuil "
|
||||||
|
f"sur le site {alerte.site_id}"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def applique_les_regles(alerte: Alert) -> list[NouvelleRecommandation]:
|
||||||
|
contexte = _contexte_de_mesure(alerte)
|
||||||
|
return [
|
||||||
|
NouvelleRecommandation(
|
||||||
|
alert_id=alerte.alert_id,
|
||||||
|
action=regle.action,
|
||||||
|
explanation=f"{regle.motif(alerte)}{contexte}.",
|
||||||
|
rule_reference=regle.reference,
|
||||||
|
)
|
||||||
|
for regle in REGLES
|
||||||
|
if regle.declencheur(alerte)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _contexte_de_mesure(alerte: Alert) -> str:
|
||||||
|
if alerte.value is None:
|
||||||
|
return ""
|
||||||
|
grandeur = alerte.metric or "valeur"
|
||||||
|
if alerte.threshold is None:
|
||||||
|
return f" ({grandeur} mesurée à {alerte.value})"
|
||||||
|
return f" ({grandeur} mesurée à {alerte.value}, seuil {alerte.threshold})"
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
from app.models.energy import Reading, Site
|
||||||
|
from app.repositories.reading import ReadingRepository
|
||||||
|
from app.repositories.site import SiteRepository
|
||||||
|
from app.services.data_quality import qualite_ou_critique
|
||||||
|
|
||||||
|
CapteurStatus = Literal["ok", "failing"]
|
||||||
|
OverallStatus = Literal["ok", "degraded", "critical"]
|
||||||
|
|
||||||
|
RAISON_VERS_CAPTEUR: dict[str, str] = {
|
||||||
|
"consumption_sensor_failure": "consumption",
|
||||||
|
"electrical_sensor_failure": "electrical",
|
||||||
|
"temperature_sensor_failure": "temperature",
|
||||||
|
"humidity_sensor_failure": "humidity",
|
||||||
|
"network_loss": "network",
|
||||||
|
}
|
||||||
|
|
||||||
|
CHAMPS_PAR_CAPTEUR: dict[str, tuple[str, ...]] = {
|
||||||
|
"consumption": ("consumption_kw",),
|
||||||
|
"electrical": ("voltage_v", "current_a", "power_factor"),
|
||||||
|
"temperature": ("temperature_celsius",),
|
||||||
|
"humidity": ("humidity_percent",),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class DiagnosticCapteur:
|
||||||
|
status: CapteurStatus
|
||||||
|
since: datetime | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SanteCapteurs:
|
||||||
|
consumption: DiagnosticCapteur
|
||||||
|
electrical: DiagnosticCapteur
|
||||||
|
temperature: DiagnosticCapteur
|
||||||
|
humidity: DiagnosticCapteur
|
||||||
|
network: DiagnosticCapteur
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SanteSite:
|
||||||
|
site_id: str
|
||||||
|
site_name: str
|
||||||
|
sensors: SanteCapteurs
|
||||||
|
overall: OverallStatus
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class EtatCapteurs:
|
||||||
|
timestamp: datetime
|
||||||
|
sites: list[SanteSite]
|
||||||
|
|
||||||
|
|
||||||
|
class SensorService:
|
||||||
|
def __init__(self, sites: SiteRepository, readings: ReadingRepository) -> None:
|
||||||
|
self._sites = sites
|
||||||
|
self._readings = readings
|
||||||
|
|
||||||
|
async def status(self) -> EtatCapteurs:
|
||||||
|
sites = await self._sites.list_all()
|
||||||
|
dernieres = {lecture.site_id: lecture for lecture in await self._readings.latest_by_site()}
|
||||||
|
|
||||||
|
return EtatCapteurs(
|
||||||
|
timestamp=datetime.now(UTC),
|
||||||
|
sites=[_sante_site(site, dernieres.get(site.site_id)) for site in sites],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _sante_site(site: Site, derniere: Reading | None) -> SanteSite:
|
||||||
|
if derniere is None:
|
||||||
|
return SanteSite(
|
||||||
|
site_id=site.site_id,
|
||||||
|
site_name=site.site_name,
|
||||||
|
sensors=_tout_en_echec(since=None),
|
||||||
|
overall="critical",
|
||||||
|
)
|
||||||
|
|
||||||
|
qualite = qualite_ou_critique(derniere.data_quality)
|
||||||
|
overall = _overall_depuis_qualite(qualite)
|
||||||
|
|
||||||
|
if overall == "critical":
|
||||||
|
return SanteSite(
|
||||||
|
site_id=site.site_id,
|
||||||
|
site_name=site.site_name,
|
||||||
|
sensors=_tout_en_echec(since=derniere.timestamp),
|
||||||
|
overall="critical",
|
||||||
|
)
|
||||||
|
|
||||||
|
raisons_signalees = {
|
||||||
|
RAISON_VERS_CAPTEUR[raison]
|
||||||
|
for raison in (derniere.null_reasons or [])
|
||||||
|
if raison in RAISON_VERS_CAPTEUR
|
||||||
|
}
|
||||||
|
|
||||||
|
return SanteSite(
|
||||||
|
site_id=site.site_id,
|
||||||
|
site_name=site.site_name,
|
||||||
|
sensors=SanteCapteurs(
|
||||||
|
consumption=_diagnostic("consumption", derniere, raisons_signalees),
|
||||||
|
electrical=_diagnostic("electrical", derniere, raisons_signalees),
|
||||||
|
temperature=_diagnostic("temperature", derniere, raisons_signalees),
|
||||||
|
humidity=_diagnostic("humidity", derniere, raisons_signalees),
|
||||||
|
network=_diagnostic("network", derniere, raisons_signalees),
|
||||||
|
),
|
||||||
|
overall=overall,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _overall_depuis_qualite(qualite: str) -> OverallStatus:
|
||||||
|
if qualite == "good":
|
||||||
|
return "ok"
|
||||||
|
if qualite in ("partial", "degraded"):
|
||||||
|
return "degraded"
|
||||||
|
return "critical"
|
||||||
|
|
||||||
|
|
||||||
|
def _diagnostic(capteur: str, derniere: Reading, raisons_signalees: set[str]) -> DiagnosticCapteur:
|
||||||
|
champs = CHAMPS_PAR_CAPTEUR.get(capteur, ())
|
||||||
|
en_echec = capteur in raisons_signalees or any(
|
||||||
|
getattr(derniere, champ) is None for champ in champs
|
||||||
|
)
|
||||||
|
return DiagnosticCapteur(
|
||||||
|
status="failing" if en_echec else "ok",
|
||||||
|
since=derniere.timestamp if en_echec else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _tout_en_echec(since: datetime | None) -> SanteCapteurs:
|
||||||
|
echec = DiagnosticCapteur(status="failing", since=since)
|
||||||
|
return SanteCapteurs(
|
||||||
|
consumption=echec, electrical=echec, temperature=echec, humidity=echec, network=echec
|
||||||
|
)
|
||||||
@@ -1,7 +1,11 @@
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from app.models.energy import Site
|
from app.models.energy import Site
|
||||||
|
from app.repositories.reading import ReadingRepository
|
||||||
from app.repositories.site import SiteRepository
|
from app.repositories.site import SiteRepository
|
||||||
|
from app.services.data_quality import DataQuality, qualite_ou_critique
|
||||||
|
|
||||||
|
|
||||||
class SiteError(Exception):
|
class SiteError(Exception):
|
||||||
@@ -12,9 +16,26 @@ class SiteNotFoundError(SiteError):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SiteCurrentReading:
|
||||||
|
timestamp: datetime | None
|
||||||
|
site_id: str
|
||||||
|
site_type: str
|
||||||
|
consumption_kw: float | None
|
||||||
|
consumption_kwh: float | None
|
||||||
|
voltage_v: float | None
|
||||||
|
current_a: float | None
|
||||||
|
power_factor: float | None
|
||||||
|
temperature_celsius: float | None
|
||||||
|
humidity_percent: float | None
|
||||||
|
null_reasons: list[str]
|
||||||
|
data_quality: DataQuality
|
||||||
|
|
||||||
|
|
||||||
class SiteService:
|
class SiteService:
|
||||||
def __init__(self, *, sites: SiteRepository) -> None:
|
def __init__(self, *, sites: SiteRepository, readings: ReadingRepository) -> None:
|
||||||
self._sites = sites
|
self._sites = sites
|
||||||
|
self._readings = readings
|
||||||
|
|
||||||
async def list_all(self) -> Sequence[Site]:
|
async def list_all(self) -> Sequence[Site]:
|
||||||
return await self._sites.list_all()
|
return await self._sites.list_all()
|
||||||
@@ -24,3 +45,38 @@ class SiteService:
|
|||||||
if site is None:
|
if site is None:
|
||||||
raise SiteNotFoundError(site_id)
|
raise SiteNotFoundError(site_id)
|
||||||
return site
|
return site
|
||||||
|
|
||||||
|
async def current(self, site_id: str) -> SiteCurrentReading:
|
||||||
|
site = await self.get_by_id(site_id)
|
||||||
|
derniere = await self._readings.latest_for_site(site_id)
|
||||||
|
|
||||||
|
if derniere is None:
|
||||||
|
return SiteCurrentReading(
|
||||||
|
timestamp=None,
|
||||||
|
site_id=site.site_id,
|
||||||
|
site_type=site.site_type,
|
||||||
|
consumption_kw=None,
|
||||||
|
consumption_kwh=None,
|
||||||
|
voltage_v=None,
|
||||||
|
current_a=None,
|
||||||
|
power_factor=None,
|
||||||
|
temperature_celsius=None,
|
||||||
|
humidity_percent=None,
|
||||||
|
null_reasons=[],
|
||||||
|
data_quality="critical",
|
||||||
|
)
|
||||||
|
|
||||||
|
return SiteCurrentReading(
|
||||||
|
timestamp=derniere.timestamp,
|
||||||
|
site_id=site.site_id,
|
||||||
|
site_type=site.site_type,
|
||||||
|
consumption_kw=derniere.consumption_kw,
|
||||||
|
consumption_kwh=derniere.consumption_kwh,
|
||||||
|
voltage_v=derniere.voltage_v,
|
||||||
|
current_a=derniere.current_a,
|
||||||
|
power_factor=derniere.power_factor,
|
||||||
|
temperature_celsius=derniere.temperature_celsius,
|
||||||
|
humidity_percent=derniere.humidity_percent,
|
||||||
|
null_reasons=derniere.null_reasons or [],
|
||||||
|
data_quality=qualite_ou_critique(derniere.data_quality),
|
||||||
|
)
|
||||||
|
|||||||
@@ -1,14 +1,10 @@
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from typing import Literal
|
|
||||||
|
|
||||||
from app.models.energy import Reading, Site
|
from app.models.energy import Reading, Site
|
||||||
from app.repositories.reading import ReadingRepository
|
from app.repositories.reading import ReadingRepository
|
||||||
from app.repositories.site import SiteRepository
|
from app.repositories.site import SiteRepository
|
||||||
|
from app.services.data_quality import QUALITES_CONNUES, DataQuality, qualite_ou_critique
|
||||||
DataQuality = Literal["good", "partial", "degraded", "critical"]
|
|
||||||
|
|
||||||
QUALITES_CONNUES: frozenset[str] = frozenset({"good", "partial", "degraded", "critical"})
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -58,13 +54,10 @@ class StatsService:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def _resume_site(site: Site, derniere: Reading | None) -> SiteConsumption:
|
def _resume_site(site: Site, derniere: Reading | None) -> SiteConsumption:
|
||||||
capacite = site.capacity_kw or 0
|
capacite = site.capacity_kw or 0
|
||||||
# Piège : `data_quality` est nul dès qu'un site n'a jamais reçu de lecture, ou que le
|
|
||||||
# producteur n'a pas su la qualifier. Le contrat frontend n'a pas de valeur pour ce cas,
|
|
||||||
# `critical` est la seule des quatre qui n'induit pas une confiance qu'on n'a pas.
|
|
||||||
qualite: DataQuality = "critical"
|
qualite: DataQuality = "critical"
|
||||||
consommation = None
|
consommation = None
|
||||||
if derniere is not None and derniere.data_quality in QUALITES_CONNUES:
|
if derniere is not None and derniere.data_quality in QUALITES_CONNUES:
|
||||||
qualite = derniere.data_quality # type: ignore[assignment]
|
qualite = qualite_ou_critique(derniere.data_quality)
|
||||||
consommation = derniere.consumption_kw
|
consommation = derniere.consumption_kw
|
||||||
|
|
||||||
charge = (
|
charge = (
|
||||||
|
|||||||
Binary file not shown.
|
After Width: | Height: | Size: 36 KiB |
File diff suppressed because it is too large
Load Diff
+1817
-19
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,10 @@ dependencies = [
|
|||||||
"pyjwt>=2.10",
|
"pyjwt>=2.10",
|
||||||
"argon2-cffi>=23.1",
|
"argon2-cffi>=23.1",
|
||||||
"anyio>=4.0",
|
"anyio>=4.0",
|
||||||
|
"aiosmtplib>=5.1.3",
|
||||||
|
"httpx>=0.28.1",
|
||||||
|
"pandas>=3.0.5",
|
||||||
|
"boto3>=1.43.101",
|
||||||
]
|
]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
@@ -25,7 +29,8 @@ dev = [
|
|||||||
"pytest>=9.1.1",
|
"pytest>=9.1.1",
|
||||||
"pytest-asyncio>=1.4.0",
|
"pytest-asyncio>=1.4.0",
|
||||||
"pytest-cov>=7.1.0",
|
"pytest-cov>=7.1.0",
|
||||||
"httpx>=0.28.1",
|
"pandas-stubs>=3.0.5.260914",
|
||||||
|
"types-boto3[s3]>=1.43.101",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
@@ -84,8 +89,11 @@ disallow_untyped_defs = false
|
|||||||
testpaths = ["tests"]
|
testpaths = ["tests"]
|
||||||
asyncio_mode = "auto"
|
asyncio_mode = "auto"
|
||||||
asyncio_default_fixture_loop_scope = "function"
|
asyncio_default_fixture_loop_scope = "function"
|
||||||
addopts = "-q --strict-markers -m 'not integration' --cov=app --cov-report=term-missing"
|
addopts = "-q --strict-markers -m 'not integration and not chaine' --cov=app --cov-report=term-missing"
|
||||||
markers = ["integration: requiert une base PostgreSQL joignable, hors `make test`"]
|
markers = [
|
||||||
|
"integration: requiert une base PostgreSQL joignable, hors `make test`",
|
||||||
|
"chaine: requiert en plus l'environnement uv de ml/, hors `make test` et hors `-m integration`",
|
||||||
|
]
|
||||||
|
|
||||||
[tool.coverage.run]
|
[tool.coverage.run]
|
||||||
source = ["app"]
|
source = ["app"]
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
# Pourquoi : classification unique des routes du contrat, lue par test_route_protection.py,
|
||||||
|
# test_openapi.py et test_matrice_acces.py. Trois listes séparées dérivaient auparavant chacune
|
||||||
|
# de leur côté, et deux entrées de ROUTES_A_ROLE ne correspondaient plus à aucune route sans que
|
||||||
|
# rien ne le signale.
|
||||||
|
# Piège : les trois ensembles doivent rester disjoints et couvrir tout le schéma. C'est
|
||||||
|
# `test_every_declared_route_is_classified` qui le vérifie, pas la relecture.
|
||||||
|
|
||||||
|
from typing import Final
|
||||||
|
|
||||||
|
from app.core.roles import Role
|
||||||
|
|
||||||
|
Route = tuple[str, str]
|
||||||
|
|
||||||
|
ROUTES_PUBLIQUES: Final[frozenset[Route]] = frozenset(
|
||||||
|
{
|
||||||
|
("GET", "/api/v1/health/live"),
|
||||||
|
("GET", "/api/v1/health/ready"),
|
||||||
|
("POST", "/api/v1/auth/login"),
|
||||||
|
# Sans cookie, la déconnexion ne fait rien et répond 204 : elle est idempotente.
|
||||||
|
("POST", "/api/v1/auth/logout"),
|
||||||
|
("POST", "/api/v1/auth/forgot-password"),
|
||||||
|
# Protégée par le jeton dans le corps de la requête, pas par un `Principal` : aucune
|
||||||
|
# authentification préalable ne s'applique, c'est la validité du jeton qui tranche.
|
||||||
|
("POST", "/api/v1/auth/reset-password"),
|
||||||
|
# Même raison : lecture seule, protégée par le jeton passé en paramètre, pas par un
|
||||||
|
# `Principal`. Le jeton est un secret de 256 bits, non brute-forçable.
|
||||||
|
("GET", "/api/v1/auth/reset-password/validate"),
|
||||||
|
("GET", "/metrics"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# Le cookie opaque porte seul l'autorisation : sans lui la route rend 401, mais aucun `Principal`
|
||||||
|
# n'est construit et `require_role` n'entre jamais en jeu.
|
||||||
|
ROUTE_COOKIE: Final[frozenset[Route]] = frozenset({("POST", "/api/v1/auth/refresh")})
|
||||||
|
|
||||||
|
# Authentifiées par `CurrentPrincipalDep` nu, donc hors de `require_role` et, avec lui, hors du
|
||||||
|
# refus `password_change_required`. Volontaire pour `/auth/password`, qui est la sortie de l'état
|
||||||
|
# provisoire ; subi pour `/auth/logout-all`, cf. test_matrice_acces.py.
|
||||||
|
ROUTES_SANS_ROLE: Final[frozenset[Route]] = frozenset(
|
||||||
|
{
|
||||||
|
("GET", "/api/v1/auth/me"),
|
||||||
|
("POST", "/api/v1/auth/password"),
|
||||||
|
("POST", "/api/v1/auth/logout-all"),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
ROLE_MINIMUM: Final[dict[Route, Role]] = {
|
||||||
|
("GET", "/api/v1/sites"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/sites/{site_id}"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/sites/{site_id}/current"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/alerts"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/recommendations"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/recommendations/{recommendation_id}"): Role.LECTEUR,
|
||||||
|
("POST", "/api/v1/recommendations/generate"): Role.ADMIN,
|
||||||
|
("GET", "/api/v1/stats/summary"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/readings"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/predictions"): Role.LECTEUR,
|
||||||
|
("GET", "/api/v1/sensors/status"): Role.ADMIN,
|
||||||
|
("GET", "/api/v1/monitoring/drift"): Role.OPERATEUR,
|
||||||
|
("GET", "/api/v1/users"): Role.ADMIN,
|
||||||
|
("POST", "/api/v1/users"): Role.ADMIN,
|
||||||
|
("PATCH", "/api/v1/users/{user_id}"): Role.ADMIN,
|
||||||
|
("POST", "/api/v1/users/{user_id}/password-reset"): Role.ADMIN,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Piège : `{recommendation_id}` est typé `int` et `{user_id}` est un UUID. Une substitution
|
||||||
|
# uniforme par une chaîne quelconque rendrait 422 avant d'atteindre la garde de rôle, et le test
|
||||||
|
# passerait en prouvant autre chose que ce qu'il annonce.
|
||||||
|
SUBSTITUTIONS: Final[dict[str, str]] = {
|
||||||
|
"{user_id}": "00000000-0000-0000-0000-000000000000",
|
||||||
|
"{site_id}": "site-absent-du-jeu-de-donnees",
|
||||||
|
"{recommendation_id}": "999999999",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def chemin_concret(chemin: str) -> str:
|
||||||
|
for gabarit, valeur in SUBSTITUTIONS.items():
|
||||||
|
chemin = chemin.replace(gabarit, valeur)
|
||||||
|
return chemin
|
||||||
|
|
||||||
|
|
||||||
|
def routes_du_schema(schema: dict[str, object]) -> list[Route]:
|
||||||
|
chemins: dict[str, dict[str, object]] = schema["paths"] # type: ignore[assignment]
|
||||||
|
return [
|
||||||
|
(methode.upper(), chemin)
|
||||||
|
for chemin, operations in chemins.items()
|
||||||
|
for methode in operations
|
||||||
|
if methode.upper() in {"GET", "POST", "PATCH", "PUT", "DELETE"}
|
||||||
|
]
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Piège : ces fixtures valident leurs écritures, contrairement à celles de tests/repositories.
|
||||||
|
|
||||||
|
Un endpoint ouvre sa propre session par `get_session` : il ne verrait pas une ligne semée dans
|
||||||
|
une transaction en cours. Lui passer la session de la fixture par `dependency_overrides`
|
||||||
|
supprimerait justement ce que ces tests prouvent, et `RecommendationService.generate` valide de
|
||||||
|
toute façon lui-même. L'isolation vient donc de la marque portée par chaque `site_id`, et le
|
||||||
|
nettoyage est explicite, dans l'ordre imposé par les clés étrangères `RESTRICT`.
|
||||||
|
|
||||||
|
Contrainte : toutes ces fixtures sont à portée fonction. `engine_per_test` vide le cache du
|
||||||
|
moteur après chaque test ; une fixture de module verrait un moteur déjà fermé à son démontage,
|
||||||
|
et ses lignes resteraient en base.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import AsyncIterator, Callable, Iterator
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from sqlalchemy import delete, select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.api.deps import get_current_principal
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from app.models.energy import Alert, Prediction, Reading, Recommendation, Site
|
||||||
|
from tests.repositories.test_alert import creer_alerte
|
||||||
|
from tests.repositories.test_prediction import creer_prediction
|
||||||
|
from tests.repositories.test_reading import creer_lecture
|
||||||
|
from tests.repositories.test_site import creer as creer_site
|
||||||
|
|
||||||
|
INSTANT = datetime(2026, 9, 16, 12, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class JeuMetier:
|
||||||
|
"""Identifiants seuls, jamais d'instance ORM : un attribut relu sur une session fermée
|
||||||
|
déclenche un `MissingGreenlet`."""
|
||||||
|
|
||||||
|
site_id: str
|
||||||
|
site_voisin: str
|
||||||
|
alert_id: int
|
||||||
|
prediction_id: int
|
||||||
|
instant: datetime
|
||||||
|
|
||||||
|
|
||||||
|
async def _supprime(session: AsyncSession, sites: list[str]) -> None:
|
||||||
|
# La suppression des recommandations est inconditionnelle : `POST /generate` en cree hors du
|
||||||
|
# controle de la fixture, et `alert` les retient par une cle etrangere `RESTRICT`.
|
||||||
|
alertes = select(Alert.alert_id).where(Alert.site_id.in_(sites))
|
||||||
|
await session.execute(delete(Recommendation).where(Recommendation.alert_id.in_(alertes)))
|
||||||
|
await session.execute(delete(Alert).where(Alert.site_id.in_(sites)))
|
||||||
|
await session.execute(delete(Prediction).where(Prediction.site_id.in_(sites)))
|
||||||
|
await session.execute(delete(Reading).where(Reading.site_id.in_(sites)))
|
||||||
|
await session.execute(delete(Site).where(Site.site_id.in_(sites)))
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def marque() -> str:
|
||||||
|
return uuid4().hex[:12]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def jeu_metier(marque: str) -> AsyncIterator[JeuMetier]:
|
||||||
|
"""Un site instrumenté, un site voisin, trois lectures horaires, une prédiction, une alerte.
|
||||||
|
|
||||||
|
Le voisin existe pour que les tests de filtre prouvent qu'ils écartent quelque chose.
|
||||||
|
"""
|
||||||
|
site_id = f"SITE-{marque}"
|
||||||
|
voisin = f"SITE-{marque}-VOISIN"
|
||||||
|
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await creer_site(session, site_id=site_id, capacity_kw=100.0)
|
||||||
|
await creer_site(session, site_id=voisin, capacity_kw=100.0)
|
||||||
|
for decalage in range(3):
|
||||||
|
await creer_lecture(
|
||||||
|
session,
|
||||||
|
site_id=site_id,
|
||||||
|
timestamp=INSTANT - timedelta(hours=decalage),
|
||||||
|
consumption_kw=10.0 + decalage,
|
||||||
|
)
|
||||||
|
prediction = await creer_prediction(session, site_id=site_id, target_at=INSTANT)
|
||||||
|
alerte = await creer_alerte(session, site_id=site_id, timestamp=INSTANT)
|
||||||
|
jeu = JeuMetier(
|
||||||
|
site_id=site_id,
|
||||||
|
site_voisin=voisin,
|
||||||
|
alert_id=alerte.alert_id,
|
||||||
|
prediction_id=prediction.prediction_id,
|
||||||
|
instant=INSTANT,
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield jeu
|
||||||
|
finally:
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await _supprime(session, [site_id, voisin])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def site_nu(marque: str) -> AsyncIterator[str]:
|
||||||
|
"""Un site sans lecture ni prédiction : le cas que seul un vrai `LEFT JOIN` distingue."""
|
||||||
|
site_id = f"SITE-{marque}-NU"
|
||||||
|
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await creer_site(session, site_id=site_id, capacity_kw=100.0)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield site_id
|
||||||
|
finally:
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await _supprime(session, [site_id])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def principal_injecte(app: FastAPI) -> Iterator[Callable[[Role], None]]:
|
||||||
|
def installe(role: Role = Role.LECTEUR) -> None:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: Principal(
|
||||||
|
id=uuid4(),
|
||||||
|
email="parcours@enervision.fr",
|
||||||
|
role=role,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
@@ -0,0 +1,137 @@
|
|||||||
|
from collections.abc import Callable, Iterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.api.deps import get_alert_service, get_current_principal
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role
|
||||||
|
from app.models.energy import Alert
|
||||||
|
from app.schemas.alert import AlertSeverity
|
||||||
|
|
||||||
|
|
||||||
|
def principal(role: Role = Role.LECTEUR) -> Principal:
|
||||||
|
return Principal(
|
||||||
|
id=uuid4(),
|
||||||
|
email=f"{role.value}@enervision.fr",
|
||||||
|
role=role,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def alert(alert_id: int = 1, site_id: str = "site-1", severity: str = "high") -> Alert:
|
||||||
|
return Alert(
|
||||||
|
alert_id=alert_id,
|
||||||
|
source_alert_id=f"ALR-{alert_id}",
|
||||||
|
site_id=site_id,
|
||||||
|
source="enervision",
|
||||||
|
timestamp=datetime(2026, 9, 16, tzinfo=UTC),
|
||||||
|
type="threshold",
|
||||||
|
severity=severity,
|
||||||
|
message="Dépassement du seuil configuré",
|
||||||
|
value=812.5,
|
||||||
|
threshold=720.0,
|
||||||
|
metric="consumption_kw",
|
||||||
|
prediction_id=None,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FauxService:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.alert = alert()
|
||||||
|
self.appels: list[tuple[str | None, str | None]] = []
|
||||||
|
|
||||||
|
async def list_all(
|
||||||
|
self, *, site_id: str | None = None, severity: str | None = None
|
||||||
|
) -> list[Alert]:
|
||||||
|
self.appels.append((site_id, severity))
|
||||||
|
return [self.alert]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def lecteur_connecte(app: FastAPI) -> Iterator[None]:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: principal()
|
||||||
|
yield
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def servi(app: FastAPI, lecteur_connecte: None) -> Iterator[Callable[[], FauxService]]:
|
||||||
|
def installe() -> FauxService:
|
||||||
|
service = FauxService()
|
||||||
|
app.dependency_overrides[get_alert_service] = lambda: service
|
||||||
|
return service
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_alert_service, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_alerts_returns_the_alerts(
|
||||||
|
servi: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/alerts")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
corps = response.json()
|
||||||
|
assert corps == [
|
||||||
|
{
|
||||||
|
"alert_id": 1,
|
||||||
|
"site_id": "site-1",
|
||||||
|
"timestamp": "2026-09-16T00:00:00Z",
|
||||||
|
"type": "threshold",
|
||||||
|
"severity": "high",
|
||||||
|
"message": "Dépassement du seuil configuré",
|
||||||
|
"value": 812.5,
|
||||||
|
"threshold": 720.0,
|
||||||
|
"metric": "consumption_kw",
|
||||||
|
"prediction_id": None,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_alerts_transmits_the_site_id_filter(
|
||||||
|
servi: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
service = servi()
|
||||||
|
|
||||||
|
await client.get("/api/v1/alerts?site_id=site-1")
|
||||||
|
|
||||||
|
assert service.appels == [("site-1", None)]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_alerts_transmits_the_severity_filter(
|
||||||
|
servi: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
service = servi()
|
||||||
|
|
||||||
|
await client.get("/api/v1/alerts?severity=critical")
|
||||||
|
|
||||||
|
assert service.appels == [(None, AlertSeverity.CRITICAL)]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_alerts_returns_422_for_an_unknown_severity(
|
||||||
|
servi: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/alerts?severity=invalide")
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_alerts_returns_an_empty_list_when_there_is_nothing(
|
||||||
|
lecteur_connecte: None, fake_session: Callable[..., None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
fake_session(result=[])
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/alerts")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == []
|
||||||
@@ -11,6 +11,7 @@ from app.core.roles import AccountKind, Role
|
|||||||
from app.services.auth import (
|
from app.services.auth import (
|
||||||
AuthenticatedSession,
|
AuthenticatedSession,
|
||||||
InvalidCredentialsError,
|
InvalidCredentialsError,
|
||||||
|
InvalidOrExpiredResetTokenError,
|
||||||
RateLimitedError,
|
RateLimitedError,
|
||||||
SessionRejectedError,
|
SessionRejectedError,
|
||||||
)
|
)
|
||||||
@@ -27,15 +28,27 @@ PRINCIPAL = Principal(
|
|||||||
|
|
||||||
|
|
||||||
class FauxService:
|
class FauxService:
|
||||||
def __init__(self, erreur: Exception | None = None) -> None:
|
def __init__(self, erreur: Exception | None = None, *, jeton_valide: bool = True) -> None:
|
||||||
self._erreur = erreur
|
self._erreur = erreur
|
||||||
|
self._jeton_valide = jeton_valide
|
||||||
|
|
||||||
async def refresh(self, **_: object) -> AuthenticatedSession:
|
async def refresh(self, **_: object) -> AuthenticatedSession:
|
||||||
return await self.authenticate()
|
return await self.authenticate()
|
||||||
|
|
||||||
|
async def is_reset_token_valid(self, **_: object) -> bool:
|
||||||
|
return self._jeton_valide
|
||||||
|
|
||||||
async def logout(self, **_: object) -> None:
|
async def logout(self, **_: object) -> None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
async def request_password_reset(self, **_: object) -> None:
|
||||||
|
if self._erreur is not None:
|
||||||
|
raise self._erreur
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def confirm_password_reset(self, **_: object) -> AuthenticatedSession:
|
||||||
|
return await self.authenticate()
|
||||||
|
|
||||||
async def authenticate(self, **_: object) -> AuthenticatedSession:
|
async def authenticate(self, **_: object) -> AuthenticatedSession:
|
||||||
if self._erreur is not None:
|
if self._erreur is not None:
|
||||||
raise self._erreur
|
raise self._erreur
|
||||||
@@ -206,3 +219,126 @@ async def test_a_cookie_bearing_route_accepts_a_request_without_origin(
|
|||||||
response = await client.post("/api/v1/auth/logout")
|
response = await client.post("/api/v1/auth/logout")
|
||||||
|
|
||||||
assert response.status_code != 403
|
assert response.status_code != 403
|
||||||
|
|
||||||
|
|
||||||
|
async def test_forgot_password_answers_202_when_the_account_exists(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/forgot-password", json={"email": "operateur@enervision.fr"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 202
|
||||||
|
assert response.headers["cache-control"] == "no-store"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_forgot_password_answers_202_identically_when_the_account_is_unknown(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/forgot-password", json={"email": "inconnu@enervision.fr"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 202
|
||||||
|
|
||||||
|
|
||||||
|
async def test_forgot_password_returns_429_with_a_retry_after_when_the_rate_limit_is_reached(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
fake_auth_service[0] = RateLimitedError(900)
|
||||||
|
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/forgot-password", json={"email": "operateur@enervision.fr"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 429
|
||||||
|
assert response.headers["retry-after"] == "900"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_forgot_password_rejects_a_malformed_email(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.post("/api/v1/auth/forgot-password", json={"email": "pas-un-email"})
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def fake_auth_service_reset_validity(app: FastAPI) -> Iterator[list[bool]]:
|
||||||
|
programme = [True]
|
||||||
|
app.dependency_overrides[get_auth_service] = lambda: FauxService(jeton_valide=programme[0])
|
||||||
|
yield programme
|
||||||
|
app.dependency_overrides.pop(get_auth_service, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_validate_reset_token_reports_a_living_token(
|
||||||
|
fake_auth_service_reset_validity: list[bool], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.get(
|
||||||
|
"/api/v1/auth/reset-password/validate", params={"token": "un-secret-opaque"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == {"valid": True}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_validate_reset_token_reports_an_invalid_or_expired_token(
|
||||||
|
fake_auth_service_reset_validity: list[bool], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
fake_auth_service_reset_validity[0] = False
|
||||||
|
|
||||||
|
response = await client.get(
|
||||||
|
"/api/v1/auth/reset-password/validate", params={"token": "un-secret-perime"}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == {"valid": False}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reset_password_returns_the_token_and_the_cookie_on_success(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/reset-password",
|
||||||
|
json={"token": "un-secret-opaque", "new_password": "Un-nouveau-mot-de-passe1!"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.cookies.get("ev_refresh") is not None
|
||||||
|
assert "refresh_secret" not in response.text
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reset_password_rejects_an_invalid_or_expired_token(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
fake_auth_service[0] = InvalidOrExpiredResetTokenError("Lien invalide ou expiré")
|
||||||
|
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/reset-password",
|
||||||
|
json={"token": "un-secret-perime", "new_password": "Un-nouveau-mot-de-passe1!"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reset_password_rejects_a_weak_password(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/reset-password",
|
||||||
|
json={"token": "un-secret-opaque", "new_password": "trop-simple"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reset_password_refuses_a_foreign_origin(
|
||||||
|
fake_auth_service: list[Exception | None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/auth/reset-password",
|
||||||
|
json={"token": "un-secret-opaque", "new_password": "Un-nouveau-mot-de-passe1!"},
|
||||||
|
headers={"Origin": "https://malveillant.example"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 403
|
||||||
|
|||||||
@@ -23,8 +23,9 @@ async def interroge(
|
|||||||
("x-content-type-options", "nosniff"),
|
("x-content-type-options", "nosniff"),
|
||||||
("x-frame-options", "DENY"),
|
("x-frame-options", "DENY"),
|
||||||
("referrer-policy", "no-referrer"),
|
("referrer-policy", "no-referrer"),
|
||||||
|
("cross-origin-resource-policy", "same-origin"),
|
||||||
],
|
],
|
||||||
ids=["nosniff", "anti_iframe", "referrer"],
|
ids=["nosniff", "anti_iframe", "referrer", "corp"],
|
||||||
)
|
)
|
||||||
async def test_every_response_carries_the_security_headers(
|
async def test_every_response_carries_the_security_headers(
|
||||||
client: AsyncClient, entete: str, valeur: str
|
client: AsyncClient, entete: str, valeur: str
|
||||||
@@ -71,6 +72,20 @@ async def test_metrics_stay_open_when_no_token_is_configured(client: AsyncClient
|
|||||||
assert response.status_code == 200
|
assert response.status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
async def test_an_empty_metrics_token_means_no_token() -> None:
|
||||||
|
assert (await interroge({"metrics_token": ""}, "/metrics")).status_code == 200
|
||||||
|
|
||||||
|
|
||||||
|
async def test_metrics_ignore_health_probes_but_count_business_routes(client: AsyncClient) -> None:
|
||||||
|
await client.get("/api/v1/health/live")
|
||||||
|
await client.get("/api/v1/sites")
|
||||||
|
|
||||||
|
exposition = (await client.get("/metrics")).text
|
||||||
|
|
||||||
|
assert 'handler="/api/v1/health/live"' not in exposition
|
||||||
|
assert 'handler="/api/v1/sites"' in exposition
|
||||||
|
|
||||||
|
|
||||||
async def test_metrics_demand_the_token_once_one_is_configured() -> None:
|
async def test_metrics_demand_the_token_once_one_is_configured() -> None:
|
||||||
surcharges = {"metrics_token": "un-jeton-de-supervision-assez-long"}
|
surcharges = {"metrics_token": "un-jeton-de-supervision-assez-long"}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,281 @@
|
|||||||
|
# Pourquoi : la matrice rôle x route sur les routes réelles. `test_authorization.py` la joue déjà,
|
||||||
|
# mais contre une route jetable montée par une fixture, ce qui ne dit rien du niveau effectivement
|
||||||
|
# posé sur `/sites` ou `/users`. `ROLE_MINIMUM` (tests/api/acces.py) est la référence, et ce
|
||||||
|
# fichier est ce qui la confronte au comportement observé.
|
||||||
|
# Piège : l'assertion porte sur le refus de la garde, pas sur un 200. Un rôle suffisant peut
|
||||||
|
# légitimement recevoir 404 ou 422 selon les données ; ce qui compte est qu'il ne reçoive pas le
|
||||||
|
# 403 `Droits insuffisants`. Sans cette nuance, le test dépendrait du contenu de la base.
|
||||||
|
# Les tests `integration` en fin de fichier rejouent la même matrice avec de vrais jetons, donc en
|
||||||
|
# traversant le décodage du JWT et la relecture du compte, ce que l'override court-circuite.
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from collections.abc import AsyncIterator, Callable, Iterator
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from httpx import AsyncClient, Response
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
from app.api.deps import get_current_principal
|
||||||
|
from app.core.hashing import build_hasher
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role, has_at_least
|
||||||
|
from app.db.session import get_session, get_session_factory
|
||||||
|
from app.repositories.user import UserRepository
|
||||||
|
from tests.api.acces import ROLE_MINIMUM, chemin_concret
|
||||||
|
|
||||||
|
ROLES = [Role.LECTEUR, Role.OPERATEUR, Role.ADMIN]
|
||||||
|
IDS_DE_ROLE = ["lecteur", "operateur", "admin"]
|
||||||
|
REFUS_DE_DROITS = "Droits insuffisants"
|
||||||
|
REFUS_DE_MOT_DE_PASSE = "password_change_required"
|
||||||
|
MOT_DE_PASSE = "un-mot-de-passe-de-recette"
|
||||||
|
|
||||||
|
|
||||||
|
# `FakeSession` de tests/factories.py rend un unique objet pour les trois formes d'appel, ce qui
|
||||||
|
# suffit à un test d'endpoint ciblé mais pas à balayer 13 routes qui interrogent chacune la base
|
||||||
|
# à sa façon. Ce double rend un résultat vide quelle que soit la forme demandée, pour que la
|
||||||
|
# réponse observée vienne de la garde de rôle et jamais d'un double mal ajusté.
|
||||||
|
class ResultatVide:
|
||||||
|
def scalars(self) -> ResultatVide:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def all(self) -> list[object]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def first(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def one_or_none(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def scalar_one_or_none(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def mappings(self) -> ResultatVide:
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __iter__(self) -> Iterator[object]:
|
||||||
|
return iter(())
|
||||||
|
|
||||||
|
|
||||||
|
class SessionMuette:
|
||||||
|
async def scalar(self, *_: object, **__: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def execute(self, *_: object, **__: object) -> ResultatVide:
|
||||||
|
return ResultatVide()
|
||||||
|
|
||||||
|
async def scalars(self, *_: object, **__: object) -> ResultatVide:
|
||||||
|
return ResultatVide()
|
||||||
|
|
||||||
|
async def get(self, *_: object, **__: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def flush(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def commit(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def rollback(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def add(self, *_: object, **__: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def base_muette(app: FastAPI) -> None:
|
||||||
|
async def override() -> AsyncIterator[SessionMuette]:
|
||||||
|
yield SessionMuette()
|
||||||
|
|
||||||
|
app.dependency_overrides[get_session] = override
|
||||||
|
|
||||||
|
|
||||||
|
def principal(role: Role, *, must_change_password: bool = False) -> Principal:
|
||||||
|
return Principal(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
email=f"matrice-{role.value}@enervision.fr",
|
||||||
|
role=role,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=must_change_password,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def connecte(app: FastAPI) -> Iterator[Callable[[Principal], None]]:
|
||||||
|
def installe(acteur: Principal) -> None:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: acteur
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def appelle(client: AsyncClient, methode: str, chemin: str, **kwargs: object) -> Response:
|
||||||
|
return await client.request(methode, chemin_concret(chemin), json={}, **kwargs) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def motif_du_refus(response: Response) -> str | None:
|
||||||
|
if response.status_code != 403:
|
||||||
|
return None
|
||||||
|
detail = response.json().get("detail")
|
||||||
|
return detail if isinstance(detail, str) else None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("role", ROLES, ids=IDS_DE_ROLE)
|
||||||
|
async def test_a_role_below_the_minimum_is_refused_on_every_guarded_route(
|
||||||
|
connecte: Callable[[Principal], None],
|
||||||
|
client: AsyncClient,
|
||||||
|
base_muette: None,
|
||||||
|
role: Role,
|
||||||
|
) -> None:
|
||||||
|
connecte(principal(role))
|
||||||
|
laissees_passer: list[tuple[str, str, int]] = []
|
||||||
|
|
||||||
|
for (methode, chemin), minimum in ROLE_MINIMUM.items():
|
||||||
|
if has_at_least(role, minimum):
|
||||||
|
continue
|
||||||
|
response = await appelle(client, methode, chemin)
|
||||||
|
if motif_du_refus(response) != REFUS_DE_DROITS:
|
||||||
|
laissees_passer.append((methode, chemin, response.status_code))
|
||||||
|
|
||||||
|
assert laissees_passer == []
|
||||||
|
|
||||||
|
|
||||||
|
# Le pendant du test précédent : sans lui, une garde posée trop haut, par exemple `AdminDep` sur
|
||||||
|
# `/sites`, ne ferait échouer aucun test du dépôt.
|
||||||
|
@pytest.mark.parametrize("role", ROLES, ids=IDS_DE_ROLE)
|
||||||
|
async def test_a_role_at_or_above_the_minimum_is_never_refused_by_the_guard(
|
||||||
|
connecte: Callable[[Principal], None],
|
||||||
|
client: AsyncClient,
|
||||||
|
base_muette: None,
|
||||||
|
role: Role,
|
||||||
|
) -> None:
|
||||||
|
connecte(principal(role))
|
||||||
|
refusees: list[tuple[str, str]] = []
|
||||||
|
|
||||||
|
for (methode, chemin), minimum in ROLE_MINIMUM.items():
|
||||||
|
if not has_at_least(role, minimum):
|
||||||
|
continue
|
||||||
|
response = await appelle(client, methode, chemin)
|
||||||
|
if motif_du_refus(response) == REFUS_DE_DROITS:
|
||||||
|
refusees.append((methode, chemin))
|
||||||
|
|
||||||
|
assert refusees == []
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_pending_password_change_is_refused_on_every_guarded_route(
|
||||||
|
connecte: Callable[[Principal], None],
|
||||||
|
client: AsyncClient,
|
||||||
|
base_muette: None,
|
||||||
|
) -> None:
|
||||||
|
connecte(principal(Role.ADMIN, must_change_password=True))
|
||||||
|
laissees_passer: list[tuple[str, str, int]] = []
|
||||||
|
|
||||||
|
for methode, chemin in ROLE_MINIMUM:
|
||||||
|
response = await appelle(client, methode, chemin)
|
||||||
|
if motif_du_refus(response) != REFUS_DE_MOT_DE_PASSE:
|
||||||
|
laissees_passer.append((methode, chemin, response.status_code))
|
||||||
|
|
||||||
|
assert laissees_passer == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def comptes_par_role() -> AsyncIterator[dict[Role, str]]:
|
||||||
|
marque = uuid.uuid4().hex[:12]
|
||||||
|
hacheur = build_hasher(time_cost=1, memory_cost_kib=8192, parallelism=1, max_concurrency=2)
|
||||||
|
empreinte = await hacheur.hash(MOT_DE_PASSE)
|
||||||
|
adresses = {role: f"matrice-{marque}-{role.value}@enervision.fr" for role in ROLES}
|
||||||
|
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
depot = UserRepository(session)
|
||||||
|
for role, email in adresses.items():
|
||||||
|
await depot.create(email=email, password_hash=empreinte, role=role)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
yield adresses
|
||||||
|
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await session.execute(
|
||||||
|
text("delete from app_user where email like :motif"), {"motif": f"matrice-{marque}-%"}
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
|
||||||
|
async def authentifie(client: AsyncClient, email: str) -> dict[str, str]:
|
||||||
|
reponse = await client.post(
|
||||||
|
"/api/v1/auth/login", json={"email": email, "password": MOT_DE_PASSE}
|
||||||
|
)
|
||||||
|
assert reponse.status_code == 200, reponse.text
|
||||||
|
return {"Authorization": f"Bearer {reponse.json()['access_token']}"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.integration
|
||||||
|
@pytest.mark.parametrize("role", ROLES, ids=IDS_DE_ROLE)
|
||||||
|
async def test_a_real_token_reaches_exactly_the_routes_of_its_rank(
|
||||||
|
comptes_par_role: dict[Role, str], client: AsyncClient, role: Role
|
||||||
|
) -> None:
|
||||||
|
entetes = await authentifie(client, comptes_par_role[role])
|
||||||
|
ecarts: list[tuple[str, str, int, str]] = []
|
||||||
|
|
||||||
|
for (methode, chemin), minimum in ROLE_MINIMUM.items():
|
||||||
|
response = await appelle(client, methode, chemin, headers=entetes)
|
||||||
|
refuse = motif_du_refus(response) == REFUS_DE_DROITS
|
||||||
|
if refuse is has_at_least(role, minimum):
|
||||||
|
ecarts.append((methode, chemin, response.status_code, response.text[:120]))
|
||||||
|
|
||||||
|
assert ecarts == []
|
||||||
|
|
||||||
|
|
||||||
|
# Contrainte : les deux rangs ne se séparent que sur les routes que `ROLE_MINIMUM` réserve à
|
||||||
|
# `operateur`. Une route d'opérateur ajoutée sans être classée fait diverger les statuts sans
|
||||||
|
# qu'aucune entrée ne l'annonce, et une garde d'opérateur posée par erreur sur une route de
|
||||||
|
# lecture fait diverger ce qui devait rester identique.
|
||||||
|
@pytest.mark.integration
|
||||||
|
async def test_the_operator_rank_diverges_from_the_reader_rank_only_where_declared(
|
||||||
|
comptes_par_role: dict[Role, str], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
lecteur = await authentifie(client, comptes_par_role[Role.LECTEUR])
|
||||||
|
operateur = await authentifie(client, comptes_par_role[Role.OPERATEUR])
|
||||||
|
ecarts: list[tuple[str, str]] = []
|
||||||
|
|
||||||
|
for (methode, chemin), minimum in ROLE_MINIMUM.items():
|
||||||
|
cote_lecteur = await appelle(client, methode, chemin, headers=lecteur)
|
||||||
|
cote_operateur = await appelle(client, methode, chemin, headers=operateur)
|
||||||
|
diverge = cote_lecteur.status_code != cote_operateur.status_code
|
||||||
|
if diverge is not (minimum is Role.OPERATEUR):
|
||||||
|
ecarts.append((methode, chemin))
|
||||||
|
|
||||||
|
assert ecarts == []
|
||||||
|
|
||||||
|
|
||||||
|
# Piège : `/auth/logout-all` prend un `CurrentPrincipalDep` nu, donc elle échappe au gate
|
||||||
|
# `must_change_password` que seul `require_role` applique. Comportement figé ici, pas corrigé.
|
||||||
|
@pytest.mark.integration
|
||||||
|
async def test_a_temporary_password_blocks_the_business_routes_but_not_logout_all(
|
||||||
|
client: AsyncClient,
|
||||||
|
) -> None:
|
||||||
|
marque = uuid.uuid4().hex[:12]
|
||||||
|
email = f"matrice-{marque}-provisoire@enervision.fr"
|
||||||
|
hacheur = build_hasher(time_cost=1, memory_cost_kib=8192, parallelism=1, max_concurrency=2)
|
||||||
|
empreinte = await hacheur.hash(MOT_DE_PASSE)
|
||||||
|
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await UserRepository(session).create(
|
||||||
|
email=email, password_hash=empreinte, role=Role.ADMIN, must_change_password=True
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
try:
|
||||||
|
entetes = await authentifie(client, email)
|
||||||
|
sites = await client.get("/api/v1/sites", headers=entetes)
|
||||||
|
identite = await client.get("/api/v1/auth/me", headers=entetes)
|
||||||
|
fermeture = await client.post("/api/v1/auth/logout-all", headers=entetes)
|
||||||
|
|
||||||
|
assert motif_du_refus(sites) == REFUS_DE_MOT_DE_PASSE
|
||||||
|
assert identite.status_code == 200
|
||||||
|
assert fermeture.status_code == 204
|
||||||
|
finally:
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await session.execute(text("delete from app_user where email = :e"), {"e": email})
|
||||||
|
await session.commit()
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
from collections.abc import Iterator, Sequence
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.api.deps import get_current_principal, get_drift_service
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role
|
||||||
|
from app.models.energy import DriftReport
|
||||||
|
|
||||||
|
INSTANT = datetime(2026, 9, 22, 12, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def operateur() -> Principal:
|
||||||
|
return Principal(
|
||||||
|
id=uuid4(),
|
||||||
|
email="operateur@enervision.fr",
|
||||||
|
role=Role.OPERATEUR,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def rapport(*, site_id: str | None) -> DriftReport:
|
||||||
|
return DriftReport(
|
||||||
|
drift_report_id=1,
|
||||||
|
computed_at=INSTANT,
|
||||||
|
site_id=site_id,
|
||||||
|
window_start=INSTANT - timedelta(hours=168),
|
||||||
|
window_end=INSTANT,
|
||||||
|
reference_start=None,
|
||||||
|
reference_end=None,
|
||||||
|
n_observations=48,
|
||||||
|
mae=1.5,
|
||||||
|
mape=12.0,
|
||||||
|
bias=0.3,
|
||||||
|
reference_mae=1.2,
|
||||||
|
coverage_ratio=0.95,
|
||||||
|
insufficient_data_ratio=0.0,
|
||||||
|
model_references=["lightgbm-aaa"],
|
||||||
|
status="stable",
|
||||||
|
reason=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FauxService:
|
||||||
|
def __init__(self, rapports: Sequence[DriftReport]) -> None:
|
||||||
|
self.rapports = list(rapports)
|
||||||
|
self.site_demande: str | None = None
|
||||||
|
|
||||||
|
async def derniers(self, *, site_id: str | None = None) -> Sequence[DriftReport]:
|
||||||
|
self.site_demande = site_id
|
||||||
|
return self.rapports
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def servi(app: FastAPI) -> Iterator[list[DriftReport]]:
|
||||||
|
rapports = [rapport(site_id="SITE001"), rapport(site_id=None)]
|
||||||
|
service = FauxService(rapports)
|
||||||
|
app.dependency_overrides[get_current_principal] = operateur
|
||||||
|
app.dependency_overrides[get_drift_service] = lambda: service
|
||||||
|
yield rapports
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_returns_the_latest_report_of_every_site(
|
||||||
|
servi: list[DriftReport], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
reponse = await client.get("/api/v1/monitoring/drift")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
assert [ligne["site_id"] for ligne in reponse.json()] == ["SITE001", None]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_exposes_the_metrics_of_the_stored_report(
|
||||||
|
servi: list[DriftReport], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
reponse = await client.get("/api/v1/monitoring/drift")
|
||||||
|
|
||||||
|
premier = reponse.json()[0]
|
||||||
|
assert premier["status"] == "stable"
|
||||||
|
assert premier["mae"] == 1.5
|
||||||
|
assert premier["model_references"] == ["lightgbm-aaa"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_returns_an_empty_list_when_no_report_exists(
|
||||||
|
app: FastAPI, client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
app.dependency_overrides[get_current_principal] = operateur
|
||||||
|
app.dependency_overrides[get_drift_service] = lambda: FauxService([])
|
||||||
|
|
||||||
|
reponse = await client.get("/api/v1/monitoring/drift")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
assert reponse.json() == []
|
||||||
|
app.dependency_overrides.clear()
|
||||||
@@ -8,6 +8,7 @@ from typing import Any
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app import cli
|
from app import cli
|
||||||
|
from tests.api.acces import ROLE_MINIMUM
|
||||||
|
|
||||||
METHODES = {"get", "post", "patch", "put", "delete"}
|
METHODES = {"get", "post", "patch", "put", "delete"}
|
||||||
|
|
||||||
@@ -24,17 +25,11 @@ ORIGINE_VERIFIEE = {
|
|||||||
|
|
||||||
# Toute route derrière `require_role` (LecteurDep, OperateurDep, AdminDep) peut rendre 403 pour
|
# Toute route derrière `require_role` (LecteurDep, OperateurDep, AdminDep) peut rendre 403 pour
|
||||||
# `password_change_required`, pas seulement les routes `admin`.
|
# `password_change_required`, pas seulement les routes `admin`.
|
||||||
ROUTES_A_ROLE = {
|
# Piège : cette liste était recopiée ici, et deux de ses entrées portaient `{id}` là où le contrat
|
||||||
("GET", "/api/v1/users"),
|
# expose `{user_id}`. Elles ne correspondaient donc à aucune opération, et le test ci-dessous
|
||||||
("POST", "/api/v1/users"),
|
# passait au vert sans rien vérifier sur ces deux routes. Elle est maintenant dérivée, et
|
||||||
("PATCH", "/api/v1/users/{id}"),
|
# `test_every_declared_route_is_classified` interdit l'entrée morte.
|
||||||
("POST", "/api/v1/users/{id}/password-reset"),
|
ROUTES_A_ROLE = frozenset(ROLE_MINIMUM)
|
||||||
("GET", "/api/v1/sites"),
|
|
||||||
("GET", "/api/v1/sites/{site_id}"),
|
|
||||||
("GET", "/api/v1/recommendations"),
|
|
||||||
("GET", "/api/v1/recommendations/{recommendation_id}"),
|
|
||||||
("GET", "/api/v1/stats/summary"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope="module")
|
@pytest.fixture(scope="module")
|
||||||
@@ -109,8 +104,8 @@ def test_the_rate_limit_documents_the_delay_header(schema: dict[str, Any]) -> No
|
|||||||
def test_the_refresh_cookie_appears_in_the_security_schemes(schema: dict[str, Any]) -> None:
|
def test_the_refresh_cookie_appears_in_the_security_schemes(schema: dict[str, Any]) -> None:
|
||||||
schemes = schema["components"]["securitySchemes"]
|
schemes = schema["components"]["securitySchemes"]
|
||||||
|
|
||||||
assert schemes["Cookie de rafraîchissement"]["in"] == "cookie"
|
assert schemes["CookieRafraichissement"]["in"] == "cookie"
|
||||||
assert schemes["Cookie de rafraîchissement"]["name"] == "ev_refresh"
|
assert schemes["CookieRafraichissement"]["name"] == "ev_refresh"
|
||||||
|
|
||||||
|
|
||||||
def test_each_tag_used_by_a_route_is_described(schema: dict[str, Any]) -> None:
|
def test_each_tag_used_by_a_route_is_described(schema: dict[str, Any]) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.core.roles import Role
|
||||||
|
from tests.api.conftest import JeuMetier
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
async def genere(client: AsyncClient, site_id: str) -> dict[str, int]:
|
||||||
|
# Toujours borne a un site : sans `site_id`, le service examine toutes les alertes de la
|
||||||
|
# base, y compris celles d'un autre test, et le rapport cesse d'etre deterministe.
|
||||||
|
reponse = await client.post(f"/api/v1/recommendations/generate?site_id={site_id}")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
return dict(reponse.json())
|
||||||
|
|
||||||
|
|
||||||
|
async def test_generate_creates_a_recommendation_for_the_alert_of_the_requested_site(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.ADMIN)
|
||||||
|
|
||||||
|
rapport = await genere(client, jeu_metier.site_id)
|
||||||
|
|
||||||
|
assert rapport["alerts_examined"] == 1
|
||||||
|
assert rapport["recommendations_created"] >= 1
|
||||||
|
assert rapport["already_present"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
async def test_generate_creates_nothing_more_when_it_runs_twice_on_the_same_alerts(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.ADMIN)
|
||||||
|
premier = await genere(client, jeu_metier.site_id)
|
||||||
|
|
||||||
|
second = await genere(client, jeu_metier.site_id)
|
||||||
|
|
||||||
|
assert second["recommendations_created"] == 0
|
||||||
|
assert second["already_present"] == premier["recommendations_created"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_generate_examines_no_alert_when_the_requested_site_has_none(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.ADMIN)
|
||||||
|
|
||||||
|
rapport = await genere(client, jeu_metier.site_voisin)
|
||||||
|
|
||||||
|
assert rapport["alerts_examined"] == 0
|
||||||
|
assert rapport["recommendations_created"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_recommendations_returns_what_generate_persisted_in_another_session(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.ADMIN)
|
||||||
|
await genere(client, jeu_metier.site_id)
|
||||||
|
|
||||||
|
reponse = await client.get("/api/v1/recommendations")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
miennes = [r for r in reponse.json() if r["alert_id"] == jeu_metier.alert_id]
|
||||||
|
assert miennes != []
|
||||||
|
assert all(r["rule_reference"] for r in miennes)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_recommendation_returns_the_row_created_by_generate(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.ADMIN)
|
||||||
|
await genere(client, jeu_metier.site_id)
|
||||||
|
liste = await client.get("/api/v1/recommendations")
|
||||||
|
creee = next(r for r in liste.json() if r["alert_id"] == jeu_metier.alert_id)
|
||||||
|
|
||||||
|
reponse = await client.get(f"/api/v1/recommendations/{creee['recommendation_id']}")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
assert reponse.json() == creee
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_recommendation_returns_404_when_the_identifier_is_unknown(
|
||||||
|
principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.LECTEUR)
|
||||||
|
|
||||||
|
reponse = await client.get("/api/v1/recommendations/9999999")
|
||||||
|
|
||||||
|
assert reponse.status_code == 404
|
||||||
|
assert reponse.json()["detail"] == "Recommandation introuvable"
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
from collections.abc import Callable
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.core.roles import Role
|
||||||
|
from app.db.session import get_session_factory
|
||||||
|
from tests.api.conftest import JeuMetier
|
||||||
|
from tests.repositories.test_reading import creer_lecture
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_sites_returns_the_seeded_site_with_its_stored_attributes(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.LECTEUR)
|
||||||
|
|
||||||
|
reponse = await client.get("/api/v1/sites")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
mien = next(site for site in reponse.json() if site["site_id"] == jeu_metier.site_id)
|
||||||
|
assert mien["capacity_kw"] == 100.0
|
||||||
|
assert mien["site_name"] == "Site de test"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_site_returns_404_when_the_identifier_is_absent_from_the_database(
|
||||||
|
principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.LECTEUR)
|
||||||
|
|
||||||
|
reponse = await client.get("/api/v1/sites/SITE-JAMAIS-INSERE")
|
||||||
|
|
||||||
|
assert reponse.status_code == 404
|
||||||
|
assert reponse.json()["detail"] == "Site introuvable"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_current_returns_the_most_recent_reading_when_several_hours_are_stored(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.LECTEUR)
|
||||||
|
|
||||||
|
reponse = await client.get(f"/api/v1/sites/{jeu_metier.site_id}/current")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
corps = reponse.json()
|
||||||
|
assert corps["consumption_kw"] == 10.0
|
||||||
|
assert corps["timestamp"].startswith("2026-09-16T12:00")
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_current_keeps_the_highest_reading_id_when_two_sources_share_the_timestamp(
|
||||||
|
jeu_metier: JeuMetier, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.LECTEUR)
|
||||||
|
async with get_session_factory()() as session:
|
||||||
|
await creer_lecture(
|
||||||
|
session,
|
||||||
|
site_id=jeu_metier.site_id,
|
||||||
|
timestamp=jeu_metier.instant,
|
||||||
|
source="api_history",
|
||||||
|
consumption_kw=999.0,
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
reponse = await client.get(f"/api/v1/sites/{jeu_metier.site_id}/current")
|
||||||
|
|
||||||
|
assert reponse.json()["consumption_kw"] == 999.0
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_current_reports_a_critical_quality_when_the_site_has_no_reading(
|
||||||
|
site_nu: str, principal_injecte: Callable[[Role], None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
principal_injecte(Role.LECTEUR)
|
||||||
|
|
||||||
|
reponse = await client.get(f"/api/v1/sites/{site_nu}/current")
|
||||||
|
|
||||||
|
assert reponse.status_code == 200
|
||||||
|
corps = reponse.json()
|
||||||
|
assert corps["timestamp"] is None
|
||||||
|
assert corps["data_quality"] == "critical"
|
||||||
@@ -0,0 +1,85 @@
|
|||||||
|
from collections.abc import Callable, Iterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.api.deps import get_current_principal, get_prediction_service
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role
|
||||||
|
from app.services.prediction import PredictionSummary, SitePrediction, SitePredictionSummary
|
||||||
|
|
||||||
|
TARGET_AT = datetime(2026, 9, 16, 13, 0, tzinfo=UTC)
|
||||||
|
CREATED_AT = datetime(2026, 9, 16, 12, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def lecteur() -> Principal:
|
||||||
|
# Le garde-fou de rôle (`lecteur` minimum) est déjà couvert par l'ensemble `ROUTES_A_ROLE`
|
||||||
|
# de `tests/api/test_openapi.py` : pas besoin ici d'un paramètre de rôle jamais appelé avec
|
||||||
|
# autre chose que sa valeur par défaut.
|
||||||
|
return Principal(
|
||||||
|
id=uuid4(),
|
||||||
|
email="lecteur@enervision.fr",
|
||||||
|
role=Role.LECTEUR,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FauxService:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.resume = PredictionSummary(
|
||||||
|
timestamp=datetime.now(UTC),
|
||||||
|
sites=[
|
||||||
|
SitePredictionSummary(
|
||||||
|
site_id="SITE001",
|
||||||
|
site_name="Bureau Paris La Défense",
|
||||||
|
prediction=SitePrediction(
|
||||||
|
target_at=TARGET_AT,
|
||||||
|
target_metric="consumption_kwh",
|
||||||
|
period_minutes=60,
|
||||||
|
predicted_value=812.5,
|
||||||
|
status="available",
|
||||||
|
failure_reason=None,
|
||||||
|
model_reference="lightgbm-abc123",
|
||||||
|
created_at=CREATED_AT,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
SitePredictionSummary(site_id="SITE002", site_name="Usine Lyon", prediction=None),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
async def summary(self) -> PredictionSummary:
|
||||||
|
return self.resume
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def servi(app: FastAPI) -> Iterator[Callable[[], FauxService]]:
|
||||||
|
def installe() -> FauxService:
|
||||||
|
service = FauxService()
|
||||||
|
app.dependency_overrides[get_prediction_service] = lambda: service
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: lecteur()
|
||||||
|
return service
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_prediction_service, None)
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_predictions_returns_the_service_result(
|
||||||
|
servi: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/predictions")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
corps = response.json()
|
||||||
|
premier, second = corps["sites"]
|
||||||
|
assert premier["site_id"] == "SITE001"
|
||||||
|
assert premier["prediction"]["predicted_value"] == 812.5
|
||||||
|
assert premier["prediction"]["status"] == "available"
|
||||||
|
assert second["site_id"] == "SITE002"
|
||||||
|
assert second["prediction"] is None
|
||||||
@@ -0,0 +1,198 @@
|
|||||||
|
from collections.abc import Callable, Iterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.api.deps import get_current_principal, get_reading_service
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role
|
||||||
|
from app.models.energy import Reading
|
||||||
|
from app.services.reading import FenetreInverseeError, FenetreTropLargeError
|
||||||
|
|
||||||
|
|
||||||
|
def principal(role: Role = Role.LECTEUR) -> Principal:
|
||||||
|
return Principal(
|
||||||
|
id=uuid4(),
|
||||||
|
email=f"{role.value}@enervision.fr",
|
||||||
|
role=role,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def reading(reading_id: int = 1, site_id: str = "site-1") -> Reading:
|
||||||
|
return Reading(
|
||||||
|
reading_id=reading_id,
|
||||||
|
site_id=site_id,
|
||||||
|
timestamp=datetime(2026, 9, 16, tzinfo=UTC),
|
||||||
|
source="api_current",
|
||||||
|
consumption_kw=42.5,
|
||||||
|
consumption_kwh=None,
|
||||||
|
consumption_euros=None,
|
||||||
|
voltage_v=230.0,
|
||||||
|
current_a=None,
|
||||||
|
power_factor=None,
|
||||||
|
temperature_celsius=None,
|
||||||
|
humidity_percent=None,
|
||||||
|
solar_irradiance_wm2=None,
|
||||||
|
is_working_hours=True,
|
||||||
|
data_quality="good",
|
||||||
|
null_reasons=None,
|
||||||
|
imputed_values=None,
|
||||||
|
imputation_method=None,
|
||||||
|
raw_data={},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FauxService:
|
||||||
|
def __init__(self, leve: Exception | None = None) -> None:
|
||||||
|
self.reading = reading()
|
||||||
|
self.leve = leve
|
||||||
|
self.appels: list[tuple[str | None, str | None, str | None, int, int]] = []
|
||||||
|
|
||||||
|
async def list_history(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
site_id: str | None = None,
|
||||||
|
start: datetime | None = None,
|
||||||
|
end: datetime | None = None,
|
||||||
|
limit: int,
|
||||||
|
offset: int,
|
||||||
|
) -> list[Reading]:
|
||||||
|
self.appels.append((site_id, start, end, limit, offset))
|
||||||
|
if self.leve is not None:
|
||||||
|
raise self.leve
|
||||||
|
return [self.reading]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def lecteur_connecte(app: FastAPI) -> Iterator[None]:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: principal()
|
||||||
|
yield
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def servi(app: FastAPI, lecteur_connecte: None) -> Iterator[Callable[..., FauxService]]:
|
||||||
|
def installe(*, leve: Exception | None = None) -> FauxService:
|
||||||
|
service = FauxService(leve=leve)
|
||||||
|
app.dependency_overrides[get_reading_service] = lambda: service
|
||||||
|
return service
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_reading_service, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_returns_the_readings(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/readings")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
corps = response.json()
|
||||||
|
assert corps == [
|
||||||
|
{
|
||||||
|
"reading_id": 1,
|
||||||
|
"site_id": "site-1",
|
||||||
|
"timestamp": "2026-09-16T00:00:00Z",
|
||||||
|
"source": "api_current",
|
||||||
|
"consumption_kw": 42.5,
|
||||||
|
"consumption_kwh": None,
|
||||||
|
"consumption_euros": None,
|
||||||
|
"voltage_v": 230.0,
|
||||||
|
"current_a": None,
|
||||||
|
"power_factor": None,
|
||||||
|
"temperature_celsius": None,
|
||||||
|
"humidity_percent": None,
|
||||||
|
"solar_irradiance_wm2": None,
|
||||||
|
"is_working_hours": True,
|
||||||
|
"data_quality": "good",
|
||||||
|
"null_reasons": None,
|
||||||
|
"imputed_values": None,
|
||||||
|
"imputation_method": None,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_transmits_the_filters_and_pagination(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
service = servi()
|
||||||
|
|
||||||
|
response = await client.get(
|
||||||
|
"/api/v1/readings",
|
||||||
|
params={
|
||||||
|
"site_id": "site-1",
|
||||||
|
"start": "2026-09-01T00:00:00Z",
|
||||||
|
"end": "2026-09-02T00:00:00Z",
|
||||||
|
"limit": 50,
|
||||||
|
"offset": 10,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert service.appels == [
|
||||||
|
(
|
||||||
|
"site-1",
|
||||||
|
datetime(2026, 9, 1, tzinfo=UTC),
|
||||||
|
datetime(2026, 9, 2, tzinfo=UTC),
|
||||||
|
50,
|
||||||
|
10,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_returns_400_when_the_window_is_inverted(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi(leve=FenetreInverseeError())
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/readings")
|
||||||
|
|
||||||
|
assert response.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_returns_400_when_the_window_is_too_large(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi(leve=FenetreTropLargeError())
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/readings")
|
||||||
|
|
||||||
|
assert response.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_returns_422_for_a_limit_above_the_maximum(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/readings", params={"limit": 5000})
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_returns_422_for_a_negative_offset(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/readings", params={"offset": -1})
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
async def test_list_readings_returns_an_empty_list_when_there_is_nothing(
|
||||||
|
lecteur_connecte: None, fake_session: Callable[..., None], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
fake_session(result=[])
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/readings")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == []
|
||||||
@@ -10,7 +10,7 @@ from app.api.deps import get_current_principal, get_recommendation_service
|
|||||||
from app.core.principal import Principal
|
from app.core.principal import Principal
|
||||||
from app.core.roles import AccountKind, Role
|
from app.core.roles import AccountKind, Role
|
||||||
from app.models.energy import Recommendation
|
from app.models.energy import Recommendation
|
||||||
from app.services.recommendation import RecommendationNotFoundError
|
from app.services.recommendation import RapportGeneration, RecommendationNotFoundError
|
||||||
|
|
||||||
MOMENT = datetime(2024, 1, 1, tzinfo=UTC)
|
MOMENT = datetime(2024, 1, 1, tzinfo=UTC)
|
||||||
|
|
||||||
@@ -40,6 +40,7 @@ class FauxService:
|
|||||||
def __init__(self, erreur: Exception | None = None) -> None:
|
def __init__(self, erreur: Exception | None = None) -> None:
|
||||||
self._erreur = erreur
|
self._erreur = erreur
|
||||||
self.recommendation = recommendation()
|
self.recommendation = recommendation()
|
||||||
|
self.site_demande: str | None = None
|
||||||
|
|
||||||
async def list_all(self) -> list[Recommendation]:
|
async def list_all(self) -> list[Recommendation]:
|
||||||
return [self.recommendation]
|
return [self.recommendation]
|
||||||
@@ -49,6 +50,10 @@ class FauxService:
|
|||||||
raise self._erreur
|
raise self._erreur
|
||||||
return self.recommendation
|
return self.recommendation
|
||||||
|
|
||||||
|
async def generate(self, *, site_id: str | None = None) -> RapportGeneration:
|
||||||
|
self.site_demande = site_id
|
||||||
|
return RapportGeneration(alertes_examinees=2, recommandations_creees=3, deja_presentes=1)
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def lecteur_connecte(app: FastAPI) -> Iterator[None]:
|
def lecteur_connecte(app: FastAPI) -> Iterator[None]:
|
||||||
@@ -142,3 +147,56 @@ async def test_get_recommendation_returns_404_when_the_session_finds_nothing(
|
|||||||
response = await client.get("/api/v1/recommendations/404")
|
response = await client.get("/api/v1/recommendations/404")
|
||||||
|
|
||||||
assert response.status_code == 404
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def admin_connecte(app: FastAPI) -> Iterator[None]:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: principal(Role.ADMIN)
|
||||||
|
yield
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def servi_en_admin(app: FastAPI, admin_connecte: None) -> Iterator[Callable[[], FauxService]]:
|
||||||
|
def installe() -> FauxService:
|
||||||
|
service = FauxService()
|
||||||
|
app.dependency_overrides[get_recommendation_service] = lambda: service
|
||||||
|
return service
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_recommendation_service, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_generate_recommendations_returns_the_generation_report(
|
||||||
|
servi_en_admin: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi_en_admin()
|
||||||
|
|
||||||
|
response = await client.post("/api/v1/recommendations/generate")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json() == {
|
||||||
|
"alerts_examined": 2,
|
||||||
|
"recommendations_created": 3,
|
||||||
|
"already_present": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_generate_recommendations_forwards_the_requested_site(
|
||||||
|
servi_en_admin: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
service = servi_en_admin()
|
||||||
|
|
||||||
|
await client.post("/api/v1/recommendations/generate", params={"site_id": "SITE002"})
|
||||||
|
|
||||||
|
assert service.site_demande == "SITE002"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_generate_recommendations_refuses_a_reader(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.post("/api/v1/recommendations/generate")
|
||||||
|
|
||||||
|
assert response.status_code == 403
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Ce test est le garde-fou de l'autorisation : rendre une route publique oblige à modifier
|
# Ce test est le garde-fou de l'autorisation : rendre une route publique oblige à modifier
|
||||||
# `ROUTES_PUBLIQUES` ci-dessous, ce qui apparaît en clair dans la diff d'une pull request et
|
# `ROUTES_PUBLIQUES` dans `tests/api/acces.py`, ce qui apparaît en clair dans la diff d'une pull
|
||||||
# demande une justification au relecteur.
|
# request et demande une justification au relecteur.
|
||||||
# Pourquoi : il interroge réellement chaque route sans jeton au lieu d'inspecter l'arbre de
|
# Pourquoi : il interroge réellement chaque route sans jeton au lieu d'inspecter l'arbre de
|
||||||
# dépendances. L'arbre n'est accessible que par l'API privée de FastAPI, et surtout une route
|
# dépendances. L'arbre n'est accessible que par l'API privée de FastAPI, et surtout une route
|
||||||
# peut porter la bonne dépendance tout en répondant quand même.
|
# peut porter la bonne dépendance tout en répondant quand même.
|
||||||
@@ -11,51 +11,64 @@ import pytest
|
|||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from httpx import AsyncClient
|
from httpx import AsyncClient
|
||||||
|
|
||||||
ROUTES_PUBLIQUES = frozenset(
|
from tests.api.acces import (
|
||||||
{
|
ROLE_MINIMUM,
|
||||||
("GET", "/api/v1/health/live"),
|
ROUTE_COOKIE,
|
||||||
("GET", "/api/v1/health/ready"),
|
ROUTES_PUBLIQUES,
|
||||||
("POST", "/api/v1/auth/login"),
|
ROUTES_SANS_ROLE,
|
||||||
# Sans cookie, la déconnexion ne fait rien et répond 204 : elle est idempotente.
|
Route,
|
||||||
("POST", "/api/v1/auth/logout"),
|
chemin_concret,
|
||||||
("GET", "/metrics"),
|
routes_du_schema,
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
|
||||||
VALEURS_DE_SUBSTITUTION = "00000000-0000-0000-0000-000000000000"
|
|
||||||
STATUTS_DE_REFUS = {401, 403}
|
STATUTS_DE_REFUS = {401, 403}
|
||||||
|
HORS_SCHEMA = {("GET", "/metrics")}
|
||||||
|
|
||||||
|
|
||||||
def routes_declarees(app: FastAPI) -> list[tuple[str, str]]:
|
def routes_declarees(app: FastAPI) -> list[Route]:
|
||||||
schema: dict[str, Any] = app.openapi()
|
schema: dict[str, Any] = app.openapi()
|
||||||
return [
|
return routes_du_schema(schema)
|
||||||
(methode.upper(), chemin)
|
|
||||||
for chemin, operations in schema["paths"].items()
|
|
||||||
for methode in operations
|
|
||||||
if methode.upper() in {"GET", "POST", "PATCH", "PUT", "DELETE"}
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def routes_protegees(app: FastAPI) -> list[tuple[str, str]]:
|
def routes_protegees(app: FastAPI) -> list[Route]:
|
||||||
return [route for route in routes_declarees(app) if route not in ROUTES_PUBLIQUES]
|
return [route for route in routes_declarees(app) if route not in ROUTES_PUBLIQUES]
|
||||||
|
|
||||||
|
|
||||||
def test_the_public_allow_list_has_no_stale_entry(app: FastAPI) -> None:
|
def test_the_public_allow_list_has_no_stale_entry(app: FastAPI) -> None:
|
||||||
declarees = set(routes_declarees(app)) | {("GET", "/metrics")}
|
declarees = set(routes_declarees(app)) | HORS_SCHEMA
|
||||||
|
|
||||||
inconnues = ROUTES_PUBLIQUES - declarees
|
inconnues = ROUTES_PUBLIQUES - declarees
|
||||||
|
|
||||||
assert inconnues == set()
|
assert inconnues == set()
|
||||||
|
|
||||||
|
|
||||||
|
# Sans lui, une route ajoutée sans être classée n'est vue par aucun test de rôle : elle hérite
|
||||||
|
# du seul contrôle anonyme, et une garde posée au mauvais niveau passe inaperçue.
|
||||||
|
def test_every_declared_route_is_classified(app: FastAPI) -> None:
|
||||||
|
classees = ROUTES_PUBLIQUES | ROUTE_COOKIE | ROUTES_SANS_ROLE | set(ROLE_MINIMUM)
|
||||||
|
|
||||||
|
non_classees = set(routes_declarees(app)) - classees
|
||||||
|
fantomes = classees - set(routes_declarees(app)) - HORS_SCHEMA
|
||||||
|
|
||||||
|
assert non_classees == set(), "classer la route dans tests/api/acces.py"
|
||||||
|
assert fantomes == set(), "entrée morte : la route n'existe plus sous ce chemin"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_four_classes_of_routes_stay_disjoint() -> None:
|
||||||
|
classes = [ROUTES_PUBLIQUES, ROUTE_COOKIE, ROUTES_SANS_ROLE, frozenset(ROLE_MINIMUM)]
|
||||||
|
|
||||||
|
for rang, classe in enumerate(classes):
|
||||||
|
for autre in classes[rang + 1 :]:
|
||||||
|
assert classe & autre == frozenset()
|
||||||
|
|
||||||
|
|
||||||
async def test_every_route_rejects_an_anonymous_caller_unless_explicitly_public(
|
async def test_every_route_rejects_an_anonymous_caller_unless_explicitly_public(
|
||||||
app: FastAPI, client: AsyncClient
|
app: FastAPI, client: AsyncClient
|
||||||
) -> None:
|
) -> None:
|
||||||
ouvertes: list[tuple[str, str, int]] = []
|
ouvertes: list[tuple[str, str, int]] = []
|
||||||
|
|
||||||
for methode, chemin in routes_protegees(app):
|
for methode, chemin in routes_protegees(app):
|
||||||
concret = chemin.replace("{user_id}", VALEURS_DE_SUBSTITUTION)
|
response = await client.request(methode, chemin_concret(chemin), json={})
|
||||||
response = await client.request(methode, concret, json={})
|
|
||||||
if response.status_code not in STATUTS_DE_REFUS:
|
if response.status_code not in STATUTS_DE_REFUS:
|
||||||
ouvertes.append((methode, chemin, response.status_code))
|
ouvertes.append((methode, chemin, response.status_code))
|
||||||
|
|
||||||
@@ -74,3 +87,19 @@ async def test_the_declared_routes_are_actually_reachable(app: FastAPI) -> None:
|
|||||||
)
|
)
|
||||||
def test_the_health_probes_stay_public(app: FastAPI, chemin: str) -> None:
|
def test_the_health_probes_stay_public(app: FastAPI, chemin: str) -> None:
|
||||||
assert ("GET", chemin) in ROUTES_PUBLIQUES
|
assert ("GET", chemin) in ROUTES_PUBLIQUES
|
||||||
|
|
||||||
|
|
||||||
|
# Piège : ni les routes `include_in_schema=False` (/docs, /redoc) ni un `Mount` Starlette
|
||||||
|
# (/static) n'apparaissent dans `app.openapi()["paths"]`. `routes_declarees()` ne les voit
|
||||||
|
# donc jamais, et elles échapperaient silencieusement au garde-fou ci-dessus.
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"chemin",
|
||||||
|
["/docs", "/redoc", "/static/logo-icon.png"],
|
||||||
|
ids=["swagger_ui", "redoc", "logo_statique"],
|
||||||
|
)
|
||||||
|
async def test_the_documentation_routes_are_public_by_design(
|
||||||
|
app: FastAPI, client: AsyncClient, chemin: str
|
||||||
|
) -> None:
|
||||||
|
response = await client.get(chemin)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
from collections.abc import Callable, Iterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from httpx import AsyncClient
|
||||||
|
|
||||||
|
from app.api.deps import get_current_principal, get_sensor_service
|
||||||
|
from app.core.principal import Principal
|
||||||
|
from app.core.roles import AccountKind, Role
|
||||||
|
from app.services.sensor import DiagnosticCapteur, EtatCapteurs, SanteCapteurs, SanteSite
|
||||||
|
|
||||||
|
TIMESTAMP = datetime(2026, 9, 16, 12, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def principal(role: Role = Role.ADMIN) -> Principal:
|
||||||
|
return Principal(
|
||||||
|
id=uuid4(),
|
||||||
|
email=f"{role.value}@enervision.fr",
|
||||||
|
role=role,
|
||||||
|
kind=AccountKind.HUMAIN,
|
||||||
|
must_change_password=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FauxService:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
ok = DiagnosticCapteur(status="ok", since=None)
|
||||||
|
en_echec = DiagnosticCapteur(status="failing", since=TIMESTAMP)
|
||||||
|
self.etat = EtatCapteurs(
|
||||||
|
timestamp=TIMESTAMP,
|
||||||
|
sites=[
|
||||||
|
SanteSite(
|
||||||
|
site_id="SITE001",
|
||||||
|
site_name="Bureau Paris La Défense",
|
||||||
|
sensors=SanteCapteurs(
|
||||||
|
consumption=ok,
|
||||||
|
electrical=ok,
|
||||||
|
temperature=en_echec,
|
||||||
|
humidity=ok,
|
||||||
|
network=ok,
|
||||||
|
),
|
||||||
|
overall="degraded",
|
||||||
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
async def status(self) -> EtatCapteurs:
|
||||||
|
return self.etat
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def admin_connecte(app: FastAPI) -> Iterator[None]:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: principal()
|
||||||
|
yield
|
||||||
|
app.dependency_overrides.pop(get_current_principal, None)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def servi(app: FastAPI, admin_connecte: None) -> Iterator[Callable[[], FauxService]]:
|
||||||
|
def installe() -> FauxService:
|
||||||
|
service = FauxService()
|
||||||
|
app.dependency_overrides[get_sensor_service] = lambda: service
|
||||||
|
return service
|
||||||
|
|
||||||
|
yield installe
|
||||||
|
app.dependency_overrides.pop(get_sensor_service, None)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_status_returns_the_service_result(
|
||||||
|
servi: Callable[[], FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/sensors/status")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
corps = response.json()
|
||||||
|
assert corps["sites"][0]["site_id"] == "SITE001"
|
||||||
|
assert corps["sites"][0]["overall"] == "degraded"
|
||||||
|
assert corps["sites"][0]["sensors"]["temperature"]["status"] == "failing"
|
||||||
|
assert corps["sites"][0]["sensors"]["consumption"]["status"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_status_refuses_a_reader(app: FastAPI, client: AsyncClient) -> None:
|
||||||
|
app.dependency_overrides[get_current_principal] = lambda: principal(Role.LECTEUR)
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/sensors/status")
|
||||||
|
|
||||||
|
assert response.status_code == 403
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
from collections.abc import Callable, Iterator
|
from collections.abc import Callable, Iterator
|
||||||
|
from datetime import UTC, datetime
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -9,7 +10,9 @@ from app.api.deps import get_current_principal, get_site_service
|
|||||||
from app.core.principal import Principal
|
from app.core.principal import Principal
|
||||||
from app.core.roles import AccountKind, Role
|
from app.core.roles import AccountKind, Role
|
||||||
from app.models.energy import Site
|
from app.models.energy import Site
|
||||||
from app.services.site import SiteNotFoundError
|
from app.services.site import SiteCurrentReading, SiteNotFoundError
|
||||||
|
|
||||||
|
TIMESTAMP = datetime(2026, 9, 16, 12, 0, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
def principal(role: Role = Role.LECTEUR) -> Principal:
|
def principal(role: Role = Role.LECTEUR) -> Principal:
|
||||||
@@ -33,10 +36,28 @@ def site(site_id: str = "site-1") -> Site:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def lecture_actuelle(site_id: str = "site-1") -> SiteCurrentReading:
|
||||||
|
return SiteCurrentReading(
|
||||||
|
timestamp=TIMESTAMP,
|
||||||
|
site_id=site_id,
|
||||||
|
site_type="industriel",
|
||||||
|
consumption_kw=87.34,
|
||||||
|
consumption_kwh=87.34,
|
||||||
|
voltage_v=401.2,
|
||||||
|
current_a=132.5,
|
||||||
|
power_factor=0.923,
|
||||||
|
temperature_celsius=22.1,
|
||||||
|
humidity_percent=58.4,
|
||||||
|
null_reasons=[],
|
||||||
|
data_quality="good",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class FauxService:
|
class FauxService:
|
||||||
def __init__(self, erreur: Exception | None = None) -> None:
|
def __init__(self, erreur: Exception | None = None) -> None:
|
||||||
self._erreur = erreur
|
self._erreur = erreur
|
||||||
self.site = site()
|
self.site = site()
|
||||||
|
self.actuel = lecture_actuelle()
|
||||||
|
|
||||||
async def list_all(self) -> list[Site]:
|
async def list_all(self) -> list[Site]:
|
||||||
return [self.site]
|
return [self.site]
|
||||||
@@ -46,6 +67,11 @@ class FauxService:
|
|||||||
raise self._erreur
|
raise self._erreur
|
||||||
return self.site
|
return self.site
|
||||||
|
|
||||||
|
async def current(self, site_id: str) -> SiteCurrentReading:
|
||||||
|
if self._erreur is not None:
|
||||||
|
raise self._erreur
|
||||||
|
return self.actuel
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def lecteur_connecte(app: FastAPI) -> Iterator[None]:
|
def lecteur_connecte(app: FastAPI) -> Iterator[None]:
|
||||||
@@ -109,6 +135,30 @@ async def test_get_site_returns_404_for_an_unknown_site(
|
|||||||
assert response.status_code == 404
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_current_returns_the_latest_reading(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi()
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/sites/site-1/current")
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
corps = response.json()
|
||||||
|
assert corps["site_id"] == "site-1"
|
||||||
|
assert corps["data_quality"] == "good"
|
||||||
|
assert corps["consumption_kw"] == 87.34
|
||||||
|
|
||||||
|
|
||||||
|
async def test_get_current_returns_404_for_an_unknown_site(
|
||||||
|
servi: Callable[..., FauxService], client: AsyncClient
|
||||||
|
) -> None:
|
||||||
|
servi(SiteNotFoundError("site-inconnu"))
|
||||||
|
|
||||||
|
response = await client.get("/api/v1/sites/site-inconnu/current")
|
||||||
|
|
||||||
|
assert response.status_code == 404
|
||||||
|
|
||||||
|
|
||||||
async def test_list_sites_reaches_the_repository_through_the_session(
|
async def test_list_sites_reaches_the_repository_through_the_session(
|
||||||
lecteur_connecte: None, fake_session: Callable[..., None], client: AsyncClient
|
lecteur_connecte: None, fake_session: Callable[..., None], client: AsyncClient
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime, timedelta
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -9,7 +9,15 @@ from sqlalchemy.exc import IntegrityError
|
|||||||
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine
|
||||||
|
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.models.energy import Alert, Dataset, Prediction, Reading, Recommendation, Site
|
from app.models.energy import (
|
||||||
|
Alert,
|
||||||
|
Dataset,
|
||||||
|
DriftReport,
|
||||||
|
Prediction,
|
||||||
|
Reading,
|
||||||
|
Recommendation,
|
||||||
|
Site,
|
||||||
|
)
|
||||||
|
|
||||||
pytestmark = pytest.mark.integration
|
pytestmark = pytest.mark.integration
|
||||||
MOMENT = datetime(2024, 1, 1, tzinfo=UTC)
|
MOMENT = datetime(2024, 1, 1, tzinfo=UTC)
|
||||||
@@ -112,8 +120,10 @@ async def test_duplicate_reading_is_rejected_when_key_matches(
|
|||||||
)
|
)
|
||||||
await data_connection.execute(statement)
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
with pytest.raises(IntegrityError):
|
with pytest.raises(IntegrityError):
|
||||||
async with data_connection.begin_nested():
|
async with savepoint:
|
||||||
await data_connection.execute(statement)
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
@@ -147,9 +157,12 @@ async def test_invalid_reading_is_rejected_when_constraints_fail(
|
|||||||
}
|
}
|
||||||
values.update(changes)
|
values.update(changes)
|
||||||
|
|
||||||
|
statement = insert(Reading).values(**values)
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
with pytest.raises(IntegrityError):
|
with pytest.raises(IntegrityError):
|
||||||
async with data_connection.begin_nested():
|
async with savepoint:
|
||||||
await data_connection.execute(insert(Reading).values(**values))
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
async def test_prediction_requires_period_when_energy_is_predicted(
|
async def test_prediction_requires_period_when_energy_is_predicted(
|
||||||
@@ -164,8 +177,10 @@ async def test_prediction_requires_period_when_energy_is_predicted(
|
|||||||
model_reference="test-model/1",
|
model_reference="test-model/1",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
with pytest.raises(IntegrityError):
|
with pytest.raises(IntegrityError):
|
||||||
async with data_connection.begin_nested():
|
async with savepoint:
|
||||||
await data_connection.execute(statement)
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
@@ -212,10 +227,7 @@ async def test_alert_rejects_prediction_when_site_differs(
|
|||||||
)
|
)
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
with pytest.raises(IntegrityError):
|
statement = insert(Alert).values(
|
||||||
async with data_connection.begin_nested():
|
|
||||||
await data_connection.execute(
|
|
||||||
insert(Alert).values(
|
|
||||||
source_alert_id=str(uuid4()),
|
source_alert_id=str(uuid4()),
|
||||||
site_id=other_site,
|
site_id=other_site,
|
||||||
source="enervision",
|
source="enervision",
|
||||||
@@ -226,7 +238,11 @@ async def test_alert_rejects_prediction_when_site_differs(
|
|||||||
prediction_id=prediction_id,
|
prediction_id=prediction_id,
|
||||||
raw_data={},
|
raw_data={},
|
||||||
)
|
)
|
||||||
)
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
|
with pytest.raises(IntegrityError):
|
||||||
|
async with savepoint:
|
||||||
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
async def test_recommendation_is_unique_when_alert_and_rule_match(
|
async def test_recommendation_is_unique_when_alert_and_rule_match(
|
||||||
@@ -256,6 +272,67 @@ async def test_recommendation_is_unique_when_alert_and_rule_match(
|
|||||||
)
|
)
|
||||||
await data_connection.execute(statement)
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
with pytest.raises(IntegrityError):
|
with pytest.raises(IntegrityError):
|
||||||
async with data_connection.begin_nested():
|
async with savepoint:
|
||||||
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
|
def _rapport(**remplacements: object) -> dict[str, object]:
|
||||||
|
defauts: dict[str, object] = {
|
||||||
|
"site_id": None,
|
||||||
|
"window_start": MOMENT,
|
||||||
|
"window_end": MOMENT,
|
||||||
|
"n_observations": 12,
|
||||||
|
"model_references": ["lightgbm-aaa"],
|
||||||
|
"status": "stable",
|
||||||
|
"reason": None,
|
||||||
|
}
|
||||||
|
return {**defauts, **remplacements}
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_report_rejects_an_unknown_status(data_connection: AsyncConnection) -> None:
|
||||||
|
statement = insert(DriftReport).values(**_rapport(status="douteux", reason="x"))
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
|
with pytest.raises(IntegrityError):
|
||||||
|
async with savepoint:
|
||||||
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_report_rejects_a_drift_without_a_reason(
|
||||||
|
data_connection: AsyncConnection,
|
||||||
|
) -> None:
|
||||||
|
statement = insert(DriftReport).values(**_rapport(status="derive"))
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
|
with pytest.raises(IntegrityError):
|
||||||
|
async with savepoint:
|
||||||
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_report_accepts_one_global_row_without_a_site(
|
||||||
|
data_connection: AsyncConnection,
|
||||||
|
) -> None:
|
||||||
|
identifiant = (
|
||||||
|
await data_connection.execute(
|
||||||
|
insert(DriftReport).values(**_rapport()).returning(DriftReport.drift_report_id)
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
assert identifiant is not None
|
||||||
|
|
||||||
|
|
||||||
|
async def test_drift_report_is_unique_when_window_and_site_match(
|
||||||
|
data_connection: AsyncConnection,
|
||||||
|
) -> None:
|
||||||
|
fenetre = MOMENT + timedelta(days=1)
|
||||||
|
statement = insert(DriftReport).values(**_rapport(window_end=fenetre))
|
||||||
|
await data_connection.execute(statement)
|
||||||
|
|
||||||
|
savepoint = data_connection.begin_nested()
|
||||||
|
|
||||||
|
with pytest.raises(IntegrityError):
|
||||||
|
async with savepoint:
|
||||||
await data_connection.execute(statement)
|
await data_connection.execute(statement)
|
||||||
|
|||||||
@@ -0,0 +1,245 @@
|
|||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pandas as pd
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.etl.historical_import import (
|
||||||
|
SOURCE_NAME,
|
||||||
|
build_reading_batch,
|
||||||
|
classify_quality,
|
||||||
|
compute_sha256,
|
||||||
|
load_metadata,
|
||||||
|
normalize_timestamps,
|
||||||
|
validate_source,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def make_metadata() -> dict:
|
||||||
|
return {
|
||||||
|
"total_records": 2,
|
||||||
|
"sites": {
|
||||||
|
"SITE001": {},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def make_dataframe() -> pd.DataFrame:
|
||||||
|
return pd.DataFrame(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"timestamp": "2023-01-01 00:00:00",
|
||||||
|
"site_id": "SITE001",
|
||||||
|
"site_type": "office",
|
||||||
|
"site_name": "Site 1",
|
||||||
|
"consumption_kwh": 10.5,
|
||||||
|
"consumption_euros": 2.5,
|
||||||
|
"temperature_celsius": 20.0,
|
||||||
|
"humidity_percent": 50.0,
|
||||||
|
"solar_irradiance_wm2": 0.0,
|
||||||
|
"hour": 0,
|
||||||
|
"day_of_week": 6,
|
||||||
|
"day_name": "Sunday",
|
||||||
|
"month": 1,
|
||||||
|
"is_weekend": True,
|
||||||
|
"is_working_hours": False,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"timestamp": "2023-01-01 01:00:00",
|
||||||
|
"site_id": "SITE001",
|
||||||
|
"site_type": "office",
|
||||||
|
"site_name": "Site 1",
|
||||||
|
"consumption_kwh": 11.0,
|
||||||
|
"consumption_euros": 2.7,
|
||||||
|
"temperature_celsius": 19.5,
|
||||||
|
"humidity_percent": 52.0,
|
||||||
|
"solar_irradiance_wm2": 0.0,
|
||||||
|
"hour": 1,
|
||||||
|
"day_of_week": 6,
|
||||||
|
"day_name": "Sunday",
|
||||||
|
"month": 1,
|
||||||
|
"is_weekend": True,
|
||||||
|
"is_working_hours": False,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_compute_sha256(tmp_path):
|
||||||
|
file_path = tmp_path / "dataset.csv"
|
||||||
|
content = b"hello-enervision"
|
||||||
|
|
||||||
|
file_path.write_bytes(content)
|
||||||
|
|
||||||
|
expected = hashlib.sha256(content).hexdigest()
|
||||||
|
|
||||||
|
assert compute_sha256(file_path) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_metadata(tmp_path):
|
||||||
|
metadata_path = tmp_path / "metadata.json"
|
||||||
|
|
||||||
|
metadata = {
|
||||||
|
"total_records": 2,
|
||||||
|
"sites": {
|
||||||
|
"SITE001": {},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata_path.write_text(
|
||||||
|
json.dumps(metadata),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert load_metadata(metadata_path) == metadata
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_source_accepts_valid_dataset():
|
||||||
|
frame = make_dataframe()
|
||||||
|
|
||||||
|
validate_source(
|
||||||
|
frame,
|
||||||
|
make_metadata(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_source_rejects_missing_column():
|
||||||
|
frame = make_dataframe().drop(columns=["consumption_kwh"])
|
||||||
|
|
||||||
|
metadata = make_metadata()
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
ValueError,
|
||||||
|
match="Colonnes obligatoires absentes",
|
||||||
|
):
|
||||||
|
validate_source(
|
||||||
|
frame,
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_source_rejects_duplicates():
|
||||||
|
frame = make_dataframe()
|
||||||
|
|
||||||
|
frame.loc[1, "timestamp"] = frame.loc[
|
||||||
|
0,
|
||||||
|
"timestamp",
|
||||||
|
]
|
||||||
|
|
||||||
|
metadata = make_metadata()
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
ValueError,
|
||||||
|
match="doublons",
|
||||||
|
):
|
||||||
|
validate_source(
|
||||||
|
frame,
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validate_source_rejects_unknown_site():
|
||||||
|
frame = make_dataframe()
|
||||||
|
|
||||||
|
frame.loc[1, "site_id"] = "SITE999"
|
||||||
|
|
||||||
|
metadata = make_metadata()
|
||||||
|
|
||||||
|
with pytest.raises(
|
||||||
|
ValueError,
|
||||||
|
match="Sites incohérents",
|
||||||
|
):
|
||||||
|
validate_source(
|
||||||
|
frame,
|
||||||
|
metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_timestamps_adds_timezone():
|
||||||
|
frame = make_dataframe()
|
||||||
|
|
||||||
|
normalized = normalize_timestamps(
|
||||||
|
frame,
|
||||||
|
"UTC",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert normalized["timestamp"].dt.tz is not None
|
||||||
|
|
||||||
|
assert "_source_timestamp" in normalized.columns
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_quality_good():
|
||||||
|
row = make_dataframe().iloc[0].to_dict()
|
||||||
|
|
||||||
|
quality, reasons = classify_quality(row)
|
||||||
|
|
||||||
|
assert quality == "good"
|
||||||
|
assert reasons == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_quality_degraded_when_consumption_missing():
|
||||||
|
row = make_dataframe().iloc[0].to_dict()
|
||||||
|
row["consumption_kwh"] = None
|
||||||
|
|
||||||
|
quality, reasons = classify_quality(row)
|
||||||
|
|
||||||
|
assert quality == "degraded"
|
||||||
|
|
||||||
|
assert "missing:consumption_kwh" in reasons
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_reading_batch_respects_database_contract():
|
||||||
|
frame = normalize_timestamps(
|
||||||
|
make_dataframe(),
|
||||||
|
"UTC",
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = build_reading_batch(
|
||||||
|
frame.iloc[:1],
|
||||||
|
dataset_id=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(rows) == 1
|
||||||
|
|
||||||
|
row = rows[0]
|
||||||
|
|
||||||
|
assert row["dataset_id"] == 3
|
||||||
|
|
||||||
|
# Important :
|
||||||
|
# contrainte ck_reading_dataset_source.
|
||||||
|
assert row["source"] == "csv"
|
||||||
|
assert SOURCE_NAME == "csv"
|
||||||
|
|
||||||
|
# Important :
|
||||||
|
# contrainte ck_reading_imputation.
|
||||||
|
assert row["imputed_values"] is None
|
||||||
|
assert row["imputation_method"] is None
|
||||||
|
|
||||||
|
assert row["data_quality"] == "good"
|
||||||
|
assert row["null_reasons"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_reading_batch_keeps_missing_values():
|
||||||
|
frame = make_dataframe()
|
||||||
|
|
||||||
|
frame.loc[0, "temperature_celsius"] = None
|
||||||
|
|
||||||
|
frame = normalize_timestamps(
|
||||||
|
frame,
|
||||||
|
"UTC",
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = build_reading_batch(
|
||||||
|
frame.iloc[:1],
|
||||||
|
dataset_id=3,
|
||||||
|
)
|
||||||
|
|
||||||
|
row = rows[0]
|
||||||
|
|
||||||
|
assert row["temperature_celsius"] is None
|
||||||
|
|
||||||
|
assert "missing:temperature_celsius" in row["null_reasons"]
|
||||||
|
|
||||||
|
# RAW ingestion : aucune imputation.
|
||||||
|
assert row["imputed_values"] is None
|
||||||
|
assert row["imputation_method"] is None
|
||||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user