From dc952d13aa77b1b1b4fb7924f61a81b406a363ea Mon Sep 17 00:00:00 2001 From: Johan LEROY Date: Wed, 23 Sep 2026 09:41:12 +0200 Subject: [PATCH] =?UTF-8?q?feat(monitoring):=20supervise=20l'API,=20la=20b?= =?UTF-8?q?ase=20et=20l'h=C3=B4te=20avec=20Prometheus=20et=20Grafana?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit L'API exposait /metrics, mais aucun collecteur ne le lisait : monitoring/ ne contenait que des .gitkeep. Sous le profil Compose `monitoring` : prometheus, alertmanager, grafana, postgres-exporter, node-exporter et cadvisor. Tous ont un mem_limit, pour environ 700 Mo au total sur la VM de 8 Go, et leurs interfaces n'écoutent que sur 127.0.0.1. Le profil est actif en prod via COMPOSE_PROFILES, donc à chaque déploiement, et se lance à la demande ailleurs (make monitoring-up). - Neuf règles d'alerte (API, base, hôte, cibles). Chacune a un cas dans les tests joués par `promtool test rules`, en CI comme par make monitoring-check. - Alertmanager route les alertes par courriel vers Mailpit ; un critical masque le warning de la même cible. - Grafana est provisionné : sources Prometheus et TimescaleDB, et trois tableaux de bord (API, données et dérive du modèle, infrastructure). - Le rôle PostgreSQL `supervision` est en lecture seule sur les seules tables métier (db/roles/supervision.sql), posé par make db-ensure-supervision et par stack-up quand le profil est actif. - Le jeton de /metrics passe à Prometheus en secret Compose (APP_METRICS_TOKEN) ; provision-host.sh génère ce secret et les deux autres. Backend : - un APP_METRICS_TOKEN vide vaut absent ; - les sondes de santé ne comptent plus dans les métriques ; - seaux de latence fins autour de 500 ms ; - un registre Prometheus par application, sans quoi toute application créée après la première (dans les tests) ne mesurait rien. Réf : #26 --- .env.example | 16 + .github/workflows/ci.yml | 1 + .github/workflows/e2e.yml | 4 + .github/workflows/infra.yml | 17 +- .gitignore | 2 - Makefile | 44 +- apps/backend/app/core/config.py | 9 +- apps/backend/app/main.py | 22 +- apps/backend/tests/api/test_hardening.py | 14 + db/README.md | 7 +- db/roles/supervision.sql | 15 + docker-compose.yml | 114 ++++ monitoring/README.md | 86 ++- monitoring/alertmanager/.gitkeep | 0 monitoring/alertmanager/alertmanager.yml | 31 + monitoring/grafana/dashboards/.gitkeep | 0 monitoring/grafana/dashboards/api.json | 578 +++++++++++++++++ monitoring/grafana/dashboards/donnees.json | 528 ++++++++++++++++ monitoring/grafana/dashboards/infra.json | 583 ++++++++++++++++++ .../grafana/provisioning/dashboards/.gitkeep | 0 .../provisioning/dashboards/dashboards.yml | 10 + .../grafana/provisioning/datasources/.gitkeep | 0 .../provisioning/datasources/datasources.yml | 28 + monitoring/prometheus/prometheus.yml | 43 ++ monitoring/prometheus/rules/.gitkeep | 0 monitoring/prometheus/rules/enervision.yml | 103 ++++ .../prometheus/tests/enervision.test.yml | 112 ++++ scripts/provision-host.sh | 16 +- 28 files changed, 2364 insertions(+), 19 deletions(-) create mode 100644 db/roles/supervision.sql delete mode 100644 monitoring/alertmanager/.gitkeep create mode 100644 monitoring/alertmanager/alertmanager.yml delete mode 100644 monitoring/grafana/dashboards/.gitkeep create mode 100644 monitoring/grafana/dashboards/api.json create mode 100644 monitoring/grafana/dashboards/donnees.json create mode 100644 monitoring/grafana/dashboards/infra.json delete mode 100644 monitoring/grafana/provisioning/dashboards/.gitkeep create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml delete mode 100644 monitoring/grafana/provisioning/datasources/.gitkeep create mode 100644 monitoring/grafana/provisioning/datasources/datasources.yml create mode 100644 monitoring/prometheus/prometheus.yml delete mode 100644 monitoring/prometheus/rules/.gitkeep create mode 100644 monitoring/prometheus/rules/enervision.yml create mode 100644 monitoring/prometheus/tests/enervision.test.yml diff --git a/.env.example b/.env.example index 3125b73..69dace9 100644 --- a/.env.example +++ b/.env.example @@ -74,3 +74,19 @@ PROXY_HTTPS_PORT= # nombre de workers vaut 1 par défaut, contre 4 pour le webserver d'Airflow 2. TS_TUNE_MEMORY=2GB TS_TUNE_NUM_CPUS=2 + +# Supervision (ADR 0016) : `monitoring` la démarre avec `make stack-up`, réglage de la prod. +# Vide ailleurs, où `make monitoring-up` la lance à la demande. +COMPOSE_PROFILES= +# Jeton présenté par Prometheus sur `/metrics`, exigé par l'API dès qu'il est posé. Requis dès +# que la supervision tourne ; même générateur que APP_SECRET_KEY. +APP_METRICS_TOKEN=change_me +# Compte `admin` de Grafana. Sans lui, le conteneur refuse de démarrer. +GRAFANA_ADMIN_PASSWORD=change_me +# Rôle PostgreSQL `supervision`, en lecture seule, de Grafana et de postgres-exporter +# (db/roles/supervision.sql, posé par `make db-ensure-supervision`). +SUPERVISION_DB_PASSWORD=change_me +# Interfaces publiées sur 127.0.0.1 seulement, par tunnel SSH. 3000 est pris par le frontend. +GRAFANA_PORT=3001 +PROMETHEUS_PORT=9090 +ALERTMANAGER_PORT=9093 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abac168..f342eec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,7 @@ jobs: compose: - "docker-compose*.yml" - ".env.example" + - "monitoring/**" - ".github/workflows/infra.yml" workflows: - ".github/**" diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d3c6268..1a7fec5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -44,6 +44,10 @@ jobs: - name: Applique les migrations run: docker compose exec -T backend alembic upgrade head + # Même cible que `make stack-up` en prod : les droits du rôle portent sur le schéma réel. + - name: Pose le rôle de supervision en lecture seule + run: make db-ensure-supervision + - name: Sème le jeu de démonstration run: docker compose exec -T db psql -U enervision -d enervision -v ON_ERROR_STOP=1 < db/seeds/demo.sql diff --git a/.github/workflows/infra.yml b/.github/workflows/infra.yml index a7e6a38..51884b2 100644 --- a/.github/workflows/infra.yml +++ b/.github/workflows/infra.yml @@ -54,7 +54,7 @@ jobs: done compose: - name: Validation des fichiers Compose + name: Validation des fichiers Compose et de la supervision if: inputs.compose runs-on: ubuntu-latest timeout-minutes: 10 @@ -71,7 +71,20 @@ jobs: run: docker compose config --quiet - name: Valide la stack déployée, profils compris - run: docker compose -f docker-compose.yml -f docker-compose.prod.yml --profile acme config --quiet + run: docker compose -f docker-compose.yml -f docker-compose.prod.yml --profile acme --profile monitoring --profile load config --quiet + + # Mêmes commandes que `make monitoring-check` : images et montages viennent du fichier Compose. + - name: Valide la configuration de Prometheus et ses règles + run: docker compose --profile monitoring run --rm --no-deps --entrypoint promtool prometheus check config /etc/prometheus/prometheus.yml + + - name: Joue les tests unitaires des règles d'alerte + run: docker compose --profile monitoring run --rm --no-deps --entrypoint promtool prometheus test rules /etc/prometheus/tests/enervision.test.yml + + - name: Valide la configuration d'Alertmanager + run: docker compose --profile monitoring run --rm --no-deps --entrypoint amtool alertmanager check-config /etc/alertmanager/alertmanager.yml + + - name: Valide les tableaux de bord Grafana + run: for tableau in monitoring/grafana/dashboards/*.json; do jq empty "$tableau"; done workflows: name: Analyse des workflows diff --git a/.gitignore b/.gitignore index d9dbdc3..d3b47d8 100644 --- a/.gitignore +++ b/.gitignore @@ -60,8 +60,6 @@ secrets/ data/raw/* !data/raw/.gitkeep *.sqlite3 -monitoring/grafana/data/ -monitoring/prometheus/data/ # ML : jeu de donnees, modeles entraines et suivi MLflow local, tous generes/volumineux ml/data/ diff --git a/Makefile b/Makefile index b410333..c05f0a5 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,19 @@ PG_TEST_DB ?= enervision_test TEST_DATABASE_URL ?= postgresql+asyncpg://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_TEST_DB) ML_TEST_DATABASE_URL ?= postgresql+psycopg://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_TEST_DB) +# Piege : ni make ni ces cibles ne lisent `.env` pour COMPOSE_PROFILES, que docker compose y lit +# seul. `stack-up` le relit ici pour savoir s'il doit poser le role `supervision` apres migration. +SUPERVISION := $(findstring monitoring,$(COMPOSE_PROFILES) $(call env-val,COMPOSE_PROFILES)) +SERVICES_SUPERVISION := prometheus alertmanager grafana postgres-exporter node-exporter cadvisor +GRAFANA_PORT := $(or $(strip $(call env-val,GRAFANA_PORT)),3001) +PROMETHEUS_PORT := $(or $(strip $(call env-val,PROMETHEUS_PORT)),9090) +supervision-garde = for cle in APP_METRICS_TOKEN GRAFANA_ADMIN_PASSWORD SUPERVISION_DB_PASSWORD; do \ + sed -n "s/^$$cle=//p" .env 2>/dev/null | tail -1 | grep -q . \ + || { echo "$$cle manquant dans .env, requis par la supervision (cf. .env.example)"; exit 1; }; \ + done +MONITORING := docker compose --profile monitoring +PROMTOOL := $(MONITORING) run --rm --no-deps --entrypoint promtool prometheus + # Piege : `e2e-prepare` ajoute trois sites `demo-*` et des comptes `test-*` a la base visee. Elle # vise la base de `make dev` ; ne jamais la lancer contre la recette ou la prod. E2E_COMPTES ?= $(CURDIR)/$(E2E)/.comptes.json @@ -61,7 +74,8 @@ DEMO_NOW ?= 2024-12-31T00:00:00Z ml-lint ml-typecheck ml-test ml-check ml-train ml-score mlflow-up detect-alerts recommendations \ airflow-lint airflow-test airflow-check airflow-up airflow-down airflow-logs \ tls-selfsigned tls-acme tls-renew stack-up stack-down stack-logs \ - e2e-install e2e-prepare e2e load-smoke load-test load-stress load-limits + e2e-install e2e-prepare e2e load-smoke load-test load-stress load-limits \ + db-ensure-supervision monitoring-up monitoring-down monitoring-logs monitoring-check help: ## Liste les cibles disponibles @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' @@ -197,8 +211,10 @@ stack-up: ## Démarre la stack derrière le reverse proxy, puis migre la base. P || { echo "Aucun certificat dans infra/proxy/tls. Lancer d'abord make tls-selfsigned"; exit 1; } @openssl x509 -in infra/proxy/tls/fullchain.pem -noout -checkhost "$(PUBLIC_HOST)" >/dev/null \ || { echo "Le certificat ne couvre pas $(PUBLIC_HOST). Relancer make tls-selfsigned PUBLIC_HOST=$(PUBLIC_HOST) FORCE=1"; exit 1; } + @$(if $(SUPERVISION),$(supervision-garde),true) $(COMPOSE_PROD) up -d --build $(COMPOSE_PROD) exec -T backend alembic upgrade head + @$(if $(SUPERVISION),$(MAKE) --no-print-directory db-ensure-supervision,true) stack-down: ## Arrête la stack complète en conservant les données $(COMPOSE_PROD) stop @@ -242,6 +258,32 @@ load-stress: ## Monte le débit jusqu'à la rupture de l'API. Sur la VM, la prod load-limits: ## Vérifie par le proxy que nginx limite le débit d'une même adresse (429) $(call k6-run,limitation-debit) +# Piege : le mot de passe est lu dans `.env` par le shell et passe a psql sur son entree +# standard. Developpe par make, il apparaitrait en clair dans la ligne de commande (`ps`). +db-ensure-supervision: ## Crée ou réaligne le rôle `supervision`, en lecture seule, de Grafana et de l'exportateur + @mdp="$$(sed -n 's/^SUPERVISION_DB_PASSWORD=//p' .env 2>/dev/null | tail -1)"; \ + [ -n "$$mdp" ] || { echo "SUPERVISION_DB_PASSWORD manquant dans .env"; exit 1; }; \ + { printf '\\set mot_de_passe %s\n' "$$mdp"; cat db/roles/supervision.sql; } \ + | docker compose exec -T db psql -U $(PG_USER) -d $(PG_DB) -v ON_ERROR_STOP=1 -v base=$(PG_DB) -q + +monitoring-up: ## Démarre la supervision sur la stack en cours : Prometheus, Alertmanager, Grafana, exporteurs + @$(supervision-garde) + $(MONITORING) up -d --no-deps $(SERVICES_SUPERVISION) + @$(MAKE) --no-print-directory db-ensure-supervision + @echo "grafana -> http://localhost:$(GRAFANA_PORT) prometheus -> http://localhost:$(PROMETHEUS_PORT)" + +monitoring-down: ## Arrête la supervision en conservant ses données + $(MONITORING) stop $(SERVICES_SUPERVISION) + +monitoring-logs: ## Suit les journaux de Prometheus, Alertmanager et Grafana + $(MONITORING) logs -f prometheus alertmanager grafana + +monitoring-check: ## Valide la configuration de supervision et joue les tests des règles d'alerte, comme la CI + $(PROMTOOL) check config /etc/prometheus/prometheus.yml + $(PROMTOOL) test rules /etc/prometheus/tests/enervision.test.yml + $(MONITORING) run --rm --no-deps --entrypoint amtool alertmanager check-config /etc/alertmanager/alertmanager.yml + @for tableau in monitoring/grafana/dashboards/*.json; do jq empty "$$tableau" || exit 1; done + db-up: ## Démarre la base PostgreSQL TimescaleDB docker compose up -d db diff --git a/apps/backend/app/core/config.py b/apps/backend/app/core/config.py index e622ea7..be3a63b 100644 --- a/apps/backend/app/core/config.py +++ b/apps/backend/app/core/config.py @@ -1,7 +1,7 @@ from functools import lru_cache from typing import Literal, Self -from pydantic import Field, SecretStr, model_validator +from pydantic import Field, SecretStr, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict Environment = Literal["local", "dev", "staging", "prod"] @@ -76,6 +76,13 @@ class Settings(BaseSettings): expose_api_docs: bool | None = None metrics_token: SecretStr | None = None + # Compose passe `APP_METRICS_TOKEN` vide quand aucun jeton n'est posé : vide vaut absent, sinon + # `/metrics` exigerait un `Bearer` sans valeur et plus rien ne pourrait le scruter. + @field_validator("metrics_token", mode="before") + @classmethod + def _jeton_vide_vaut_absent(cls, valeur: object) -> object: + return None if valeur == "" else valeur + @property def allowed_origins(self) -> list[str]: return [origin.strip() for origin in self.cors_origins.split(",") if origin.strip()] diff --git a/apps/backend/app/main.py b/apps/backend/app/main.py index 1380a84..c2fc685 100644 --- a/apps/backend/app/main.py +++ b/apps/backend/app/main.py @@ -6,7 +6,8 @@ from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html from fastapi.staticfiles import StaticFiles -from prometheus_fastapi_instrumentator import Instrumentator +from prometheus_client import CollectorRegistry, GCCollector, PlatformCollector, ProcessCollector +from prometheus_fastapi_instrumentator import Instrumentator, metrics from starlette.requests import Request from starlette.responses import HTMLResponse @@ -37,6 +38,16 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await get_engine().dispose() +# Pourquoi : le registre global n'accepte chaque métrique qu'une fois. Toute application créée +# après la première, dans les tests notamment, n'aurait rien mesuré. +def _registre_de_metriques() -> CollectorRegistry: + registre = CollectorRegistry() + ProcessCollector(registry=registre) + PlatformCollector(registry=registre) + GCCollector(registry=registre) + return registre + + def create_app(settings: Settings | None = None) -> FastAPI: resolved = settings or get_settings() configure_logging(resolved) @@ -102,7 +113,14 @@ def create_app(settings: Settings | None = None) -> FastAPI: register_error_handlers(application) - Instrumentator().instrument(application).expose( + # Les sondes de santé tombent toutes les 30 s : comptées, elles fausseraient latences et débit. + # Seaux fins autour du seuil de charge (p95 < 500 ms, ADR 0015), route par route. + registre = _registre_de_metriques() + Instrumentator( + excluded_handlers=["/metrics", f"{resolved.api_prefix}/health/.*"], registry=registre + ).add( + metrics.default(latency_lowr_buckets=(0.05, 0.1, 0.25, 0.5, 1, 2.5), registry=registre) + ).instrument(application).expose( application, endpoint="/metrics", include_in_schema=False, diff --git a/apps/backend/tests/api/test_hardening.py b/apps/backend/tests/api/test_hardening.py index 3ee1170..e278169 100644 --- a/apps/backend/tests/api/test_hardening.py +++ b/apps/backend/tests/api/test_hardening.py @@ -71,6 +71,20 @@ async def test_metrics_stay_open_when_no_token_is_configured(client: AsyncClient assert response.status_code == 200 +async def test_an_empty_metrics_token_means_no_token() -> None: + assert (await interroge({"metrics_token": ""}, "/metrics")).status_code == 200 + + +async def test_metrics_ignore_health_probes_but_count_business_routes(client: AsyncClient) -> None: + await client.get("/api/v1/health/live") + await client.get("/api/v1/sites") + + exposition = (await client.get("/metrics")).text + + assert 'handler="/api/v1/health/live"' not in exposition + assert 'handler="/api/v1/sites"' in exposition + + async def test_metrics_demand_the_token_once_one_is_configured() -> None: surcharges = {"metrics_token": "un-jeton-de-supervision-assez-long"} diff --git a/db/README.md b/db/README.md index fd62d6f..265cfb6 100644 --- a/db/README.md +++ b/db/README.md @@ -5,7 +5,12 @@ PostgreSQL 17 avec l'extension TimescaleDB, servie en local par le service `db` - `init` : scripts de bootstrap joues au premier demarrage du conteneur. - `migrations` : migrations SQL versionnees. -- `seeds` : jeux de donnees de reference. +- `seeds` : jeux de donnees de reference. `demo.sql` seme trois sites `demo-*`, 72 heures de + releves, des alertes et des rapports de derive pour la CI, l'e2e et les tirs de charge. Base + jetable seulement. +- `roles` : roles PostgreSQL hors schema applicatif. `supervision.sql` pose le role en lecture + seule de Grafana et de postgres-exporter, rejoue par `make db-ensure-supervision` (et par + `make stack-up` quand la supervision est active) plutot que par `init`, qui ne rejoue jamais. Les migrations du schema applicatif expose par l'API vivent dans `apps/backend/alembic`, pas ici. diff --git a/db/roles/supervision.sql b/db/roles/supervision.sql new file mode 100644 index 0000000..a946490 --- /dev/null +++ b/db/roles/supervision.sql @@ -0,0 +1,15 @@ +-- Contrainte : rôle en lecture seule de la supervision (Grafana, postgres-exporter) - +-- supervision.sql. Ses droits ne portent que sur les tables métier : jamais `app_user`, les +-- jetons ni le journal d'audit. `pg_monitor` donne à l'exportateur les vues de statistiques. +-- Rejoué par `make db-ensure-supervision`, qui passe `mot_de_passe` et `base` en variables psql : +-- crée le rôle au besoin, puis réaligne à chaque passage mot de passe et droits. +-- Piège : les tables doivent exister, d'où l'appel après `alembic upgrade head` dans `stack-up`. + +SELECT 'CREATE ROLE supervision LOGIN' +WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'supervision') \gexec + +ALTER ROLE supervision WITH LOGIN PASSWORD :'mot_de_passe'; +GRANT pg_monitor TO supervision; +GRANT CONNECT ON DATABASE :"base" TO supervision; +GRANT USAGE ON SCHEMA public TO supervision; +GRANT SELECT ON site, reading, alert, prediction, recommendation, drift_report TO supervision; diff --git a/docker-compose.yml b/docker-compose.yml index 9eab014..5365bb4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,6 +107,7 @@ services: APP_SMTP_PORT: "1025" APP_SMTP_USE_TLS: "false" APP_SMTP_FROM_ADDRESS: ${APP_SMTP_FROM_ADDRESS:-no-reply@enervision.fr} + APP_METRICS_TOKEN: ${APP_METRICS_TOKEN:-} ports: - "${BACKEND_PORT:-8000}:8000" restart: unless-stopped @@ -193,6 +194,111 @@ services: airflow-init: condition: service_completed_successfully + # Profil `monitoring` : actif en prod par COMPOSE_PROFILES, à la demande ailleurs (ADR 0016). + # Aucun `depends_on` : `make monitoring-up` démarre en `--no-deps`, sans jamais recréer `db`. + prometheus: + image: prom/prometheus:v3.14.0 + profiles: ["monitoring"] + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=15d + - --storage.tsdb.retention.size=1GB + volumes: + - ./monitoring/prometheus:/etc/prometheus:ro + - prometheus_data:/prometheus + secrets: + - metrics_token + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" + mem_limit: 512m + restart: unless-stopped + + alertmanager: + image: prom/alertmanager:v0.34.1 + profiles: ["monitoring"] + command: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + volumes: + - ./monitoring/alertmanager:/etc/alertmanager:ro + - alertmanager_data:/alertmanager + ports: + - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093" + mem_limit: 64m + restart: unless-stopped + + # Sans mot de passe, Grafana créerait un compte admin/admin : le conteneur refuse de démarrer. + grafana: + image: grafana/grafana:13.2.2 + profiles: ["monitoring"] + entrypoint: + - /bin/sh + - -c + - ': "$${GF_SECURITY_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD manquant dans .env}" && exec /run.sh' + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/api.json + POSTGRES_DB: ${POSTGRES_DB:-enervision} + SUPERVISION_DB_PASSWORD: ${SUPERVISION_DB_PASSWORD:-} + volumes: + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro + - grafana_data:/var/lib/grafana + ports: + - "127.0.0.1:${GRAFANA_PORT:-3001}:3000" + mem_limit: 256m + restart: unless-stopped + + postgres-exporter: + image: prometheuscommunity/postgres-exporter:v0.20.1 + profiles: ["monitoring"] + environment: + DATA_SOURCE_URI: db:5432/${POSTGRES_DB:-enervision}?sslmode=disable + DATA_SOURCE_USER: supervision + DATA_SOURCE_PASS: ${SUPERVISION_DB_PASSWORD:-} + mem_limit: 64m + restart: unless-stopped + + node-exporter: + image: prom/node-exporter:v1.12.1 + profiles: ["monitoring"] + command: + - --path.rootfs=/host + pid: host + volumes: + - /:/host:ro,rslave + mem_limit: 64m + restart: unless-stopped + + # Contrainte : cAdvisor lit les cgroups de tous les conteneurs de l'hôte, d'où `privileged` et + # ses montages en lecture seule. Aucun port publié : seul Prometheus le joint. + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.55.1 + profiles: ["monitoring"] + privileged: true + devices: + - /dev/kmsg + command: + - --docker_only=true + - --housekeeping_interval=30s + - --store_container_labels=false + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + mem_limit: 160m + restart: unless-stopped + # Pourquoi : sur le réseau du projet, k6 joint `backend:8000` sans passer par nginx, dont la # limite par adresse (20 req/s) fausserait la mesure de l'API. `make load-*` le lance (ADR 0015). k6: @@ -214,3 +320,11 @@ volumes: pgdata: airflow_logs: airflow_ml_state: + prometheus_data: + alertmanager_data: + grafana_data: + +# Vide tant qu'APP_METRICS_TOKEN n'est pas posé : l'API n'exige alors aucun jeton. +secrets: + metrics_token: + environment: APP_METRICS_TOKEN diff --git a/monitoring/README.md b/monitoring/README.md index d1b5254..2d841e0 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -1,10 +1,82 @@ -# Monitoring +# Supervision -Prometheus, Grafana et Alertmanager. Non initialise, voir le ticket dedie. +Prometheus, Alertmanager, Grafana et trois exporteurs, sous le profil Compose `monitoring`. +Issue #26, décisions dans l'ADR 0016, vue d'architecture dans +`docs/architecture/60-observabilite.md`. -- `prometheus` : configuration de collecte et regles d'alerte. -- `grafana/provisioning` : sources de donnees et fournisseurs de dashboards. -- `grafana/dashboards` : dashboards versionnes au format JSON. -- `alertmanager` : routage et inhibition des alertes. +| Service | Image | Rôle | Accès | +|---|---|---|---| +| `prometheus` | `prom/prometheus` | Collecte toutes les 15 s, évalue les règles, garde 15 jours (1 Go au plus) | `127.0.0.1:${PROMETHEUS_PORT:-9090}` | +| `alertmanager` | `prom/alertmanager` | Groupe les alertes et les envoie par courriel à Mailpit | `127.0.0.1:${ALERTMANAGER_PORT:-9093}` | +| `grafana` | `grafana/grafana` | Trois tableaux de bord provisionnés, dossier « EnerVision » | `127.0.0.1:${GRAFANA_PORT:-3001}` | +| `postgres-exporter` | `prometheuscommunity/postgres-exporter` | Connexions, transactions, taille des bases | réseau interne | +| `node-exporter` | `prom/node-exporter` | Processeur, mémoire et disque de l'hôte | réseau interne | +| `cadvisor` | `gcr.io/cadvisor/cadvisor` | Mémoire et processeur par conteneur | réseau interne | -Le backend expose deja ses metriques sur `/metrics` au format Prometheus. +Les interfaces n'écoutent que sur `127.0.0.1`. Depuis un poste, on passe par un tunnel SSH, +comme pour Airflow : + +```bash +ssh -L 3001:127.0.0.1:3001 -L 9090:127.0.0.1:9090 enervision@10.101.200.37 +``` + +## Démarrer + +- **Prod.** `COMPOSE_PROFILES=monitoring` dans le `.env` : `make stack-up`, donc chaque + déploiement, démarre la supervision et pose le rôle `supervision` après les migrations. +- **Recette et poste.** À la demande, sur une stack déjà démarrée : `make monitoring-up`. Les + services partent en `--no-deps`, sans toucher aux autres. + +Trois secrets sont requis, et `make stack-up` comme `make monitoring-up` refusent de démarrer +s'il en manque un. `scripts/provision-host.sh` les génère pour un nouvel environnement. + +| Variable | Rôle | +|---|---| +| `APP_METRICS_TOKEN` | Jeton que Prometheus présente sur `/metrics`, et que l'API exige dès qu'il est posé | +| `GRAFANA_ADMIN_PASSWORD` | Compte `admin` de Grafana. Sans lui, le conteneur refuse de démarrer | +| `SUPERVISION_DB_PASSWORD` | Rôle PostgreSQL `supervision`, en lecture seule (`db/roles/supervision.sql`) | + +L'API doit tourner en conteneur (`make stack-up`, ou `docker compose up -d backend`) : +Prometheus la joint en `backend:8000`, sur le réseau du projet. Une API lancée par `make dev` +sur l'hôte reste hors de sa portée, et l'alerte `ApiIndisponible` le signale. + +## Tableaux de bord + +| Tableau | Source | Contenu | +|---|---|---| +| EnerVision · API | Prometheus | Débit, erreurs 5xx, latences p50/p95/p99, globales et par route. C'est lui qu'on regarde pendant un tir k6 | +| EnerVision · Données et modèle | TimescaleDB | Fraîcheur des relevés par site, relevés ingérés, alertes par sévérité, dérive du modèle (`drift_report`) | +| EnerVision · Infrastructure | Prometheus | Hôte, mémoire et processeur par conteneur (recette et prod comprises), PostgreSQL | + +Les fichiers JSON de `grafana/dashboards` sont la source : Grafana les recharge et refuse de +les modifier depuis l'interface. Pour changer un tableau, l'exporter en JSON depuis Grafana et +remplacer le fichier. + +## Alertes + +`prometheus/rules/enervision.yml` définit les règles, et `prometheus/tests/enervision.test.yml` +porte un cas par règle, joué par `promtool test rules`. + +| Alerte | Condition | Sévérité | +|---|---|---| +| `ApiIndisponible` | `/metrics` injoignable pendant 2 min | critical | +| `ApiErreursServeur` | Plus de 5 % de 5xx sur 5 min | critical | +| `ApiLatenceElevee` | p95 au-delà d'une seconde pendant 10 min | warning | +| `BaseIndisponible` | Exportateur sans connexion pendant 2 min | critical | +| `BaseConnexionsSaturees` | Plus de 80 % de `max_connections` | warning | +| `HoteMemoireSaturee` | Mémoire au-delà de 90 % pendant 10 min | warning | +| `HoteDisquePlein` | Moins de 10 % libres sur `/` | critical | +| `HoteCpuSature` | Processeur au-delà de 90 % pendant 15 min | warning | +| `CibleInjoignable` | Un exporteur muet pendant 5 min | warning | + +Alertmanager envoie les courriels à `supervision@enervision.fr` via Mailpit, qui les capture : +ils se lisent dans son interface. Un `critical` masque le `warning` de la même cible. + +## Vérifier la configuration + +```bash +make monitoring-check # promtool check config et test rules, amtool check-config, JSON des tableaux +``` + +La CI joue les mêmes commandes (job « Validation des fichiers Compose et de la supervision ») +dès que `monitoring/` ou un fichier Compose change. diff --git a/monitoring/alertmanager/.gitkeep b/monitoring/alertmanager/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 0000000..3ddef30 --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,31 @@ +# Contrainte : Mailpit est le seul serveur SMTP de la stack, et il ne relaie rien vers l'extérieur +# - alertmanager.yml. Les alertes se lisent dans son interface (MAILPIT_UI_PORT), comme les +# courriels de réinitialisation de l'API. + +global: + smtp_smarthost: mailpit:1025 + smtp_from: supervision@enervision.fr + smtp_require_tls: false + +route: + receiver: equipe + group_by: [alertname, severity] + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + routes: + - matchers: ['severity="critical"'] + receiver: equipe + group_wait: 10s + repeat_interval: 1h + +receivers: + - name: equipe + email_configs: + - to: supervision@enervision.fr + send_resolved: true + +inhibit_rules: + - source_matchers: ['severity="critical"'] + target_matchers: ['severity="warning"'] + equal: [instance] diff --git a/monitoring/grafana/dashboards/.gitkeep b/monitoring/grafana/dashboards/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/grafana/dashboards/api.json b/monitoring/grafana/dashboards/api.json new file mode 100644 index 0000000..82e1ecc --- /dev/null +++ b/monitoring/grafana/dashboards/api.json @@ -0,0 +1,578 @@ +{ + "uid": "enervision-api", + "title": "EnerVision · API", + "description": "Débit, erreurs et latences vus par l'API elle-même (prometheus-fastapi-instrumentator).", + "tags": [ + "enervision" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "refresh": "10s", + "schemaVersion": 39, + "version": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "API", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "up{job=\"backend\"}", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Injoignable", + "color": "red" + }, + "1": { + "text": "Joignable", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "description": "Prometheus joint-il /metrics ?" + }, + { + "id": 2, + "type": "stat", + "title": "Débit", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(http_requests_total{job=\"backend\"}[5m]))", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Erreurs 5xx", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(http_requests_total{job=\"backend\", status=\"5xx\"}[5m])) / sum(rate(http_requests_total{job=\"backend\"}[5m]))", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.01 + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "description": "Seuil d'alerte : 5 % pendant 5 minutes." + }, + { + "id": 4, + "type": "stat", + "title": "Latence p95", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_highr_seconds_bucket{job=\"backend\"}[5m])))", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "description": "Seuil de charge : 500 ms (ADR 0015). Alerte au-delà d'une seconde pendant 10 minutes." + }, + { + "id": 5, + "type": "timeseries", + "title": "Débit par route", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (handler) (rate(http_requests_total{job=\"backend\"}[1m]))", + "legendFormat": "{{handler}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Latence de l'API", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.50, sum by (le) (rate(http_request_duration_highr_seconds_bucket{job=\"backend\"}[1m])))", + "legendFormat": "p50", + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_highr_seconds_bucket{job=\"backend\"}[1m])))", + "legendFormat": "p95", + "range": true + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.99, sum by (le) (rate(http_request_duration_highr_seconds_bucket{job=\"backend\"}[1m])))", + "legendFormat": "p99", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Latence p95 par route", + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (le, handler) (rate(http_request_duration_seconds_bucket{job=\"backend\"}[5m])))", + "legendFormat": "{{handler}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Estimée sur les seaux 50 ms à 2,5 s de http_request_duration_seconds." + }, + { + "id": 8, + "type": "timeseries", + "title": "Réponses par classe de statut", + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (status) (rate(http_requests_total{job=\"backend\"}[1m]))", + "legendFormat": "{{status}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Mémoire du processus API", + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "process_resident_memory_bytes{job=\"backend\"}", + "legendFormat": "résidente", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Requêtes en cours de traitement", + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum(rate(http_request_duration_highr_seconds_sum{job=\"backend\"}[1m]))", + "legendFormat": "secondes de traitement par seconde", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Loi de Little : durée moyenne multipliée par le débit, soit le nombre moyen de requêtes simultanées." + } + ] +} diff --git a/monitoring/grafana/dashboards/donnees.json b/monitoring/grafana/dashboards/donnees.json new file mode 100644 index 0000000..0098d19 --- /dev/null +++ b/monitoring/grafana/dashboards/donnees.json @@ -0,0 +1,528 @@ +{ + "uid": "enervision-donnees", + "title": "EnerVision · Données et modèle", + "description": "Fraîcheur des relevés, alertes métier et dérive du modèle, lus dans TimescaleDB par le rôle supervision.", + "tags": [ + "enervision" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "refresh": "1m", + "schemaVersion": 39, + "version": 1, + "time": { + "from": "now-7d", + "to": "now" + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Sites suivis", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT count(*) AS \"Sites\" FROM site", + "format": "table", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 2, + "type": "stat", + "title": "Relevés sur 24 h", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT count(*) AS \"Relevés\" FROM reading WHERE timestamp > now() - interval '24 hours'", + "format": "table", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "description": "Zéro : l'import de la Mock API ne tourne plus." + }, + { + "id": 3, + "type": "stat", + "title": "Alertes sur 24 h", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT count(*) AS \"Alertes\" FROM alert WHERE timestamp > now() - interval '24 hours'", + "format": "table", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "Sites en dérive", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT count(*) AS \"Sites\" FROM (SELECT DISTINCT ON (site_id) status FROM drift_report WHERE site_id IS NOT NULL ORDER BY site_id, computed_at DESC) AS derniers WHERE status = 'derive'", + "format": "table", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "description": "Dernier rapport de dérive de chaque site (ADR 0013)." + }, + { + "id": 5, + "type": "table", + "title": "Fraîcheur des relevés par site", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT s.site_name AS \"Site\", max(r.timestamp) AS \"Dernier relevé\", round(extract(epoch FROM now() - max(r.timestamp)) / 60) AS \"Âge (min)\"\nFROM site AS s\nLEFT JOIN reading AS r ON r.site_id = s.site_id AND r.timestamp > now() - interval '7 days'\nGROUP BY s.site_name\nORDER BY 3 DESC NULLS FIRST", + "format": "table", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "description": "Un site sans relevé depuis sept jours apparaît vide, en tête de liste." + }, + { + "id": 6, + "type": "table", + "title": "Dérive du modèle, dernier rapport", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT DISTINCT ON (d.site_id) coalesce(s.site_name, 'Tout le parc') AS \"Site\", d.status AS \"Statut\", round(d.mae::numeric, 2) AS \"MAE\", round(d.reference_mae::numeric, 2) AS \"MAE de référence\", round(d.bias::numeric, 2) AS \"Biais\", d.computed_at AS \"Calculé le\"\nFROM drift_report AS d\nLEFT JOIN site AS s ON s.site_id = d.site_id\nORDER BY d.site_id, d.computed_at DESC", + "format": "table", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "description": "Une ligne par site, plus la ligne globale (ADR 0013)." + }, + { + "id": 7, + "type": "timeseries", + "title": "Consommation par site", + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT $__timeGroupAlias(r.timestamp, $__interval), s.site_name AS metric, avg(r.consumption_kw) AS value\nFROM reading AS r\nJOIN site AS s ON s.site_id = r.site_id\nWHERE $__timeFilter(r.timestamp)\nGROUP BY 1, 2\nORDER BY 1", + "format": "time_series", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "kwatt" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Relevés ingérés par heure", + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT time_bucket('1 hour', timestamp) AS time, count(*) AS \"Relevés\"\nFROM reading\nWHERE $__timeFilter(timestamp)\nGROUP BY 1\nORDER BY 1", + "format": "time_series", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Alertes par sévérité", + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT time_bucket('1 hour', timestamp) AS time, severity AS metric, count(*) AS value\nFROM alert\nWHERE $__timeFilter(timestamp)\nGROUP BY 1, 2\nORDER BY 1", + "format": "time_series", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Erreur absolue moyenne du modèle", + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "timescaledb" + }, + "rawSql": "SELECT d.computed_at AS time, coalesce(s.site_name, 'Tout le parc') AS metric, d.mae AS value\nFROM drift_report AS d\nLEFT JOIN site AS s ON s.site_id = d.site_id\nWHERE $__timeFilter(d.computed_at)\nORDER BY 1", + "format": "time_series", + "rawQuery": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "kwatt" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/monitoring/grafana/dashboards/infra.json b/monitoring/grafana/dashboards/infra.json new file mode 100644 index 0000000..824af49 --- /dev/null +++ b/monitoring/grafana/dashboards/infra.json @@ -0,0 +1,583 @@ +{ + "uid": "enervision-infra", + "title": "EnerVision · Infrastructure", + "description": "Hôte (node-exporter), conteneurs (cAdvisor) et PostgreSQL (postgres-exporter). L'hôte porte la recette et la prod.", + "tags": [ + "enervision" + ], + "timezone": "browser", + "editable": false, + "graphTooltip": 1, + "refresh": "30s", + "schemaVersion": 39, + "version": 1, + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [] + }, + "annotations": { + "list": [] + }, + "links": [], + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Mémoire de l'hôte", + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.8 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + }, + "description": "Alerte au-delà de 90 % pendant 10 minutes." + }, + { + "id": 2, + "type": "stat", + "title": "Processeur de l'hôte", + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Disque libre", + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_filesystem_avail_bytes{mountpoint=\"/\", fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\", fstype!~\"tmpfs|overlay\"}", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 0.1 + }, + { + "color": "green", + "value": 0.2 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "PostgreSQL", + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "pg_up", + "legendFormat": "", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + }, + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Injoignable", + "color": "red" + }, + "1": { + "text": "Joignable", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "background", + "graphMode": "area", + "textMode": "auto" + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Mémoire par conteneur", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (name) (container_memory_working_set_bytes{name!=\"\"})", + "legendFormat": "{{name}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "description": "Recette et prod partagent l'hôte : leurs conteneurs se distinguent par le préfixe du projet Compose." + }, + { + "id": 6, + "type": "timeseries", + "title": "Processeur par conteneur", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (name) (rate(container_cpu_usage_seconds_total{name!=\"\"}[5m]))", + "legendFormat": "{{name}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "Mémoire de l'hôte", + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "legendFormat": "utilisée", + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "node_memory_MemTotal_bytes", + "legendFormat": "totale", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Connexions PostgreSQL par état", + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (state) (pg_stat_activity_count)", + "legendFormat": "{{state}}", + "range": true + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(pg_settings_max_connections)", + "legendFormat": "maximum", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Transactions validées par base", + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (datname) (rate(pg_stat_database_xact_commit{datname!~\"template.*|postgres\"}[5m]))", + "legendFormat": "{{datname}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Taille des bases", + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "pg_database_size_bytes{datname!~\"template.*|postgres\"}", + "legendFormat": "{{datname}}", + "range": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + } + ] +} diff --git a/monitoring/grafana/provisioning/dashboards/.gitkeep b/monitoring/grafana/provisioning/dashboards/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..162e451 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: enervision + folder: EnerVision + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/.gitkeep b/monitoring/grafana/provisioning/datasources/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/grafana/provisioning/datasources/datasources.yml b/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..c8d8577 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,28 @@ +# Pourquoi : Grafana lit la base avec le rôle `supervision`, en lecture seule sur les seules tables +# métier (db/roles/supervision.sql) - datasources.yml. Jamais le compte applicatif. + +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: TimescaleDB + uid: timescaledb + type: grafana-postgresql-datasource + access: proxy + url: db:5432 + user: supervision + jsonData: + database: ${POSTGRES_DB} + sslmode: disable + postgresVersion: 1700 + timescaledb: true + secureJsonData: + password: ${SUPERVISION_DB_PASSWORD} + editable: false diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..9c1769f --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,43 @@ +# Contrainte : Prometheus ne lit pas les variables d'environnement dans ce fichier - prometheus.yml. +# Le jeton de `/metrics` lui parvient en secret Compose (`metrics_token`, tiré d'APP_METRICS_TOKEN) ; +# vide, l'API n'en exige aucun (app/api/security.py). + +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - /etc/prometheus/rules/*.yml + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + +scrape_configs: + - job_name: backend + authorization: + type: Bearer + credentials_file: /run/secrets/metrics_token + static_configs: + - targets: ["backend:8000"] + + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: alertmanager + static_configs: + - targets: ["alertmanager:9093"] + + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: node + static_configs: + - targets: ["node-exporter:9100"] + + - job_name: cadvisor + static_configs: + - targets: ["cadvisor:8080"] diff --git a/monitoring/prometheus/rules/.gitkeep b/monitoring/prometheus/rules/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/monitoring/prometheus/rules/enervision.yml b/monitoring/prometheus/rules/enervision.yml new file mode 100644 index 0000000..65d659d --- /dev/null +++ b/monitoring/prometheus/rules/enervision.yml @@ -0,0 +1,103 @@ +# Contrainte : chaque règle a son cas dans tests/enervision.test.yml, joué par `promtool test +# rules` en CI - enervision.yml. Une règle modifiée sans son test fait échouer le job Infra. +# Pourquoi : `critical` réveille (mail immédiat, rappel toutes les heures), `warning` se lit le +# lendemain ; alertmanager.yml masque le warning d'une cible déjà en critical. + +groups: + - name: api + rules: + - alert: ApiIndisponible + expr: up{job="backend"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "L'API ne répond plus" + description: "Prometheus ne joint plus {{ $labels.instance }} depuis 2 minutes." + + - alert: ApiErreursServeur + expr: | + sum(rate(http_requests_total{job="backend", status="5xx"}[5m])) + / sum(rate(http_requests_total{job="backend"}[5m])) > 0.05 + for: 5m + labels: + severity: critical + annotations: + summary: "Plus de 5 % de réponses 5xx" + description: "{{ $value | humanizePercentage }} des réponses de l'API sont des erreurs serveur." + + - alert: ApiLatenceElevee + expr: | + histogram_quantile(0.95, + sum by (le) (rate(http_request_duration_highr_seconds_bucket{job="backend"}[5m])) + ) > 1 + for: 10m + labels: + severity: warning + annotations: + summary: "p95 de l'API au-dessus d'une seconde" + description: "Le p95 des réponses vaut {{ $value | humanizeDuration }} depuis 10 minutes." + + - name: base + rules: + - alert: BaseIndisponible + expr: pg_up == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "PostgreSQL ne répond plus" + description: "L'exportateur ne se connecte plus à la base depuis 2 minutes." + + - alert: BaseConnexionsSaturees + expr: | + sum by (instance) (pg_stat_activity_count) + / max by (instance) (pg_settings_max_connections) > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "Connexions PostgreSQL au-delà de 80 %" + description: "{{ $value | humanizePercentage }} des connexions autorisées sont ouvertes." + + - name: hote + rules: + - alert: HoteMemoireSaturee + expr: 1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes > 0.9 + for: 10m + labels: + severity: warning + annotations: + summary: "Mémoire de l'hôte au-delà de 90 %" + description: "{{ $value | humanizePercentage }} de la mémoire est utilisée, recette et prod comprises." + + - alert: HoteDisquePlein + expr: | + node_filesystem_avail_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} + / node_filesystem_size_bytes{mountpoint="/", fstype!~"tmpfs|overlay"} < 0.1 + for: 10m + labels: + severity: critical + annotations: + summary: "Moins de 10 % de disque libre" + description: "Il reste {{ $value | humanizePercentage }} d'espace sur la racine de l'hôte." + + - alert: HoteCpuSature + expr: 1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) > 0.9 + for: 15m + labels: + severity: warning + annotations: + summary: "Processeur de l'hôte au-delà de 90 %" + description: "Le processeur est occupé à {{ $value | humanizePercentage }} depuis 15 minutes." + + - name: supervision + rules: + - alert: CibleInjoignable + expr: up{job!="backend"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Cible de supervision injoignable" + description: "Prometheus ne joint plus {{ $labels.job }} ({{ $labels.instance }}) depuis 5 minutes." diff --git a/monitoring/prometheus/tests/enervision.test.yml b/monitoring/prometheus/tests/enervision.test.yml new file mode 100644 index 0000000..14ee0a4 --- /dev/null +++ b/monitoring/prometheus/tests/enervision.test.yml @@ -0,0 +1,112 @@ +rule_files: + - ../rules/enervision.yml + +evaluation_interval: 1m + +tests: + - name: l'API injoignable déclenche une alerte critique au bout de 2 minutes + interval: 1m + input_series: + - series: 'up{job="backend", instance="backend:8000"}' + values: "1 0 0 0 0" + alert_rule_test: + - eval_time: 2m + alertname: ApiIndisponible + exp_alerts: [] + - eval_time: 3m + alertname: ApiIndisponible + exp_alerts: + - exp_labels: + severity: critical + job: backend + instance: backend:8000 + exp_annotations: + summary: "L'API ne répond plus" + description: "Prometheus ne joint plus backend:8000 depuis 2 minutes." + + - name: une API qui répond ne déclenche rien + interval: 1m + input_series: + - series: 'up{job="backend", instance="backend:8000"}' + values: "1x10" + alert_rule_test: + - eval_time: 10m + alertname: ApiIndisponible + exp_alerts: [] + + - name: dix pour cent de 5xx déclenchent l'alerte d'erreurs serveur + interval: 1m + input_series: + - series: 'http_requests_total{job="backend", handler="/api/v1/sites", method="GET", status="5xx"}' + values: "0+10x20" + - series: 'http_requests_total{job="backend", handler="/api/v1/sites", method="GET", status="2xx"}' + values: "0+90x20" + alert_rule_test: + - eval_time: 12m + alertname: ApiErreursServeur + exp_alerts: + - exp_labels: + severity: critical + exp_annotations: + summary: "Plus de 5 % de réponses 5xx" + description: "10% des réponses de l'API sont des erreurs serveur." + + - name: un p95 au-delà d'une seconde déclenche l'alerte de latence + interval: 1m + input_series: + - series: 'http_request_duration_highr_seconds_bucket{job="backend", le="0.5"}' + values: "0x30" + - series: 'http_request_duration_highr_seconds_bucket{job="backend", le="1"}' + values: "0x30" + - series: 'http_request_duration_highr_seconds_bucket{job="backend", le="2"}' + values: "0+10x30" + - series: 'http_request_duration_highr_seconds_bucket{job="backend", le="+Inf"}' + values: "0+10x30" + alert_rule_test: + - eval_time: 20m + alertname: ApiLatenceElevee + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + summary: "p95 de l'API au-dessus d'une seconde" + description: "Le p95 des réponses vaut 1.95s depuis 10 minutes." + + - name: la mémoire de l'hôte au-delà de 90 % déclenche un avertissement + interval: 1m + input_series: + - series: 'node_memory_MemAvailable_bytes{job="node", instance="node-exporter:9100"}' + values: "500000000x15" + - series: 'node_memory_MemTotal_bytes{job="node", instance="node-exporter:9100"}' + values: "10000000000x15" + alert_rule_test: + - eval_time: 12m + alertname: HoteMemoireSaturee + exp_alerts: + - exp_labels: + severity: warning + job: node + instance: node-exporter:9100 + exp_annotations: + summary: "Mémoire de l'hôte au-delà de 90 %" + description: "95% de la mémoire est utilisée, recette et prod comprises." + + - name: un exportateur muet déclenche l'alerte de cible, pas celle de l'API + interval: 1m + input_series: + - series: 'up{job="postgres", instance="postgres-exporter:9187"}' + values: "0x10" + alert_rule_test: + - eval_time: 6m + alertname: CibleInjoignable + exp_alerts: + - exp_labels: + severity: warning + job: postgres + instance: postgres-exporter:9187 + exp_annotations: + summary: "Cible de supervision injoignable" + description: "Prometheus ne joint plus postgres (postgres-exporter:9187) depuis 5 minutes." + - eval_time: 6m + alertname: ApiIndisponible + exp_alerts: [] diff --git a/scripts/provision-host.sh b/scripts/provision-host.sh index 6bad530..1ce9cf6 100755 --- a/scripts/provision-host.sh +++ b/scripts/provision-host.sh @@ -36,6 +36,7 @@ verifier_outils() { preparer() { local env="$1" branche="$2" hote="$3" origine="$4" local port_https="$5" port_http="$6" port_pg="$7" port_mailpit="$8" port_airflow="$9" + local profils="${10}" port_grafana="${11}" port_prometheus="${12}" port_alertmanager="${13}" local dossier="$RACINE/$env" if [[ -d "$dossier/.git" ]]; then @@ -63,6 +64,13 @@ preparer() { -e "s|^COMPOSE_PROJECT_NAME=.*|COMPOSE_PROJECT_NAME=enervision-$env|" \ -e "s|^PROXY_HTTP_PORT=.*|PROXY_HTTP_PORT=$port_http|" \ -e "s|^PROXY_HTTPS_PORT=.*|PROXY_HTTPS_PORT=$port_https|" \ + -e "s|^COMPOSE_PROFILES=.*|COMPOSE_PROFILES=$profils|" \ + -e "s|^APP_METRICS_TOKEN=.*|APP_METRICS_TOKEN=$(secret)|" \ + -e "s|^GRAFANA_ADMIN_PASSWORD=.*|GRAFANA_ADMIN_PASSWORD=$(secret | cut -c1-20)|" \ + -e "s|^SUPERVISION_DB_PASSWORD=.*|SUPERVISION_DB_PASSWORD=$(secret)|" \ + -e "s|^GRAFANA_PORT=.*|GRAFANA_PORT=$port_grafana|" \ + -e "s|^PROMETHEUS_PORT=.*|PROMETHEUS_PORT=$port_prometheus|" \ + -e "s|^ALERTMANAGER_PORT=.*|ALERTMANAGER_PORT=$port_alertmanager|" \ "$dossier/.env.example" > "$brouillon" # Branche antérieure à l'ADR 0009 : ces clés manquent alors dans .env.example. for cle in "COMPOSE_PROJECT_NAME=enervision-$env" "PUBLIC_ORIGIN=$origine" \ @@ -90,9 +98,11 @@ preparer() { verifier_outils mkdir -p "$RACINE" -# env branche hôte origine https http pg mailpit airflow -preparer prod main enervision.local https://enervision.local 443 80 5433 8025 8080 -preparer rec dev rec.enervision.local https://rec.enervision.local:8443 8443 127.0.0.1:8081 5434 8026 8082 +# Supervision active en prod seulement (ADR 0016) ; les ports de la recette restent décalés au cas +# où on l'y lancerait à la demande. +# env branche hôte origine https http pg mailpit airflow profils grafana prometheus alertmanager +preparer prod main enervision.local https://enervision.local 443 80 5433 8025 8080 monitoring 3001 9090 9093 +preparer rec dev rec.enervision.local https://rec.enervision.local:8443 8443 127.0.0.1:8081 5434 8026 8082 "" 3002 9091 9094 if [[ -n "$PROPRIETAIRE" && "$(id -u)" -eq 0 ]]; then chown -R "$PROPRIETAIRE" "$RACINE"