Johan LEROY
e53c7e441c
Reprend l'amorce de la PR 164 et l'intègre à la stack : service `garage` (dxflrs/garage v2.4.1, `--single-node --default-bucket`) dans docker-compose.yml, garage.toml versionné sans secret, ports sur 127.0.0.1 décalés par environnement dans provision-host.sh, garde des six clés GARAGE_* dans le Makefile, cible Prometheus avec jeton, tests de fumée déplacés dans tests/garage et joués par le job compose d'infra.yml contre le vrai conteneur, SSE-C compris. Variables APP_S3_* et APP_READING_RETENTION_DAYS posées sur airflow-scheduler pour le DAG `retention`. ADR 0019. Closes #24
376 lines
15 KiB
YAML
376 lines
15 KiB
YAML
# Piege : PGDATA de l'image timescaledb-ha vaut /home/postgres/pgdata/data, pas le chemin
|
|
# habituel de l'image postgres. Monte ailleurs, le volume ne retient rien, sans erreur.
|
|
# Piege : db/init est monte fichier par fichier. Monter le dossier masquerait les scripts
|
|
# d'init de l'image, dont timescaledb-tune. Ajouter un fichier impose une ligne ici.
|
|
|
|
name: enervision
|
|
|
|
# Piege : LocalExecutor fait tourner les taches comme sous-processus du scheduler, jamais de
|
|
# l'api-server. `airflow_ml_state` (modele entraine, magasin MLflow) n'a donc besoin d'etre monte
|
|
# que sur `airflow-scheduler` en pratique, mais reste partage avec l'api-server pour que ce
|
|
# dernier puisse au besoin l'inspecter sans en devenir dependant.
|
|
x-airflow-common: &airflow-common
|
|
build:
|
|
context: .
|
|
dockerfile: etl/airflow/Dockerfile
|
|
environment: &airflow-common-env
|
|
AIRFLOW__CORE__EXECUTOR: LocalExecutor
|
|
AIRFLOW__CORE__LOAD_EXAMPLES: "false"
|
|
# Piege : pas de `:?` sur les secrets Airflow. Compose interpole le fichier entier avant de
|
|
# filtrer les services : une variable requise manquante casserait aussi `make db-up`,
|
|
# `make dev`... pour quiconque n'a pas encore complete son `.env`. Le refus est porte par
|
|
# `airflow-init` (ci-dessous), dont `api-server`, `dag-processor` et `scheduler` dependent.
|
|
AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW_FERNET_KEY:-}
|
|
AIRFLOW__API__SECRET_KEY: ${AIRFLOW_API_SECRET_KEY:-}
|
|
# Signe les jetons entre scheduler, tâches et api-server. Conteneurs distincts : un secret
|
|
# généré au démarrage ne serait pas partagé, il doit venir du .env.
|
|
AIRFLOW__API_AUTH__JWT_SECRET: ${AIRFLOW_JWT_SECRET:-}
|
|
AIRFLOW__CORE__EXECUTION_API_SERVER_URL: http://airflow-apiserver:8080/execution/
|
|
# FabAuthManager plutôt que le SimpleAuthManager par défaut d'Airflow 3 : seul le provider
|
|
# FAB sait créer le compte admin que `airflow-init` pose via `_AIRFLOW_WWW_USER_*`.
|
|
AIRFLOW__CORE__AUTH_MANAGER: airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
|
|
AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/airflow
|
|
# Role `enervision_ml` dedie pas encore provisionne (dette assumee, cf. ADR 0003) :
|
|
# memes identifiants que le backend en attendant.
|
|
ML_DATABASE_URL: postgresql+psycopg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
|
|
MLFLOW_TRACKING_URI: sqlite:////opt/ml/state/mlflow.db
|
|
# Les DAGs backend lisent `DATABASE_URL` en dialecte asyncpg, là où le pipeline ML
|
|
# utilise `ML_DATABASE_URL`.
|
|
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
|
|
|
|
# Clé distincte de celle de l'API : les traitements lancés par Airflow ne signent ni ne
|
|
# vérifient aucun jeton. Airflow permet d'exécuter du code depuis son interface (ADR 0008).
|
|
APP_SECRET_KEY: ${AIRFLOW_APP_SECRET_KEY:-}
|
|
|
|
volumes:
|
|
- ./etl/airflow/dags:/opt/airflow/dags
|
|
- ./etl/airflow/plugins:/opt/airflow/plugins
|
|
- ./data/raw:/opt/data/raw:ro
|
|
- airflow_logs:/opt/airflow/logs
|
|
- airflow_ml_state:/opt/ml/state
|
|
restart: unless-stopped
|
|
|
|
services:
|
|
db:
|
|
image: timescale/timescaledb-ha:pg17
|
|
environment:
|
|
POSTGRES_USER: ${POSTGRES_USER:?}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?}
|
|
POSTGRES_DB: ${POSTGRES_DB:?}
|
|
TIMESCALEDB_TELEMETRY: ${TIMESCALEDB_TELEMETRY:-off}
|
|
ports:
|
|
- "${POSTGRES_PORT:-5433}:5432"
|
|
volumes:
|
|
- pgdata:/home/postgres/pgdata/data
|
|
- ./db/init/100-extensions.sql:/docker-entrypoint-initdb.d/100-extensions.sql:ro
|
|
- ./db/init/110-test-database.sql:/docker-entrypoint-initdb.d/110-test-database.sql:ro
|
|
- ./db/init/120-airflow-database.sql:/docker-entrypoint-initdb.d/120-airflow-database.sql:ro
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
|
interval: 10s
|
|
timeout: 5s
|
|
retries: 12
|
|
start_period: 40s
|
|
restart: unless-stopped
|
|
|
|
# Piege : Mailpit ne relaie rien vers l'exterieur, il capture tout email envoye par le
|
|
# backend. Aucun acces reseau sortant n'est requis ; l'UI web (8025) sert a lire les emails.
|
|
mailpit:
|
|
image: axllent/mailpit
|
|
ports:
|
|
- "${MAILPIT_SMTP_PORT:-1025}:1025"
|
|
- "${MAILPIT_UI_PORT:-8025}:8025"
|
|
restart: unless-stopped
|
|
|
|
# Piège : image `FROM scratch`, sans shell : healthcheck en forme exec, et aucune garde shell sur
|
|
# les secrets. Un GARAGE_RPC_SECRET vide ou non hexadécimal fait échouer Garage lui-même, message
|
|
# explicite dans ses journaux ; `make services-up` et `make stack-up` vérifient le .env avant.
|
|
# Piège : GARAGE_SECRET_KEY ne se change pas sur un volume `garage_meta` déjà peuplé, Garage
|
|
# refuse alors de démarrer. Rotation par `garage key` ou par recréation du volume (ADR 0019).
|
|
garage:
|
|
image: dxflrs/garage:v2.4.1
|
|
command: ["/garage", "server", "--single-node", "--default-bucket"]
|
|
environment:
|
|
GARAGE_RPC_SECRET: ${GARAGE_RPC_SECRET:-}
|
|
GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-}
|
|
GARAGE_METRICS_TOKEN: ${GARAGE_METRICS_TOKEN:-}
|
|
GARAGE_DEFAULT_ACCESS_KEY: ${GARAGE_ACCESS_KEY:-}
|
|
GARAGE_DEFAULT_SECRET_KEY: ${GARAGE_SECRET_KEY:-}
|
|
GARAGE_DEFAULT_BUCKET: ${GARAGE_BUCKET:-enervision-archives}
|
|
volumes:
|
|
- ./infra/garage/garage.toml:/etc/garage.toml:ro
|
|
- garage_meta:/var/lib/garage/meta
|
|
- garage_data:/var/lib/garage/data
|
|
ports:
|
|
- "127.0.0.1:${GARAGE_S3_PORT:-3900}:3900"
|
|
- "127.0.0.1:${GARAGE_ADMIN_PORT:-3903}:3903"
|
|
healthcheck:
|
|
test: ["CMD", "/garage", "health", "-q"]
|
|
interval: 15s
|
|
timeout: 5s
|
|
retries: 6
|
|
start_period: 20s
|
|
mem_limit: 256m
|
|
restart: unless-stopped
|
|
|
|
backend:
|
|
build: ./apps/backend
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
mailpit:
|
|
condition: service_started
|
|
environment:
|
|
APP_ENV: ${APP_ENV:-local}
|
|
APP_DEBUG: ${APP_DEBUG:-false}
|
|
APP_LOG_LEVEL: ${APP_LOG_LEVEL:-INFO}
|
|
APP_SECRET_KEY: ${APP_SECRET_KEY:?}
|
|
APP_CORS_ORIGINS: ${APP_CORS_ORIGINS:-http://localhost:4200}
|
|
DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}
|
|
|
|
APP_MOCK_API_BASE_URL: ${APP_MOCK_API_BASE_URL:-https://api-mock.charlieandre.fr}
|
|
APP_MOCK_API_USERNAME: ${APP_MOCK_API_USERNAME:-}
|
|
APP_MOCK_API_PASSWORD: ${APP_MOCK_API_PASSWORD:-}
|
|
APP_MOCK_API_TIMEOUT_SECONDS: ${APP_MOCK_API_TIMEOUT_SECONDS:-10}
|
|
|
|
APP_FRONTEND_RESET_PASSWORD_URL: ${APP_FRONTEND_RESET_PASSWORD_URL:-http://localhost:4200/reset-password}
|
|
APP_SMTP_HOST: mailpit
|
|
APP_SMTP_PORT: "1025"
|
|
APP_SMTP_USE_TLS: "false"
|
|
APP_SMTP_FROM_ADDRESS: ${APP_SMTP_FROM_ADDRESS:-no-reply@enervision.fr}
|
|
APP_METRICS_TOKEN: ${APP_METRICS_TOKEN:-}
|
|
ports:
|
|
- "${BACKEND_PORT:-8000}:8000"
|
|
restart: unless-stopped
|
|
|
|
frontend:
|
|
build: ./apps/frontend
|
|
ports:
|
|
- "${FRONTEND_PORT:-3000}:3000"
|
|
restart: unless-stopped
|
|
|
|
# Conteneur unique, jamais redemarre. La migration et la creation du premier compte sont
|
|
# portees par l'entrypoint de l'image (`_AIRFLOW_DB_MIGRATE`, `_AIRFLOW_WWW_USER_*`), qui porte
|
|
# aussi leur code de sortie : une migration ratee (ex. base `airflow` absente sur un volume
|
|
# `pgdata` deja peuple) fait echouer ce service, et api-server, dag-processor et scheduler,
|
|
# qui attendent son succes, ne demarrent pas sur une base non migree. Le mot de passe passe
|
|
# par l'environnement, jamais par `argv` (ni `ps`, ni `docker compose config`).
|
|
# Sans mot de passe, l'entrypoint refuse lui-meme de creer le compte ; la commande ci-dessous
|
|
# refuse en plus les cles et secrets vides.
|
|
airflow-init:
|
|
<<: *airflow-common
|
|
restart: "no"
|
|
environment:
|
|
<<: *airflow-common-env
|
|
_AIRFLOW_DB_MIGRATE: "true"
|
|
_AIRFLOW_WWW_USER_CREATE: "true"
|
|
_AIRFLOW_WWW_USER_USERNAME: ${AIRFLOW_ADMIN_USERNAME:-admin}
|
|
_AIRFLOW_WWW_USER_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD:-}
|
|
_AIRFLOW_WWW_USER_EMAIL: ${AIRFLOW_ADMIN_EMAIL:-admin@enervision.fr}
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
command:
|
|
- bash
|
|
- -c
|
|
- |
|
|
set -euo pipefail
|
|
: "$${AIRFLOW__CORE__FERNET_KEY:?AIRFLOW_FERNET_KEY manquant dans .env}"
|
|
: "$${AIRFLOW__API__SECRET_KEY:?AIRFLOW_API_SECRET_KEY manquant dans .env}"
|
|
: "$${AIRFLOW__API_AUTH__JWT_SECRET:?AIRFLOW_JWT_SECRET manquant dans .env}"
|
|
: "$${APP_SECRET_KEY:?AIRFLOW_APP_SECRET_KEY manquant dans .env}"
|
|
exec airflow version
|
|
|
|
airflow-apiserver:
|
|
<<: *airflow-common
|
|
command: api-server
|
|
ports:
|
|
- "${AIRFLOW_PORT:-8080}:8080"
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/monitor/health"]
|
|
interval: 30s
|
|
timeout: 10s
|
|
retries: 5
|
|
start_period: 60s
|
|
|
|
airflow-scheduler:
|
|
<<: *airflow-common
|
|
command: scheduler
|
|
environment:
|
|
<<: *airflow-common-env
|
|
# LocalExecutor exécute les tâches dans le scheduler : lui seul a besoin des
|
|
# identifiants de l'API Mock.
|
|
APP_MOCK_API_BASE_URL: ${APP_MOCK_API_BASE_URL:-https://api-mock.charlieandre.fr}
|
|
APP_MOCK_API_USERNAME: ${APP_MOCK_API_USERNAME:-}
|
|
APP_MOCK_API_PASSWORD: ${APP_MOCK_API_PASSWORD:-}
|
|
APP_MOCK_API_TIMEOUT_SECONDS: ${APP_MOCK_API_TIMEOUT_SECONDS:-10}
|
|
# Le DAG `retention` archive les chunks de `reading` sur le Garage du projet (ADR 0019),
|
|
# chiffrés par la clé SSE-C du .env (ADR 0020). Vides, `app.etl.reading_retention` refuse seul.
|
|
APP_S3_ENDPOINT_URL: http://garage:3900
|
|
APP_S3_REGION: garage
|
|
APP_S3_ACCESS_KEY: ${GARAGE_ACCESS_KEY:-}
|
|
APP_S3_SECRET_KEY: ${GARAGE_SECRET_KEY:-}
|
|
APP_S3_BUCKET: ${GARAGE_BUCKET:-enervision-archives}
|
|
APP_S3_SSE_KEY: ${GARAGE_SSE_KEY:-}
|
|
APP_READING_RETENTION_DAYS: ${READING_RETENTION_DAYS:-1095}
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
|
|
# Obligatoire depuis Airflow 3 : le scheduler ne parse plus les fichiers de dags/ lui-même.
|
|
airflow-dag-processor:
|
|
<<: *airflow-common
|
|
command: dag-processor
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
airflow-init:
|
|
condition: service_completed_successfully
|
|
|
|
# Profil `monitoring` : actif en prod par COMPOSE_PROFILES, à la demande ailleurs (ADR 0016).
|
|
# Aucun `depends_on` : `make monitoring-up` démarre en `--no-deps`, sans jamais recréer `db`.
|
|
prometheus:
|
|
image: prom/prometheus:v3.14.0
|
|
profiles: ["monitoring"]
|
|
command:
|
|
- --config.file=/etc/prometheus/prometheus.yml
|
|
- --storage.tsdb.path=/prometheus
|
|
- --storage.tsdb.retention.time=15d
|
|
- --storage.tsdb.retention.size=1GB
|
|
volumes:
|
|
- ./monitoring/prometheus:/etc/prometheus:ro
|
|
- prometheus_data:/prometheus
|
|
secrets:
|
|
- metrics_token
|
|
- garage_metrics_token
|
|
ports:
|
|
- "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
alertmanager:
|
|
image: prom/alertmanager:v0.34.1
|
|
profiles: ["monitoring"]
|
|
command:
|
|
- --config.file=/etc/alertmanager/alertmanager.yml
|
|
- --storage.path=/alertmanager
|
|
volumes:
|
|
- ./monitoring/alertmanager:/etc/alertmanager:ro
|
|
- alertmanager_data:/alertmanager
|
|
ports:
|
|
- "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
|
|
mem_limit: 64m
|
|
restart: unless-stopped
|
|
|
|
# Sans mot de passe, Grafana créerait un compte admin/admin : le conteneur refuse de démarrer.
|
|
grafana:
|
|
image: grafana/grafana:13.2.2
|
|
profiles: ["monitoring"]
|
|
entrypoint:
|
|
- /bin/sh
|
|
- -c
|
|
- ': "$${GF_SECURITY_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD manquant dans .env}" && exec /run.sh'
|
|
environment:
|
|
GF_SECURITY_ADMIN_USER: admin
|
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-}
|
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
|
GF_AUTH_ANONYMOUS_ENABLED: "false"
|
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
|
GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: "false"
|
|
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
|
GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /etc/grafana/dashboards/api.json
|
|
POSTGRES_DB: ${POSTGRES_DB:-enervision}
|
|
SUPERVISION_DB_PASSWORD: ${SUPERVISION_DB_PASSWORD:-}
|
|
volumes:
|
|
- ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
|
|
- ./monitoring/grafana/dashboards:/etc/grafana/dashboards:ro
|
|
- grafana_data:/var/lib/grafana
|
|
ports:
|
|
- "127.0.0.1:${GRAFANA_PORT:-3001}:3000"
|
|
mem_limit: 256m
|
|
restart: unless-stopped
|
|
|
|
postgres-exporter:
|
|
image: prometheuscommunity/postgres-exporter:v0.20.1
|
|
profiles: ["monitoring"]
|
|
environment:
|
|
DATA_SOURCE_URI: db:5432/${POSTGRES_DB:-enervision}?sslmode=disable
|
|
DATA_SOURCE_USER: supervision
|
|
DATA_SOURCE_PASS: ${SUPERVISION_DB_PASSWORD:-}
|
|
mem_limit: 64m
|
|
restart: unless-stopped
|
|
|
|
node-exporter:
|
|
image: prom/node-exporter:v1.12.1
|
|
profiles: ["monitoring"]
|
|
command:
|
|
- --path.rootfs=/host
|
|
pid: host
|
|
volumes:
|
|
- /:/host:ro,rslave
|
|
mem_limit: 64m
|
|
restart: unless-stopped
|
|
|
|
# Contrainte : cAdvisor lit les cgroups de tous les conteneurs de l'hôte, d'où `privileged` et
|
|
# ses montages en lecture seule. Aucun port publié : seul Prometheus le joint.
|
|
cadvisor:
|
|
image: gcr.io/cadvisor/cadvisor:v0.55.1
|
|
profiles: ["monitoring"]
|
|
privileged: true
|
|
devices:
|
|
- /dev/kmsg
|
|
command:
|
|
- --docker_only=true
|
|
- --housekeeping_interval=30s
|
|
- --store_container_labels=false
|
|
volumes:
|
|
- /:/rootfs:ro
|
|
- /var/run:/var/run:ro
|
|
- /sys:/sys:ro
|
|
- /var/lib/docker/:/var/lib/docker:ro
|
|
- /dev/disk/:/dev/disk:ro
|
|
mem_limit: 160m
|
|
restart: unless-stopped
|
|
|
|
# Pourquoi : sur le réseau du projet, k6 joint `backend:8000` sans passer par nginx, dont la
|
|
# limite par adresse (20 req/s) fausserait la mesure de l'API. `make load-*` le lance (ADR 0015).
|
|
k6:
|
|
image: grafana/k6:2.3.0
|
|
profiles: ["load"]
|
|
volumes:
|
|
- ./tests/load:/scripts:ro
|
|
- ./tests/load/results:/results
|
|
environment:
|
|
K6_BASE_URL: ${K6_BASE_URL:-http://backend:8000}
|
|
K6_PROXY_URL: ${K6_PROXY_URL:-https://proxy}
|
|
K6_EMAIL: ${K6_EMAIL:-}
|
|
K6_PASSWORD: ${K6_PASSWORD:-}
|
|
K6_RESUME: ${K6_RESUME:-}
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
|
|
volumes:
|
|
pgdata:
|
|
garage_meta:
|
|
garage_data:
|
|
airflow_logs:
|
|
airflow_ml_state:
|
|
prometheus_data:
|
|
alertmanager_data:
|
|
grafana_data:
|
|
|
|
# Vide tant qu'APP_METRICS_TOKEN n'est pas posé : l'API n'exige alors aucun jeton.
|
|
secrets:
|
|
metrics_token:
|
|
environment: APP_METRICS_TOKEN
|
|
garage_metrics_token:
|
|
environment: GARAGE_METRICS_TOKEN
|