From 0c0bd77156e9dd02faeb8ff2a0aba4f067a3d347 Mon Sep 17 00:00:00 2001 From: Johan LEROY Date: Tue, 21 Jul 2026 14:05:19 +0200 Subject: [PATCH] TP2 Partie 1 : package lab + DVC (split full_history v1) --- .dvc/.gitignore | 2 + .dvc/config | 8 ++++ .dvcignore | 3 ++ .env.example | 15 ++++++ .gitignore | 23 +++++---- README.md | 58 +++++++++++++++++++++++ data/test.parquet.dvc | 5 ++ data/train.parquet.dvc | 5 ++ data/validation.parquet.dvc | 5 ++ lab/__init__.py | 0 lab/constants.py | 72 ++++++++++++++++++++++++++++ lab/modeling/__init__.py | 0 lab/modeling/cli.py | 77 ++++++++++++++++++++++++++++++ lab/modeling_ridge/__init__.py | 0 lab/modeling_ridge/cli.py | 85 ++++++++++++++++++++++++++++++++++ lab/split/__init__.py | 0 lab/split/cli.py | 35 ++++++++++++++ 17 files changed, 384 insertions(+), 9 deletions(-) create mode 100644 .dvc/.gitignore create mode 100644 .dvc/config create mode 100644 .dvcignore create mode 100644 .env.example create mode 100644 README.md create mode 100644 data/test.parquet.dvc create mode 100644 data/train.parquet.dvc create mode 100644 data/validation.parquet.dvc create mode 100644 lab/__init__.py create mode 100644 lab/constants.py create mode 100644 lab/modeling/__init__.py create mode 100644 lab/modeling/cli.py create mode 100644 lab/modeling_ridge/__init__.py create mode 100644 lab/modeling_ridge/cli.py create mode 100644 lab/split/__init__.py create mode 100644 lab/split/cli.py diff --git a/.dvc/.gitignore b/.dvc/.gitignore new file mode 100644 index 0000000..25e0cc7 --- /dev/null +++ b/.dvc/.gitignore @@ -0,0 +1,2 @@ +/tmp +/cache diff --git a/.dvc/config b/.dvc/config new file mode 100644 index 0000000..3fb4544 --- /dev/null +++ b/.dvc/config @@ -0,0 +1,8 @@ +[core] + analytics = false + remote = garage +['remote "garage"'] + url = s3://dvc-store + endpointurl = https://garage.192-168-122-143.nip.io + region = garage + ssl_verify = /etc/ssl/certs/ca-certificates.crt diff --git a/.dvcignore b/.dvcignore new file mode 100644 index 0000000..5197305 --- /dev/null +++ b/.dvcignore @@ -0,0 +1,3 @@ +# Add patterns of files dvc should ignore, which could improve +# the performance. Learn more at +# https://dvc.org/doc/user-guide/dvcignore diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8f2764c --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Copier en .env (gitignore) et renseigner les secrets. +# A sourcer avant de lancer les scripts : set -a; source .env; set +a + +# --- MLflow (serveur de la VM, basic auth) --- +MLFLOW_TRACKING_URI=https://mlflow.192-168-122-143.nip.io +MLFLOW_TRACKING_USERNAME=admin +MLFLOW_TRACKING_PASSWORD=change-me +MLFLOW_EXPERIMENT_NAME=tp02_electricity_consumption + +# --- CA interne ENI MLOps (les clients Python n'utilisent pas le store systeme par defaut) --- +REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + +# --- Import du package lab --- +PYTHONPATH=/home/user/tp diff --git a/.gitignore b/.gitignore index cade693..0634093 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,17 @@ -# Donnees du fil rouge (volumineuses, gerees hors git / DVC plus tard) -data/ -*.parquet -*.csv +# Donnees : gerees par DVC. Seuls les pointeurs .dvc et le .gitignore +# genere par DVC sont versionnes ; les .parquet reels vont sur le remote S3 (Garage). +/data/* +!/data/*.dvc +!/data/.gitignore + +# Secrets / environnement +.env +*.key +*.pem +.dvc/config.local + +# MLflow local eventuel (on utilise le serveur distant) +mlruns/ # Jupyter .ipynb_checkpoints/ @@ -14,8 +24,3 @@ __pycache__/ *.pyc .venv/ venv/ - -# Secrets / environnement -.env -*.key -*.pem diff --git a/README.md b/README.md new file mode 100644 index 0000000..88f584e --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# TP02 - Comparer et tracer des experimentations ML (DVC + MLflow) + +Fil rouge : prediction de la consommation electrique. Ce module compare des strategies +de features et de split, en versionnant les datasets avec **DVC** (remote S3 = Garage de la VM) +et en tracant les experiences avec **MLflow** (serveur de la VM). + +## Prerequis (sur la VM) + +- venv : `/opt/venvs/mlops` +- donnees source : `/data/modelling/features.parquet` + `target.parquet` +- `.env` renseigne (voir `.env.example`), puis : + +```bash +cd /home/user/tp +set -a; source .env; set +a +alias py=/opt/venvs/mlops/bin/python +alias dvc=/opt/venvs/mlops/bin/dvc +``` + +## Package + +- `lab/constants.py` : strategies de split (`full_history`, `recent_history`), strategies de + features (`short_memory`, `seasonality`, `tendency`, `mixed`, `full`), `RIDGE_ALPHAS`. +- `lab/split/cli.py` : lit `/data/modelling`, ecrit `data/{train,validation,test}.parquet` + selon `CHOSEN_SPLIT_STRATEGY`. +- `lab/modeling/cli.py` : `py -m lab.modeling.cli ` -> regression lineaire -> MLflow. +- `lab/modeling_ridge/cli.py` : `py -m lab.modeling_ridge.cli [--strategy ]` -> Ridge sur `RIDGE_ALPHAS`. + +## Versionner un dataset avec DVC + +```bash +py -m lab.split.cli +dvc add data/train.parquet data/validation.parquet data/test.parquet +git add data/*.dvc data/.gitignore lab .gitignore +git commit -m "split " +git tag dataset- +dvc push +``` + +Restaurer une version anterieure du dataset : + +```bash +git checkout dataset-v1-full-history -- data/train.parquet.dvc data/validation.parquet.dvc data/test.parquet.dvc +dvc checkout +``` + +## Entrainements + +```bash +for s in short_memory seasonality tendency mixed; do py -m lab.modeling.cli $s; done # Parties 1 et 2 +py -m lab.modeling_ridge.cli --strategy mixed # Partie 3 +``` + +Resultats et comparaisons : https://mlflow.192-168-122-143.nip.io (experience `tp02_electricity_consumption`). + +## Livrable + +Synthese des resultats et reponses aux questions : `SYNTHESE.md`. diff --git a/data/test.parquet.dvc b/data/test.parquet.dvc new file mode 100644 index 0000000..33dacd8 --- /dev/null +++ b/data/test.parquet.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 2f70e749c483c8c74cde844a69d52631 + size: 114542933 + hash: md5 + path: test.parquet diff --git a/data/train.parquet.dvc b/data/train.parquet.dvc new file mode 100644 index 0000000..12f2515 --- /dev/null +++ b/data/train.parquet.dvc @@ -0,0 +1,5 @@ +outs: +- md5: b08dfd4b6aa3f59d68220a35110df0b0 + size: 216998722 + hash: md5 + path: train.parquet diff --git a/data/validation.parquet.dvc b/data/validation.parquet.dvc new file mode 100644 index 0000000..9cdfd10 --- /dev/null +++ b/data/validation.parquet.dvc @@ -0,0 +1,5 @@ +outs: +- md5: 24bf315461302605f8fd229eb263f499 + size: 115206905 + hash: md5 + path: validation.parquet diff --git a/lab/__init__.py b/lab/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lab/constants.py b/lab/constants.py new file mode 100644 index 0000000..162d3d4 --- /dev/null +++ b/lab/constants.py @@ -0,0 +1,72 @@ +import datetime +from enum import StrEnum +from pathlib import Path +from typing import Literal + +# Racine du depot de travail (/home/user/tp sur la VM) : lab/constants.py -> parents[1] +REPO_ROOT = Path(__file__).resolve().parents[1] + +# Donnees source, deja preparees (hors git, volumineuses) : voir /data sur la VM +SOURCE_DIR = Path("/data/modelling") + +# Sorties de split versionnees par DVC dans le depot +DATASET_DIR = REPO_ROOT / "data" + +FEATURE_FILENAME = "features.parquet" +TARGET_FILENAME = "target.parquet" + + +class SplitStrategy(StrEnum): + FULL_HISTORY = "full_history" + RECENT_HISTORY = "recent_history" + + +# Strategie de split active : pilote a la fois le decoupage produit par split/cli.py +# et le parametre "split_strategy" logge dans MLflow. On la modifie (et on committe) +# a chaque changement de version de dataset pour synchroniser DVC et Git. +CHOSEN_SPLIT_STRATEGY = SplitStrategy.FULL_HISTORY + +DatasetPart = Literal["train", "test", "validation"] + +DATASET_SPLIT_DATES: dict[SplitStrategy, dict[DatasetPart, tuple[datetime.date, datetime.date]]] = { + # Partie 1 : tout l'historique disponible pour l'entrainement + SplitStrategy.FULL_HISTORY: { + "train": (datetime.date(2011, 1, 1), datetime.date(2012, 12, 31)), + "validation": (datetime.date(2013, 1, 1), datetime.date(2013, 12, 31)), + "test": (datetime.date(2014, 1, 1), datetime.date(2014, 12, 31)), + }, + # Partie 2 : donnees plus recentes uniquement + SplitStrategy.RECENT_HISTORY: { + "train": (datetime.date(2013, 1, 1), datetime.date(2013, 12, 31)), + "validation": (datetime.date(2014, 1, 1), datetime.date(2014, 5, 31)), + "test": (datetime.date(2014, 6, 1), datetime.date(2014, 12, 31)), + }, +} + + +class ModellingStrategy(StrEnum): + SHORT_MEMORY = "short_memory" + SEASONALITY = "seasonality" + TENDENCY = "tendency" + MIXED = "mixed" + FULL = "full" + + +Features = Literal["lag_1d", "lag_7d", "lag_30d", "lag_365d", "rolling_mean_7d", "rolling_mean_30d"] +TARGET = "consumption_kwh" + +MODELLING_FEATURES: dict[ModellingStrategy, list] = { + # La conso depend surtout de la veille + ModellingStrategy.SHORT_MEMORY: ["lag_1d"], + # La conso est plus saisonniere que journaliere + ModellingStrategy.SEASONALITY: ["lag_7d", "lag_30d"], + # La conso suit surtout une tendance + ModellingStrategy.TENDENCY: ["rolling_mean_7d", "rolling_mean_30d"], + # Melange lags + tendance + ModellingStrategy.MIXED: ["lag_1d", "lag_7d", "lag_30d", "rolling_mean_30d"], + # Toutes les features disponibles (ajoutee en Partie 2 Etape 3) + ModellingStrategy.FULL: ["lag_1d", "lag_7d", "lag_30d", "lag_365d", "rolling_mean_7d", "rolling_mean_30d"], +} + +# Valeurs d'alpha demandees par l'enonce (Partie 3) +RIDGE_ALPHAS = [1, 1e3, 1e9] diff --git a/lab/modeling/__init__.py b/lab/modeling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lab/modeling/cli.py b/lab/modeling/cli.py new file mode 100644 index 0000000..11182de --- /dev/null +++ b/lab/modeling/cli.py @@ -0,0 +1,77 @@ +import logging + +import mlflow +import pandas as pd +import typer +from sklearn import linear_model +from sklearn import metrics + +from .. import constants + +app = typer.Typer() + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@app.command() +def main( + strategy: constants.ModellingStrategy, +): + training_file_path = constants.DATASET_DIR / "train.parquet" + validation_file_path = constants.DATASET_DIR / "validation.parquet" + + features = constants.MODELLING_FEATURES[strategy] + + train_df = pd.read_parquet(training_file_path) + validation_df = pd.read_parquet(validation_file_path) + + train_df = train_df.dropna( + subset=features + [constants.TARGET] + ) + validation_df = validation_df.dropna( + subset=features + [constants.TARGET] + ) + + X_train = train_df[features] + y_train = train_df[constants.TARGET] + + X_validation = validation_df[features] + y_validation = validation_df[constants.TARGET] + + logger.info(f"Training model with strategy '{strategy}' and features {features}") + + with mlflow.start_run(run_name=f"modelling_{strategy.value}"): + mlflow.log_param("model_type", "linear") + mlflow.log_param("strategy", strategy.value) + mlflow.log_param("split_strategy", constants.CHOSEN_SPLIT_STRATEGY.value) + + mlflow.log_param("features", ",".join(features)) + model = linear_model.LinearRegression() + model.fit(X_train, y_train) + + train_predictions = model.predict(X_train) + validation_predictions = model.predict(X_validation) + + train_rmse = metrics.root_mean_squared_error(y_train, train_predictions) + validation_rmse = metrics.root_mean_squared_error(y_validation, validation_predictions) + + train_mae = metrics.mean_absolute_error(y_train, train_predictions) + validation_mae = metrics.mean_absolute_error(y_validation, validation_predictions) + + mlflow.log_metric("train_rmse", train_rmse) + mlflow.log_metric("validation_rmse", validation_rmse) + + mlflow.log_metric("train_mae", train_mae) + mlflow.log_metric("validation_mae", validation_mae) + + for feature_name, coefficient in zip( + features, + model.coef_, + strict=True, + ): + mlflow.log_metric(f"coef_{feature_name}", float(coefficient)) + + +if __name__ == "__main__": + app() diff --git a/lab/modeling_ridge/__init__.py b/lab/modeling_ridge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lab/modeling_ridge/cli.py b/lab/modeling_ridge/cli.py new file mode 100644 index 0000000..3fe235b --- /dev/null +++ b/lab/modeling_ridge/cli.py @@ -0,0 +1,85 @@ +import logging + +import mlflow +import pandas as pd +import typer +from sklearn import linear_model +from sklearn import metrics + +from .. import constants + +app = typer.Typer() + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@app.command() +def main( + strategy: constants.ModellingStrategy = constants.ModellingStrategy.MIXED, +): + training_file_path = constants.DATASET_DIR / "train.parquet" + validation_file_path = constants.DATASET_DIR / "validation.parquet" + + features = constants.MODELLING_FEATURES[strategy] + + train_df = pd.read_parquet(training_file_path) + validation_df = pd.read_parquet(validation_file_path) + + train_df = train_df.dropna( + subset=features + [constants.TARGET] + ) + validation_df = validation_df.dropna( + subset=features + [constants.TARGET] + ) + + X_train = train_df[features] + y_train = train_df[constants.TARGET] + + X_validation = validation_df[features] + y_validation = validation_df[constants.TARGET] + + logger.info(f"Training Ridge with strategy '{strategy}' and features {features}") + + for alpha in constants.RIDGE_ALPHAS: + with mlflow.start_run(run_name=f"modelling_{strategy.value}_ridge_alpha_{alpha:g}"): + mlflow.log_param("model_type", "ridge") + mlflow.log_param("strategy", strategy.value) + mlflow.log_param("split_strategy", constants.CHOSEN_SPLIT_STRATEGY.value) + + mlflow.log_param("features", ",".join(features)) + mlflow.log_param("alpha", alpha) + model = linear_model.Ridge(alpha=alpha) + model.fit(X_train, y_train) + + train_predictions = model.predict(X_train) + validation_predictions = model.predict(X_validation) + + train_rmse = metrics.root_mean_squared_error(y_train, train_predictions) + validation_rmse = metrics.root_mean_squared_error(y_validation, validation_predictions) + + train_mae = metrics.mean_absolute_error(y_train, train_predictions) + validation_mae = metrics.mean_absolute_error(y_validation, validation_predictions) + + mlflow.log_metric("train_rmse", train_rmse) + mlflow.log_metric("validation_rmse", validation_rmse) + + mlflow.log_metric("train_mae", train_mae) + mlflow.log_metric("validation_mae", validation_mae) + + for feature_name, coefficient in zip( + features, + model.coef_, + strict=True, + ): + mlflow.log_metric(f"coef_{feature_name}", float(coefficient)) + + mlflow.sklearn.log_model( + sk_model=model, + name="electricity_consumption_model", + registered_model_name="electricity_consumption_model", + ) + + +if __name__ == "__main__": + app() diff --git a/lab/split/__init__.py b/lab/split/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lab/split/cli.py b/lab/split/cli.py new file mode 100644 index 0000000..d3eab5c --- /dev/null +++ b/lab/split/cli.py @@ -0,0 +1,35 @@ +import logging + +import pandas as pd + +from .. import constants + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def main(): + # Entrees : dataset source deja prepare (hors git) + feature_file_path = constants.SOURCE_DIR / constants.FEATURE_FILENAME + target_file_path = constants.SOURCE_DIR / constants.TARGET_FILENAME + + logger.info(f"Read dataset from {feature_file_path} and {target_file_path}") + df_features = pd.read_parquet(feature_file_path) + df_target = pd.read_parquet(target_file_path) + + df = df_features.join(df_target) + timestamps = df.index.get_level_values("timestamp") + + # Sorties : splits versionnes par DVC dans le depot + constants.DATASET_DIR.mkdir(parents=True, exist_ok=True) + logger.info(f"Split strategy: {constants.CHOSEN_SPLIT_STRATEGY.value}") + for dataset_name, (start_date, end_date) in constants.DATASET_SPLIT_DATES[constants.CHOSEN_SPLIT_STRATEGY].items(): + file_path = constants.DATASET_DIR / f"{dataset_name}.parquet" + mask = (timestamps.date >= start_date) & (timestamps.date <= end_date) + df_split = df[mask] + logger.info(f"Split dataset into {dataset_name} ({start_date} -> {end_date}) with shape: {df_split.shape}") + df_split.to_parquet(file_path) + + +if __name__ == "__main__": + main()