TP2 Partie 1 : package lab + DVC (split full_history v1)

lab/constants.py

@@ -0,0 +1,72 @@
+import datetime
+from enum import StrEnum
+from pathlib import Path
+from typing import Literal
+
+# Racine du depot de travail (/home/user/tp sur la VM) : lab/constants.py -> parents[1]
+REPO_ROOT = Path(__file__).resolve().parents[1]
+
+# Donnees source, deja preparees (hors git, volumineuses) : voir /data sur la VM
+SOURCE_DIR = Path("/data/modelling")
+
+# Sorties de split versionnees par DVC dans le depot
+DATASET_DIR = REPO_ROOT / "data"
+
+FEATURE_FILENAME = "features.parquet"
+TARGET_FILENAME = "target.parquet"
+
+
+class SplitStrategy(StrEnum):
+    FULL_HISTORY = "full_history"
+    RECENT_HISTORY = "recent_history"
+
+
+# Strategie de split active : pilote a la fois le decoupage produit par split/cli.py
+# et le parametre "split_strategy" logge dans MLflow. On la modifie (et on committe)
+# a chaque changement de version de dataset pour synchroniser DVC et Git.
+CHOSEN_SPLIT_STRATEGY = SplitStrategy.FULL_HISTORY
+
+DatasetPart = Literal["train", "test", "validation"]
+
+DATASET_SPLIT_DATES: dict[SplitStrategy, dict[DatasetPart, tuple[datetime.date, datetime.date]]] = {
+    # Partie 1 : tout l'historique disponible pour l'entrainement
+    SplitStrategy.FULL_HISTORY: {
+        "train": (datetime.date(2011, 1, 1), datetime.date(2012, 12, 31)),
+        "validation": (datetime.date(2013, 1, 1), datetime.date(2013, 12, 31)),
+        "test": (datetime.date(2014, 1, 1), datetime.date(2014, 12, 31)),
+    },
+    # Partie 2 : donnees plus recentes uniquement
+    SplitStrategy.RECENT_HISTORY: {
+        "train": (datetime.date(2013, 1, 1), datetime.date(2013, 12, 31)),
+        "validation": (datetime.date(2014, 1, 1), datetime.date(2014, 5, 31)),
+        "test": (datetime.date(2014, 6, 1), datetime.date(2014, 12, 31)),
+    },
+}
+
+
+class ModellingStrategy(StrEnum):
+    SHORT_MEMORY = "short_memory"
+    SEASONALITY = "seasonality"
+    TENDENCY = "tendency"
+    MIXED = "mixed"
+    FULL = "full"
+
+
+Features = Literal["lag_1d", "lag_7d", "lag_30d", "lag_365d", "rolling_mean_7d", "rolling_mean_30d"]
+TARGET = "consumption_kwh"
+
+MODELLING_FEATURES: dict[ModellingStrategy, list] = {
+    # La conso depend surtout de la veille
+    ModellingStrategy.SHORT_MEMORY: ["lag_1d"],
+    # La conso est plus saisonniere que journaliere
+    ModellingStrategy.SEASONALITY: ["lag_7d", "lag_30d"],
+    # La conso suit surtout une tendance
+    ModellingStrategy.TENDENCY: ["rolling_mean_7d", "rolling_mean_30d"],
+    # Melange lags + tendance
+    ModellingStrategy.MIXED: ["lag_1d", "lag_7d", "lag_30d", "rolling_mean_30d"],
+    # Toutes les features disponibles (ajoutee en Partie 2 Etape 3)
+    ModellingStrategy.FULL: ["lag_1d", "lag_7d", "lag_30d", "lag_365d", "rolling_mean_7d", "rolling_mean_30d"],
+}
+
+# Valeurs d'alpha demandees par l'enonce (Partie 3)
+RIDGE_ALPHAS = [1, 1e3, 1e9]

lab/modeling/cli.py

@@ -0,0 +1,77 @@
+import logging
+
+import mlflow
+import pandas as pd
+import typer
+from sklearn import linear_model
+from sklearn import metrics
+
+from .. import constants
+
+app = typer.Typer()
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+@app.command()
+def main(
+    strategy: constants.ModellingStrategy,
+):
+    training_file_path = constants.DATASET_DIR / "train.parquet"
+    validation_file_path = constants.DATASET_DIR / "validation.parquet"
+
+    features = constants.MODELLING_FEATURES[strategy]
+
+    train_df = pd.read_parquet(training_file_path)
+    validation_df = pd.read_parquet(validation_file_path)
+
+    train_df = train_df.dropna(
+        subset=features + [constants.TARGET]
+    )
+    validation_df = validation_df.dropna(
+        subset=features + [constants.TARGET]
+    )
+
+    X_train = train_df[features]
+    y_train = train_df[constants.TARGET]
+
+    X_validation = validation_df[features]
+    y_validation = validation_df[constants.TARGET]
+
+    logger.info(f"Training model with strategy '{strategy}' and features {features}")
+
+    with mlflow.start_run(run_name=f"modelling_{strategy.value}"):
+        mlflow.log_param("model_type", "linear")
+        mlflow.log_param("strategy", strategy.value)
+        mlflow.log_param("split_strategy", constants.CHOSEN_SPLIT_STRATEGY.value)
+
+        mlflow.log_param("features", ",".join(features))
+        model = linear_model.LinearRegression()
+        model.fit(X_train, y_train)
+
+        train_predictions = model.predict(X_train)
+        validation_predictions = model.predict(X_validation)
+
+        train_rmse = metrics.root_mean_squared_error(y_train, train_predictions)
+        validation_rmse = metrics.root_mean_squared_error(y_validation, validation_predictions)
+
+        train_mae = metrics.mean_absolute_error(y_train, train_predictions)
+        validation_mae = metrics.mean_absolute_error(y_validation, validation_predictions)
+
+        mlflow.log_metric("train_rmse", train_rmse)
+        mlflow.log_metric("validation_rmse", validation_rmse)
+
+        mlflow.log_metric("train_mae", train_mae)
+        mlflow.log_metric("validation_mae", validation_mae)
+
+        for feature_name, coefficient in zip(
+            features,
+            model.coef_,
+            strict=True,
+        ):
+            mlflow.log_metric(f"coef_{feature_name}", float(coefficient))
+
+
+if __name__ == "__main__":
+    app()

lab/modeling_ridge/cli.py

@@ -0,0 +1,85 @@
+import logging
+
+import mlflow
+import pandas as pd
+import typer
+from sklearn import linear_model
+from sklearn import metrics
+
+from .. import constants
+
+app = typer.Typer()
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+@app.command()
+def main(
+    strategy: constants.ModellingStrategy = constants.ModellingStrategy.MIXED,
+):
+    training_file_path = constants.DATASET_DIR / "train.parquet"
+    validation_file_path = constants.DATASET_DIR / "validation.parquet"
+
+    features = constants.MODELLING_FEATURES[strategy]
+
+    train_df = pd.read_parquet(training_file_path)
+    validation_df = pd.read_parquet(validation_file_path)
+
+    train_df = train_df.dropna(
+        subset=features + [constants.TARGET]
+    )
+    validation_df = validation_df.dropna(
+        subset=features + [constants.TARGET]
+    )
+
+    X_train = train_df[features]
+    y_train = train_df[constants.TARGET]
+
+    X_validation = validation_df[features]
+    y_validation = validation_df[constants.TARGET]
+
+    logger.info(f"Training Ridge with strategy '{strategy}' and features {features}")
+
+    for alpha in constants.RIDGE_ALPHAS:
+        with mlflow.start_run(run_name=f"modelling_{strategy.value}_ridge_alpha_{alpha:g}"):
+            mlflow.log_param("model_type", "ridge")
+            mlflow.log_param("strategy", strategy.value)
+            mlflow.log_param("split_strategy", constants.CHOSEN_SPLIT_STRATEGY.value)
+
+            mlflow.log_param("features", ",".join(features))
+            mlflow.log_param("alpha", alpha)
+            model = linear_model.Ridge(alpha=alpha)
+            model.fit(X_train, y_train)
+
+            train_predictions = model.predict(X_train)
+            validation_predictions = model.predict(X_validation)
+
+            train_rmse = metrics.root_mean_squared_error(y_train, train_predictions)
+            validation_rmse = metrics.root_mean_squared_error(y_validation, validation_predictions)
+
+            train_mae = metrics.mean_absolute_error(y_train, train_predictions)
+            validation_mae = metrics.mean_absolute_error(y_validation, validation_predictions)
+
+            mlflow.log_metric("train_rmse", train_rmse)
+            mlflow.log_metric("validation_rmse", validation_rmse)
+
+            mlflow.log_metric("train_mae", train_mae)
+            mlflow.log_metric("validation_mae", validation_mae)
+
+            for feature_name, coefficient in zip(
+                features,
+                model.coef_,
+                strict=True,
+            ):
+                mlflow.log_metric(f"coef_{feature_name}", float(coefficient))
+
+            mlflow.sklearn.log_model(
+                sk_model=model,
+                name="electricity_consumption_model",
+                registered_model_name="electricity_consumption_model",
+            )
+
+
+if __name__ == "__main__":
+    app()

lab/split/cli.py

@@ -0,0 +1,35 @@
+import logging
+
+import pandas as pd
+
+from .. import constants
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def main():
+    # Entrees : dataset source deja prepare (hors git)
+    feature_file_path = constants.SOURCE_DIR / constants.FEATURE_FILENAME
+    target_file_path = constants.SOURCE_DIR / constants.TARGET_FILENAME
+
+    logger.info(f"Read dataset from {feature_file_path} and {target_file_path}")
+    df_features = pd.read_parquet(feature_file_path)
+    df_target = pd.read_parquet(target_file_path)
+
+    df = df_features.join(df_target)
+    timestamps = df.index.get_level_values("timestamp")
+
+    # Sorties : splits versionnes par DVC dans le depot
+    constants.DATASET_DIR.mkdir(parents=True, exist_ok=True)
+    logger.info(f"Split strategy: {constants.CHOSEN_SPLIT_STRATEGY.value}")
+    for dataset_name, (start_date, end_date) in constants.DATASET_SPLIT_DATES[constants.CHOSEN_SPLIT_STRATEGY].items():
+        file_path = constants.DATASET_DIR / f"{dataset_name}.parquet"
+        mask = (timestamps.date >= start_date) & (timestamps.date <= end_date)
+        df_split = df[mask]
+        logger.info(f"Split dataset into {dataset_name} ({start_date} -> {end_date}) with shape: {df_split.shape}")
+        df_split.to_parquet(file_path)
+
+
+if __name__ == "__main__":
+    main()