Skip to content

flexcv.results_handling

This module provides a CrossValidationResults class which can be used to summarize the results of CrossValidation.perform().

flexcv.results_handling.CrossValidationResults

Bases: dict

A summary of the results of CrossValidation.perform(). Cross validate returns a dictionary of results for each model with the form: 

{
    "model_name_1": {
        "model": [model_1_fold_1, model_1_fold_2, ...],
        "parameters": [params_1_fold_1, params_1_fold_2, ...],
        "metrics": [
            {
                "metric_1_fold_1": metric_value_1_fold_1,
                "metric_2_fold_1": metric_value_2_fold_1,
                    ...
            },
            {
                "metric_1_fold_2": metric_value_1_fold_2,
                "metric_2_fold_2": metric_value_2_fold_2,
                ...
            },
        ],
        "y_pred": [y_pred_1_fold_1, y_pred_1_fold_2, ...],
        "y_test": [y_test_1_fold_1, y_test_1_fold_2, ...],
        "y_pred_train": [y_pred_train_1_fold_1, y_pred_train_1_fold_2, ...],
        "y_train": [y_train_1_fold_1, y_train_1_fold_2, ...],
    },
    "model_name_2": {
        ...
    },
    ...
}
This class is a wrapper around this dictionary which provides a summary of the results.

  • _make_summary computes the mean, median and standard deviation of the metrics for each model.
  • _make_summary is called the first time the summary property is accessed and the result is cached.
  • _get_model returns the model instance corresponding to the given model name.
Properties

summary (pd.DataFrame): Summary of the results.

Source code in flexcv/results_handling.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
class CrossValidationResults(dict):
    """A summary of the results of CrossValidation.perform().
    Cross validate returns a dictionary of results for each model with the form:
    ```python
    {
        "model_name_1": {
            "model": [model_1_fold_1, model_1_fold_2, ...],
            "parameters": [params_1_fold_1, params_1_fold_2, ...],
            "metrics": [
                {
                    "metric_1_fold_1": metric_value_1_fold_1,
                    "metric_2_fold_1": metric_value_2_fold_1,
                        ...
                },
                {
                    "metric_1_fold_2": metric_value_1_fold_2,
                    "metric_2_fold_2": metric_value_2_fold_2,
                    ...
                },
            ],
            "y_pred": [y_pred_1_fold_1, y_pred_1_fold_2, ...],
            "y_test": [y_test_1_fold_1, y_test_1_fold_2, ...],
            "y_pred_train": [y_pred_train_1_fold_1, y_pred_train_1_fold_2, ...],
            "y_train": [y_train_1_fold_1, y_train_1_fold_2, ...],
        },
        "model_name_2": {
            ...
        },
        ...
    }
    ```
    This class is a wrapper around this dictionary which provides a summary of the results.

    - `_make_summary` computes the mean, median and standard deviation of the metrics for each model.
    - `_make_summary` is called the first time the summary property is accessed and the result is cached.
    - `_get_model` returns the model instance corresponding to the given model name.

    Properties:
        summary (pd.DataFrame): Summary of the results.

    """

    def __init__(self, results_dict):
        super().__init__(results_dict)
        self._summary = None
        # TODO add some kind of id or description

    def __repr__(self):
        return f"CrossValidationResults {pformat_dict(self)}"

    @property
    def summary(self):
        """Property that returns a pandas dataframe with the fold values, mean, median and standard deviation of the metrics for each model."""
        if self._summary is None:
            self._summary = self._make_summary()
        return self._summary

    def _make_summary(self):
        """Creates pandas dataframe with the fold values, mean, median and standard deviation of the metrics for each model.
        Columns: model names
        Multiindex from tuples: (fold id, metric)
        1. It reorders the data from
        ```python
        "metrics": [
                {
                    "metric_1_fold_1": metric_value_1_fold_1,
                    "metric_2_fold_1": metric_value_2_fold_1,
                        ...
                },
                {
                    "metric_1_fold_2": metric_value_1_fold_2,
                    "metric_2_fold_2": metric_value_2_fold_2,
                    ...
                },
            ],
        ```
        to the form
        ```python
        "metrics": {
            "metric_1": [metric_value_1_fold_1, metric_value_1_fold_2, ...],
            "metric_2": [metric_value_2_fold_1, metric_value_2_fold_2, ...],
            ...
        }
        ```

        """
        model_dfs = []
        for model in self.keys():
            metrics_dfs = []
            for metric in self[model]["folds_by_metrics"].keys():
                # collect the values for each fold for single metric
                values_df = pd.DataFrame(
                    self[model]["folds_by_metrics"][metric], columns=[metric]
                )
                values_df.index.name = "Fold"
                values_df = add_summary_stats(values_df)
                metrics_dfs.append(values_df)
            # concatenate the dataframes for each metric
            model_df = pd.concat(metrics_dfs, axis=1)
            # reshape to long format and add model name as column
            new_df = model_df.reset_index().melt(
                id_vars="Fold", var_name="Metric", value_name=model
            )
            # set the index to fold and metric
            new_df = new_df.set_index(["Fold", "Metric"])
            model_dfs.append(new_df)
        # concatenate the dataframes for each model
        summary_df = pd.concat(model_dfs, axis=1)
        return summary_df

    def get_best_model_by_metric(
        self, model_name=None, metric_name="mse", direction="min"
    ):
        """Returns the model with the best metric value for the given metric.
        Direction can be "min" or "max" and determines whether the best model is the one with the lowest or highest metric value.
        E.g. for MSE, direction should be "min" and for R2, direction should be "max".

        Args:
          model_name (str): Name of the model (Default value = None)
          metric_name (str): Name for the metric (Default value = "mse")
          direction (str): Minimize or maximize. (Default value = "min")

        Returns:
            (object): The model with the best metric value for the given metric.
        """
        assert direction in [
            "min",
            "max",
        ], f"direction must be 'min' or 'max', got {direction}"
        arg_func = np.argmin if direction == "min" else np.argmax
        op_func = operator.lt if direction == "min" else operator.gt

        best_model_name = None
        best_metric_value = None
        best_metric_index = None

        iter_dict = (
            self.items() if model_name is None else [(model_name, self[model_name])]
        )

        for model_name, model_results in iter_dict:
            metric_values = model_results["metrics"][metric_name]
            metric_index = arg_func(metric_values)
            metric_value = metric_values[metric_index]

            if best_metric_value is None or op_func(metric_value, best_metric_value):
                best_metric_value = metric_value
                best_metric_index = metric_index
                best_model_name = model_name

        return self[best_model_name]["model"][best_metric_index]

    def get_predictions(self, model_name, fold_id):
        """Returns the predictions for the given model and fold.

        Args:
          model_name (str): The name of the model.
          fold_id (int): The id of the fold.

        Returns:
          (array-like): The predictions for the given model and fold.

        """
        return self[model_name]["y_pred"][fold_id]

    def get_true_values(self, model_name, fold_id):
        """Returns the true values for the given model and fold.

        Args:
          model_name (str): The name of the model.
          fold_id (int): The id of the fold.

        Returns:
            (array-like): The true values for the given model and fold.
        """
        return self[model_name]["y_test"][fold_id]

    def get_training_predictions(self, model_name, fold_id):
        """Returns the predictions for the given model and fold.

        Args:
          model_name (str): The name of the model.
          fold_id (int): The id of the fold.

        Returns:
            (array-like): The training predictions for the given model and fold.
        """
        return self[model_name]["y_pred_train"][fold_id]

    def get_training_true_values(self, model_name, fold_id):
        """Returns the true values for the given model and fold.

        Args:
          model_name (str): The name of the model.
          fold_id (int): The id of the fold.

        Returns:
            (array-like): The training true values for the given model and fold.
        """
        return self[model_name]["y_train"][fold_id]

    def get_params(self, model_name, fold_id):
        """Returns the parameters for the given model and fold.
        If the key is not found, returns None and will not raise an error.

        Args:
          model_name (str): The name of the model.
          fold_id (int): The id of the fold.

        Returns:
            (dict): The parameters for the given model and fold.
        """
        return self[model_name].get("parameters", [None])[fold_id]

flexcv.results_handling.CrossValidationResults.summary property

Property that returns a pandas dataframe with the fold values, mean, median and standard deviation of the metrics for each model.

flexcv.results_handling.CrossValidationResults.get_best_model_by_metric(model_name=None, metric_name='mse', direction='min')

Returns the model with the best metric value for the given metric. Direction can be "min" or "max" and determines whether the best model is the one with the lowest or highest metric value. E.g. for MSE, direction should be "min" and for R2, direction should be "max".

Parameters:

Name Type Description Default
model_name str

Name of the model (Default value = None)

 None
metric_name str

Name for the metric (Default value = "mse")

 'mse'
direction str

Minimize or maximize. (Default value = "min")

 'min'

Returns:

Type Description
object

The model with the best metric value for the given metric.
Source code in flexcv/results_handling.py 
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
def get_best_model_by_metric(
    self, model_name=None, metric_name="mse", direction="min"
):
    """Returns the model with the best metric value for the given metric.
    Direction can be "min" or "max" and determines whether the best model is the one with the lowest or highest metric value.
    E.g. for MSE, direction should be "min" and for R2, direction should be "max".

    Args:
      model_name (str): Name of the model (Default value = None)
      metric_name (str): Name for the metric (Default value = "mse")
      direction (str): Minimize or maximize. (Default value = "min")

    Returns:
        (object): The model with the best metric value for the given metric.
    """
    assert direction in [
        "min",
        "max",
    ], f"direction must be 'min' or 'max', got {direction}"
    arg_func = np.argmin if direction == "min" else np.argmax
    op_func = operator.lt if direction == "min" else operator.gt

    best_model_name = None
    best_metric_value = None
    best_metric_index = None

    iter_dict = (
        self.items() if model_name is None else [(model_name, self[model_name])]
    )

    for model_name, model_results in iter_dict:
        metric_values = model_results["metrics"][metric_name]
        metric_index = arg_func(metric_values)
        metric_value = metric_values[metric_index]

        if best_metric_value is None or op_func(metric_value, best_metric_value):
            best_metric_value = metric_value
            best_metric_index = metric_index
            best_model_name = model_name

    return self[best_model_name]["model"][best_metric_index]

flexcv.results_handling.CrossValidationResults.get_params(model_name, fold_id)

Returns the parameters for the given model and fold. If the key is not found, returns None and will not raise an error.

Parameters:

Name Type Description Default
model_name str

The name of the model.

 required
fold_id int

The id of the fold.

 required

Returns:

Type Description
dict

The parameters for the given model and fold.
Source code in flexcv/results_handling.py 
232
233
234
235
236
237
238
239
240
241
242
243
def get_params(self, model_name, fold_id):
    """Returns the parameters for the given model and fold.
    If the key is not found, returns None and will not raise an error.

    Args:
      model_name (str): The name of the model.
      fold_id (int): The id of the fold.

    Returns:
        (dict): The parameters for the given model and fold.
    """
    return self[model_name].get("parameters", [None])[fold_id]

flexcv.results_handling.CrossValidationResults.get_predictions(model_name, fold_id)

Returns the predictions for the given model and fold.

Parameters:

Name Type Description Default
model_name str

The name of the model.

 required
fold_id int

The id of the fold.

 required

Returns:

Type Description
array - like

The predictions for the given model and fold.
Source code in flexcv/results_handling.py 
183
184
185
186
187
188
189
190
191
192
193
194
def get_predictions(self, model_name, fold_id):
    """Returns the predictions for the given model and fold.

    Args:
      model_name (str): The name of the model.
      fold_id (int): The id of the fold.

    Returns:
      (array-like): The predictions for the given model and fold.

    """
    return self[model_name]["y_pred"][fold_id]

flexcv.results_handling.CrossValidationResults.get_training_predictions(model_name, fold_id)

Returns the predictions for the given model and fold.

Parameters:

Name Type Description Default
model_name str

The name of the model.

 required
fold_id int

The id of the fold.

 required

Returns:

Type Description
array - like

The training predictions for the given model and fold.
Source code in flexcv/results_handling.py 
208
209
210
211
212
213
214
215
216
217
218
def get_training_predictions(self, model_name, fold_id):
    """Returns the predictions for the given model and fold.

    Args:
      model_name (str): The name of the model.
      fold_id (int): The id of the fold.

    Returns:
        (array-like): The training predictions for the given model and fold.
    """
    return self[model_name]["y_pred_train"][fold_id]

flexcv.results_handling.CrossValidationResults.get_training_true_values(model_name, fold_id)

Returns the true values for the given model and fold.

Parameters:

Name Type Description Default
model_name str

The name of the model.

 required
fold_id int

The id of the fold.

 required

Returns:

Type Description
array - like

The training true values for the given model and fold.
Source code in flexcv/results_handling.py 
220
221
222
223
224
225
226
227
228
229
230
def get_training_true_values(self, model_name, fold_id):
    """Returns the true values for the given model and fold.

    Args:
      model_name (str): The name of the model.
      fold_id (int): The id of the fold.

    Returns:
        (array-like): The training true values for the given model and fold.
    """
    return self[model_name]["y_train"][fold_id]

flexcv.results_handling.CrossValidationResults.get_true_values(model_name, fold_id)

Returns the true values for the given model and fold.

Parameters:

Name Type Description Default
model_name str

The name of the model.

 required
fold_id int

The id of the fold.

 required

Returns:

Type Description
array - like

The true values for the given model and fold.
Source code in flexcv/results_handling.py 
196
197
198
199
200
201
202
203
204
205
206
def get_true_values(self, model_name, fold_id):
    """Returns the true values for the given model and fold.

    Args:
      model_name (str): The name of the model.
      fold_id (int): The id of the fold.

    Returns:
        (array-like): The true values for the given model and fold.
    """
    return self[model_name]["y_test"][fold_id]

flexcv.results_handling.MergedSummary

Bases: CrossValidationResults

Source code in flexcv/results_handling.py 
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
class MergedSummary(CrossValidationResults):
    """ """

    def __init__(
        self,
        cv_results_1,
        cv_results_2,
    ):
        self.cv_results_1 = cv_results_1
        self.cv_results_2 = cv_results_2

    @property
    def summary(self):
        """ """
        return self._merge()

    def _merge(self):
        """Merges the two summaries dataframes into one."""
        return pd.concat([self.cv_results_1.summary, self.cv_results_2.summary], axis=1)

    def __add__(self, other):
        raise NotImplementedError

flexcv.results_handling.MergedSummary.summary property

flexcv.results_handling.add_summary_stats(df)

Add summary statistics to a pandas DataFrame. Calculates the mean, median and standard deviation on copies slices of the original data and adds them as rows to the DataFrame. This makes sure, that the summary statistics are not sequentially dependent on each other.

Parameters:

Name Type Description Default
df DataFrame

pd.DataFrame: Input data.

 required

Returns:

Type Description
DataFrame

Data with added summary statistics
Source code in flexcv/results_handling.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
def add_summary_stats(df: pd.DataFrame) -> pd.DataFrame:
    """Add summary statistics to a pandas DataFrame.
    Calculates the mean, median and standard deviation on copies slices of the original data and adds them as rows to the DataFrame.
    This makes sure, that the summary statistics are not sequentially dependent on each other.

    Args:
      df: pd.DataFrame: Input data.

    Returns:
      (pd.DataFrame): Data with added summary statistics

    """
    original_fold_slice = df.copy(deep=True)
    df.loc["mean"] = original_fold_slice.mean(skipna=True)
    df.loc["median"] = original_fold_slice.median(skipna=True)
    df.loc["std"] = original_fold_slice.std(skipna=True)
    return df

flexcv.fold_results_handling

flexcv.fold_results_handling.SingleModelFoldResult dataclass

This dataclass is used to store the fold data as well as the predictions of a single model in a single fold. It's make_results method is used to evaluate the model with the metrics and log the results to Neptune.

Attributes:

Name Type Description
k int

The fold number.
model_name str

The name of the model.
best_model object

The best model after inner cv or the model when skipping inner cv.
best_params dict | Any

The best parameters.
y_pred Series

The predictions of the model.
y_test Series

The test data.
X_test DataFrame

The test data.
y_train Series

The train data.
y_pred_train Series

The predictions of the model.
X_train DataFrame

The train data.
fit_result Any

The result of the fit method of the model.
fit_kwargs dict

Additional keyword arguments to pass to the fit method. (default: None)
Source code in flexcv/fold_results_handling.py 
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
@dataclass
class SingleModelFoldResult:
    """This dataclass is used to store the fold data as well as the predictions of a single model in a single fold.
    It's make_results method is used to evaluate the model with the metrics and log the results to Neptune.

    Attributes:
        k (int): The fold number.
        model_name (str): The name of the model.
        best_model (object): The best model after inner cv or the model when skipping inner cv.
        best_params (dict | Any): The best parameters.
        y_pred (pd.Series): The predictions of the model.
        y_test (pd.Series): The test data.
        X_test (pd.DataFrame): The test data.
        y_train (pd.Series): The train data.
        y_pred_train (pd.Series): The predictions of the model.
        X_train (pd.DataFrame): The train data.
        fit_result (Any): The result of the fit method of the model.
        fit_kwargs (dict): Additional keyword arguments to pass to the fit method. (default: None)

    """

    k: int
    model_name: str
    best_model: object
    best_params: dict | Any
    y_pred: pd.Series
    y_test: pd.Series
    X_test: pd.DataFrame
    y_train: pd.Series
    y_pred_train: pd.Series
    X_train: pd.DataFrame
    fit_result: Any
    fit_kwargs: dict = None

    def make_results(
        self,
        run,
        results_all_folds,
        study: Study | None,
        metrics: MetricsDict = METRICS,
    ):
        """This method is used to evaluate the model with the metrics and log the results to Neptune.

        Args:
          run (neptune.run): Neptune run object.
          results_all_folds (dict): Dictionary containing the results of all models and folds.
          study (optuna.study): Optuna study object.
          metrics (dict): Dictionary containing the metrics to be evaluated.

        Returns:
          (dict): Dictionary containing the results of all models and folds.
        """

        def res_vs_fitted_plot(y_test, y_pred):
            fig = plt.figure()
            residuals = y_test - y_pred
            plt.scatter(y_pred, residuals)
            plt.xlabel("Fitted values")
            plt.ylabel("Residuals")
            plt.title("Residuals vs Fitted Values")
            return fig

        if metrics is None:
            metrics = METRICS

        inner_cv_exists = True
        if not study is None:
            df = study.trials_dataframe()
            best_idx = study.best_trial.number
            mse_in_train = -df.iloc[best_idx]["user_attrs_mean_train_score"]
            mse_in_test = -df.iloc[best_idx]["user_attrs_mean_test_score"]
            of = -df.iloc[best_idx]["user_attrs_mean_OF_score"]
        else:
            inner_cv_exists = False
            mse_in_train = np.nan
            mse_in_test = np.nan
            of = np.nan

        eval_metrics = {}
        for metric_name, metric_func in metrics.items():
            eval_metrics[metric_name] = metric_func(self.y_test, self.y_pred)
            eval_metrics[metric_name + "_train"] = metric_func(
                self.y_train, self.y_pred_train
            )

        eval_metrics["mse_in_test"] = mse_in_test
        eval_metrics["mse_in_train"] = mse_in_train

        if self.model_name not in results_all_folds:
            results_all_folds[self.model_name] = {
                "model": [],
                "parameters": [],
                "metrics": [],
                "folds_by_metrics": {},
                "y_pred": [],
                "y_test": [],
                "y_pred_train": [],
                "y_train": [],
            }

        for metric_name in eval_metrics.keys():
            results_all_folds[self.model_name]["folds_by_metrics"].setdefault(
                metric_name, []
            ).append(eval_metrics[metric_name])

        results_all_folds[self.model_name]["metrics"].append(eval_metrics)
        results_all_folds[self.model_name]["model"].append(self.best_model)
        results_all_folds[self.model_name]["parameters"].append(self.best_params)
        results_all_folds[self.model_name]["y_pred"].append(self.y_pred)
        results_all_folds[self.model_name]["y_test"].append(self.y_test)
        results_all_folds[self.model_name]["y_pred_train"].append(self.y_pred_train)
        results_all_folds[self.model_name]["y_train"].append(self.y_train)

        for key, value in eval_metrics.items():
            run[f"{self.model_name}/{key}"].append(value)

        run[f"{self.model_name}/ObjectiveValue"].append(of)

        # saving the model
        # check if model has method save_raw -> important for xgboost

        try:
            self.best_model.save_model(f"{self.model_name}_{self.k}.json")
            run[f"{self.model_name}/Model/{self.k}"].upload(f"{self.model_name}_{self.k}.json")
            Path(f"{self.model_name}_{self.k}.json").unlink()
        except (AttributeError, KeyError):
            # AttributeError is raised when model has no method save_raw
            # KeyError is raised when model has method save_raw but no raw_format='json'
            run[f"{self.model_name}/Model/{self.k}"].upload(File.as_pickle(self.best_model))

        try:
            run[f"{self.model_name}/Parameters/"] = stringify_unsupported(
                npt_utils.get_estimator_params(self.best_model)
            )
        except (RuntimeError, TypeError):
            # is raised when model is not a scikit-learn model
            run[f"{self.model_name}/Parameters/"].append(pformat(self.best_params))

        run[f"{self.model_name}/ResidualPlot/"].append(
            res_vs_fitted_plot(self.y_test, self.y_pred)
        )

        try:
            run[
                f"{self.model_name}/RegressionSummary/{self.k}"
            ] = npt_utils.create_regressor_summary(
                self.best_model, self.X_train, self.X_test, self.y_train, self.y_test
            )
        except (KeyError, TypeError, RuntimeError, AssertionError) as e:
            # is raised when model is not a scikit-learn model
            logger.info(
                f"Regression summary not available for model {self.model_name}. Skipping.\n{e}")
            run[f"{self.model_name}/RegressionSummary/{self.k}"] = f"Not available: {e}"

        plt.close()
        return results_all_folds

flexcv.fold_results_handling.SingleModelFoldResult.make_results(run, results_all_folds, study, metrics=METRICS)

This method is used to evaluate the model with the metrics and log the results to Neptune.

Parameters:

Name Type Description Default
run run

Neptune run object.

 required
results_all_folds dict

Dictionary containing the results of all models and folds.

 required
study study

Optuna study object.

 required
metrics dict

Dictionary containing the metrics to be evaluated.

 METRICS

Returns:

Type Description
dict

Dictionary containing the results of all models and folds.
Source code in flexcv/fold_results_handling.py 
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
def make_results(
    self,
    run,
    results_all_folds,
    study: Study | None,
    metrics: MetricsDict = METRICS,
):
    """This method is used to evaluate the model with the metrics and log the results to Neptune.

    Args:
      run (neptune.run): Neptune run object.
      results_all_folds (dict): Dictionary containing the results of all models and folds.
      study (optuna.study): Optuna study object.
      metrics (dict): Dictionary containing the metrics to be evaluated.

    Returns:
      (dict): Dictionary containing the results of all models and folds.
    """

    def res_vs_fitted_plot(y_test, y_pred):
        fig = plt.figure()
        residuals = y_test - y_pred
        plt.scatter(y_pred, residuals)
        plt.xlabel("Fitted values")
        plt.ylabel("Residuals")
        plt.title("Residuals vs Fitted Values")
        return fig

    if metrics is None:
        metrics = METRICS

    inner_cv_exists = True
    if not study is None:
        df = study.trials_dataframe()
        best_idx = study.best_trial.number
        mse_in_train = -df.iloc[best_idx]["user_attrs_mean_train_score"]
        mse_in_test = -df.iloc[best_idx]["user_attrs_mean_test_score"]
        of = -df.iloc[best_idx]["user_attrs_mean_OF_score"]
    else:
        inner_cv_exists = False
        mse_in_train = np.nan
        mse_in_test = np.nan
        of = np.nan

    eval_metrics = {}
    for metric_name, metric_func in metrics.items():
        eval_metrics[metric_name] = metric_func(self.y_test, self.y_pred)
        eval_metrics[metric_name + "_train"] = metric_func(
            self.y_train, self.y_pred_train
        )

    eval_metrics["mse_in_test"] = mse_in_test
    eval_metrics["mse_in_train"] = mse_in_train

    if self.model_name not in results_all_folds:
        results_all_folds[self.model_name] = {
            "model": [],
            "parameters": [],
            "metrics": [],
            "folds_by_metrics": {},
            "y_pred": [],
            "y_test": [],
            "y_pred_train": [],
            "y_train": [],
        }

    for metric_name in eval_metrics.keys():
        results_all_folds[self.model_name]["folds_by_metrics"].setdefault(
            metric_name, []
        ).append(eval_metrics[metric_name])

    results_all_folds[self.model_name]["metrics"].append(eval_metrics)
    results_all_folds[self.model_name]["model"].append(self.best_model)
    results_all_folds[self.model_name]["parameters"].append(self.best_params)
    results_all_folds[self.model_name]["y_pred"].append(self.y_pred)
    results_all_folds[self.model_name]["y_test"].append(self.y_test)
    results_all_folds[self.model_name]["y_pred_train"].append(self.y_pred_train)
    results_all_folds[self.model_name]["y_train"].append(self.y_train)

    for key, value in eval_metrics.items():
        run[f"{self.model_name}/{key}"].append(value)

    run[f"{self.model_name}/ObjectiveValue"].append(of)

    # saving the model
    # check if model has method save_raw -> important for xgboost

    try:
        self.best_model.save_model(f"{self.model_name}_{self.k}.json")
        run[f"{self.model_name}/Model/{self.k}"].upload(f"{self.model_name}_{self.k}.json")
        Path(f"{self.model_name}_{self.k}.json").unlink()
    except (AttributeError, KeyError):
        # AttributeError is raised when model has no method save_raw
        # KeyError is raised when model has method save_raw but no raw_format='json'
        run[f"{self.model_name}/Model/{self.k}"].upload(File.as_pickle(self.best_model))

    try:
        run[f"{self.model_name}/Parameters/"] = stringify_unsupported(
            npt_utils.get_estimator_params(self.best_model)
        )
    except (RuntimeError, TypeError):
        # is raised when model is not a scikit-learn model
        run[f"{self.model_name}/Parameters/"].append(pformat(self.best_params))

    run[f"{self.model_name}/ResidualPlot/"].append(
        res_vs_fitted_plot(self.y_test, self.y_pred)
    )

    try:
        run[
            f"{self.model_name}/RegressionSummary/{self.k}"
        ] = npt_utils.create_regressor_summary(
            self.best_model, self.X_train, self.X_test, self.y_train, self.y_test
        )
    except (KeyError, TypeError, RuntimeError, AssertionError) as e:
        # is raised when model is not a scikit-learn model
        logger.info(
            f"Regression summary not available for model {self.model_name}. Skipping.\n{e}")
        run[f"{self.model_name}/RegressionSummary/{self.k}"] = f"Not available: {e}"

    plt.close()
    return results_all_folds