plot_calibration with examples#

An example showing the plot_calibration function used by a scikit-learn classifier.

 9 # Authors: The scikit-plots developers
10 # SPDX-License-Identifier: BSD-3-Clause
11
12 # run: Python scripts and shows any outputs directly in the notebook.
13 # %run ./examples/calibration/plot_calibration_script.py

Import scikit-plots#

19 from sklearn.calibration import CalibratedClassifierCV
20 from sklearn.datasets import make_classification
21 from sklearn.ensemble import RandomForestClassifier
22 from sklearn.linear_model import LogisticRegression
23 from sklearn.naive_bayes import GaussianNB
24 from sklearn.svm import LinearSVC
25
26 import numpy as np
27
28 np.random.seed(0)  # reproducibility
29 # importing pylab or pyplot
30 import matplotlib.pyplot as plt
31
32 # Import scikit-plot
33 import scikitplot as sp

Loading the dataset#

39 # Load the data
40 X, y = make_classification(
41     n_samples=100000,
42     n_features=20,
43     n_informative=4,
44     n_redundant=2,
45     n_repeated=0,
46     n_classes=3,
47     n_clusters_per_class=2,
48     random_state=0,
49 )
50 X_train, y_train, X_val, y_val = X[:1000], y[:1000], X[1000:], y[1000:]

Model Training#

56 # Create an instance of the LogisticRegression
57 lr_probas = (
58     LogisticRegression(max_iter=int(1e5), random_state=0)
59     .fit(X_train, y_train)
60     .predict_proba(X_val)
61 )
62 nb_probas = GaussianNB().fit(X_train, y_train).predict_proba(X_val)
63 svc_scores = LinearSVC().fit(X_train, y_train).decision_function(X_val)
64 svc_isotonic = (
65     CalibratedClassifierCV(LinearSVC(), cv=2, method="isotonic")
66     .fit(X_train, y_train)
67     .predict_proba(X_val)
68 )
69 svc_sigmoid = (
70     CalibratedClassifierCV(LinearSVC(), cv=2, method="sigmoid")
71     .fit(X_train, y_train)
72     .predict_proba(X_val)
73 )
74 rf_probas = (
75     RandomForestClassifier(random_state=0).fit(X_train, y_train).predict_proba(X_val)
76 )
77
78 probas_dict = {
79     LogisticRegression(): lr_probas,
80     # GaussianNB(): nb_probas,
81     "LinearSVC() + MinMax": svc_scores,
82     "LinearSVC() + Isotonic": svc_isotonic,
83     "LinearSVC() + Sigmoid": svc_sigmoid,
84     # RandomForestClassifier(): rf_probas,
85 }
86 probas_dict
{LogisticRegression(): array([[0.81630981, 0.09296828, 0.09072191],
       [0.67725691, 0.0160746 , 0.30666849],
       [0.25378422, 0.00096865, 0.74524713],
       ...,
       [0.01335815, 0.62376013, 0.36288171],
       [0.03551495, 0.0508954 , 0.91358965],
       [0.00664527, 0.70371215, 0.28964258]]), 'LinearSVC() + MinMax': array([[ 0.3380658 , -0.70059113, -0.84551216],
       [ 0.32189335, -1.54567754, -0.11575068],
       [ 0.22549259, -2.49509036,  0.69223215],
       ...,
       [-1.4690199 ,  0.16087262, -0.22137806],
       [-1.00072177, -0.95593185,  0.42770417],
       [-1.65162242,  0.34262892, -0.20019956]]), 'LinearSVC() + Isotonic': array([[0.75192088, 0.18665461, 0.06142451],
       [0.55808311, 0.        , 0.44191689],
       [0.34438275, 0.        , 0.65561725],
       ...,
       [0.01302864, 0.60399643, 0.38297494],
       [0.05420294, 0.10488174, 0.84091532],
       [0.        , 0.58988428, 0.41011572]]), 'LinearSVC() + Sigmoid': array([[0.73559196, 0.13937371, 0.12503433],
       [0.6086243 , 0.02233023, 0.36904547],
       [0.37500556, 0.00263819, 0.62235624],
       ...,
       [0.02215638, 0.60023759, 0.37760603],
       [0.07558115, 0.1089974 , 0.81542145],
       [0.01489511, 0.64718593, 0.33791896]])}

Plot!#

 92 # Plot!
 93 fig, ax = plt.subplots(figsize=(12, 6))
 94 ax = sp.metrics.plot_calibration(
 95     y_val,
 96     y_probas_list=probas_dict.values(),
 97     estimator_names=probas_dict.keys(),
 98     ax=ax,
 99     save_fig=True,
100     save_fig_filename="",
101     overwrite=True,
102     add_timestamp=True,
103     verbose=True,
104 )
Calibration Curves (Reliability Diagrams)
2025-06-29 13:41:58.281026: E scikitplot 140126653119360 _calibration.py:318:plot_calibration] 3
[INFO] Saving path to: /home/circleci/repo/galleries/examples/calibration/result_images/plot_calibration_20250629_134158Z.png
[INFO] Plot saved to: /home/circleci/repo/galleries/examples/calibration/result_images/plot_calibration_20250629_134158Z.png

Interpretation

Primary Use: Evaluating probabilistic classifiers by comparing predicted probabilities to observed frequencies of the positive class.

Goal: To assess how well the predicted probabilities align with the actual outcomes, identifying if a model is well-calibrated, overconfident, or underconfident.

Typical Characteristics:

  • X-axis: Predicted probability (e.g., in bins from 0 to 1).

  • Y-axis: Observed frequency of the positive class within each bin.

  • Reference line (diagonal at 45°): Represents perfect calibration, where predicted probabilities match observed frequencies.

Tags: model-type: classification model-workflow: model evaluation component: fitted model plot-type: line plot-type: calibration plot level: beginner purpose: showcase

Total running time of the script: (0 minutes 2.589 seconds)

Related examples

plot_precision_recall with examples

plot_precision_recall with examples

plot_roc_curve with examples

plot_roc_curve with examples

plot_confusion_matrix with examples

plot_confusion_matrix with examples

plot_classifier_eval with examples

plot_classifier_eval with examples

Gallery generated by Sphinx-Gallery