plot_calibration with examples#
An example showing the plot_calibration function
used by a scikit-learn classifier.
9 # Authors: The scikit-plots developers
10 # SPDX-License-Identifier: BSD-3-Clause
11
12 # run: Python scripts and shows any outputs directly in the notebook.
13 # %run ./examples/calibration/plot_calibration_script.py
Import scikit-plots#
19 from sklearn.calibration import CalibratedClassifierCV
20 from sklearn.datasets import make_classification
21 from sklearn.ensemble import RandomForestClassifier
22 from sklearn.linear_model import LogisticRegression
23 from sklearn.naive_bayes import GaussianNB
24 from sklearn.svm import LinearSVC
25
26 import numpy as np
27
28 np.random.seed(0) # reproducibility
29 # importing pylab or pyplot
30 import matplotlib.pyplot as plt
31
32 # Import scikit-plot
33 import scikitplot as sp
Loading the dataset#
39 # Load the data
40 X, y = make_classification(
41 n_samples=100000,
42 n_features=20,
43 n_informative=4,
44 n_redundant=2,
45 n_repeated=0,
46 n_classes=3,
47 n_clusters_per_class=2,
48 random_state=0,
49 )
50 X_train, y_train, X_val, y_val = X[:1000], y[:1000], X[1000:], y[1000:]
Model Training#
56 # Create an instance of the LogisticRegression
57 lr_probas = (
58 LogisticRegression(max_iter=int(1e5), random_state=0)
59 .fit(X_train, y_train)
60 .predict_proba(X_val)
61 )
62 nb_probas = GaussianNB().fit(X_train, y_train).predict_proba(X_val)
63 svc_scores = LinearSVC().fit(X_train, y_train).decision_function(X_val)
64 svc_isotonic = (
65 CalibratedClassifierCV(LinearSVC(), cv=2, method="isotonic")
66 .fit(X_train, y_train)
67 .predict_proba(X_val)
68 )
69 svc_sigmoid = (
70 CalibratedClassifierCV(LinearSVC(), cv=2, method="sigmoid")
71 .fit(X_train, y_train)
72 .predict_proba(X_val)
73 )
74 rf_probas = (
75 RandomForestClassifier(random_state=0).fit(X_train, y_train).predict_proba(X_val)
76 )
77
78 probas_dict = {
79 LogisticRegression(): lr_probas,
80 # GaussianNB(): nb_probas,
81 "LinearSVC() + MinMax": svc_scores,
82 "LinearSVC() + Isotonic": svc_isotonic,
83 "LinearSVC() + Sigmoid": svc_sigmoid,
84 # RandomForestClassifier(): rf_probas,
85 }
86 probas_dict
{LogisticRegression(): array([[0.81630981, 0.09296828, 0.09072191],
[0.67725691, 0.0160746 , 0.30666849],
[0.25378422, 0.00096865, 0.74524713],
...,
[0.01335815, 0.62376013, 0.36288171],
[0.03551495, 0.0508954 , 0.91358965],
[0.00664527, 0.70371215, 0.28964258]]), 'LinearSVC() + MinMax': array([[ 0.3380658 , -0.70059113, -0.84551216],
[ 0.32189335, -1.54567754, -0.11575068],
[ 0.22549259, -2.49509036, 0.69223215],
...,
[-1.4690199 , 0.16087262, -0.22137806],
[-1.00072177, -0.95593185, 0.42770417],
[-1.65162242, 0.34262892, -0.20019956]]), 'LinearSVC() + Isotonic': array([[0.75192088, 0.18665461, 0.06142451],
[0.55808311, 0. , 0.44191689],
[0.34438275, 0. , 0.65561725],
...,
[0.01302864, 0.60399643, 0.38297494],
[0.05420294, 0.10488174, 0.84091532],
[0. , 0.58988428, 0.41011572]]), 'LinearSVC() + Sigmoid': array([[0.73559196, 0.13937371, 0.12503433],
[0.6086243 , 0.02233023, 0.36904547],
[0.37500556, 0.00263819, 0.62235624],
...,
[0.02215638, 0.60023759, 0.37760603],
[0.07558115, 0.1089974 , 0.81542145],
[0.01489511, 0.64718593, 0.33791896]])}
Plot!#
92 # Plot!
93 fig, ax = plt.subplots(figsize=(12, 6))
94 ax = sp.metrics.plot_calibration(
95 y_val,
96 y_probas_list=probas_dict.values(),
97 estimator_names=probas_dict.keys(),
98 ax=ax,
99 save_fig=True,
100 save_fig_filename="",
101 overwrite=True,
102 add_timestamp=True,
103 verbose=True,
104 )
2025-06-29 13:41:58.281026: E scikitplot 140126653119360 _calibration.py:318:plot_calibration] 3
[INFO] Saving path to: /home/circleci/repo/galleries/examples/calibration/result_images/plot_calibration_20250629_134158Z.png
[INFO] Plot saved to: /home/circleci/repo/galleries/examples/calibration/result_images/plot_calibration_20250629_134158Z.png
Interpretation
Primary Use: Evaluating probabilistic classifiers by comparing predicted probabilities to observed frequencies of the positive class.
Goal: To assess how well the predicted probabilities align with the actual outcomes, identifying if a model is well-calibrated, overconfident, or underconfident.
Typical Characteristics:
X-axis: Predicted probability (e.g., in bins from 0 to 1).
Y-axis: Observed frequency of the positive class within each bin.
Reference line (diagonal at 45°): Represents perfect calibration, where predicted probabilities match observed frequencies.
Total running time of the script: (0 minutes 2.589 seconds)
Related examples
