# SPDX-FileCopyrightText: Copyright © 2026 Idiap Research Institute <contact@idiap.ch>
#
# SPDX-License-Identifier: GPL-3.0-or-later
"""Implementation of :py:mod:`Scikit-Learn compatible measures
<sklearn.metrics>` with confidence intervals estimated via non-parametric bootstrapping.
"""
import typing
import numpy
import numpy.typing
import sklearn.metrics
from .utils import bootstrap_metric
NUMBER_BOOTSTRAPS = 1000
"""Default number of bootstrap replicates used to estimate confidence
intervals throughout the package. A value of 1000 generally provides
a good trade-off between statistical stability and computational cost
for most applications."""
[docs]
def precision_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float, float]:
r"""Precision **binary** classification score.
AKA positive predictive value (PPV). It corresponds arithmetically to
``tp/(tp+fp)``. This function only supports **binary** classification problems.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.precision_score``.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual precision, as would be returned by scikit-learn
* The mode of the bootstrap distribution, estimated as the peak
of a Gaussian kernel density estimate (KDE) fitted to the
bootstrap samples.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, mode, lower, upper = bootstrap_metric(
y_true,
y_pred,
metric_func=sklearn.metrics.precision_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, mode, lower, upper
[docs]
def recall_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float, float]:
r"""Recall **binary** classification score.
AKA sensitivity, hit rate, or true positive rate (TPR). It corresponds
arithmetically to ``tp/(tp+fn)``.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.recall_score``.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual recall, as would be returned by scikit-learn
* The mode of the bootstrap distribution, estimated as the peak
of a Gaussian kernel density estimate (KDE) fitted to the
bootstrap samples.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, mode, lower, upper = bootstrap_metric(
y_true,
y_pred,
metric_func=sklearn.metrics.recall_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, mode, lower, upper
[docs]
def specificity_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float, float]:
r"""Specificity **binary** classification score.
AKA selectivity or true negative rate (TNR). It corresponds arithmetically to
``tn/(tn+fp)``.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.recall_score`` (this because
the specificity is computed as the recall of the negative class).
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual specificity, as would be returned by scikit-learn
* The mode of the bootstrap distribution, estimated as the peak
of a Gaussian kernel density estimate (KDE) fitted to the
bootstrap samples.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, mode, lower, upper = bootstrap_metric(
y_true,
y_pred,
metric_func=sklearn.metrics.recall_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
pos_label=0,
**kwargs,
)
return point, mode, lower, upper
[docs]
def accuracy_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float, float]:
r"""Accuracy **binary** classification score.
See `Accuracy
<https://en.wikipedia.org/wiki/Evaluation_of_binary_classifiers>`_. is the
proportion of correct predictions (both true positives and true negatives)
among the total number of pixels examined. It corresponds arithmetically
to ``(tp+tn)/(tp+tn+fp+fn)``. This measure includes both true-negatives
and positives in the numerator, what makes it sensitive to data or regions
without annotations. AKA selectivity or true negative rate (TNR), mean,
mode and credible intervals. It corresponds arithmetically to
``tn/(tn+fp)``.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.accuracy_score``.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual accuracy, as would be returned by scikit-learn
* The mode of the bootstrap distribution, estimated as the peak
of a Gaussian kernel density estimate (KDE) fitted to the
bootstrap samples.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, mode, lower, upper = bootstrap_metric(
y_true,
y_pred,
metric_func=sklearn.metrics.accuracy_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, mode, lower, upper
[docs]
def jaccard_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float, float]:
r"""Jaccard **binary** classification score.
See `Jaccard Index or Similarity
<https://en.wikipedia.org/wiki/Jaccard_index>`_. It corresponds
arithmetically to ``tp/(tp+fp+fn)``. The Jaccard index depends on a
TP-only numerator, similarly to the F1 score. For regions where there are
no annotations, the Jaccard index will always be zero, irrespective of the
model output. Accuracy may be a better proxy if one needs to consider the
true abscence of annotations in a region as part of the measure.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.jaccard_score``.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual jaccard score, as would be returned by scikit-learn
* The mode of the bootstrap distribution, estimated as the peak
of a Gaussian kernel density estimate (KDE) fitted to the
bootstrap samples.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, mode, lower, upper = bootstrap_metric(
y_true,
y_pred,
metric_func=sklearn.metrics.jaccard_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, mode, lower, upper
[docs]
def f1_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float, float]:
r"""Return the mean, mode, upper and lower bounds of the credible region of
the F1 score.
See `F1-score <https://en.wikipedia.org/wiki/F1_score>`_. It corresponds
arithmetically to ``2*P*R/(P+R)`` or ``2*tp/(2*tp+fp+fn)``. The F1 or Dice
score depends on a TP-only numerator, similarly to the Jaccard index. For
regions where there are no annotations, the F1-score will always be zero,
irrespective of the model output. Accuracy may be a better proxy if one
needs to consider the true abscence of annotations in a region as part of
the measure.
This implementation is based on [GOUTTE-2005]_.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.f1_score``.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual F1 score, as would be returned by scikit-learn
* The mode of the bootstrap distribution, estimated as the peak
of a Gaussian kernel density estimate (KDE) fitted to the
bootstrap samples.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, mode, lower, upper = bootstrap_metric(
y_true,
y_pred,
metric_func=sklearn.metrics.f1_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, mode, lower, upper
[docs]
def roc_curve():
r"""Compute Receiver operating characteristic (ROC)."""
raise NotImplementedError
[docs]
def roc_auc_score(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float]:
r"""Calculate the area under the ROC (FPR vs TPR) curve.
This function mimics the scikit-learn API, except it also returns lower and
upper bounds computed from empirical quantiles of the bootstrap distribution
(e.g., the 2.5th and 97.5th percentiles for a 95% confidence interval).
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.roc_auc_score``.
Returns
-------
tuple[float, float, float]
A tuple with 3 floats:
* The area under the ROC (FPR vs. TPR) curve
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, _, lower, upper = bootstrap_metric(
y_true,
y_score,
metric_func=sklearn.metrics.roc_auc_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, lower, upper
[docs]
def det_curve():
r"""Compute the Detection Error-Tradeoff (DET) curve."""
raise NotImplementedError
[docs]
def precision_recall_curve():
r"""Compute Precision-Recall (PR) curve."""
raise NotImplementedError
[docs]
def average_precision_score(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
n_bootstraps: int = NUMBER_BOOTSTRAPS,
require_all_classes: bool = True,
max_resample_attempts: int = 100,
rng: numpy.random.Generator = numpy.random.default_rng(),
**kwargs,
) -> tuple[float, float, float]:
r"""Compute average precision (AP) from prediction scores.
This function mimics the scikit-learn API, except it also returns lower and
upper bounds computed from empirical quantiles of the bootstrap distribution
(e.g., the 2.5th and 97.5th percentiles for a 95% confidence interval).
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1 indicating the desired
confidence level. A value of ``0.95`` will compute a bootstrap
confidence interval containing the central 95% of the bootstrap
distribution.
n_bootstraps
Number of bootstrapping steps to evaluate.
require_all_classes
If set to True, each accepted bootstrap sample must contain all classes
present in y_true. This is required for metrics such as ROC AUC.
max_resample_attempts
Maximum number of redraw attempts for a given bootstrap replicate when
`require_all_classes=True`.
rng
An initialized numpy random number generator.
**kwargs
Additional parameters for ``sklearn.metrics.average_precision_score``.
Returns
-------
tuple[float, float, float]
A tuple with 3 floats:
* The area under the Precision-Recall curve.
* The lower bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
* The upper bound of the percentile bootstrap confidence interval
defined by the ``coverage`` parameter.
"""
point, _, lower, upper = bootstrap_metric(
y_true,
y_score,
metric_func=sklearn.metrics.average_precision_score,
rng=rng,
coverage=coverage,
n_bootstraps=n_bootstraps,
require_all_classes=require_all_classes,
max_resample_attempts=max_resample_attempts,
**kwargs,
)
return point, lower, upper