# SPDX-FileCopyrightText: Copyright © 2026 Idiap Research Institute <contact@idiap.ch>
#
# SPDX-License-Identifier: GPL-3.0-or-later
"""Common interface for :py:mod:`Scikit-Learn compatible measures <sklearn.metrics>`
with uncertainity estimates.
This module provides wrappers around :py:mod:`sklearn.metrics` that augment
standard performance measures with uncertainty quantification through either
Bayesian credible intervals or frequentist confidence intervals, depending on the selected
``method`` argument.
.. include:: ../links.rst
"""
import typing
import numpy.typing
import credible.bayesian.metrics
import credible.frequentist.metrics
[docs]
def precision_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float, float]:
r"""Precision **binary** classification score.
AKA positive predictive value (PPV). It corresponds arithmetically to ``tp/(tp+fp)``.
This function only supports **binary** classification problems.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.precision_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual precision, as would be returned by scikit-learn
* The mode of the estimated distribution. For the Bayesian method, this
is the mode of the posterior distribution. For the frequentist method,
this is the peak of a Gaussian KDE fitted to the bootstrap samples.
* The lower value of the credible region/confidence interval
* The upper value of the credible region/confidence interval
"""
match method:
case "frequentist":
return credible.frequentist.metrics.precision_score(
y_true, y_pred, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.precision_score(
y_true, y_pred, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def recall_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float, float]:
r"""Recall **binary** classification score.
AKA sensitivity, hit rate, or true positive rate (TPR).
It corresponds arithmetically to ``tp/(tp+fn)``.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.recall_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual recall, as would be returned by scikit-learn
* The mode of the estimated distribution. For the Bayesian method, this
is the mode of the posterior distribution. For the frequentist method,
this is the peak of a Gaussian KDE fitted to the bootstrap samples.
* The lower value of the credible region/confidence interval
* The upper value of the credible region/confidence interval
"""
match method:
case "frequentist":
return credible.frequentist.metrics.recall_score(
y_true, y_pred, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.recall_score(
y_true, y_pred, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def specificity_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float, float]:
r"""Specificity **binary** classification score.
AKA selectivity or true negative rate (TNR).
It corresponds arithmetically to ``tn/(tn+fp)``.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.recall_score`` (this because
the specificity is computed as the recall of the negative class).
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual specificity score computed via scikit-learn recall_score
setting ``pos_label=0``
* The mode of the estimated distribution. For the Bayesian method, this
is the mode of the posterior distribution. For the frequentist method,
this is the peak of a Gaussian KDE fitted to the bootstrap samples.
* The lower value of the credible region/confidence interval
* The upper value of the credible region/confidence interval
"""
match method:
case "frequentist":
return credible.frequentist.metrics.specificity_score(
y_true, y_pred, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.specificity_score(
y_true, y_pred, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def accuracy_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float, float]:
r"""Accuracy **binary** classification score.
See `Accuracy
<https://en.wikipedia.org/wiki/Evaluation_of_binary_classifiers>`_. is the
proportion of correct predictions (both true positives and true negatives)
among the total number of pixels examined. It corresponds arithmetically
to ``(tp+tn)/(tp+tn+fp+fn)``. This measure includes both true-negatives
and positives in the numerator, what makes it sensitive to data or regions
without annotations. AKA selectivity or true negative rate (TNR).
It corresponds arithmetically to ``tn/(tn+fp)``.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.accuracy_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual accuracy, as would be returned by scikit-learn
* The mode of the estimated distribution. For the Bayesian method, this
is the mode of the posterior distribution. For the frequentist method,
this is the peak of a Gaussian KDE fitted to the bootstrap samples.
* The lower value of the credible region/confidence interval
* The upper value of the credible region/confidence interval
"""
match method:
case "frequentist":
return credible.frequentist.metrics.accuracy_score(
y_true, y_pred, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.accuracy_score(
y_true, y_pred, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def jaccard_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float, float]:
r"""Jaccard **binary** classification score.
See `Jaccard Index or Similarity
<https://en.wikipedia.org/wiki/Jaccard_index>`_. It corresponds
arithmetically to ``tp/(tp+fp+fn)``. The Jaccard index depends on a
TP-only numerator, similarly to the F1 score. For regions where there are
no annotations, the Jaccard index will always be zero, irrespective of the
model output. Accuracy may be a better proxy if one needs to consider the
true abscence of annotations in a region as part of the measure.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.jaccard_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual jaccard score, as would be returned by scikit-learn
* The mode of the estimated distribution. For the Bayesian method, this
is the mode of the posterior distribution. For the frequentist method,
this is the peak of a Gaussian KDE fitted to the bootstrap samples.
* The lower value of the credible region/confidence interval
* The upper value of the credible region/confidence interval
"""
match method:
case "frequentist":
return credible.frequentist.metrics.jaccard_score(
y_true, y_pred, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.jaccard_score(
y_true, y_pred, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def f1_score(
y_true: typing.Iterable[int],
y_pred: typing.Iterable[int],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float, float]:
r"""Return the mean, mode, upper and lower bounds of the credible region of
the F1 score.
See `F1-score <https://en.wikipedia.org/wiki/F1_score>`_. It corresponds
arithmetically to ``2*P*R/(P+R)`` or ``2*tp/(2*tp+fp+fn)``. The F1 or Dice
score depends on a TP-only numerator, similarly to the Jaccard index. For
regions where there are no annotations, the F1-score will always be zero,
irrespective of the model output. Accuracy may be a better proxy if one
needs to consider the true abscence of annotations in a region as part of
the measure.
This implementation is based on [GOUTTE-2005]_.
Parameters
----------
y_true
Ground truth (correct) labels.
y_pred
Predicted labels, as returned by a classifier.
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.f1_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
tuple[float, float, float, float]
Tuple with 4 floating-point numbers:
* The actual f1 score, as would be returned by scikit-learn
* The mode of the estimated distribution. For the Bayesian method, this
is the mode of the posterior distribution. For the frequentist method,
this is the peak of a Gaussian KDE fitted to the bootstrap samples.
* The lower value of the credible region/confidence interval
* The upper value of the credible region/confidence interval
"""
match method:
case "frequentist":
return credible.frequentist.metrics.f1_score(
y_true, y_pred, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.f1_score(
y_true, y_pred, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def roc_curve(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[
numpy.typing.NDArray[numpy.double], # fpr
numpy.typing.NDArray[numpy.double], # tpr
numpy.typing.NDArray[numpy.double], # thresholds
numpy.typing.NDArray[numpy.double], # fpr lower ci
numpy.typing.NDArray[numpy.double], # tpr lower ci
numpy.typing.NDArray[numpy.double], # fpr upper ci
numpy.typing.NDArray[numpy.double], # tpr upper ci
]:
r"""Compute Receiver operating characteristic (ROC).
Approximately follows API of :py:func:`sklearn.metrics.roc_curve`.
.. note::
The ``frequentist`` implementation is currently not available for this
metric. At present, only the ``bayesian`` method is supported.
.. important::
The returned credible regions are not immediately usable for plots or
the evaluation of the area under the curve, only as point estimates for
individual thresholds. To plot, feed the output of this funtion to
:py:func:`.curves.curve_ci_hull` and use the lower and upper estimates
provided by that function instead.
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1.0 indicating the coverage
you are expecting. A value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``rng`` (frequentist): An initialized numpy random number generator.
Returns
-------
Seven 1-D floating point arrays corresponding to:
* FPR (false positive rates)
* TPR (true positive rates)
* The thresholds used to evaluated the selected metrics
* The lower confidence interval for the FPR
* The lower confidence interval for the TPR
* The upper confidence interval for the FPR
* The upper confidence interval for the TPR
"""
match method:
case "frequentist":
return credible.frequentist.metrics.roc_curve()
case "bayesian":
return credible.bayesian.metrics.roc_curve(
y_true, y_score, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def roc_auc_score(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float]:
r"""Calculate the area under the ROC (FPR vs TPR) curve.
This function mimics the scikit-learn API, except it also returns lower and
upper bounds of uncertainity.
For ``method="bayesian"``, the returned bounds corrspond to credible regions
defined in each threshold. For ``method="frequentist"``, they correspond to confidence
intervals computed via percentile bootstrap.
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.roc_auc_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
A tuple with 3 floats:
* The area under the ROC (FPR vs. TPR) curve
* The lower bound of the estimated interval. For
``method="bayesian"``, this corresponds to the lower bound of the
credible interval. For ``method="frequentist"``, this corresponds
to the lower bound of the bootstrap confidence interval.
* The upper bound of the estimated interval. For
``method="bayesian"``, this corresponds to the upper bound of the
credible interval. For ``method="frequentist"``, this corresponds
to the upper bound of the bootstrap confidence interval.
"""
match method:
case "frequentist":
return credible.frequentist.metrics.roc_auc_score(
y_true, y_score, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.roc_auc_score(
y_true, y_score, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def det_curve(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[
numpy.typing.NDArray[numpy.double], # fpr
numpy.typing.NDArray[numpy.double], # fnr
numpy.typing.NDArray[numpy.double], # thresholds
numpy.typing.NDArray[numpy.double], # fpr lower ci
numpy.typing.NDArray[numpy.double], # fnr lower ci
numpy.typing.NDArray[numpy.double], # fpr upper ci
numpy.typing.NDArray[numpy.double], # fnr upper ci
]:
r"""Compute the Detection Error-Tradeoff (DET) curve.
Approximately follows API of :py:func:`sklearn.metrics.det_curve`.
.. note::
The ``frequentist`` implementation is currently not available for this
metric. At present, only the ``bayesian`` method is supported.
.. important::
The returned credible regions are not immediately usable for plots or
the evaluation of the area under the curve, only as point estimates for
individual thresholds. To plot, feed the output of this funtion to
:py:func:`.curves.curve_ci_hull` and use the lower and upper estimates
provided by that function instead.
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1.0 indicating the coverage
you are expecting. A value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``rng`` (frequentist): An initialized numpy random number generator.
The rest are passed to the scikit-learn metric function.
For additional details, see the corresponding methods in the respective modules.
Returns
-------
Seven 1-D floating point arrays corresponding to:
* FPR (false positive rates)
* FNR (false negative rates)
* The thresholds used to evaluated the selected metrics
* The lower confidence interval for the FPR
* The lower confidence interval for the FNR
* The upper confidence interval for the FPR
* The upper confidence interval for the FNR
"""
match method:
case "frequentist":
return credible.frequentist.metrics.det_curve()
case "bayesian":
return credible.bayesian.metrics.det_curve(
y_true, y_score, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def precision_recall_curve(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[
numpy.typing.NDArray[numpy.double], # precision
numpy.typing.NDArray[numpy.double], # redcall
numpy.typing.NDArray[numpy.double], # thresholds
numpy.typing.NDArray[numpy.double], # precision lower ci
numpy.typing.NDArray[numpy.double], # recall lower ci
numpy.typing.NDArray[numpy.double], # precision upper ci
numpy.typing.NDArray[numpy.double], # recall upper ci
]:
r"""Compute Precision-Recall (PR) curve.
Approximately follows API of
:py:func:`sklearn.metrics.precision_recall_curve`.
.. note::
The ``frequentist`` implementation is currently not available for this
metric. At present, only the ``bayesian`` method is supported.
.. note::
This package computes the precision-recall curve in a similar, but
slightly different way than scikit-learn. It does not add an extra
(1.0, 0.0) at the end of the PR curve. (c.f.: documentation for
:py:func:`sklearn.metrics.precision_recall_curve`).
.. important::
The returned credible regions are not immediately usable for plots or
the evaluation of the area under the curve, only as point estimates for
individual thresholds. To plot, feed the output of this funtion to
:py:func:`.curves.curve_ci_hull` and use the lower and upper estimates
provided by that function instead.
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1.0 indicating the coverage
you are expecting. A value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``rng`` (frequentist): An initialized numpy random number generator.
The rest are passed to the scikit-learn metric function.
For additional details, see the corresponding methods in the respective modules.
Returns
-------
Seven 1-D floating point arrays corresponding to:
* Precision
* Recall
* The thresholds used to evaluated the selected metrics
* The lower confidence interval for the Precision
* The lower confidence interval for the Recall
* The upper confidence interval for the Precision
* The upper confidence interval for the Recall
"""
match method:
case "frequentist":
return credible.frequentist.metrics.precision_recall_curve()
case "bayesian":
return credible.bayesian.metrics.precision_recall_curve(
y_true, y_score, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)
[docs]
def average_precision_score(
y_true: typing.Iterable[int],
y_score: typing.Iterable[float],
coverage: float = 0.95,
method: str = "bayesian",
**kwargs,
) -> tuple[float, float, float]:
r"""Compute average precision (AP) from prediction scores.
This function mimics the scikit-learn API, except it also returns lower and
upper bounds of uncertainity.
For ``method="bayesian"``, the returned bounds corrspond to credible regions
defined in each threshold. For ``method="frequentist"``, they correspond to confidence
intervals computed via percentile bootstrap.
Parameters
----------
y_true
Ground truth (correct) labels.
y_score
Target scores, can either be probability estimates of the positive
class, confidence values, or non-thresholded measure of decisions (as
returned by “decision_function” on some classifiers).
coverage
A floating-point number between 0 and 1 indicating the desired interval
coverage.
* For the ``bayesian`` method, a value of 0.95 will ensure 95% of the area under
the probability density of the posterior is covered by the returned
equal-tailed interval.
* For the ``frequentist`` method, a value of 0.95 returns a
percentile bootstrap confidence interval containing the central 95% of
the bootstrap distribution.
method
The method used to compute the results, either `frequentist` or
`bayesian`.
**kwargs
Additional arguments to be passed. In particular:
* ``lambda_`` (bayesian): The parameterisation of the Beta prior to consider.
* ``n_bootstraps`` (frequentist): Number of bootstrap replicates.
* ``require_all_classes`` (frequentist): Whether each accepted bootstrap sample must
contain all classes present in ``y_true``.
* ``max_resample_attempts`` (frequentist): Maximum number of resampling attempts. If
``require_all_classes`` is True, this parameter limits the number of times the function will
attempt to resample the data to obtain a bootstrap sample that contains all classes.
* ``rng`` (frequentist): An initialized numpy random number generator.
Any remaining keyword arguments are passed to ``sklearn.metrics.average_precision_score``.
For additional details on the supported arguments and their behavior, refer to the scikit-learn
documentation.
Returns
-------
A tuple with 3 floats:
* The area under the Precision-Recall curve.
* The lower bound of the estimated interval. For
``method="bayesian"``, this corresponds to the lower bound of the
credible interval. For ``method="frequentist"``, this corresponds
to the lower bound of the bootstrap confidence interval.
* The upper bound of the estimated interval. For
``method="bayesian"``, this corresponds to the upper bound of the
credible interval. For ``method="frequentist"``, this corresponds
to the upper bound of the bootstrap confidence interval.
"""
match method:
case "frequentist":
return credible.frequentist.metrics.average_precision_score(
y_true, y_score, coverage, **kwargs
)
case "bayesian":
return credible.bayesian.metrics.average_precision_score(
y_true, y_score, coverage, **kwargs
)
case _:
raise ValueError(
f"Method `{method}` not supported. Available options are `frequentist`, `bayesian`."
)