Skip to content

🐛 Fix documentation #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions auto_tutorials_source/tutorial_scaler.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@

from torch_uncertainty.datamodules import CIFAR100DataModule
from torch_uncertainty.metrics import CalibrationError
from torch_uncertainty.models.resnet import resnet18
from torch_uncertainty.models.resnet import resnet
from torch_uncertainty.post_processing import TemperatureScaler
from torch_uncertainty.utils import load_hf

Expand All @@ -40,7 +40,7 @@
# This can be done in a one liner:

# Build the model
model = resnet18(in_channels=3, num_classes=100, style="cifar", conv_bias=False)
model = resnet(in_channels=3, num_classes=100, arch=18, style="cifar", conv_bias=False)

# Download the weights (the config is not used here)
weights, config = load_hf("resnet18_c100")
Expand Down
38 changes: 30 additions & 8 deletions docs/source/api.rst
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@ Segmentation
DeepLabBaseline
SegFormerBaseline

.. currentmodule:: torch_uncertainty.baselines.depth

Monocular Depth Estimation
^^^^^^^^^^^^^^^^^^^^^^^^^^

Expand Down Expand Up @@ -173,29 +175,49 @@ Monte Carlo Dropout
Metrics
-------

.. currentmodule:: torch_uncertainty.metrics
Classification
^^^^^^^^^^^^^^

.. currentmodule:: torch_uncertainty.metrics.classification

.. autosummary::
:toctree: generated/
:nosignatures:
:template: class.rst

AUSE
AURC
AUSE
FPR95
AdaptiveCalibrationError
BrierScore
CategoricalNLL
CalibrationError
CovAt5Risk,
CategoricalNLL
CovAt5Risk
Disagreement
DistributionNLL
Entropy
FPR95
GroupingLoss
MeanIntersectionOverUnion
MutualInformation
RiskAt80Cov
VariationRatio

Regression
^^^^^^^^^^

.. currentmodule:: torch_uncertainty.metrics.regression

.. autosummary::
:toctree: generated/
:nosignatures:
:template: class.rst

DistributionNLL
Log10
MeanAbsoluteErrorInverse
MeanGTRelativeAbsoluteError
MeanGTRelativeSquaredError
MutualInformation
RiskAt80Cov,
MeanSquaredErrorInverse
MeanSquaredLogError
SILog
ThresholdAccuracy

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ def _ace_compute(


class BinaryAdaptiveCalibrationError(Metric):
r"""`Adaptive Top-label Calibration Error` for binary tasks."""
r"""Adaptive Top-label Calibration Error for binary tasks."""

is_differentiable: bool = False
higher_is_better: bool = False
Expand Down Expand Up @@ -140,7 +140,7 @@ def compute(self) -> Tensor:


class MulticlassAdaptiveCalibrationError(Metric):
r"""`Adaptive Top-label Calibration Error` for multiclass tasks."""
r"""Adaptive Top-label Calibration Error for multiclass tasks."""

is_differentiable: bool = False
higher_is_better: bool = False
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def _ce_plot(self, ax: _AX_TYPE | None = None) -> _PLOT_OUT_TYPE:


class CalibrationError:
r"""`Top-label Calibration Error`_.
r"""Top-label Calibration Error.

See
`CalibrationError <https://torchmetrics.readthedocs.io/en/stable/classification/calibration_error.html>`_
Expand Down
16 changes: 8 additions & 8 deletions torch_uncertainty/metrics/classification/risk_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,15 @@ class AURC(Metric):
errors: list[Tensor]

def __init__(self, **kwargs) -> None:
r"""`Area Under the Risk-Coverage curve`_.
r"""Area Under the Risk-Coverage curve.

The Area Under the Risk-Coverage curve (AURC) is the main metric for
Selective Classification (SC) performance assessment. It evaluates the
quality of uncertainty estimates by measuring the ability to
discriminate between correct and incorrect predictions based on their
rank (and not their values in contrast with calibration).

As input to ``forward`` and ``update`` the metric accepts the following
input:
As input to ``forward`` and ``update`` the metric accepts the following input:

- ``preds`` (:class:`~torch.Tensor`): A float tensor of shape
``(N, ...)`` containing probabilities for each observation.
Expand All @@ -37,6 +36,7 @@ def __init__(self, **kwargs) -> None:

As output to ``forward`` and ``compute`` the metric returns the
following output:

- ``aurc`` (:class:`~torch.Tensor`): A scalar tensor containing the
area under the risk-coverage curve

Expand Down Expand Up @@ -231,7 +231,7 @@ def compute(self) -> Tensor:

class CovAt5Risk(CovAtxRisk):
def __init__(self, **kwargs) -> None:
r"""`Coverage at 5% Risk`_.
r"""Coverage at 5% Risk.

If there are multiple coverage values corresponding to 5% risk, the
coverage at 5% risk is the maximum coverage value corresponding to 5%
Expand All @@ -250,7 +250,7 @@ class RiskAtxCov(Metric):
errors: list[Tensor]

def __init__(self, cov_threshold: float, **kwargs) -> None:
r"""`Risk at x Coverage`_.
r"""Risk at given Coverage.

Args:
cov_threshold (float): The coverage threshold at which to compute
Expand All @@ -276,10 +276,10 @@ def update(self, probs: Tensor, targets: Tensor) -> None:
self.errors.append((probs.argmax(-1) != targets) * 1.0)

def compute(self) -> Tensor:
"""Compute the risk at x coverage.
"""Compute the risk at given coverage.

Returns:
Tensor: The risk at x coverage.
Tensor: The risk at given coverage.
"""
scores = dim_zero_cat(self.scores)
errors = dim_zero_cat(self.errors)
Expand All @@ -289,7 +289,7 @@ def compute(self) -> Tensor:

class RiskAt80Cov(RiskAtxCov):
def __init__(self, **kwargs) -> None:
r"""`Risk at 80% Coverage`_."""
r"""Risk at 80% Coverage."""
super().__init__(cov_threshold=0.8, **kwargs)


Expand Down
10 changes: 4 additions & 6 deletions torch_uncertainty/metrics/regression/inverse.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def _unit_to_factor(unit: Literal["mm", "m", "km"]) -> float:


class MeanSquaredErrorInverse(MeanSquaredError):
r"""Compute the `Mean Squared Error of the inverse predictions`_ (iMSE).
r"""Mean Squared Error of the inverse predictions (iMSE).

.. math:: \text{iMSE} = \frac{1}{N}\sum_i^N(\frac{1}{y_i} - \frac{1}{\hat{y_i}})^2

Expand All @@ -47,8 +47,7 @@ class MeanSquaredErrorInverse(MeanSquaredError):
num_outputs: Number of outputs in multioutput setting.
unit: Unit for the computation of the metric. Must be one of 'mm', 'm',
'km'. Defauts to 'km'.
kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more
info.
kwargs: Additional keyword arguments.
"""

def __init__(
Expand All @@ -69,7 +68,7 @@ def update(self, preds: Tensor, target: Tensor) -> None:


class MeanAbsoluteErrorInverse(MeanAbsoluteError):
r"""`Compute the Mean Absolute Error of the inverse predictions`_ (iMAE).
r"""Mean Absolute Error of the inverse predictions (iMAE).

.. math:: \text{iMAE} = \frac{1}{N}\sum_i^N | \frac{1}{y_i} - \frac{1}{\hat{y_i}} |

Expand All @@ -93,8 +92,7 @@ class MeanAbsoluteErrorInverse(MeanAbsoluteError):
Args:
unit: Unit for the computation of the metric. Must be one of 'mm', 'm',
'km'. Defauts to 'km'.
kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for
more info.
kwargs: Additional keyword arguments.
"""

def __init__(self, unit: str = "km", **kwargs) -> None:
Expand Down
2 changes: 1 addition & 1 deletion torch_uncertainty/metrics/regression/mse_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

class MeanSquaredLogError(MeanSquaredError):
def __init__(self, squared: bool = True, **kwargs) -> None:
r"""Compute MeanSquaredLogError (MSELog).
r"""MeanSquaredLogError (MSELog) regression metric.

.. math:: \text{MSELog} = \frac{1}{N}\sum_i^N (\log \hat{y_i} - \log y_i)^2

Expand Down
22 changes: 9 additions & 13 deletions torch_uncertainty/metrics/regression/silog.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,9 @@ def __init__(
.. math:: \text{SILog} = \frac{1}{N} \sum_{i=1}^{N} \left(\log(y_i) - \log(\hat{y_i})\right)^2 - \left(\frac{1}{N} \sum_{i=1}^{N} \log(y_i) \right)^2,

where :math:`N` is the batch size, :math:`y_i` is a tensor of target
values and :math:`\hat{y_i}` is a tensor of prediction.
values and :math:`\hat{y_i}` is a tensor of prediction.
Return the square root of SILog by setting :attr:`sqrt` to `True`.

Inputs:
- :attr:`pred`: :math:`(N)`
- :attr:`target`: :math:`(N)`

Args:
sqrt: If `True`, return the square root of the metric. Defaults to
False.
Expand All @@ -31,13 +27,8 @@ def __init__(
<https://torchmetrics.readthedocs.io/en/stable/pages/overview.html#metric-kwargs>`_.

Reference:
Depth Map Prediction from a Single Image using a Multi-Scale Deep
Network.
David Eigen, Christian Puhrsch, Rob Fergus. NeurIPS 2014.
From Big to Small: Multi-Scale Local Planar Guidance for Monocular
Depth Estimation.
Jin Han Lee, Myung-Kyu Han, Dong Wook Ko and Il Hong Suh. (For
:attr:`lmbda`)
Depth Map Prediction from a Single Image using a Multi-Scale Deep Network. David Eigen, Christian Puhrsch, Rob Fergus. NeurIPS 2014.
From Big to Small: Multi-Scale Local Planar Guidance for Monocular Depth Estimation. Jin Han Lee, Myung-Kyu Han, Dong Wook Ko and Il Hong Suh. (For :attr:`lmbda`)
"""
super().__init__(**kwargs)
self.sqrt = sqrt
Expand All @@ -55,7 +46,12 @@ def __init__(
self.add_state("total", default=torch.tensor(0), dist_reduce_fx="sum")

def update(self, pred: Tensor, target: Tensor) -> None:
"""Update state with predictions and targets."""
"""Update state with predictions and targets.

Args:
pred (Tensor): A prediction tensor of shape (batch)
target (Tensor): A tensor of ground truth labels of shape (batch)
"""
self.log_dists += torch.sum(pred.log() - target.log())
self.sq_log_dists += torch.sum((pred.log() - target.log()) ** 2)
self.total += target.size(0)
Expand Down
4 changes: 2 additions & 2 deletions torch_uncertainty/routines/segmentation.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,14 @@ def __init__(
log_plots: bool = False,
num_calibration_bins: int = 15,
) -> None:
"""Routine for training & testing on **segmentation tasks**.
"""Routine for training & testing on segmentation tasks.

Args:
model (torch.nn.Module): Model to train.
num_classes (int): Number of classes in the segmentation task.
loss (torch.nn.Module): Loss function to optimize the :attr:`model`.
num_estimators (int, optional): The number of estimators for the
ensemble. Defaults to ̀`` (single model).
ensemble. Defaults to ̀`1` (single model).
optim_recipe (dict or Optimizer, optional): The optimizer and
optionally the scheduler to use. Defaults to ``None``.
format_batch_fn (torch.nn.Module, optional): The function to format the
Expand Down
Loading