From 3c978dadb05d928d3723c509a3b8e052e8acf76b Mon Sep 17 00:00:00 2001 From: Olivier Date: Wed, 29 May 2024 15:09:15 +0200 Subject: [PATCH] :bug: Fix documentation --- auto_tutorials_source/tutorial_scaler.py | 4 +- docs/source/api.rst | 38 +++++++++++++++---- .../adaptive_calibration_error.py | 4 +- .../classification/calibration_error.py | 2 +- .../metrics/classification/risk_coverage.py | 16 ++++---- .../metrics/regression/inverse.py | 10 ++--- .../metrics/regression/mse_log.py | 2 +- torch_uncertainty/metrics/regression/silog.py | 22 +++++------ torch_uncertainty/routines/segmentation.py | 4 +- 9 files changed, 59 insertions(+), 43 deletions(-) diff --git a/auto_tutorials_source/tutorial_scaler.py b/auto_tutorials_source/tutorial_scaler.py index e9a31969..75f1953c 100644 --- a/auto_tutorials_source/tutorial_scaler.py +++ b/auto_tutorials_source/tutorial_scaler.py @@ -28,7 +28,7 @@ from torch_uncertainty.datamodules import CIFAR100DataModule from torch_uncertainty.metrics import CalibrationError -from torch_uncertainty.models.resnet import resnet18 +from torch_uncertainty.models.resnet import resnet from torch_uncertainty.post_processing import TemperatureScaler from torch_uncertainty.utils import load_hf @@ -40,7 +40,7 @@ # This can be done in a one liner: # Build the model -model = resnet18(in_channels=3, num_classes=100, style="cifar", conv_bias=False) +model = resnet(in_channels=3, num_classes=100, arch=18, style="cifar", conv_bias=False) # Download the weights (the config is not used here) weights, config = load_hf("resnet18_c100") diff --git a/docs/source/api.rst b/docs/source/api.rst index 1a415798..d4f99acf 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -98,6 +98,8 @@ Segmentation DeepLabBaseline SegFormerBaseline +.. currentmodule:: torch_uncertainty.baselines.depth + Monocular Depth Estimation ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -173,29 +175,49 @@ Monte Carlo Dropout Metrics ------- -.. currentmodule:: torch_uncertainty.metrics +Classification +^^^^^^^^^^^^^^ + +.. currentmodule:: torch_uncertainty.metrics.classification .. autosummary:: :toctree: generated/ :nosignatures: :template: class.rst - AUSE AURC + AUSE + FPR95 AdaptiveCalibrationError BrierScore - CategoricalNLL CalibrationError - CovAt5Risk, + CategoricalNLL + CovAt5Risk Disagreement - DistributionNLL Entropy - FPR95 + GroupingLoss + MeanIntersectionOverUnion + MutualInformation + RiskAt80Cov + VariationRatio + +Regression +^^^^^^^^^^ + +.. currentmodule:: torch_uncertainty.metrics.regression + +.. autosummary:: + :toctree: generated/ + :nosignatures: + :template: class.rst + + DistributionNLL Log10 + MeanAbsoluteErrorInverse MeanGTRelativeAbsoluteError MeanGTRelativeSquaredError - MutualInformation - RiskAt80Cov, + MeanSquaredErrorInverse + MeanSquaredLogError SILog ThresholdAccuracy diff --git a/torch_uncertainty/metrics/classification/adaptive_calibration_error.py b/torch_uncertainty/metrics/classification/adaptive_calibration_error.py index 8c5de1b1..4f4c4850 100644 --- a/torch_uncertainty/metrics/classification/adaptive_calibration_error.py +++ b/torch_uncertainty/metrics/classification/adaptive_calibration_error.py @@ -92,7 +92,7 @@ def _ace_compute( class BinaryAdaptiveCalibrationError(Metric): - r"""`Adaptive Top-label Calibration Error` for binary tasks.""" + r"""Adaptive Top-label Calibration Error for binary tasks.""" is_differentiable: bool = False higher_is_better: bool = False @@ -140,7 +140,7 @@ def compute(self) -> Tensor: class MulticlassAdaptiveCalibrationError(Metric): - r"""`Adaptive Top-label Calibration Error` for multiclass tasks.""" + r"""Adaptive Top-label Calibration Error for multiclass tasks.""" is_differentiable: bool = False higher_is_better: bool = False diff --git a/torch_uncertainty/metrics/classification/calibration_error.py b/torch_uncertainty/metrics/classification/calibration_error.py index 7577b740..5def4d3c 100644 --- a/torch_uncertainty/metrics/classification/calibration_error.py +++ b/torch_uncertainty/metrics/classification/calibration_error.py @@ -70,7 +70,7 @@ def _ce_plot(self, ax: _AX_TYPE | None = None) -> _PLOT_OUT_TYPE: class CalibrationError: - r"""`Top-label Calibration Error`_. + r"""Top-label Calibration Error. See `CalibrationError `_ diff --git a/torch_uncertainty/metrics/classification/risk_coverage.py b/torch_uncertainty/metrics/classification/risk_coverage.py index f10fcdc5..46c40c6d 100644 --- a/torch_uncertainty/metrics/classification/risk_coverage.py +++ b/torch_uncertainty/metrics/classification/risk_coverage.py @@ -19,7 +19,7 @@ class AURC(Metric): errors: list[Tensor] def __init__(self, **kwargs) -> None: - r"""`Area Under the Risk-Coverage curve`_. + r"""Area Under the Risk-Coverage curve. The Area Under the Risk-Coverage curve (AURC) is the main metric for Selective Classification (SC) performance assessment. It evaluates the @@ -27,8 +27,7 @@ def __init__(self, **kwargs) -> None: discriminate between correct and incorrect predictions based on their rank (and not their values in contrast with calibration). - As input to ``forward`` and ``update`` the metric accepts the following - input: + As input to ``forward`` and ``update`` the metric accepts the following input: - ``preds`` (:class:`~torch.Tensor`): A float tensor of shape ``(N, ...)`` containing probabilities for each observation. @@ -37,6 +36,7 @@ def __init__(self, **kwargs) -> None: As output to ``forward`` and ``compute`` the metric returns the following output: + - ``aurc`` (:class:`~torch.Tensor`): A scalar tensor containing the area under the risk-coverage curve @@ -231,7 +231,7 @@ def compute(self) -> Tensor: class CovAt5Risk(CovAtxRisk): def __init__(self, **kwargs) -> None: - r"""`Coverage at 5% Risk`_. + r"""Coverage at 5% Risk. If there are multiple coverage values corresponding to 5% risk, the coverage at 5% risk is the maximum coverage value corresponding to 5% @@ -250,7 +250,7 @@ class RiskAtxCov(Metric): errors: list[Tensor] def __init__(self, cov_threshold: float, **kwargs) -> None: - r"""`Risk at x Coverage`_. + r"""Risk at given Coverage. Args: cov_threshold (float): The coverage threshold at which to compute @@ -276,10 +276,10 @@ def update(self, probs: Tensor, targets: Tensor) -> None: self.errors.append((probs.argmax(-1) != targets) * 1.0) def compute(self) -> Tensor: - """Compute the risk at x coverage. + """Compute the risk at given coverage. Returns: - Tensor: The risk at x coverage. + Tensor: The risk at given coverage. """ scores = dim_zero_cat(self.scores) errors = dim_zero_cat(self.errors) @@ -289,7 +289,7 @@ def compute(self) -> Tensor: class RiskAt80Cov(RiskAtxCov): def __init__(self, **kwargs) -> None: - r"""`Risk at 80% Coverage`_.""" + r"""Risk at 80% Coverage.""" super().__init__(cov_threshold=0.8, **kwargs) diff --git a/torch_uncertainty/metrics/regression/inverse.py b/torch_uncertainty/metrics/regression/inverse.py index d80a730c..d661d9d2 100644 --- a/torch_uncertainty/metrics/regression/inverse.py +++ b/torch_uncertainty/metrics/regression/inverse.py @@ -21,7 +21,7 @@ def _unit_to_factor(unit: Literal["mm", "m", "km"]) -> float: class MeanSquaredErrorInverse(MeanSquaredError): - r"""Compute the `Mean Squared Error of the inverse predictions`_ (iMSE). + r"""Mean Squared Error of the inverse predictions (iMSE). .. math:: \text{iMSE} = \frac{1}{N}\sum_i^N(\frac{1}{y_i} - \frac{1}{\hat{y_i}})^2 @@ -47,8 +47,7 @@ class MeanSquaredErrorInverse(MeanSquaredError): num_outputs: Number of outputs in multioutput setting. unit: Unit for the computation of the metric. Must be one of 'mm', 'm', 'km'. Defauts to 'km'. - kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for more - info. + kwargs: Additional keyword arguments. """ def __init__( @@ -69,7 +68,7 @@ def update(self, preds: Tensor, target: Tensor) -> None: class MeanAbsoluteErrorInverse(MeanAbsoluteError): - r"""`Compute the Mean Absolute Error of the inverse predictions`_ (iMAE). + r"""Mean Absolute Error of the inverse predictions (iMAE). .. math:: \text{iMAE} = \frac{1}{N}\sum_i^N | \frac{1}{y_i} - \frac{1}{\hat{y_i}} | @@ -93,8 +92,7 @@ class MeanAbsoluteErrorInverse(MeanAbsoluteError): Args: unit: Unit for the computation of the metric. Must be one of 'mm', 'm', 'km'. Defauts to 'km'. - kwargs: Additional keyword arguments, see :ref:`Metric kwargs` for - more info. + kwargs: Additional keyword arguments. """ def __init__(self, unit: str = "km", **kwargs) -> None: diff --git a/torch_uncertainty/metrics/regression/mse_log.py b/torch_uncertainty/metrics/regression/mse_log.py index c182c5cf..82f083fc 100644 --- a/torch_uncertainty/metrics/regression/mse_log.py +++ b/torch_uncertainty/metrics/regression/mse_log.py @@ -4,7 +4,7 @@ class MeanSquaredLogError(MeanSquaredError): def __init__(self, squared: bool = True, **kwargs) -> None: - r"""Compute MeanSquaredLogError (MSELog). + r"""MeanSquaredLogError (MSELog) regression metric. .. math:: \text{MSELog} = \frac{1}{N}\sum_i^N (\log \hat{y_i} - \log y_i)^2 diff --git a/torch_uncertainty/metrics/regression/silog.py b/torch_uncertainty/metrics/regression/silog.py index a0ac3152..b7c4cf0d 100644 --- a/torch_uncertainty/metrics/regression/silog.py +++ b/torch_uncertainty/metrics/regression/silog.py @@ -15,13 +15,9 @@ def __init__( .. math:: \text{SILog} = \frac{1}{N} \sum_{i=1}^{N} \left(\log(y_i) - \log(\hat{y_i})\right)^2 - \left(\frac{1}{N} \sum_{i=1}^{N} \log(y_i) \right)^2, where :math:`N` is the batch size, :math:`y_i` is a tensor of target - values and :math:`\hat{y_i}` is a tensor of prediction. + values and :math:`\hat{y_i}` is a tensor of prediction. Return the square root of SILog by setting :attr:`sqrt` to `True`. - Inputs: - - :attr:`pred`: :math:`(N)` - - :attr:`target`: :math:`(N)` - Args: sqrt: If `True`, return the square root of the metric. Defaults to False. @@ -31,13 +27,8 @@ def __init__( `_. Reference: - Depth Map Prediction from a Single Image using a Multi-Scale Deep - Network. - David Eigen, Christian Puhrsch, Rob Fergus. NeurIPS 2014. - From Big to Small: Multi-Scale Local Planar Guidance for Monocular - Depth Estimation. - Jin Han Lee, Myung-Kyu Han, Dong Wook Ko and Il Hong Suh. (For - :attr:`lmbda`) + Depth Map Prediction from a Single Image using a Multi-Scale Deep Network. David Eigen, Christian Puhrsch, Rob Fergus. NeurIPS 2014. + From Big to Small: Multi-Scale Local Planar Guidance for Monocular Depth Estimation. Jin Han Lee, Myung-Kyu Han, Dong Wook Ko and Il Hong Suh. (For :attr:`lmbda`) """ super().__init__(**kwargs) self.sqrt = sqrt @@ -55,7 +46,12 @@ def __init__( self.add_state("total", default=torch.tensor(0), dist_reduce_fx="sum") def update(self, pred: Tensor, target: Tensor) -> None: - """Update state with predictions and targets.""" + """Update state with predictions and targets. + + Args: + pred (Tensor): A prediction tensor of shape (batch) + target (Tensor): A tensor of ground truth labels of shape (batch) + """ self.log_dists += torch.sum(pred.log() - target.log()) self.sq_log_dists += torch.sum((pred.log() - target.log()) ** 2) self.total += target.size(0) diff --git a/torch_uncertainty/routines/segmentation.py b/torch_uncertainty/routines/segmentation.py index c4fc02f2..6e1e11e2 100644 --- a/torch_uncertainty/routines/segmentation.py +++ b/torch_uncertainty/routines/segmentation.py @@ -30,14 +30,14 @@ def __init__( log_plots: bool = False, num_calibration_bins: int = 15, ) -> None: - """Routine for training & testing on **segmentation tasks**. + """Routine for training & testing on segmentation tasks. Args: model (torch.nn.Module): Model to train. num_classes (int): Number of classes in the segmentation task. loss (torch.nn.Module): Loss function to optimize the :attr:`model`. num_estimators (int, optional): The number of estimators for the - ensemble. Defaults to ̀`1̀` (single model). + ensemble. Defaults to ̀`1` (single model). optim_recipe (dict or Optimizer, optional): The optimizer and optionally the scheduler to use. Defaults to ``None``. format_batch_fn (torch.nn.Module, optional): The function to format the