Refactoring Objective tuple to a class with lazy properties

Author: jindrahelcl

neuralmonkey/trainers/__init__.py

@@ -1 +1,3 @@
 from .cross_entropy_trainer import CrossEntropyTrainer
+from .delayed_update_trainer import DelayedUpdateTrainer
+from .multitask_trainer import MultitaskTrainer

neuralmonkey/trainers/cross_entropy_trainer.py

@@ -3,23 +3,21 @@
 import tensorflow as tf
 from typeguard import check_argument_types
 
-from neuralmonkey.trainers.generic_trainer import (
-    GenericTrainer, Objective, ObjectiveWeight)
+from neuralmonkey.logging import warn
+from neuralmonkey.trainers.generic_trainer import GenericTrainer
+from neuralmonkey.trainers.objective import (
+    Objective, CostObjective, ObjectiveWeight)
 
 
+# for compatibility reasons
 def xent_objective(decoder, weight=None) -> Objective:
     """Get XENT objective from decoder with cost."""
-    return Objective(
-        name="{} - cross-entropy".format(decoder.name),
-        decoder=decoder,
-        loss=decoder.cost,
-        gradients=None,
-        weight=weight,
-    )
-
-# pylint: disable=too-few-public-methods,too-many-arguments
+    warn("Using deprecated xent_objective function. Use the CostObjective "
+         "class directly.")
+    return CostObjective(decoder, weight)
 
 
+# pylint: disable=too-many-arguments
 class CrossEntropyTrainer(GenericTrainer):
 
     def __init__(self,
@@ -41,7 +39,7 @@ def __init__(self,
                 "decoder_weights (length {}) do not match decoders (length {})"
                 .format(len(decoder_weights), len(decoders)))
 
-        objectives = [xent_objective(dec, w)
+        objectives = [CostObjective(dec, w)
                       for dec, w in zip(decoders, decoder_weights)]
 
         GenericTrainer.__init__(

neuralmonkey/trainers/generic_trainer.py

@@ -1,42 +1,18 @@
-from typing import Dict, List, NamedTuple, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Sequence
 import re
 
 import tensorflow as tf
 from typeguard import check_argument_types
 
 from neuralmonkey.decorators import tensor
-from neuralmonkey.model.model_part import GenericModelPart
+from neuralmonkey.logging import warn
 from neuralmonkey.runners.base_runner import GraphExecutor, NextExecute
-
-# pylint: disable=invalid-name
-Gradients = List[Tuple[tf.Tensor, tf.Variable]]
-ObjectiveWeight = Union[tf.Tensor, float, None]
-# pylint: enable=invalid-name
+from neuralmonkey.trainers.objective import (
+    Objective, Gradients, ObjectiveWeight)
 
 BIAS_REGEX = re.compile(r"[Bb]ias")
 
 
-class Objective(NamedTuple(
-        "Objective",
-        [("name", str),
-         ("decoder", GenericModelPart),
-         ("loss", tf.Tensor),
-         ("gradients", Optional[Gradients]),
-         ("weight", ObjectiveWeight)])):
-    """The training objective.
-
-    Attributes:
-        name: The name for the objective. Used in TensorBoard.
-        decoder: The decoder which generates the value to optimize.
-        loss: The loss tensor fetched by the trainer.
-        gradients: Manually specified gradients. Useful for reinforcement
-            learning.
-        weight: The weight of this objective. The loss will be multiplied by
-            this so the gradients can be controled in case of multiple
-            objectives.
-    """
-
-
 # pylint: disable=too-few-public-methods,too-many-locals,too-many-arguments
 class GenericTrainer(GraphExecutor):
 
@@ -78,7 +54,7 @@ def default_optimizer():
         return tf.train.AdamOptimizer(learning_rate=1e-4)
 
     def __init__(self,
-                 objectives: List[Objective],
+                 objectives: Sequence[Objective],
                  l1_weight: float = 0.0,
                  l2_weight: float = 0.0,
                  clip_norm: float = None,
@@ -110,6 +86,10 @@ def regularization_losses(self) -> Tuple[tf.Tensor, tf.Tensor]:
                          and not v.name.startswith("Inception")
                          and not v.name.startswith("resnet")]
 
+        if not regularizable:
+            warn("It seems that there are no trainable variables in the model")
+            return tf.zeros([]), tf.zeros([])
+
         with tf.name_scope("regularization"):
             l1_norm = sum(tf.reduce_sum(abs(v)) for v in regularizable)
             l2_norm = sum(tf.reduce_sum(v ** 2) for v in regularizable)
@@ -120,11 +100,15 @@ def regularization_losses(self) -> Tuple[tf.Tensor, tf.Tensor]:
     @tensor
     def objective_values(self) -> List[tf.Tensor]:
         """Compute unweighted losses for fetching."""
+        # Note here we need to call the losses first, in case the model is
+        # being built. We need to compute the regularizers after that.
+        losses = [o.loss for o in self.objectives]
+
         # pylint: disable=unpacking-non-sequence
         l1_norm, l2_norm = self.regularization_losses
         # pylint: disable=unpacking-non-sequence
 
-        return [o.loss for o in self.objectives] + [l1_norm, l2_norm]
+        return losses + [l1_norm, l2_norm]
 
     @tensor
     def differentiable_loss_sum(self) -> tf.Tensor:

neuralmonkey/trainers/multitask_trainer.py

@@ -25,7 +25,9 @@ def __init__(self,
         self.trainers = trainers
         self.trainer_idx = 0
 
-        self.var_list = list(set.union(*[set(t.var_list) for t in trainers]))
+    @property
+    def var_list(self) -> List[tf.Variable]:
+        return list(set.union(*[set(t.var_list) for t in self.trainers]))
 
     def get_executable(
             self, compute_losses: bool = True, summaries: bool = True,

neuralmonkey/trainers/objective.py

@@ -0,0 +1,81 @@
+from abc import abstractproperty
+from typing import TypeVar, Union, Tuple, List, Optional, Generic
+import tensorflow as tf
+from typeguard import check_argument_types
+
+from neuralmonkey.decorators import tensor
+from neuralmonkey.model.model_part import GenericModelPart
+
+# pylint: disable=invalid-name
+ObjectiveWeight = Union[tf.Tensor, float, None]
+Gradients = List[Tuple[tf.Tensor, tf.Variable]]
+MP = TypeVar("MP", bound=GenericModelPart)
+# pylint: enable=invalid-name
+
+
+class Objective(Generic[MP]):
+    """The training objective.
+
+    Attributes:
+        name: The name for the objective. Used in TensorBoard.
+        decoder: The decoder which generates the value to optimize.
+        loss: The loss tensor fetched by the trainer.
+        gradients: Manually specified gradients. Useful for reinforcement
+            learning.
+        weight: The weight of this objective. The loss will be multiplied by
+            this so the gradients can be controled in case of multiple
+            objectives.
+    """
+
+    def __init__(self, name: str, decoder: MP) -> None:
+        self._name = name
+        self._decoder = decoder
+
+    @property
+    def decoder(self) -> MP:
+        return self._decoder
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @abstractproperty
+    def loss(self) -> tf.Tensor:
+        raise NotImplementedError()
+
+    @property
+    def gradients(self) -> Optional[Gradients]:
+        return None
+
+    @property
+    def weight(self) -> Optional[tf.Tensor]:
+        return None
+
+
+class CostObjective(Objective[GenericModelPart]):
+
+    def __init__(self, decoder: GenericModelPart,
+                 weight: ObjectiveWeight = None) -> None:
+        check_argument_types()
+
+        if not hasattr(decoder, "cost"):
+            raise TypeError("The decoder does not have a `cost` attribute")
+
+        name = "{} - cost".format(str(decoder))
+        Objective[GenericModelPart].__init__(self, name, decoder)
+        self._weight = weight
+
+    @tensor
+    def loss(self) -> tf.Tensor:
+        assert hasattr(self.decoder, "cost")
+        return getattr(self.decoder, "cost")
+
+    @tensor
+    def weight(self) -> Optional[tf.Tensor]:
+        if self._weight is None:
+            return None
+
+        if isinstance(self._weight, float):
+            return tf.constant(self._weight)
+
+        return self._weight