Lazifying most of the model parts

Author: jindrahelcl

neuralmonkey/attention/combination.py

@@ -24,6 +24,7 @@
 from neuralmonkey.attention.namedtuples import HierarchicalLoopState
 from neuralmonkey.checking import assert_shape
 from neuralmonkey.decorators import tensor
+from neuralmonkey.logging import debug
 from neuralmonkey.model.model_part import ModelPart
 from neuralmonkey.model.parameterized import InitializerSpecs
 from neuralmonkey.tf_utils import get_variable
@@ -138,53 +139,63 @@ def __init__(self,
             load_checkpoint=load_checkpoint,
             initializers=initializers)
         self._encoders = encoders
+    # pylint: enable=too-many-arguments
 
-        # pylint: disable=protected-access
-        self._encoders_tensors = [
-            get_attention_states(e) for e in self._encoders]
-        self._encoders_masks = [get_attention_mask(e) for e in self._encoders]
-        # pylint: enable=protected-access
+    @tensor
+    def _encoders_tensors(self) -> List[tf.Tensor]:
+        tensors = [get_attention_states(e) for e in self._encoders]
+        for e_t in tensors:
+            assert_shape(e_t, [-1, -1, -1])
+        return tensors
 
-        for e_m in self._encoders_masks:
+    @tensor
+    def _encoders_masks(self) -> List[tf.Tensor]:
+        masks = [get_attention_mask(e) for e in self._encoders]
+        for e_m in masks:
             assert_shape(e_m, [-1, -1])
 
-        for e_t in self._encoders_tensors:
-            assert_shape(e_t, [-1, -1, -1])
+        if self._use_sentinels:
+            masks.append(tf.ones([tf.shape(masks[0])[0], 1]))
+        return masks
 
-        with self.use_scope():
-            self.encoder_projections_for_logits = \
-                self.get_encoder_projections("logits_projections")
+    @tensor
+    def encoder_projections_for_logits(self) -> List[tf.Tensor]:
+        return self.get_encoder_projections("logits_projections")
 
-            self.encoder_attn_biases = [
-                get_variable(name="attn_bias_{}".format(i),
-                             shape=[],
+    @tensor
+    def encoder_attn_biases(self) -> List[tf.Variable]:
+        return [get_variable(name="attn_bias_{}".format(i), shape=[],
                              initializer=tf.zeros_initializer())
                 for i in range(len(self._encoders_tensors))]
 
-            if self._share_projections:
-                self.encoder_projections_for_ctx = \
-                    self.encoder_projections_for_logits
-            else:
-                self.encoder_projections_for_ctx = \
-                    self.get_encoder_projections("context_projections")
-
-            if self._use_sentinels:
-                self._encoders_masks.append(
-                    tf.ones([tf.shape(self._encoders_masks[0])[0], 1]))
+    @tensor
+    def encoder_projections_for_ctx(self) -> List[tf.Tensor]:
+        if self._share_projections:
+            return self.encoder_projections_for_logits
+        return self.get_encoder_projections("context_projections")
 
-            self.masks_concat = tf.concat(self._encoders_masks, 1)
-    # pylint: enable=too-many-arguments
+    @tensor
+    def masks_concat(self) -> tf.Tensor:
+        return tf.concat(self._encoders_masks, 1)
 
     def initial_loop_state(self) -> AttentionLoopState:
 
+        # pylint: disable=not-an-iterable
+        # TODO blessing
+        for val in self.encoder_projections_for_logits:
+            debug(val)
+        debug(self.masks_concat)
+
         length = sum(tf.shape(s)[1] for s in self._encoders_tensors)
+        # pylint: enable=not-an-iterable
+
         if self._use_sentinels:
             length += 1
 
         return empty_attention_loop_state(self.batch_size, length,
                                           self.context_vector_size)
 
-    def get_encoder_projections(self, scope):
+    def get_encoder_projections(self, scope) -> List[tf.Tensor]:
         encoder_projections = []
         with tf.variable_scope(scope):
             for i, encoder_tensor in enumerate(self._encoders_tensors):
@@ -216,9 +227,11 @@ def get_encoder_projections(self, scope):
                 encoder_projections.append(projection)
             return encoder_projections
 
+    # pylint: disable=unsubscriptable-object
     @property
     def context_vector_size(self) -> int:
         return self.encoder_projections_for_ctx[0].get_shape()[2].value
+    # pylint: enable=unsubscriptable-object
 
     # pylint: disable=too-many-locals
     def attention(self,
@@ -280,6 +293,7 @@ def attention(self,
             return contexts, next_loop_state
     # pylint: enable=too-many-locals
 
+    # pylint: disable=not-an-iterable,unsubscriptable-object
     def _tile_encoders_for_beamsearch(self, projected_sentinel):
         sentinel_batch_size = tf.shape(projected_sentinel)[0]
         encoders_batch_size = tf.shape(
@@ -293,6 +307,7 @@ def _tile_encoders_for_beamsearch(self, projected_sentinel):
 
         return [tf.tile(proj, [beam_size, 1, 1])
                 for proj in self.encoder_projections_for_ctx]
+    # pylint: enable=not-an-iterable,unsubscriptable-object
 
     def _renorm_softmax(self, logits):
         """Renormalized softmax wrt. attention mask."""

neuralmonkey/attention/coverage.py

@@ -10,8 +10,10 @@
 
 from neuralmonkey.attention.base_attention import Attendable
 from neuralmonkey.attention.feed_forward import Attention
+from neuralmonkey.decorators import tensor
 from neuralmonkey.model.model_part import ModelPart
 from neuralmonkey.model.parameterized import InitializerSpecs
+from neuralmonkey.tf_utils import get_variable
 
 
 class CoverageAttention(Attention):
@@ -30,17 +32,22 @@ def __init__(self,
         Attention.__init__(self, name, encoder, dropout_keep_prob, state_size,
                            reuse, save_checkpoint, load_checkpoint,
                            initializers)
-
         self.max_fertility = max_fertility
+    # pylint: enable=too-many-arguments
 
-        self.coverage_weights = tf.get_variable(
-            "coverage_matrix", [1, 1, 1, self.state_size])
-        self.fertility_weights = tf.get_variable(
+    @tensor
+    def coverage_weights(self) -> tf.Variable:
+        return get_variable("coverage_matrix", [1, 1, 1, self.state_size])
+
+    @tensor
+    def fertility_weights(self) -> tf.Variable:
+        return get_variable(
             "fertility_matrix", [1, 1, self.context_vector_size])
 
-        self.fertility = 1e-8 + self.max_fertility * tf.sigmoid(
+    @tensor
+    def fertility(self) -> tf.Tensor:
+        return 1e-8 + self.max_fertility * tf.sigmoid(
             tf.reduce_sum(self.fertility_weights * self.attention_states, [2]))
-    # pylint: enable=too-many-arguments
 
     def get_energies(self, y: tf.Tensor, weights_in_time: tf.Tensor):
         weight_sum = tf.cond(

neuralmonkey/attention/scaled_dot_product.py

@@ -12,12 +12,13 @@
 import tensorflow as tf
 from typeguard import check_argument_types
 
-from neuralmonkey.nn.utils import dropout
-from neuralmonkey.model.model_part import ModelPart
-from neuralmonkey.model.parameterized import InitializerSpecs
 from neuralmonkey.attention.base_attention import (
     BaseAttention, Attendable, get_attention_states, get_attention_mask)
 from neuralmonkey.attention.namedtuples import MultiHeadLoopState
+from neuralmonkey.decorators import tensor
+from neuralmonkey.model.model_part import ModelPart
+from neuralmonkey.model.parameterized import InitializerSpecs
+from neuralmonkey.nn.utils import dropout
 
 
 def split_for_heads(x: tf.Tensor, n_heads: int, head_dim: int) -> tf.Tensor:
@@ -263,23 +264,35 @@ def __init__(self,
         self.n_heads = n_heads
         self.dropout_keep_prob = dropout_keep_prob
 
+        self.keys_encoder = keys_encoder
+
+        if values_encoder is not None:
+            self.values_encoder = values_encoder
+        else:
+            self.values_encoder = self.keys_encoder
+
         if self.n_heads <= 0:
             raise ValueError("Number of heads must be greater than zero.")
 
         if self.dropout_keep_prob <= 0.0 or self.dropout_keep_prob > 1.0:
             raise ValueError("Dropout keep prob must be inside (0,1].")
 
-        if values_encoder is None:
-            values_encoder = keys_encoder
-
-        self.attention_keys = get_attention_states(keys_encoder)
-        self.attention_mask = get_attention_mask(keys_encoder)
-        self.attention_values = get_attention_states(values_encoder)
-
         self._variable_scope.set_initializer(tf.variance_scaling_initializer(
             mode="fan_avg", distribution="uniform"))
     # pylint: enable=too-many-arguments
 
+    @tensor
+    def attention_keys(self) -> tf.Tensor:
+        return get_attention_states(self.keys_encoder)
+
+    @tensor
+    def attention_mask(self) -> tf.Tensor:
+        return get_attention_mask(self.keys_encoder)
+
+    @tensor
+    def attention_values(self) -> tf.Tensor:
+        return get_attention_states(self.values_encoder)
+
     def attention(self,
                   query: tf.Tensor,
                   decoder_prev_state: tf.Tensor,
@@ -346,9 +359,11 @@ def finalize_loop(self, key: str,
             head_weights = last_loop_state.head_weights[i]
             self.histories["{}_head{}".format(key, i)] = head_weights
 
+    # pylint: disable=no-member
     @property
     def context_vector_size(self) -> int:
         return self.attention_values.get_shape()[-1].value
+    # pylint: enable=no-member
 
     def visualize_attention(self, key: str, max_outputs: int = 16) -> None:
         for i in range(self.n_heads):

neuralmonkey/decoders/beam_search_decoder.py

@@ -157,7 +157,7 @@ def __init__(self,
         # Create a placeholder for maximum number of steps that is necessary
         # during ensembling, when the decoder is called repetitively with the
         # max_steps attribute set to one.
-        self.max_steps = tf.placeholder_with_default(max_steps, [])
+        self.max_steps = tf.placeholder_with_default(self.max_steps_int, [])
 
         # This is an ugly hack for handling the whole graph when expanding to
         # the beam. We need to access all the inner states of the network in

neuralmonkey/decoders/classifier.py

@@ -4,13 +4,13 @@
 from typeguard import check_argument_types
 
 from neuralmonkey.dataset import Dataset
-from neuralmonkey.vocabulary import Vocabulary
+from neuralmonkey.decorators import tensor
 from neuralmonkey.model.feedable import FeedDict
 from neuralmonkey.model.parameterized import InitializerSpecs
 from neuralmonkey.model.model_part import ModelPart
 from neuralmonkey.model.stateful import Stateful
 from neuralmonkey.nn.mlp import MultilayerPerceptron
-from neuralmonkey.decorators import tensor
+from neuralmonkey.vocabulary import Vocabulary
 
 
 class Classifier(ModelPart):
@@ -60,35 +60,42 @@ def __init__(self,
         self.activation_fn = activation_fn
         self.dropout_keep_prob = dropout_keep_prob
         self.max_output_len = 1
-
-        with self.use_scope():
-            self.gt_inputs = [tf.placeholder(tf.int32, [None], "targets")]
-
-            mlp_input = tf.concat([enc.output for enc in self.encoders], 1)
-            self._mlp = MultilayerPerceptron(
-                mlp_input, self.layers,
-                self.dropout_keep_prob, len(self.vocabulary),
-                activation_fn=self.activation_fn, train_mode=self.train_mode)
-
-        tf.summary.scalar(
-            "train_optimization_cost",
-            self.cost, collections=["summary_train"])
     # pylint: enable=too-many-arguments
 
+    # pylint: disable=no-self-use
     @tensor
-    def loss_with_gt_ins(self) -> tf.Tensor:
-        return tf.reduce_mean(
-            tf.nn.sparse_softmax_cross_entropy_with_logits(
-                logits=self._mlp.logits, labels=self.gt_inputs[0]))
+    def gt_inputs(self) -> tf.Tensor:
+        return tf.placeholder(tf.int32, [None], "targets")
+    # pylint: enable=no-self-use
+
+    @tensor
+    def _mlp(self) -> MultilayerPerceptron:
+        mlp_input = tf.concat([enc.output for enc in self.encoders], 1)
+        return MultilayerPerceptron(
+            mlp_input, self.layers, self.dropout_keep_prob,
+            len(self.vocabulary), activation_fn=self.activation_fn,
+            train_mode=self.train_mode)
 
     @property
     def loss_with_decoded_ins(self) -> tf.Tensor:
         return self.loss_with_gt_ins
 
     @property
     def cost(self) -> tf.Tensor:
+        tf.summary.scalar(
+            "train_optimization_cost",
+            self.loss_with_gt_ins, collections=["summary_train"])
+
         return self.loss_with_gt_ins
 
+    # pylint: disable=no-member
+    # this is for the _mlp attribute (pylint property bug)
+    @tensor
+    def loss_with_gt_ins(self) -> tf.Tensor:
+        return tf.reduce_mean(
+            tf.nn.sparse_softmax_cross_entropy_with_logits(
+                logits=self._mlp.logits, labels=self.gt_inputs))
+
     @tensor
     def decoded_seq(self) -> tf.Tensor:
         return tf.expand_dims(self._mlp.classification, 0)
@@ -100,6 +107,7 @@ def decoded_logits(self) -> tf.Tensor:
     @tensor
     def runtime_logprobs(self) -> tf.Tensor:
         return tf.expand_dims(tf.nn.log_softmax(self._mlp.logits), 0)
+    # pylint: enable=no-member
 
     @property
     def train_loss(self):
@@ -120,6 +128,6 @@ def feed_dict(self, dataset: Dataset, train: bool = False) -> FeedDict:
         if sentences is not None:
             label_tensors, _ = self.vocabulary.sentences_to_tensor(
                 list(sentences), self.max_output_len)
-            fd[self.gt_inputs[0]] = label_tensors[0]
+            fd[self.gt_inputs] = label_tensors[0]
 
         return fd

neuralmonkey/decoders/sequence_labeler.py

@@ -4,14 +4,14 @@
 from typeguard import check_argument_types
 
 from neuralmonkey.dataset import Dataset
+from neuralmonkey.decorators import tensor
+from neuralmonkey.encoders.recurrent import RecurrentEncoder
+from neuralmonkey.encoders.facebook_conv import SentenceEncoder
 from neuralmonkey.model.feedable import FeedDict
 from neuralmonkey.model.parameterized import InitializerSpecs
 from neuralmonkey.model.model_part import ModelPart
-from neuralmonkey.encoders.recurrent import RecurrentEncoder
-from neuralmonkey.encoders.facebook_conv import SentenceEncoder
-from neuralmonkey.vocabulary import Vocabulary
-from neuralmonkey.decorators import tensor
 from neuralmonkey.tf_utils import get_variable
+from neuralmonkey.vocabulary import Vocabulary
 
 
 class SequenceLabeler(ModelPart):
@@ -36,15 +36,21 @@ def __init__(self,
         self.vocabulary = vocabulary
         self.data_id = data_id
         self.dropout_keep_prob = dropout_keep_prob
+    # pylint: enable=too-many-arguments
 
-        self.rnn_size = int(self.encoder.temporal_states.get_shape()[-1])
+    # pylint: disable=no-self-use
+    @tensor
+    def train_targets(self) -> tf.Tensor:
+        return tf.placeholder(tf.int32, [None, None], "targets")
 
-        with self.use_scope():
-            self.train_targets = tf.placeholder(
-                tf.int32, [None, None], "labeler_targets")
-            self.train_weights = tf.placeholder(
-                tf.float32, [None, None], "labeler_padding_weights")
-    # pylint: enable=too-many-arguments
+    @tensor
+    def train_weights(self) -> tf.Tensor:
+        return tf.placeholder(tf.float32, [None, None], "padding")
+    # pylint: enable=no-self-use
+
+    @property
+    def rnn_size(self) -> int:
+        return int(self.encoder.temporal_states.get_shape()[-1])
 
     @tensor
     def decoding_w(self) -> tf.Variable: