keras backend dot product

[Richard Liao]

Hi,

I am trying to build a custom layer...

from keras import backend as K

from keras.engine.topology import Layer

from keras import initializations

import numpy as np

class AttLayer(Layer):

    def __init__(self, **kwargs):

        self.init = initializations.get('normal')

        super(AttLayer, self).__init__(**kwargs)

    def build(self, input_shape):

        assert len(input_shape)==3

        assert input_shape[-1] == 100

        self.W = self.init((input_shape[-1],))

        self.trainable_weights = [self.W]

        super(AttLayer, self).build(input_shape)  # be sure you call this somewhere!

    def call(self, x, mask=None):

        return K.dot(x, self.W)

        # more to do after this....

    def get_output_shape_for(self, input_shape):

        return (input_shape[0], input_shape[-1])

Then running the code below

sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')

embedded_sequences = embedding_layer(sequence_input)

l_lstm = LSTM(100, return_sequences=True)(embedded_sequences)

l_att = AttLayer()(l_lstm)

this AttLayer is sitting on top of LSTM layer so the tensor dimension is 3 (batch, time steps, features). There are a lot to implement but I am stuck at the first tensor operator K.dot. The idea is the I need to train this Self.W so that the dot product of x and self.W becomes 2 dimension (batch, time setps). However, I am getting the error as below

IndexError Traceback (most recent call last)

<ipython-input-62-aa2f76b896ac> in <module>()
      2 embedded_sequences = embedding_layer(sequence_input)
      3 l_lstm = LSTM(100, return_sequences=True)(embedded_sequences)
----> 4 l_att = AttLayer()(l_lstm)

/ext/home/analyst/jupyter/py2_kernel/local/lib/python2.7/site-packages/keras/engine/topology.pyc in __call__(self, x, mask)
    512         if inbound_layers:
    513             # this will call layer.build() if necessary
--> 514             self.add_inbound_node(inbound_layers, node_indices, tensor_indices)
    515             input_added = True
    516 

/ext/home/analyst/jupyter/py2_kernel/local/lib/python2.7/site-packages/keras/engine/topology.pyc in add_inbound_node(self, inbound_layers, node_indices, tensor_indices)
    570         # creating the node automatically updates self.inbound_nodes
    571         # as well as outbound_nodes on inbound layers.
--> 572         Node.create_node(self, inbound_layers, node_indices, tensor_indices)
    573 
    574     def get_output_shape_for(self, input_shape):

/ext/home/analyst/jupyter/py2_kernel/local/lib/python2.7/site-packages/keras/engine/topology.pyc in create_node(cls, outbound_layer, inbound_layers, node_indices, tensor_indices)
    147 
    148         if len(input_tensors) == 1:
--> 149             output_tensors = to_list(outbound_layer.call(input_tensors[0], mask=input_masks[0]))
    150             output_masks = to_list(outbound_layer.compute_mask(input_tensors[0], input_masks[0]))
    151             # TODO: try to auto-infer shape if exception is raised by get_output_shape_for

<ipython-input-61-323782152668> in call(self, x, mask)
     29 
     30     def call(self, x, mask=None):
---> 31         return K.dot(x, self.W)
     32 
     33     def get_output_shape_for(self, input_shape):

/ext/home/analyst/jupyter/py2_kernel/local/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc in dot(x, y)
    340         y_shape = int_shape(y)
    341         y_permute_dim = list(range(ndim(y)))
--> 342         y_permute_dim = [y_permute_dim.pop(-2)] + y_permute_dim
    343         xt = tf.reshape(x, [-1, x_shape[-1]])
    344         yt = tf.reshape(tf.transpose(y, perm=y_permute_dim), [y_shape[-2], -1])

IndexError: pop index out of range

Looking at the keras backend implementation, I am not sure I understand the logic - the Y, which is self.w, is only one dimension. That' why I got pop index out of range. 

Then the question, If I simply want to do dot product like this

x = np.random.random((2,3,4))
y = np.random.random((4,))
print np.dot(x,y)

How can I do that in Keras? 

Thanks much

[Pinkesh Badjatiya]

Hi,

I also tried the same example and got the same error. Did you find what caused the problem? May be it might help me in fixing it.

Regards,

Pinkesh Badjatiya

[Pinkesh Badjatiya]

Hi,

Ah, i could make it work! :D

Looks like the code does not work with tensorflow as the backed, worked as expected with theano.

Regards,

Pinkesh Badjatiya