training time explodes if model created inside python function rather than at module level

41 views

Skip to first unread message

jcavali...@gmail.com

unread,

Jul 30, 2016, 9:17:45 AM7/30/16

to Keras-users

This is driving me crazy. I have a model that is built inside a python function where I pass the model inputs to that function and then the model is created, compiled and trained within the function. This process leads to a training for an epoch of 25000 seconds. However, when I run the exact same model at the module level (not inside any function) then training for an epoch is 1000 seconds.

The only difference is that I left out some dropout layers in the module level code and I replaced the named variables with their actual numeric values in the module level code but the named variables and numeric values are identical as can be seen in the function level code where the named parameter values are given.

Here is the code within the functions:

from __future__ import print_function

import datetime
import os

from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.engine.topology import merge
from keras.layers import Convolution1D, MaxPooling1D
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Input
from keras.layers.embeddings import Embedding
from keras.models import Model

import modelParameters
from CNN_model_fixed import save_CNNmodel_specs
from convert_review import construct_designmatrix_pairs
from loss_functions import contrastiveLoss
from siamese_activations import vectorDifference, squaredl2

USEWORDS = True

if USEWORDS:
    VocabSize = modelParameters.VocabSize_w
    maxReviewLen = modelParameters.MaxLen_w
    skipTop = modelParameters.skip_top
else:
    VocabSize = modelParameters.VocabSize_c
    maxReviewLen = modelParameters.MaxLen_c
    skipTop = 0

DEVSPLIT = modelParameters.devset_split

basename = "siamese"
suffix = datetime.datetime.now().strftime("%m%d_%I%M")
filename = "_".join([basename, suffix])

batch_size = 20
num_epochs = 4

region = 'same'

embedding_dims = 200

num_filters1 = 1300
filter_length1 = 2
pool_len1 = 2

num_filters2 = 800
filter_length2 = 3
pool_len2 = 2

num_filters3 = 500
filter_length3 = 4
pool_len3 = 2

num_filters4 = 300
filter_length4 = 5
pool_len4 = 2

dense_dims1 = 1250
dense_dims2 = 800
dense_dims3 = 250


def build_siamese_input(verbose=True,**kwargs):
    """

    :param verbose:
    :return:
    """

    if verbose:
        print('building pairs of reviews for siamese model input')

    trainCutoff = kwargs.get('trainingSet_cutoff',50000)

    ((trainingSets), (devSets), (devKNNsets), (testSets)) = construct_designmatrix_pairs(VocabSize,
                                                                                         useWords=USEWORDS,
                                                                                         skipTop=skipTop,
                                                                                         trainingSet_cutoff=trainCutoff,
                                                                                         devSplit=DEVSPLIT)

    if verbose:
        print(len(trainingSets[0]), 'train sequences length')
        print(len(devSets[0]), 'dev sequences length')

        print(len(devKNNsets[0]), 'devKNN sequences length')
        print(len(testSets[0]), 'test sequences length')

        print('train shape:', trainingSets[0].shape)
        print('dev shape:', devSets[0].shape)
        print('devKNN shape:', devKNNsets[0].shape)
        print('test shape:', testSets[0].shape)

    return {'training': trainingSets, 'dev': devSets, 'KNN': devKNNsets, 'test': testSets}


def build_siamese_model(inputType, do_training=False, model_inputs=None, weight_path=None, verbose=True, **kwargs):
    """

    :param do_training:
    :param model_inputs:
    :param weight_path:
    :param verbose:
    :param kwargs:
    :return:
    """

    def merged_outshape(inputShapes):
        """
        dynamically computes output shape of merge layer
        :param inputShapes: list of input shapes
        :return:
        """
        shape = list(inputShapes)
        return shape[0]

    if verbose:
        print('Building siamese model')

    EMBEDDING_TYPE = 'embeddingMatrix'
    ONEHOT_TYPE = '1hotVector'

    defined_input_types = {EMBEDDING_TYPE, ONEHOT_TYPE}

    assert inputType in defined_input_types, "unknown input type"

    if inputType is ONEHOT_TYPE:

        review_input = Input(shape=(modelParameters.MaxLen_w,), dtype='int32', name="1hot_review")

        layer = Embedding(modelParameters.VocabSize_w + 1, embedding_dims,
                          input_length=modelParameters.MaxLen_w, name='embeddingLayer')(review_input)

    else:
        review_input = Input(shape=(modelParameters.MaxLen_w, embedding_dims), dtype="float32", name="embedding_review")
        layer = review_input

    layer = Convolution1D(nb_filter=num_filters1,
                          filter_length=filter_length1,
                          border_mode=region,
                          activation='relu', input_length=modelParameters.MaxLen_w,
                          subsample_length=1, input_shape=(modelParameters.MaxLen_w, embedding_dims),
                          name='ConvLayer1')(layer, )

    layer = Dropout(0.25, )(layer)

    layer = MaxPooling1D(pool_length=pool_len1)(layer)

    layer = Convolution1D(nb_filter=num_filters2,
                          filter_length=filter_length2,
                          border_mode=region,
                          activation='relu',
                          subsample_length=1,
                          name='ConvLayer2'
                          )(layer, )

    layer = Dropout(0.30)(layer)

    layer = MaxPooling1D(pool_length=pool_len2)(layer)

    layer = Convolution1D(nb_filter=num_filters3,
                          filter_length=num_filters3,
                          border_mode=region,
                          activation='relu',
                          subsample_length=1,
                          name='ConvLayer3'
                          )(layer, )

    layer = Dropout(0.35)(layer)

    layer = MaxPooling1D(pool_length=pool_len3)(layer)

    layer = Convolution1D(nb_filter=num_filters4,
                          filter_length=filter_length4,
                          border_mode=region,
                          activation='relu',
                          subsample_length=1,
                          name='ConvLayer4',
                          )(layer, )

    layer = Dropout(0.35)(layer)

    layer = MaxPooling1D(pool_length=pool_len4)(layer)

    layer = Flatten()(layer)

    layer = Dense(dense_dims1, activation='relu',
                  name='dense1')(layer)

    layer = Dropout(0.35)(layer)

    layer = Dense(dense_dims2, activation='relu',
                  name='dense2')(layer)

    layer = Dropout(0.35)(layer)

    out_A = Dense(dense_dims3, activation='relu',
                  name='dense3_outA')(layer)


    CNN_model = Model(input=[review_input], output=[out_A], name="CNN_model")

    Lreview = Input(shape=(maxReviewLen,), dtype='int32', name="Lreview")
    Rreview = Input(shape=(maxReviewLen,), dtype='int32', name="Rreview")

    rightbranch = CNN_model([Lreview])
    leftbranch = CNN_model([Rreview])

    # first take the difference of the final feature representations from the CNN_model
    # represented by leftbranch and rightbranch
    merged_vector = merge([leftbranch, rightbranch], mode=vectorDifference, output_shape=merged_outshape,
                          name='merged_vector')

    # then that difference vector is fed into the final fully connected layer that
    # outputs the energy i.e. squared euclidian distance ||leftbranch-rightbranch||
    energy = Dense(1, activation=squaredl2,
                   name='energy_output')(merged_vector)

    siamese_model = Model(input=[Lreview, Rreview], output=[energy],
                          name="siamese_model")

    siamese_model.compile(optimizer='adam', loss=contrastiveLoss, )

    models = {'siamese': siamese_model, 'CNN': CNN_model}

    if weight_path:
        siamese_model.load_weights(weight_path)

    if do_training:
        X_left, y_left, X_right, y_right, similarity = model_inputs['training']

        Xdev_left, ydev_left, Xdev_right, ydev_right, dev_similarity = model_inputs['dev']

        weightPath = os.path.join(modelParameters.WEIGHT_PATH, filename)

        checkpoint = ModelCheckpoint(weightPath + '_W.{epoch:02d}-{val_loss:.2f}.hdf5',
                                     verbose=1, monitor='val_loss', save_best_only=True)

        earlyStop = EarlyStopping(patience=1, verbose=1, monitor='val_loss')

        call_backs = [checkpoint, earlyStop]

        hist = siamese_model.fit({'Lreview': X_left, 'Rreview': X_right,},
                                 {'energy_output': similarity},
                                 batch_size=batch_size,
                                 nb_epoch=num_epochs,
                                 verbose=1,
                                 validation_data=
                                 ({'Lreview': Xdev_left, 'Rreview': Xdev_right,},
                                  {'energy_output': dev_similarity}),
                                 callbacks=call_backs)

        return models, hist

    return models

And here is the code at module level.

from __future__ import print_function

import datetime
import os

from keras.layers.embeddings import Embedding
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Convolution1D, MaxPooling1D
from CNN_model_fixed import build_CNN_model,save_CNNmodel_specs
from convert_review import construct_designmatrix_pairs
from loss_functions import contrastiveLoss
from siamese_activations import vectorDifference, squaredl2
from keras.engine.topology import merge
from keras.engine.training import Model
from keras.layers import Input
from siamese_model import build_siamese_input
from CNN_model_fixed import build_CNN_input
import modelParameters


def merged_outshape(inputShapes):
        """
        dynamically computes output shape of merge layer
        :param inputShapes: list of input shapes
        :return:
        """
        shape = list(inputShapes)
        return shape[0]

basename = "siamese"
suffix = datetime.datetime.now().strftime("%m%d_%I%M")
filename = "_".join([basename, suffix])

batch_size = 20

num_epochs = 4

review_input = Input(shape=(540,), dtype='int32', name="1hot_review")

layer = Embedding((modelParameters.VocabSize_w + 3), 200,input_length=540, name='embeddingLayer')(review_input)

layer = Convolution1D(nb_filter=1300,
                          filter_length=2,
                          border_mode='same',
                          activation='relu', 
                          subsample_length=1,
                          name='ConvLayer1')(layer, )

layer = MaxPooling1D(pool_length=2)(layer)

layer = Convolution1D(nb_filter=800,
                          filter_length=3,
                          border_mode='same',
                          activation='relu',
                          subsample_length=1,
                          name='ConvLayer2'
                          )(layer, )

layer = MaxPooling1D(pool_length=2)(layer)

layer = Convolution1D(nb_filter=500,
                          filter_length=4,
                          border_mode='same',
                          activation='relu',
                          subsample_length=1,
                          name='ConvLayer3'
                          )(layer, )



layer = MaxPooling1D(pool_length=2)(layer)

layer = Convolution1D(nb_filter=300,
                          filter_length=5,
                          border_mode='same',
                          activation='relu',
                          subsample_length=1,
                          name='ConvLayer4',
                          )(layer, )



layer = MaxPooling1D(pool_length=2)(layer)

layer = Flatten()(layer)


layer = Dense(1250, activation='relu', 
                  name='d1')(layer)

layer = Dense(800, activation='relu', 
                  name='d2')(layer)

output = Dense(250, activation='relu', 
                  name='output')(layer)


CNN_model = Model(input=[review_input], output=[output], name="CNN_model")


Lreview = Input(shape=(540,), dtype='int32', name="Lreview")
Rreview = Input(shape=(540,), dtype='int32', name="Rreview")

rightbranch = CNN_model([Lreview])
leftbranch = CNN_model([Rreview])

# first take the difference of the final feature representations from the CNN_model
# represented by leftbranch and rightbranch
merged_vector = merge([leftbranch, rightbranch], mode=vectorDifference, output_shape=merged_outshape,
                  name='merged_vector')

# then that difference vector is fed into the final fully connected layer that
# outputs the energy i.e. squared euclidian distance ||leftbranch-rightbranch||
energy = Dense(1, activation=squaredl2,
           name='energy_output')(merged_vector)

siamese_model = Model(input=[Lreview, Rreview], output=[energy],
                  name="siamese_model")

siamese_model.compile(optimizer='adam', loss=contrastiveLoss, )


model_inputs = build_siamese_input()

X_left, y_left, X_right, y_right, similarity = model_inputs['training']

Xdev_left, ydev_left, Xdev_right, ydev_right, dev_similarity = model_inputs['dev']

weightPath = os.path.join(modelParameters.WEIGHT_PATH, filename)

checkpoint = ModelCheckpoint(weightPath + '_W.{epoch:02d}-{val_loss:.2f}.hdf5',
                             verbose=1, monitor='val_loss', save_best_only=True)

earlyStop = EarlyStopping(patience=1, verbose=1, monitor='val_loss')

call_backs = [checkpoint, earlyStop]

hist = siamese_model.fit({'Lreview': X_left, 'Rreview': X_right,},
                 {'energy_output': similarity},
                 batch_size=batch_size,
                 nb_epoch=num_epochs,
                 verbose=1,
                 validation_data=
                 ({'Lreview': Xdev_left, 'Rreview': Xdev_right,},
                  {'energy_output': dev_similarity}),
                 callbacks=call_backs)