How to solve ValueError in computing scores when training a custom model with TFRS?

117 views

Skip to first unread message

anush bharathi

unread,

Mar 27, 2023, 6:40:49 AM3/27/23

to Keras-users

I'm new to TensorFlow Recommenders (TFRS) and trying to build a custom recommendation model using this library. I am encountering an issue when computing the scores in my custom model's compute_loss method. I have a user model and a stim model, both of which produce embeddings that I want to use to compute scores.

However, when I try to compute the scores using tf.matmul() or tf.einsum(), I keep getting a ValueError related to the dimensions of the embeddings. I have tried various methods to reshape and adjust the dimensions, but I can't seem to find a working solution.

Here's the relevant code for my custom model:

from tensorflow.keras import Sequential
from tensorflow.keras.layers import Embedding, Dense, GlobalAveragePooling1D, Flatten, Reshape, Concatenate
from tensorflow.keras.layers.experimental.preprocessing import StringLookup, Normalization, TextVectorization

# Preprocessing layers
gender_lookup = StringLookup()
gender_lookup.adapt(users_df['gender'])

sexuality_lookup = StringLookup()
sexuality_lookup.adapt(users_df['sexuality'])

nd_condition_lookup = StringLookup()
nd_condition_lookup.adapt(np.unique(np.concatenate(users_df['neurodivergent_conditions'].values)))

hobby_lookup = StringLookup()
hobby_lookup.adapt(np.unique(np.concatenate(users_df['hobbies'].values)))

age_normalizer = Normalization()
age_normalizer.adapt(users_df['age'])

score_normalizer = Normalization()
score_normalizer.adapt(users_df['stimming_essentiality_score'])

# TextVectorization layer for stim names
max_tokens = 2000  # Maximum number of unique tokens, adjust this value according to your dataset
output_sequence_length = 16 # Adjust this value based on the expected number of tokens per stim_name
embedding_dim = 32  # Dimension of the dense vectors

text_vectorization = TextVectorization(max_tokens=max_tokens, output_sequence_length=output_sequence_length)
text_vectorization.adapt(list(stim_id_mapping.keys()))

description_vectorization = TextVectorization(max_tokens=max_tokens, output_sequence_length=output_sequence_length)
description_vectorization.adapt(stims_df['description'])

# Create a Normalization layer for harmfulness_score
harmfulness_score_normalizer = Normalization()
harmfulness_score_normalizer.adapt(stims_df['harmfulness_score'])

def create_user_input_layers():
    age_input = tf.keras.Input(shape=(1,), name="age", dtype=tf.int32)
    gender_input = tf.keras.Input(shape=(1,), name="gender", dtype=tf.string)
    sexuality_input = tf.keras.Input(shape=(1,), name="sexuality", dtype=tf.string)
    neurodivergent_conditions_input = tf.keras.Input(shape=(None,), ragged=True, name="neurodivergent_conditions", dtype=tf.string)
    hobbies_input = tf.keras.Input(shape=(None,), ragged=True , name="hobbies", dtype=tf.string)
    stimming_essentiality_score_input = tf.keras.Input(shape=(1,), name="stimming_essentiality_score", dtype=tf.int32)

    return {
        "age": age_input,
        "gender": gender_input,
        "sexuality": sexuality_input,
        "neurodivergent_conditions": neurodivergent_conditions_input,
        "hobbies": hobbies_input,
        "stimming_essentiality_score": stimming_essentiality_score_input,
    }
def create_stim_input_layers():
    name_input = tf.keras.Input(shape=(1,), name="name", dtype=tf.string)
    description_input = tf.keras.Input(shape=(1,), name="description", dtype=tf.string)
    harmfulness_score_input = tf.keras.Input(shape=(1,), name="harmfulness_score", dtype=tf.int32)
    return {"name": name_input, "description": description_input, "harmfulness_score": harmfulness_score_input}


def create_user_model(user_input_layers):
    # Preprocessing layers
    age_normalized = tf.expand_dims(age_normalizer(user_input_layers["age"]), -1)
    gender_embedded = gender_lookup(user_input_layers["gender"])
    sexuality_embedded = sexuality_lookup(user_input_layers["sexuality"])
    neurodivergent_conditions_embedded = nd_condition_lookup(user_input_layers["neurodivergent_conditions"])
    hobbies_embedded = hobby_lookup(user_input_layers["hobbies"])
    stimming_essentiality_score_normalized = tf.expand_dims(score_normalizer(user_input_layers["stimming_essentiality_score"]),-1)

    # Embedding layers
    embedding_dim = 8
    gender_embedding = Embedding(input_dim=len(gender_lookup.get_vocabulary()), output_dim=embedding_dim)(gender_embedded)
    sexuality_embedding = Embedding(input_dim=len(sexuality_lookup.get_vocabulary()), output_dim=embedding_dim)(sexuality_embedded)
    neurodivergent_conditions_dense = Embedding(input_dim=len(nd_condition_lookup.get_vocabulary()), output_dim=embedding_dim)(neurodivergent_conditions_embedded)
    hobbies_dense = Embedding(input_dim=len(hobby_lookup.get_vocabulary()), output_dim=embedding_dim)(hobbies_embedded)
    

    # Dense layers
    age_dense = Dense(4, activation="relu")(tf.reshape(age_normalized, (-1, 1)))
    stimming_essentiality_dense = Dense(6, activation="relu")(tf.reshape(stimming_essentiality_score_normalized, (-1, 1)))

    gender_dense = Dense(2, activation='relu')(Flatten()(gender_embedding))
    sexuality_dense = Dense(2, activation='relu')(Flatten()(sexuality_embedded))

    neurodivergent_conditions_pooled = GlobalAveragePooling1D()(neurodivergent_conditions_dense)
    hobbies_pooled = GlobalAveragePooling1D()(hobbies_dense)
    nd_condition_dense = Dense(9, activation="relu")(neurodivergent_conditions_pooled)
    hobbies_dense = Dense(9, activation="relu")(hobbies_pooled)

    # Concatenate dense layers
    concatenated = Concatenate(axis=-1)([
        age_dense,
        gender_dense,
        sexuality_dense,
        nd_condition_dense,
        hobbies_dense,
        stimming_essentiality_dense
    ])

    flat_embeddings = Flatten()(concatenated)
    # User Dense layers
    dense_1 = Dense(64, activation='relu')(flat_embeddings)
    dense_2 = Dense(32, activation='relu')(dense_1)

    model = tf.keras.Model(inputs=user_input_layers, outputs=dense_2, name="user_model")
    return model

def create_stim_model(stim_input_layers):
    # TextVectorization layer
    text_vectorized = text_vectorization(stim_input_layers["name"])
    description_vectorized = description_vectorization(stim_input_layers["description"])

    # Embedding layer
    embedding_layer = Embedding(input_dim=max_tokens, output_dim=embedding_dim, input_length=output_sequence_length)
    embedded = embedding_layer(text_vectorized)

    description_embedding_layer = Embedding(input_dim=max_tokens, output_dim=embedding_dim, input_length=output_sequence_length)
    description_embedded = description_embedding_layer(description_vectorized)

    name_flattened = Flatten()(embedded)
    description_flattened = Flatten()(description_embedded)

    harmfulness_score_normalized = tf.expand_dims(harmfulness_score_normalizer(stim_input_layers["harmfulness_score"]), -1)

    name_dense=Dense(8, activation="relu")(name_flattened)
    description_dense=Dense(16, activation="relu")(description_flattened)
    harmfulness_score_dense=Dense(8, activation="relu")(tf.reshape(harmfulness_score_normalized,(-1,1)))

    concatenated = Concatenate(axis=-1)([
        name_dense,
        description_dense,
        harmfulness_score_dense
    ])
    flat_embeddings = Flatten()(concatenated)
    # Flatten and Dense layers
    dense_1 = Dense(64, activation='relu')(flat_embeddings)
    dense_2 = Dense(32, activation='relu')(dense_1)

    return tf.keras.Model(inputs=stim_input_layers, outputs=dense_2, name="stim_model")


user_input_layers = create_user_input_layers()
stim_input_layers = create_stim_input_layers()

user_model = create_user_model( user_input_layers)
stim_model = create_stim_model( stim_input_layers)

class StimmingRecommenderModel(tfrs.Model):

    def __init__(self, user_model, stim_model):
        super().__init__()
        self.user_model = user_model
        self.stim_model = stim_model
        self.task = tfrs.tasks.Retrieval(
            metrics=tfrs.metrics.FactorizedTopK(
                candidates=tf.data.Dataset.from_tensor_slices(dict(stims_df)).batch(128).map(stim_model)))

        
    def compute_loss(self, inputs, training=False):
        if isinstance(inputs, tuple):
            features = inputs[0]
        else:
            features = inputs
        
        user_id, stim_id = features['user_id'], features['stim_id']
        print(user_id)
        user_features = {
            k: tf.gather(v, user_id)
            for k, v in {
                col: tf.constant(users_df[col].apply(lambda x: x if users_df[col].dtype != object or isinstance(x, str) else ','.join(x)).values, dtype=tf.string if users_df[col].dtype == object else tf.int32)
                for col in users_df.columns
            }.items()
        }
        stim_features = {
            k: tf.gather(v, stim_id)
            for k, v in {
                col: tf.constant(stims_df[col].values, dtype=tf.string if stims_df[col].dtype == object else tf.int32)
                for col in stims_df.columns
            }.items()
        }

        user_embeddings = user_model({
            "age": user_features["age"],
            "gender": user_features["gender"],
            "sexuality": user_features["sexuality"],
            "neurodivergent_conditions": tf.strings.split(user_features["neurodivergent_conditions"], sep=","),
            "hobbies": tf.strings.split(user_features["hobbies"], sep=','),
            "stimming_essentiality_score": user_features["stimming_essentiality_score"]
        })

        stim_embeddings = stim_model({"name": stim_features["name"],
                                      'description':stim_features['description'],
                                      'harmfulness_score':stim_features['harmfulness_score']})

        print(f"user_embeddings shape: {user_embeddings.shape}")
        print(f"stim_embeddings shape: {stim_embeddings.shape}")

        # Compute the scores using tf.matmul()
        scores = tf.matmul(user_embeddings, stim_embeddings, transpose_b=False)
        #scores= tf.transpose(user_embeddings*stim_embeddings)
        scores=tf.reduce_sum(scores,axis=1)
        print(f"scores shape: {scores.shape}")

        # Use the interaction scores as labels
        labels = features['usefulness_score'] + features['frequency_score'] + 2 * features['sensory_regulation_score'] - features['discomfort_score']
        labels=tf.expand_dims(labels, axis=1)
        labels=tf.reduce_sum(labels,axis=1)
        print(labels, scores)

        # Call the retrieval task with adjusted shapes

return self.task(labels, scores)

#compile the model

model = StimmingRecommenderModel(user_model=user_model, stim_model=stim_model)
model.compile(optimizer=tf.keras.optimizers.Adagrad(learning_rate=0.1))
num_epochs=10

def to_tf_dataset(df):
    data_dict = {
        "user_id": tf.constant(df["user_id"].values, dtype=tf.int32),
        "stim_id": tf.constant(df["stim_id"].values, dtype=tf.int32),
        "usefulness_score": tf.constant(df["usefulness_score"].values, dtype=tf.float32),
        "frequency_score": tf.constant(df["frequency_score"].values, dtype=tf.float32),
        "sensory_regulation_score": tf.constant(df["sensory_regulation_score"].values, dtype=tf.float32),
        "discomfort_score": tf.constant(df["discomfort_score"].values, dtype=tf.float32)
    }
    return data_dict

train_tf_dataset = to_tf_dataset(train_interactions)
history = model.fit(train_tf_dataset, epochs=num_epochs, steps_per_epoch= len(train_interactions) // 32, verbose=2)

Here's the error I get:

ValueError Traceback (most recent call last)
<ipython-input-48-443920f9e2cb> in <module>
36
37 # Train the model
---> 38 history = model.fit(train_tf_dataset, epochs=num_epochs, steps_per_epoch= len(train_interactions) // 32, verbose=2)

4 frames
/usr/local/lib/python3.9/dist-packages/tensorflow_recommenders/tasks/retrieval.py in tf__call(self, query_embeddings, candidate_embeddings, sample_weight, candidate_sampling_probability, candidate_ids, compute_metrics, compute_batch_metrics)
40 do_return = False
41 retval_ = ag__.UndefinedReturnValue()
---> 42 scores = ag__.converted_call(ag__.ld(tf).linalg.matmul, (ag__.ld(query_embeddings), ag__.ld(candidate_embeddings)), dict(transpose_b=True), fscope)
43 num_queries = ag__.converted_call(ag__.ld(tf).shape, (ag__.ld(scores),), None, fscope)[0]
44 num_candidates = ag__.converted_call(ag__.ld(tf).shape, (ag__.ld(scores),), None, fscope)[1]

ValueError: in user code:

ValueError: Exception encountered when calling layer 'retrieval_14' (type Retrieval).

in user code:

File "/usr/local/lib/python3.9/dist-packages/tensorflow_recommenders/tasks/retrieval.py", line 160, in call *
scores = tf.linalg.matmul(

ValueError: Shape must be rank 2 but is rank 1 for '{{node retrieval_14/MatMul}} = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=true](Sum_1, Sum)' with input shapes: [?], [?].

Call arguments received by layer 'retrieval_14' (type Retrieval):
• query_embeddings=tf.Tensor(shape=(None,), dtype=float32)
• candidate_embeddings=tf.Tensor(shape=(None,), dtype=float32)
• sample_weight=None
• candidate_sampling_probability=None
• candidate_ids=None
• compute_metrics=True
• compute_batch_metrics=True

I expected to be able to compute the scores using tf.matmul() or tf.einsum() with the appropriate reshaping of the embeddings. I tried the following methods:

1 Reshaping the embeddings with tf.expand_dims(), tf.squeeze(), and tf.reshape().

2 Using tf.einsum() with different combinations of indices.

However, none of these methods have resolved the error. I was expecting to compute the scores without encountering the ValueError related to the dimensions of the embeddings.