How to concatenate embeddings with variable length inputs in Keras? - machine-learning

Question

Here is the network diagram I am working on and data is tabular and structured,
On the left, we have some abilities which are continuous features and on the right, we could have 'N' number of modifiers. Each modifier has modifier_type which is categorical and some statistics which are continuous features.
If it was only one modifier here is the code that works just fine!
import keras.backend as K
from keras.models import Model
from keras.layers import Input, Embedding, concatenate
from keras.layers import Dense, GlobalMaxPooling1D, Reshape
from keras.optimizers import Adam
K.clear_session()
# Using embeddings for categorical features
modifier_type_embedding_in=[]
modifier_type_embedding_out=[]
# sample categorical features
categorical_features = ['modifier_type']
modifier_input_ = Input(shape=(1,), name='modifier_type_in')
# Let's assume 10 unique type of modifiers and let's have embedding dimension as 6
modifier_output_ = Embedding(input_dim=10, output_dim=6, name='modifier_type')(modifier_input_)
modifier_output_ = Reshape(target_shape=(6,))(modifier_output_)
modifier_type_embedding_in.append(modifier_input_)
modifier_type_embedding_out.append(modifier_output_)
# sample continuous features
statistics = ['duration']
statistics_inputs =[Input(shape=(len(statistics),), name='statistics')] # Input(shape=(1,))
# sample continuous features
abilities = ['buyback_cost', 'cooldown', 'number_of_deaths', 'ability', 'teleport', 'team', 'level', 'max_mana', 'intelligence']
abilities_inputs=[Input(shape=(len(abilities),), name='abilities')] # Input(shape=(9,))
concat = concatenate(modifier_type_embedding_out + statistics_inputs)
FC_relu = Dense(128, activation='relu', name='fc_relu_1')(concat)
FC_relu = Dense(128, activation='relu', name='fc_relu_2')(FC_relu)
model = concatenate(abilities_inputs + [FC_relu])
model = Dense(64, activation='relu', name='fc_relu_3')(model)
model_out = Dense(1, activation='sigmoid', name='fc_sigmoid')(model)
model_in = abilities_inputs + modifier_type_embedding_in + statistics_inputs
model = Model(inputs=model_in, outputs=model_out)
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=2e-05, decay=1e-3), metrics=['accuracy'])
However, while compiling for 'N' number of modifiers I get below error and below are the changes I made in code,
modifier_input_ = Input(shape=(None, 1,), name='modifier_type_in')
statistics_inputs =[Input(shape=(None, len(statistics),), name='statistics')] # Input(shape=(None, 1,))
FC_relu = Dense(128, activation='relu', name='fc_relu_2')(FC_relu)
max_pool = GlobalMaxPooling1D()(FC_relu)
model = concatenate(abilities_inputs + [max_pool])
Here is what I get,
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-3-7703088b1d24> in <module>
22 abilities_inputs=[Input(shape=(len(abilities),), name='abilities')] # Input(shape=(9,))
23
---> 24 concat = concatenate(modifier_type_embedding_out + statistics_inputs)
25 FC_relu = Dense(128, activation='relu', name='fc_relu_1')(concat)
26 FC_relu = Dense(128, activation='relu', name='fc_relu_2')(FC_relu)
e:\Miniconda3\lib\site-packages\keras\layers\merge.py in concatenate(inputs, axis, **kwargs)
647 A tensor, the concatenation of the inputs alongside axis `axis`.
648 """
--> 649 return Concatenate(axis=axis, **kwargs)(inputs)
650
651
e:\Miniconda3\lib\site-packages\keras\engine\base_layer.py in __call__(self, inputs, **kwargs)
423 'You can build it manually via: '
424 '`layer.build(batch_input_shape)`')
--> 425 self.build(unpack_singleton(input_shapes))
426 self.built = True
427
e:\Miniconda3\lib\site-packages\keras\layers\merge.py in build(self, input_shape)
360 'inputs with matching shapes '
361 'except for the concat axis. '
--> 362 'Got inputs shapes: %s' % (input_shape))
363
364 def _merge_function(self, inputs):
ValueError: A `Concatenate` layer requires inputs with matching shapes except for the concat axis. Got inputs shapes: [(None, 6), (None, None, 1)]
How do I use the embedding layer within a neural network designed to accept variable input length features?

Answer 1

The answer is,
import keras.backend as K
from keras.models import Model
from keras.layers import Input, Embedding, concatenate
from keras.layers import Dense, GlobalMaxPooling1D, Reshape
from keras.optimizers import Adam
K.clear_session()
# Using embeddings for categorical features
modifier_type_embedding_in=[]
modifier_type_embedding_out=[]
# sample categorical features
categorical_features = ['modifier_type']
modifier_input_ = Input(shape=(None,), name='modifier_type_in')
# Let's assume 10 unique type of modifiers and let's have embedding dimension as 6
modifier_output_ = Embedding(input_dim=10, output_dim=6, name='modifier_type')(modifier_input_)
modifier_type_embedding_in.append(modifier_input_)
modifier_type_embedding_out.append(modifier_output_)
# sample continuous features
statistics = ['duration']
statistics_inputs =[Input(shape=(None, len(statistics),), name='statistics')] # Input(shape=(1,))
# sample continuous features
abilities = ['buyback_cost', 'cooldown', 'number_of_deaths', 'ability', 'teleport', 'team', 'level', 'max_mana', 'intelligence']
abilities_inputs=[Input(shape=(len(abilities),), name='abilities')] # Input(shape=(9,))
concat = concatenate(modifier_type_embedding_out + statistics_inputs)
FC_relu = Dense(128, activation='relu', name='fc_relu_1')(concat)
FC_relu = Dense(128, activation='relu', name='fc_relu_2')(FC_relu)
max_pool = GlobalMaxPooling1D()(FC_relu)
model = concatenate(abilities_inputs + [max_pool])
model = Dense(64, activation='relu', name='fc_relu_3')(model)
model_out = Dense(1, activation='sigmoid', name='fc_sigmoid')(model)
model_in = abilities_inputs + modifier_type_embedding_in + statistics_inputs
model = Model(inputs=model_in, outputs=model_out)
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=2e-05, decay=1e-3), metrics=['accuracy'])