Layer model_1 expects 2 input but it received 1 tensors received:[<tf.Tensor IteratorGetNext:0 shape=(None, None, None, None, None) dtype=float32>] - time-series

Encountered the error while trying to fit model of encoder-decoder using ConvLSTM2D. the x_train is of shape (31567, 7, 210, 203, 1)(batch_size,framelength,H,W,C).
The encoder part works when executed in isolation but the error occurs when i add the decoder part, seems like the problem is in the input part of decoder but not sure.
tried reshaping the encoder_state_c_1 and encoder_state_h_1 to 5D before passing it to the decoder ConvLSTM2D but doesn't help either.
Please find the code and error here:
MODEL
def define_models_1_moving_1(framelength, n_filter, filter_size):
# define training encoder
encoder_inputs = Input(name = "encoder_input",
shape=(x_train.shape[1], x_train.shape[2], x_train.shape[3], x_train.shape[4]))
encoder_1 = ConvLSTM2D(name = "encoder_ConvLSTM",
filters = n_filter, kernel_size=filter_size, padding='same', return_sequences=True, return_state=True,
kernel_regularizer=l2(0.0005), recurrent_regularizer=l2(0.0005), bias_regularizer=l2(0.0005))
# input_shape=(x_train.shape[1], x_train.shape[2], x_train.shape[3], x_train.shape[4]))
encoder_outputs_1, encoder_state_h_1, encoder_state_c_1 = encoder_1(encoder_inputs)
# define training decoder
decoder_inputs = Input(name = "decoder_input",
shape=(x_train.shape[1], x_train.shape[2], x_train.shape[3], x_train.shape[4]))
decoder_1 = ConvLSTM2D(name = "decoder_ConvLSTM",
filters=n_filter, kernel_size=filter_size, padding='same', return_sequences=True, return_state=True,
kernel_regularizer=l2(0.0005), recurrent_regularizer=l2(0.0005), bias_regularizer=l2(0.0005))
decoder_outputs_1, _, _ = decoder_1([decoder_inputs, encoder_state_h_1, encoder_state_c_1]) #### This line is giving Error
model = Model([encoder_inputs, decoder_inputs], decoder_outputs_1)
return model
Error
Traceback (most recent call last):
File "D:\Chintan\Dataset\model.py", line 155, in
training_history = train_1_moving_1.fit(
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 70, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\Users\Admin\AppData\Local\Temp_autograph_generated_filernuwcygs.py", line 15, in tf__train_function
retval = ag__.converted_call(ag__.ld(step_function), (ag__.ld(self), ag__.ld(iterator)), None, fscope)
ValueError: in user code:
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1249, in train_function *
return step_function(self, iterator)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1233, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1222, in run_step **
outputs = model.train_step(data)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1023, in train_step
y_pred = self(x, training=True)
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 70, in error_handler
raise e.with_traceback(filtered_tb) from None
File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\input_spec.py", line 216, in assert_input_compatibility
raise ValueError(
ValueError: Layer "model_120" expects 2 input(s), but it received 1 input tensors. Inputs received: [<tf.Tensor 'IteratorGetNext:0' shape=(None, None, None, None, None) dtype=float32>]

Related

How to use model.fit() on a dataset generated using Keras ImageDataGenerator

My directory consists of two classes as shown here.
enter image description here
I want to apply this model I found online for cifar10 on my own dataset but I am having trouble training it. Any help would be greatly appreciated as I am very new to this.
This is the model I am trying to train.
import pathlib
import tensorflow as tf
from tensorflow import keras
from keras.preprocessing.image import ImageDataGenerator
import pathlib
import numpy as np
data_dir = pathlib.Path('/content/drive/My Drive/Images/images')
from keras.models import Sequential
from keras.optimizers import SGD
from keras.regularizers import l2
from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, Dropout
NUM_CLASSES = 2
NUM_EPOCHS = 20
WEIGHT_DECAY = 5e-4
image_generator = ImageDataGenerator(rescale=1/255, validation_split=0.2)
train_dataset = image_generator.flow_from_directory(batch_size=32,
directory=data_dir,
shuffle=True,
target_size=(280, 280),
subset="training",
class_mode='categorical')
validation_dataset = image_generator.flow_from_directory(batch_size=32,
directory=data_dir,
shuffle=True,
target_size=(280, 280),
subset="validation",
class_mode='categorical')
model = Sequential()
model.add(Conv2D(96, (11,11),activation='relu', padding = 'same', input_shape=(32,32,3),kernel_regularizer = l2(WEIGHT_DECAY), name='conv1'))
model.add(MaxPooling2D((2,2), name='pool1'))
model.add(Conv2D(256, (5,5),activation='relu', padding = 'same', kernel_regularizer = l2(WEIGHT_DECAY), name ='conv2'))
model.add(MaxPooling2D((2,2),name='pool2'))
model.add(Conv2D(384, (3,3),activation='relu', padding = 'same', kernel_regularizer = l2(WEIGHT_DECAY), name ='conv3'))
model.add(Conv2D(384, (5,5),activation='relu', padding = 'same', kernel_regularizer = l2(WEIGHT_DECAY), name ='conv4'))
model.add(Conv2D(256, (5,5),activation='relu', padding = 'same', kernel_regularizer = l2(WEIGHT_DECAY), name ='conv5'))
model.add(MaxPooling2D((2,2),name='pool3'))
model.add(Flatten())
model.add(Dense(4096,activation='relu', kernel_regularizer= l2(WEIGHT_DECAY), name='fc1'))
model.add(Dropout(0.5))
model.add(Dense(4096,activation='relu', kernel_regularizer= l2(WEIGHT_DECAY), name='fc2'))
model.add(Dropout(0.5))
model.add(Dense(NUM_CLASSES, activation='softmax', kernel_regularizer = l2(WEIGHT_DECAY), name ='output'))
print(model.summary())
model.compile(loss ='categorical_crossentropy',optimizer=SGD(learning_rate=1e-2, momentum=0.9),metrics=['accuracy'])
tensorboard = keras.callbacks.TensorBoard(log_dir = 'alexnet')
model.fit(train_dataset, epochs=20, validation_data= validation_dataset,callbacks=[tensorboard])
score = model.evaluate(validation_dataset)
print('\nTest accuracy: {0:.4f}'.format(score[1]))
However when I run it, I get this error
enter image description here
enter image description here
Detected at node 'categorical_crossentropy/softmax_cross_entropy_with_logits' defined at (most recent call last):
File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "/usr/local/lib/python3.7/dist-packages/traitlets/config/application.py", line 846, in launch_instance
app.start()
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelapp.py", line 612, in start
self.io_loop.start()
File "/usr/local/lib/python3.7/dist-packages/tornado/platform/asyncio.py", line 149, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.7/asyncio/base_events.py", line 541, in run_forever
self._run_once()
File "/usr/lib/python3.7/asyncio/base_events.py", line 1786, in _run_once
handle._run()
File "/usr/lib/python3.7/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.7/dist-packages/tornado/ioloop.py", line 690, in
lambda f: self._run_callback(functools.partial(callback, future))
File "/usr/local/lib/python3.7/dist-packages/tornado/ioloop.py", line 743, in _run_callback
ret = callback()
File "/usr/local/lib/python3.7/dist-packages/tornado/gen.py", line 787, in inner
self.run()
File "/usr/local/lib/python3.7/dist-packages/tornado/gen.py", line 748, in run
yielded = self.gen.send(value)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 365, in process_one
yield gen.maybe_future(dispatch(*args))
File "/usr/local/lib/python3.7/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 268, in dispatch_shell
yield gen.maybe_future(handler(stream, idents, msg))
File "/usr/local/lib/python3.7/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 545, in execute_request
user_expressions, allow_stdin,
File "/usr/local/lib/python3.7/dist-packages/tornado/gen.py", line 209, in wrapper
yielded = next(result)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/ipkernel.py", line 306, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/zmqshell.py", line 536, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2855, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2881, in _run_cell
return runner(coro)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/async_helpers.py", line 68, in pseudo_sync_runner
coro.send(None)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 3058, in run_cell_async
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 3249, in run_ast_nodes
if (await self.run_code(code, result, async=asy)):
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 3326, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 60, in
model.fit(train_dataset, epochs=20, validation_data= validation_dataset,callbacks=[tensorboard])
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1409, in fit
tmp_logs = self.train_function(iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1051, in train_function
return step_function(self, iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1040, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1030, in run_step
outputs = model.train_step(data)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 890, in train_step
loss = self.compute_loss(x, y, y_pred, sample_weight)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 949, in compute_loss
y, y_pred, sample_weight, regularization_losses=self.losses)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/compile_utils.py", line 201, in call
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 139, in call
losses = call_fn(y_true, y_pred)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 243, in call
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 1788, in categorical_crossentropy
y_true, y_pred, from_logits=from_logits, axis=axis)
File "/usr/local/lib/python3.7/dist-packages/keras/backend.py", line 5135, in categorical_crossentropy
labels=target, logits=output, axis=axis)
Node: 'categorical_crossentropy/softmax_cross_entropy_with_logits'
logits and labels must be broadcastable: logits_size=[2450,2] labels_size=[32,2]
[[{{node categorical_crossentropy/softmax_cross_entropy_with_logits}}]] [Op:__inference_train_function_11909]

Custom Loss function error in low light image enhancement model

I'm trying to create a custom loss function for an image enhancement model using SSIM loss.
def mirnet_model(num_rrg, num_mrb, channels):
input_tensor = keras.Input(shape=[None, None, 3])
x1 = layers.Conv2D(channels, kernel_size=(3, 3), padding="same")(input_tensor)
for _ in range(num_rrg):
x1 = recursive_residual_group(x1, num_mrb, channels)
conv = layers.Conv2D(3, kernel_size=(3, 3), padding="same")(x1)
output_tensor = layers.Add()([input_tensor, conv])
return keras.Model(input_tensor, output_tensor)
model = mirnet_model(num_rrg=3, num_mrb=2, channels=64)
Loss function -
def ssim_loss(y_true, y_pred):
return tf.reduce_mean(tf.image.ssim(y_true, y_pred, 2.0))
model training -
model.compile(
# optimizer=optimizer, loss=charbonnier_loss, metrics=[peak_signal_noise_ratio]
optimizer=optimizer, loss=ssim_loss, metrics=[peak_signal_noise_ratio]
)
history = model.fit(
train_dataset,
validation_data=val_dataset,
epochs=50,
callbacks=[
keras.callbacks.ReduceLROnPlateau(
monitor="val_peak_signal_noise_ratio",
factor=0.5,
patience=5,
verbose=1,
min_delta=1e-7,
mode="max",
# callbacks=[cp_callback]
)
],
)
I'm getting good results with charbonnier loss but it might overfit because the LOL dataset is small.
I'm getting this error with the ssim loss -
ValueError: in user code:
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1021, in train_function *
return step_function(self, iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1010, in step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1000, in run_step **
outputs = model.train_step(data)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 859, in train_step
y_pred = self(x, training=True)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 67, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/usr/local/lib/python3.7/dist-packages/keras/engine/input_spec.py", line 249, in assert_input_compatibility
f'Input {input_index} of layer "{layer_name}" is '
ValueError: Exception encountered when calling layer "sequential" (type Sequential).
Input 0 of layer "conv2d_662" is incompatible with the layer: expected axis -1 of input shape to have value 1, but received input with shape (4, None, None, 3)
Call arguments received:
• inputs=tf.Tensor(shape=(4, None, None, 3), dtype=float32)
• training=True
• mask=None
How to remove this error? And also tell me if there are any good loss functions that I can use in my case.

Where is this dimension error getting generated in my ADAM algorithm?

I am working on main.py in this BRATS Unet
https://github.com/pykao/Modified-3D-UNet-Pytorch/blob/master/main.py
# create your optimizer
print ("Creating Optimizer")
##optimizer = optim.adam(net.parameteres(), lr=)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
print ("Created! \n")
trainloader = torch.utils.data.DataLoader(train_idx, batch_size=2, shuffle=True)
testloader = torch.utils.data.DataLoader(test_idx, batch_size=2, shuffle=False)
for epoch in range(2): # loop over the dataset multiple times
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
print("inside for")
# get the inputs THIS ERRORS OUT
inputs, labels = data
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
loss = criterion(outputs, target)
loss.backward()
optimizer.step()
# print statistics
running_loss += loss.item()
if i % 2000 == 1999: # print every 2000 mini-batches
print('[%d, %5d] loss: %.3f' %
(epoch + 1, i + 1, running_loss / 2000))
running_loss = 0.0
print('Finished Training')
I get this output:
Creating Optimizer
Created!
inside for
Traceback (most recent call last):
File "main.py", line 109, in <module>
outputs = model(inputs)
File "/home/MAHEUNIX/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/mnt/c/Users/MAHE/Modified Unet3D Master -TestRun/model.py", line 99, in forward
out = self.conv3d_c1_1(x)
File "/home/MAHEUNIX/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__
result = self.forward(*input, **kwargs)
File "/home/MAHEUNIX/anaconda3/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 448, in forward
self.padding, self.dilation, self.groups)
RuntimeError: Expected 5-dimensional input for 5-dimensional weight [16, 4, 3, 3, 3], but got 0-dimensional input of size [] instead
I am unfamiliar with PyTorch, and so trainloader, testloader are probably incorrectly used. Please assume I don't know much while you help me. Thanks.
New error:
Traceback (most recent call last):
File "/mnt/c/Users/MAHE/Modified Unet3D Master -TestRun/main.py", line 91, in <module>
for id, info in enumerate(trainloader,0):
File "/home/MAHEUNIX/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 615, in __next__
batch = self.collate_fn([self.dataset[i] for i in indices])
File "/home/MAHEUNIX/anaconda3/lib/python3.6/site-packages/torch/utils/data/dataloader.py", line 615, in <listcomp>
batch = self.collate_fn([self.dataset[i] for i in indices])
KeyError: 0
You should pass the dataset to the data loader API. So, pass train_data and test_data instead of train_idx and test_idx to torch.utils.data.DataLoader.

Viewing layer activations with Keras

In NVIDIA's paper "End to End Learning for Self-Driving Cars" there's an illustration showing the activation of first-layer feature maps:
I'm trying to recreate this with the Comma AI model, but the only visualisation tools I've found are Keras' instructions for gradient ascent and descent, rather than simply viewing activations. what should I be looking for?
EDIT IN RESPONSE TO COMMENT
I tried implementing the code in this answer using the below code:
from keras import backend as K
import json
from keras.models import model_from_json
with open('outputs/steering_model/steering_angle.json', 'r') as jfile:
z = json.load(jfile)
model = model_from_json(z)
print("Loaded model")
model.load_weights('outputs/steering_model/steering_angle.keras')
print("Loaded weights")
img_width = 320
img_height = 160
outputs = [layer.output for layer in model.layers] # all layer outputs
functors = [K.function([inp]+ [K.learning_phase()], [out]) for out in outputs] # evaluation functions
# Testing
test = np.random.random((1, 3, img_width, img_height))
layer_outs = [func([test, 1.]) for func in functors]
print layer_outs
This give the following output error:
Using Theano backend.
Loaded model
Loaded weights
Traceback (most recent call last):
File "vis-layers.py", line 22, in <module>
layer_outs = [func([test, 1.]) for func in functors]
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/backend/theano_backend.py", line 959, in __call__
return self.function(*inputs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function_module.py", line 871, in __call__
storage_map=getattr(self.fn, 'storage_map', None))
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/link.py", line 314, in raise_with_op
reraise(exc_type, exc_value, exc_trace)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function_module.py", line 859, in __call__
outputs = self.fn()
ValueError: Shape mismatch: x has 49152 cols (and 1 rows) but y has 12800 rows (and 512 cols)
Apply node that caused the error: Dot22(Elemwise{Composite{Switch(GT(i0, i1), i0, expm1(i0))}}[(0, 0)].0, dense_1_W)
Toposort index: 50
Inputs types: [TensorType(float32, matrix), TensorType(float32, matrix)]
Inputs shapes: [(1, 49152), (12800, 512)]
Inputs strides: [(196608, 4), (2048, 4)]
Inputs values: ['not shown', 'not shown']
Outputs clients: [[Elemwise{Add}[(0, 0)](Dot22.0, InplaceDimShuffle{x,0}.0)]]
I thought this might be a problem with th vs tf dimensions, so tried changing the test input to:
test = np.random.random((1, img_height, img_width, 3))
which gave the following error:
Using Theano backend.
Loaded model
Loaded weights
Traceback (most recent call last):
File "vis-layers.py", line 22, in <module>
layer_outs = [func([test, 1.]) for func in functors]
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/backend/theano_backend.py", line 959, in __call__
return self.function(*inputs)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function_module.py", line 871, in __call__
storage_map=getattr(self.fn, 'storage_map', None))
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/gof/link.py", line 314, in raise_with_op
reraise(exc_type, exc_value, exc_trace)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/theano/compile/function_module.py", line 859, in __call__
outputs = self.fn()
ValueError: CorrMM images and kernel must have the same stack size
Apply node that caused the error: CorrMM{half, (4, 4)}(Elemwise{Composite{(i0 + (i1 * i2))}}.0, Subtensor{::, ::, ::int64, ::int64}.0)
Toposort index: 9
Inputs types: [TensorType(float32, 4D), TensorType(float32, 4D)]
Inputs shapes: [(1, 320, 160, 3), (16, 3, 8, 8)]
Inputs strides: [(2250000, 6000, 12, 4), (768, 256, -32, -4)]
Inputs values: ['not shown', 'not shown']
Outputs clients: [[Subtensor{int64:int64:int8, int64:int64:int8, int64:int64:int8, int64:int64:int8}(CorrMM{half, (4, 4)}.0, ScalarFromTensor.0, ScalarFromTensor.0, Constant{1}, Constant{0}, Constant{16}, Constant{1}, ScalarFromTensor.0, ScalarFromTensor.0, Constant{1}, ScalarFromTensor.0, ScalarFromTensor.0, Constant{1})]]
Backtrace when the node is created(use Theano flag traceback.limit=N to make it longer):
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/utils/layer_utils.py", line 43, in layer_from_config
return layer_class.from_config(config['config'])
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.py", line 1091, in from_config
model.add(layer)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/models.py", line 332, in add
output_tensor = layer(self.outputs[0])
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 572, in __call__
self.add_inbound_node(inbound_layers, node_indices, tensor_indices)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 635, in add_inbound_node
Node.create_node(self, inbound_layers, node_indices, tensor_indices)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/engine/topology.py", line 166, in create_node
output_tensors = to_list(outbound_layer.call(input_tensors[0], mask=input_masks[0]))
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/layers/convolutional.py", line 475, in call
filter_shape=self.W_shape)
File "/home/ubuntu/anaconda2/lib/python2.7/site-packages/keras/backend/theano_backend.py", line 1508, in conv2d
filter_shape=filter_shape)
EDIT: Output of model.summary()
____________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
====================================================================================================
lambda_1 (Lambda) (None, 3, 160, 320) 0 lambda_input_1[0][0]
____________________________________________________________________________________________________
convolution2d_1 (Convolution2D) (None, 16, 40, 80) 3088 lambda_1[0][0]
____________________________________________________________________________________________________
elu_1 (ELU) (None, 16, 40, 80) 0 convolution2d_1[0][0]
____________________________________________________________________________________________________
convolution2d_2 (Convolution2D) (None, 32, 20, 40) 12832 elu_1[0][0]
____________________________________________________________________________________________________
elu_2 (ELU) (None, 32, 20, 40) 0 convolution2d_2[0][0]
____________________________________________________________________________________________________
convolution2d_3 (Convolution2D) (None, 64, 10, 20) 51264 elu_2[0][0]
____________________________________________________________________________________________________
flatten_1 (Flatten) (None, 12800) 0 convolution2d_3[0][0]
____________________________________________________________________________________________________
dropout_1 (Dropout) (None, 12800) 0 flatten_1[0][0]
____________________________________________________________________________________________________
elu_3 (ELU) (None, 12800) 0 dropout_1[0][0]
____________________________________________________________________________________________________
dense_1 (Dense) (None, 512) 6554112 elu_3[0][0]
____________________________________________________________________________________________________
dropout_2 (Dropout) (None, 512) 0 dense_1[0][0]
____________________________________________________________________________________________________
elu_4 (ELU) (None, 512) 0 dropout_2[0][0]
____________________________________________________________________________________________________
dense_2 (Dense) (None, 1) 513 elu_4[0][0]
====================================================================================================
Total params: 6,621,809
Trainable params: 6,621,809
Non-trainable params: 0
____________________________________________________________________________________________________
EDIT: DEBUGGING WITH SINGLE LAYER
In order to debug the issue with input shapes, I rewrote the script for a single layer:
from keras import backend as K
import numpy as np
import json
from keras.models import model_from_json
with open('outputs/steering_model/steering_angle.json', 'r') as jfile:
z = json.load(jfile)
model = model_from_json(z)
print("Loaded model")
model.load_weights('outputs/steering_model/steering_angle.keras')
print("Loaded weights")
layer_name = 'lambda_1'
img_width = 160
img_height = 320
inp = model.input
layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0]
output = model.layers[layer_idx].output
functor = K.function([inp]+ [K.learning_phase()], output) # evaluation function
# Testing
test = np.random.random((1, img_height, img_width, 3))
layer_out = functor([test, 1.])
print layer_out
The output from this is as follows:
Using Theano backend.
Loaded model
Loaded weights
[[[[-0.99223709 -0.99468529 -0.99318016]
[-0.99725926 -0.9924705 -0.9994905 ]
[-0.99380279 -0.99291307 -0.99927235]
...,
[-0.99361622 -0.99258155 -0.99954134]
[-0.99748689 -0.99217939 -0.99918425]
[-0.99475586 -0.99366009 -0.992827 ]]
[[-0.99330682 -0.99756712 -0.99795902]
[-0.99421203 -0.99240923 -0.99438184]
[-0.99222761 -0.99425066 -0.99886942]
...,
[-0.99329156 -0.99460274 -0.99994165]
[-0.99763876 -0.99870259 -0.9998613 ]
[-0.99962425 -0.99702215 -0.9943046 ]]
[[-0.99947125 -0.99577188 -0.99294066]
[-0.99582225 -0.99568367 -0.99345332]
[-0.99823713 -0.99376178 -0.99432898]
...,
[-0.99828976 -0.99264622 -0.99669623]
[-0.99485278 -0.99353015 -0.99411404]
[-0.99832171 -0.99390954 -0.99620205]]
...,
[[-0.9980613 -0.99474132 -0.99680966]
[-0.99378282 -0.99288809 -0.99248403]
[-0.99375945 -0.9966079 -0.99440354]
...,
[-0.99634677 -0.99931824 -0.99611002]
[-0.99781156 -0.99990571 -0.99249381]
[-0.9996115 -0.99991143 -0.99486816]]
[[-0.99839222 -0.99690026 -0.99410695]
[-0.99551272 -0.99262673 -0.99934679]
[-0.99432331 -0.99822938 -0.99294668]
...,
[-0.99515969 -0.99867356 -0.9926796 ]
[-0.99478716 -0.99883151 -0.99760127]
[-0.9982425 -0.99547088 -0.99658638]]
[[-0.99240851 -0.99792403 -0.99360847]
[-0.99226022 -0.99546915 -0.99411654]
[-0.99558711 -0.9960795 -0.9993062 ]
...,
[-0.99745959 -0.99276334 -0.99800634]
[-0.99249429 -0.99748743 -0.99576926]
[-0.99531293 -0.99618822 -0.99997312]]]]
However, attempting the same on the first convolutional layer (convolution2d_1) with an 80x40 image returns the same error:
ValueError: CorrMM images and kernel must have the same stack size
Apply node that caused the error: CorrMM{half, (4, 4)}(Elemwise{Composite{(i0 + (i1 * i2))}}.0, Subtensor{::, ::, ::int64, ::int64}.0)
Toposort index: 9
Inputs types: [TensorType(float32, 4D), TensorType(float32, 4D)]
Inputs shapes: [(1, 40, 80, 3), (16, 3, 8, 8)]
Inputs strides: [(38400, 960, 12, 4), (768, 256, -32, -4)]
Inputs values: ['not shown', 'not shown']
Outputs clients: [[Subtensor{int64:int64:int8, int64:int64:int8, int64:int64:int8, int64:int64:int8}(CorrMM{half, (4, 4)}.0, ScalarFromTensor.0, ScalarFromTensor.0, Constant{1}, Constant{0}, Constant{16}, Constant{1}, ScalarFromTensor.0, ScalarFromTensor.0, Constant{1}, ScalarFromTensor.0, ScalarFromTensor.0, Constant{1})]]
EDIT: OUTPUT LAYER DATA AS IMAGE
The following code replaces the random image with a loaded one, and takes the layer output and saves it as an image:
input_img_data = imread(impath+'.png').astype(np.float32)
# change image to 4d theano array
test = np.expand_dims(input_img_data,axis=0)
print test.shape
layer_out = functor([test, 1.])
img = Image.fromarray(layer_out[0,:,:,:], 'RGB')
img.save('activ_%s_%s.png' % (layer_name,impath))
print("Created Image")
Here is the final code that does what I want it to do, still rough and in need of tidying up:
from keras import backend as K
from PIL import Image
from scipy.misc import imread
from scipy.misc import imsave
import numpy as np
import json
from keras.models import model_from_json
with open('outputs/steering_model/steering_angle.json', 'r') as jfile:
z = json.load(jfile)
model = model_from_json(z)
print("Loaded model")
model.load_weights('outputs/steering_model/steering_angle.keras')
print("Loaded weights")
layer_name = 'lambda_1'
#layer_name = 'convolution2d_1'
#layer_name = 'elu_1'
#layer_name = 'convolution2d_2'
impaths = ['track','road','mway']
img_width = 500
img_height = 375
inp = model.input
layer_idx = [idx for idx, layer in enumerate(model.layers) if layer.name == layer_name][0]
output = model.layers[layer_idx].output
functor = K.function([inp]+ [K.learning_phase()], output) # evaluation function
for impath in impaths:
input_img_data = imread('testimages/'+impath+'.png').astype(np.float32)
input_img_data = np.rollaxis(input_img_data,2,0) # change to (channels,h,w)
test = np.expand_dims(input_img_data,axis=0) # change to (dims,channels,h,w)
print("Test Shape: %s" % (test.shape,)) # check shape
layer_out = functor([test, 1.])
print ("Output Shape: %s" % (layer_out.shape,)) # check output shape
# save multiple greyscale images
layer_out = np.rollaxis(layer_out,0,4)
print ("Output Image Shape: %s" % (layer_out.shape,)) # check output shape
count = 1
for x in layer_out:
x = np.rollaxis(x,2,0)
print ("Final Image Shape: %s" % (x.shape,)) # check output shape
imsave('activationimages/activ_%s_%s_%d.png' % (layer_name,impath,count),x[0,:,:])
count = count + 1
The main issue was wrangling the shapes of the various input and output layers - hence all the print commands in the above code, for debugging.
A second confusion was that I was interpreting an array shape of (3,w,h) as a single RGB (3-channel) image, rather than one greyscale image.
The version above tests an array of images at a time (hardcoded image path). The lambda_1 level outputs a single RGB image per test image, convolution2d_1 and elu_1 output sixteen smaller (25%) greyscale images - one for each filter. And, I hope, so on.
I will add a Github link to a tidied gist with image stitching when I've done this. I've learned a lot.

Keras Wrappers for Scikit Learn - AUC scorer is not working

I'm trying to use Keras Scikit Learn Wrapper in order to make random search for parameters easier. I wrote an example code here where :
I generate an artificial dataset:
I am using moons from scikit learn
from sklearn.datasets import make_moons
dataset = make_moons(1000)
Model builder definition:
I define build_fn function needed:
def build_fn(nr_of_layers = 2,
first_layer_size = 10,
layers_slope_coeff = 0.8,
dropout = 0.5,
activation = "relu",
weight_l2 = 0.01,
act_l2 = 0.01,
input_dim = 2):
result_model = Sequential()
result_model.add(Dense(first_layer_size,
input_dim = input_dim,
activation=activation,
W_regularizer= l2(weight_l2),
activity_regularizer=activity_l2(act_l2)
))
current_layer_size = int(first_layer_size * layers_slope_coeff) + 1
for index_of_layer in range(nr_of_layers - 1):
result_model.add(BatchNormalization())
result_model.add(Dropout(dropout))
result_model.add(Dense(current_layer_size,
W_regularizer= l2(weight_l2),
activation=activation,
activity_regularizer=activity_l2(act_l2)
))
current_layer_size = int(current_layer_size * layers_slope_coeff) + 1
result_model.add(Dense(1,
activation = "sigmoid",
W_regularizer = l2(weight_l2)))
result_model.compile(optimizer="rmsprop", metrics = ["accuracy"], loss = "binary_crossentropy")
return result_model
NeuralNet = KerasClassifier(build_fn)
Parameter grid definition :
Then I defined a parameter grid :
param_grid = {
"nr_of_layers" : [2, 3, 4, 5],
"first_layer_size" : [5, 10, 15],
"layers_slope_coeff" : [0.4, 0.6, 0.8],
"dropout" : [0.3, 0.5, 0.8],
"weight_l2" : [0.01, 0.001, 0.0001],
"verbose" : [0],
"batch_size" : [1],
"nb_epoch" : [30]
}
RandomizedSearchCV phase:
I defined RandomizedSearchCV object and fitted with values from artificial dataset:
random_search = RandomizedSearchCV(NeuralNet,
param_distributions=param_grid, verbose=2, n_iter=1, scoring="roc_auc")
random_search.fit(dataset[0], dataset[1])
What I got (after running this code in console) is :
Traceback (most recent call last):
File "C:\Anaconda2\lib\site-packages\IPython\core\interactiveshell.py", line 2885, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-3-c5bdbc2770b7>", line 2, in <module>
random_search.fit(dataset[0], dataset[1])
File "C:\Anaconda2\lib\site-packages\sklearn\grid_search.py", line 996, in fit
return self._fit(X, y, sampled_params)
File "C:\Anaconda2\lib\site-packages\sklearn\grid_search.py", line 553, in _fit
for parameters in parameter_iterable
File "C:\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 800, in __call__
while self.dispatch_one_batch(iterator):
File "C:\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 658, in dispatch_one_batch
self._dispatch(tasks)
File "C:\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 566, in _dispatch
job = ImmediateComputeBatch(batch)
File "C:\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 180, in __init__
self.results = batch()
File "C:\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py", line 72, in __call__
return [func(*args, **kwargs) for func, args, kwargs in self.items]
File "C:\Anaconda2\lib\site-packages\sklearn\cross_validation.py", line 1550, in _fit_and_score
test_score = _score(estimator, X_test, y_test, scorer)
File "C:\Anaconda2\lib\site-packages\sklearn\cross_validation.py", line 1606, in _score
score = scorer(estimator, X_test, y_test)
File "C:\Anaconda2\lib\site-packages\sklearn\metrics\scorer.py", line 175, in __call__
y_pred = y_pred[:, 1]
IndexError: index 1 is out of bounds for axis 1 with size 1
This code work fine when instead of using scoring = "roc_auc" I used accuracy metric. Can anyone explain me what's wrong? Have anyone had similiar problem?
There is a bug in the KerasClassifier that is causing this issue. I have opened an issue for it on the repo. https://github.com/fchollet/keras/issues/2864
The fix is also in there. You can define your own KerasClassifier in the mean time as a temporary workaround.
class FixedKerasClassifier(KerasClassifier):
def predict_proba(self, X, **kwargs):
kwargs = self.filter_sk_params(Sequential.predict_proba, kwargs)
probs = self.model.predict_proba(X, **kwargs)
if(probs.shape[1] == 1):
probs = np.hstack([1-probs,probs])
return probs

Resources