Related
Currently, I'm doing research to create object detection model using YOLOv2 from scratch. I have target data with shape (13, 13, 32), where 32 is become from 4 anchor box, 5 prediction value (confidence_score, delta_x, delta_y, delta_w, delta_h) and 3 class probability (my dataset has 3 class).
I'm stuck in a loss function and have built and tried several versions, and still got loss with nan value.
This is the code :
class YOLOLoss(tf.keras.losses.Loss):
def __init__(self, anchor_box, lambda_coord, lambda_noobj, name='yolo_loss'):
super(YOLOLoss, self).__init__(name=name)
self.anchor_box = anchor_box
self.lambda_coord = lambda_coord
self.lambda_noobj = lambda_noobj
cy, cx = tf.meshgrid(tf.range(13), tf.range(13))
self.cell = tf.stack([cx, cy], axis = -1)
self.cell = tf.expand_dims(self.cell, axis = -2)
self.cell = tf.tile(self.cell, [1, 1, 4, 1])
self.cell = tf.cast(self.cell, tf.float32)
def call(self, y_true, y_pred):
y_true = tf.reshape(y_true, (-1, 13, 13, 4, 8))
y_pred = tf.reshape(y_pred, (-1, 13, 13, 4, 8))
true_conf = y_true[..., 0]
pred_conf = y_pred[..., 0]
true_coord = y_true[..., 1:5]
pred_coord = y_pred[..., 1:5]
true_prob = y_true[..., 5:]
pred_prob = y_pred[..., 5:]
objectness = tf.where(true_conf == 1, 1., 0.)
ious = self.iou(true_coord, pred_coord)
coord_loss = tf.reduce_sum(tf.square(pred_coord - true_coord), axis = -1)
coord_loss = self.lambda_coord * tf.reduce_sum(objectness * coord_loss)
object_loss = tf.reduce_sum(objectness * tf.square(tf.math.sigmoid(pred_conf) - ious))
no_object_loss = self.lambda_noobj * tf.reduce_sum((1 - objectness) * tf.square((tf.math.sigmoid(pred_conf) - 0)))
class_loss = tf.reduce_sum(tf.square(tf.nn.softmax(pred_prob) - true_prob), axis = -1)
class_loss = tf.reduce_sum(objectness * class_loss)
print('')
print(tf.math.sigmoid(pred_conf))
print(f'COORDINATE LOSS\t: {coord_loss.numpy()}')
print(f'OBJECT LOSS\t: {object_loss.numpy()}')
print(f'NO OBJECT LOSS\t: {no_object_loss.numpy()}')
print(f'CLASS LOSS\t: {class_loss.numpy()}')
print('')
total_loss = coord_loss + object_loss + no_object_loss + class_loss
return total_loss
def convXY(self, delta_xy):
xy_grid = delta_xy * self.anchor_box + 0.5
xy = 32 * xy_grid + 32 * self.cell
return tf.round(xy)
def convWH(self, delta_wh):
wh_grid = self.anchor_box * tf.math.exp(delta_wh)
wh = wh_grid * 32
return tf.round(wh)
def iou(self, true_coord, pred_coord):
true_delta_xy = true_coord[..., :2]
pred_delta_xy = pred_coord[..., :2]
true_delta_wh = true_coord[..., 2:]
pred_delta_wh = pred_coord[..., 2:]
true_xy = self.convXY(true_delta_xy)
true_wh = self.convWH(true_delta_wh)
pred_xy = self.convXY(pred_delta_xy)
pred_wh = self.convWH(pred_delta_wh)
x1, y1 = true_xy[..., 0], true_xy[..., 1]
w1, h1 = true_wh[..., 0], true_wh[..., 1]
x2, y2 = pred_xy[..., 0], pred_xy[..., 1]
w2, h2 = pred_wh[..., 0], pred_wh[..., 1]
intersection = tf.math.minimum(x1 + w1, x2 + w2) - tf.math.minimum(x1, x2)
intersection *= tf.math.minimum(y1 + h1, y2 + h2) - tf.math.minimum(y1, y2)
area1 = w1 * h1
area2 = w2 * h2
union = area1 + area2 - intersection
iou = intersection / union
return iou
and this is the Darknet-19 architecture that I created, I don't know if this is correct or not :
input_layer = tf.keras.layers.Input((416, 416, 3))
# Convolution 1
x = tf.keras.layers.Conv2D(32, (3, 3), padding = 'same', name = 'conv_1')(input_layer)
x = tf.keras.layers.BatchNormalization(name = 'norm_1')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
#Maxpool 1
x = tf.keras.layers.MaxPooling2D((2, 2), strides = 2)(x)
# Convolution 2
x = tf.keras.layers.Conv2D(64, (3, 3), padding = 'same', name = 'conv_2')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_2')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
#Maxpool 2
x = tf.keras.layers.MaxPooling2D((2, 2), strides = 2)(x)
# Convolution 3
x = tf.keras.layers.Conv2D(128, (3, 3), padding = 'same', name = 'conv_3')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_3')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 4
x = tf.keras.layers.Conv2D(64, (1, 1), padding = 'same', name = 'conv_4')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_4')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 5
x = tf.keras.layers.Conv2D(128, (3, 3), padding = 'same', name = 'conv_5')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_5')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
#Maxpool 3
x = tf.keras.layers.MaxPooling2D((2, 2), strides = 2)(x)
# Convolution 6
x = tf.keras.layers.Conv2D(256, (3, 3), padding = 'same', name = 'conv_6')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_6')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 7
x = tf.keras.layers.Conv2D(128, (1, 1), padding = 'same', name = 'conv_7')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_7')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 8
x = tf.keras.layers.Conv2D(256, (3, 3), padding = 'same', name = 'conv_8')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_8')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
#Maxpool 4
x = tf.keras.layers.MaxPooling2D((2, 2), strides = 2)(x)
# Convolution 9
x = tf.keras.layers.Conv2D(512, (3, 3), padding = 'same', name = 'conv_9')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_9')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 10
x = tf.keras.layers.Conv2D(256, (1, 1), padding = 'same', name = 'conv_10')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_10')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 11
x = tf.keras.layers.Conv2D(512, (3, 3), padding = 'same', name = 'conv_11')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_11')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 12
x = tf.keras.layers.Conv2D(256, (1, 1), padding = 'same', name = 'conv_12')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_12')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 13
x = tf.keras.layers.Conv2D(512, (3, 3), padding = 'same', name = 'conv_13')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_13')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
#Maxpool 5
x = tf.keras.layers.MaxPooling2D((2, 2), strides = 2)(x)
# Convolution 14
x = tf.keras.layers.Conv2D(1024, (3, 3), padding = 'same', name = 'conv_14')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_14')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 15
x = tf.keras.layers.Conv2D(512, (3, 3), padding = 'same', name = 'conv_15')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_15')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 16
x = tf.keras.layers.Conv2D(1024, (3, 3), padding = 'same', name = 'conv_16')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_16')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 17
x = tf.keras.layers.Conv2D(512, (3, 3), padding = 'same', name = 'conv_17')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_17')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 18
x = tf.keras.layers.Conv2D(1024, (3, 3), padding = 'same', name = 'conv_18')(x)
x = tf.keras.layers.BatchNormalization(name = 'norm_18')(x)
x = tf.keras.layers.LeakyReLU(alpha = 0.1)(x)
# Convolution 19
output = tf.keras.layers.Conv2D(32, (1, 1), padding = 'same', activation = 'linear', name = 'conv_19')(x)
for the optimizer I use SGD like this :
optimizer = tf.keras.optimizers.SGD(
learning_rate = 0.001,
momentum = 0.9,
decay = 0.0005
)
model.compile(
optimizer = optimizer,
loss = YOLOLoss(anchor_box = anchor_grid, lambda_coord = 5.0, lambda_noobj = 0.5),
metrics = ['accuracy'],
run_eagerly = True
)
when I try to train a model, I get nan for loss
enter image description here
Please someone tell me what's wrong with my code. I Appreciate it! :)
You can see the full jupyter notebook right here : https://colab.research.google.com/drive/19T3geZakP2Wc6oaEqCJX_3tHwBMt60qG?usp=sharing
I try to train YOLOv2 object detection model, I hope the model have a good accuracy, but when I try to train the model I got the loss with nan value.
I am trying to get the HoltWinters Alpha function added.
I have a Table called Sales1 and the code should refer to this table.
Is there anyone who can correct or amend my code below ,so i get the Holtwinters Alpha instead of the Chronbachs Alpha?
Holt winters calc (need this probably amended)
library(forecast)
library(Metrics
)
read_file(sales1)
x <- sales
x = c(Sales1)
mSES = HoltWinters(x, alpha = 0.5, beta = FALSE, gamma = FALSE)
mHW = HoltWinters(x, alpha = 0.5, beta = FALSE, gamma = FALSE)
mSES$SSE
mHW$SSE
HoltWinters(x, alpha = NULL, beta = NULL, gamma = NULL,
seasonal = c("additive", "multiplicative"),
start.periods = 2, l.start = NULL, b.start = NULL,
s.start = NULL,
optim.start = c(alpha = 0.3, beta = 0.1, gamma = 0.1),
optim.control = list())
chronbachs alpha calc
read_file(sales1)
library(tidyverse)
library(psy)
Number of rows before to take into account
rolling = 2
sales1 <- sales::sales( ~date, ~sales,)
#Lag
sales1 = sales1 %>% mutate(lagsales = lag(sales))
#Rolling Chronbachs Alpha.:( I need the Holtwinter Alpha here )
sales1$alpha = c( rep(NA, rolling),
map_dbl((rolling + 1):nrow(sales1), function(x){
cronbach(sales1 %>% select(sales, lagsales) %>% slice((x-rolling):x))$alpha
})
)
sales1
tibbet from Sales1 table:
df <- tibble::tribble(
~seq, ~date, ~sales,
1, "3/01/2017", 40,
2, "4/01/2017", 2,
3, "5/01/2017", 2,
4, "6/01/2017", 2,
5, "7/01/2017", 30,
6, "8/01/2017", 2,
7, "1/02/2017", 9,
8, "2/02/2017", 5,
9, "3/02/2017", 65,
10, "4/02/2017", 3,
11, "5/02/2017", 65
Not sure whether my question is TF specific or just NNs in general but i have created a CNN using tensorflow. and im having trouble understanding why the size of the output on my fully connected layer is what it is.
X = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.int64, [None])
is_training = tf.placeholder(tf.bool)
# define model
def complex_model(X,y,is_training):
# conv layer
wconv_1 = tf.get_variable('wconv_1', [7 ,7 ,3, 32])
bconv_1 = tf.get_variable('bconv_1', [32])
# affine layer 1
w1 = tf.get_variable('w1', [26*26*32//4, 1024]) #LINE 13
b1 = tf.get_variable('b1', [1024])
# batchnorm params
bn_gamma = tf.get_variable('bn_gamma', shape=[32]) #scale
bn_beta = tf.get_variable('bn_beta', shape=[32] ) #shift
# affine layer 2
w2 = tf.get_variable('w2', [1024, 10])
b2 = tf.get_variable('b2', [10])
c1_out = tf.nn.conv2d(X, wconv_1, strides=[1, 1, 1, 1], padding="VALID") + bconv_1
activ_1 = tf.nn.relu(c1_out)
mean, var = tf.nn.moments(activ_1, axes=[0,1,2], keep_dims=False)
bn = tf.nn.batch_normalization(act_1, mean, var, bn_gamma, bn_beta, 1e-6)
mp = tf.nn.max_pool(bn, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')
affine_in_flat = tf.reshape(mp, [-1, 26*26*32//4])
affine_1 = tf.matmul(affine_in_flat, w1) + b1
activ_2 = tf.nn.relu(affine_1)
affine_2 = tf.matmul(activ_2, w2) + b2
return affine_2
#print(affine_2.shape)
In line 13 where i set the value of w1 i would have expected to just put:
w1 = tf.get_variable('w1', [26*26*32, 1024])
however if i run the code with the line shown above and with
affine_in_flat = tf.reshape(mp, [-1, 26*26*32])
my output size is 16,10 instead of 64,10 which is what i would expect given the initialisations below:
x = np.random.randn(64, 32, 32,3)
with tf.Session() as sess:
with tf.device("/cpu:0"): #"/cpu:0" or "/gpu:0"
tf.global_variables_initializer().run()
#print("train", x.size, is_training, y_out)
ans = sess.run(y_out,feed_dict={X:x,is_training:True})
%timeit sess.run(y_out,feed_dict={X:x,is_training:True})
print(ans.shape)
print(np.array_equal(ans.shape, np.array([64, 10])))
can anybody tell me why i need to divide the size of w1[0] by 4?
Adding print statements for bn and mp I get:
bn: <tf.Tensor 'batchnorm/add_1:0' shape=(?, 26, 26, 32) dtype=float32>
mp: <tf.Tensor 'MaxPool:0' shape=(?, 13, 13, 32) dtype=float32>
Which would seem to be due to the strides=[1, 2, 2, 1] on the max pooling (but to maintain 26, 26 you'd also need padding='SAME').
So I've been implementing the categorical generative adversarial networks which is described in here.
[Jost T. Springenberg. Unsupervised and semi-supervised learning with
categorical generative adversarial networks, April 2016.]
formula
This is the loss function introduced in page 6 and the thing is that the formula uses arg_max which is odd because most of the optimizers I can use on various frameworks such as Tensorflow only work in arg_min.
So would you please guys tell me how to implement this formula?
Here is the code I implemented.
import tensorflow as tf
import numpy as np
import PIL.Image as Image
# constants
X_dim = 256
Y_dim = 2
Z_dim = 256 * 256
value_lambda = 1.0
X = tf.placeholder(tf.float32, shape=[None, X_dim, X_dim, 1])
Y = tf.placeholder(tf.float32, shape=[None, Y_dim])
Z = tf.placeholder(tf.float32, shape=[None, Z_dim])
initializer = tf.contrib.layers.variance_scaling_initializer
activation_function = tf.nn.elu
regularizer = tf.contrib.layers.l2_regularizer(0.5)
custom_filter = np.ones(shape=[32, 256, 256, 1], dtype=np.float)
custom_filter[:, 255, :, :] = 0
custom_filter[:, :, 255, :] = 0
custom_filter = tf.constant(custom_filter, dtype=tf.float32)
def discriminator(x, name=None):
with tf.name_scope(name, "discriminator", [x]) as scope:
D_conv_1 = tf.layers.conv2d(inputs=x, filters=16, kernel_size=[
5, 5], padding='SAME', activation=activation_function, kernel_regularizer=regularizer)
# [256, 256]
D_mean_pool_1 = tf.nn.pool(D_conv_1, window_shape=[
2, 2], pooling_type='AVG', padding='VALID', strides=[2, 2])
# [128, 128]
D_conv_2 = tf.layers.conv2d(D_mean_pool_1, filters=32, kernel_size=[
3, 3], padding='SAME', activation=activation_function, kernel_regularizer=regularizer)
# [128, 128]
D_mean_pool_2 = tf.nn.pool(D_conv_2, window_shape=[
2, 2], pooling_type='AVG', padding='VALID', strides=[2, 2])
# [64, 64]
D_conv_3 = tf.layers.conv2d(D_mean_pool_2, filters=64, kernel_size=[
3, 3], padding='SAME', activation=activation_function, kernel_regularizer=regularizer)
# [64, 64]
D_mean_pool_3 = tf.nn.pool(D_conv_3, window_shape=[
2, 2], pooling_type='AVG', padding='VALID', strides=[2, 2])
# [32, 32]
D_conv_4 = tf.layers.conv2d(D_mean_pool_3, filters=128, kernel_size=[
3, 3], padding='SAME', activation=activation_function, kernel_regularizer=regularizer)
# [32, 32]
D_mean_pool_4 = tf.nn.pool(D_conv_4, window_shape=[
2, 2], pooling_type='AVG', padding='VALID', strides=[2, 2])
# [16, 16]
D_conv_5 = tf.layers.conv2d(D_mean_pool_4, filters=256, kernel_size=[
3, 3], padding='SAME', activation=activation_function, kernel_regularizer=regularizer)
# [16, 16]
D_mean_pool_5 = tf.nn.pool(D_conv_5, window_shape=[
4, 4], pooling_type='AVG', padding='VALID', strides=[4, 4])
# [4, 4]
D_conv_6 = tf.layers.conv2d(D_mean_pool_5, filters=2, kernel_size=[
3, 3], padding='SAME', activation=activation_function, kernel_regularizer=regularizer)
# [4, 4]
D_mean_pool_6 = tf.nn.pool(D_conv_6, window_shape=[
4, 4], pooling_type='AVG', padding='VALID', strides=[4, 4])
# [1, 1], and finally, [batch_size][1][1][2]
D_logit = tf.reshape(D_mean_pool_6, shape=[32, 2])
# [batch_size][2]
return D_logit
'''
D_hidden_layer_1 = tf.layers.dense(
inputs=x, units=255, activation=activation_function)
D_hidden_layer_2 = tf.layers.dense(
inputs=D_hidden_layer_1, units=16, activation=activation_function)
D_logit = tf.layers.dense(inputs=D_hidden_layer_2, units=Y_dim,
activation=activation_function)
return D_logit
'''
def generator(z, name=None):
with tf.name_scope(name, "generator", [z]) as scope:
# z[32, 4096]
input = tf.reshape(z, shape=[32, 256, 256, 1])
# input[32, 64, 64, 1]
G_conv_1 = tf.layers.conv2d(input, filters=96, kernel_size=[
8, 8], padding='SAME', activation=activation_function)
# [32, 64, 64, 96]
# G_upscaled_1 = tf.image.resize_bicubic(images=G_conv_1, size=[128, 128])
# [32, 128, 128, 96]
G_conv_2 = tf.layers.conv2d(G_conv_1, filters=64, kernel_size=[
5, 5], padding='SAME', activation=activation_function)
# [32, 128, 128, 64]
# G_upscaled_2 = tf.image.resize_bicubic(G_conv_2, size=[256, 256])
# [32, 256, 256, 64]
G_conv_3 = tf.layers.conv2d(G_conv_2, filters=64, kernel_size=[
5, 5], padding='SAME', activation=activation_function)
# [32, 256, 256, 64]
G_conv_4 = tf.layers.conv2d(G_conv_3, filters=1, kernel_size=[
5, 5], padding='SAME', activation=activation_function)
# [32, 256, 256, 1]
G_logit = G_conv_4 * custom_filter
# [32, 256, 256, 1], but filtered out the last column and row
return G_logit
'''
G_hidden_layer_1 = tf.layers.dense(
inputs=z, units=255, activation=activation_function)
G_outputs = tf.layers.dense(inputs=G_hidden_layer_1, units=X_dim,
activation=activation_function)
return G_outputs
'''
with tf.name_scope("training") as scope:
# Getting samples from random data
G_sample = generator(Z)
# Getting logits
D_logit_real = discriminator(X)
D_logit_fake = discriminator(G_sample)
# Applying softmax
D_proba_real = tf.nn.softmax(logits=D_logit_real)
D_proba_real = tf.clip_by_value(
D_proba_real, clip_value_min=1e-4, clip_value_max=1.0)
D_proba_fake = tf.nn.softmax(logits=D_logit_fake)
D_proba_fake = tf.clip_by_value(
D_proba_fake, clip_value_min=1e-4, clip_value_max=1.0)
with tf.name_scope("category_1") as sub_scope:
# Getting Shannon's entrophy in X's distribution
D_log_real = tf.log(D_proba_real)
D_entrophy_real = D_proba_real * D_log_real
D_mean_real = tf.reduce_sum(D_entrophy_real, axis=1)
D_mean_real = -D_mean_real
D_entrophy_real_mean = tf.reduce_mean(D_mean_real, axis=0)
D_entrophy_real_mean = tf.reshape(D_entrophy_real_mean, shape=[1])
with tf.name_scope("category_2") as sub_scope:
# Gettning Shannon's entrophy in Z's distribution
G_log_fake = tf.log(D_proba_fake)
G_entrophy_fake = D_proba_fake * G_log_fake
G_mean = tf.reduce_sum(G_entrophy_fake, axis=1)
G_mean = -G_mean
G_entrophy_fake_mean = tf.reduce_mean(G_mean, axis=0)
G_entrophy_fake_mean = tf.reshape(G_entrophy_fake_mean, shape=[1])
with tf.name_scope("category_3") as sub_scope:
# Getting Shannon's entrophy between classes
D_class_mean = tf.reduce_mean(D_proba_real, axis=0, keep_dims=True)
D_class_mean_log = tf.log(D_class_mean)
D_class_entropy = D_class_mean * D_class_mean_log
D_class = tf.reduce_sum(D_class_entropy, axis=1)
D_class = -D_class
D_class = tf.reshape(D_class, shape=[1])
G_class_mean = tf.reduce_mean(D_proba_fake, axis=0, keep_dims=True)
G_class_mean_log = tf.log(G_class_mean)
G_class_entrophy = G_class_mean * G_class_mean_log
G_class = tf.reduce_sum(G_class_entrophy, axis=1)
G_class = -G_class
G_class = tf.reshape(G_class, shape=[1])
with tf.name_scope("supervised") as sub_scope:
# Getting cross entrophy for labeled data
D_labeled = Y * D_log_real
D_cross_entrophy = tf.reduce_sum(D_labeled, axis=1)
D_cross_entrophy = -D_cross_entrophy
D_supervised = tf.reduce_mean(D_cross_entrophy, axis=0)
D_supervised_weighted = value_lambda * D_supervised
D_supervised_weighted = tf.reshape(D_supervised_weighted, shape=[1])
D_loss = D_class - D_entrophy_real_mean + \
G_entrophy_fake_mean + D_supervised_weighted
G_loss = -G_class + G_entrophy_fake_mean
D_loss = -D_loss
D_solver = tf.train.AdamOptimizer().minimize(D_loss)
G_solver = tf.train.AdamOptimizer().minimize(G_loss)
# with tf.name_scope("testing") as scope:
I've done some research and asked some questions to my friends who works in a big company doing research on deep learning. As it turns out that the generative adversarial networks is not good at classification jobs. So I changed my mind and implemented it with GoogLenet. Problem solved!
Why is my convolutional autoencoder not converging properly? I have a very simple layer stack.
Encoder: Conv/ReLU(Kernel size: 7x7, stride = 1, padding = 0) => maxPool(kernel size=2x2, stride = 2) => Conv/ReLU(Kernel size: 5x5, stride = 1, padding = 0) => MaxPool(kernel size=2x2, stride = 2)
Decoder: Nearest Neighbour Upsampling => Deconv/ReLU => Nearest Neighbour Upsampling => Deconv/ReLU
Training Images are of size 30x30x1.
I tried to train it with 1000 images over 1000 epoch, but the error (MSE) is still 120.
BATCH_SIZE = 100
IMAGE_SIZE = 30
NUM_CHANNELS = 1
num_images = 1000
def init_weights(shape):
return tf.Variable(tf.random_normal(shape, stddev=0.01))
def encoder(X, w, w2, wd, wd2):
l1a = tf.nn.relu(tf.nn.conv2d(X, w,strides=[1, 1, 1, 1], padding='VALID'))
l1 = tf.nn.max_pool(l1a, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
l2a = tf.nn.relu(tf.nn.conv2d(l1, w2,strides=[1, 1, 1, 1], padding='VALID'))
l2 = tf.nn.max_pool(l2a, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
l1da = tf.image.resize_images(l2, 8, 8, 1, align_corners=False)
output_shapel1d = tf.convert_to_tensor([BATCH_SIZE, 12, 12, 32], dtype=tf.int32);
l1d = tf.nn.relu(tf.nn.conv2d_transpose(l1da, wd, output_shapel1d, strides=[1, 1, 1, 1], padding='VALID'))
l2da = tf.image.resize_images(l1d, 24, 24, 1, align_corners=False)
output_shapel2d = tf.convert_to_tensor([BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS], dtype=tf.int32);
l2d = tf.nn.relu(tf.nn.conv2d_transpose(l2da, wd2, output_shapel2d, strides=[1, 1, 1, 1], padding='VALID'))
return l2d
complete_image = extract_data(0, 1000)
trX = complete_image[0:900]
trY = trX
teX = complete_image[900:1000]
teY = teX
X = tf.placeholder("float", [BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS])
Y = tf.placeholder("float", [BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, NUM_CHANNELS])
w = init_weights([7, 7, 1, 32])
w2 = init_weights([5, 5, 32, 64])
wd = init_weights([5, 5, 32, 64])
wd2 = init_weights([7, 7, 1, 32])
py_x = encoder(X, w, w2, wd, wd2)
cost = tf.reduce_mean(tf.squared_difference(py_x, Y, name = None))
train_op = tf.train.RMSPropOptimizer(0.001, 0.9).minimize(cost)
predict_op = py_x;
global_step = tf.Variable(0, name='global_step', trainable=False)
saver = tf.train.Saver()
with tf.Session() as sess:
tf.initialize_all_variables().run()
start = global_step.eval() # get last global_step
print "Start from:", start
if FLAGS.output == "train":
for i in range(start, 500):
training_batch = zip(range(0, num_images - BATCH_SIZE, batch_size),
range(batch_size, num_images - BATCH_SIZE, batch_size))
for start, end in training_batch:
sess.run(train_op, feed_dict={X: trX[start:end], Y: trY[start:end]})
total_epoch_cost += sess.run(cost, feed_dict={X: trX[start:end], Y: trY[start:end]})
avg_epoch_cost = total_epoch_cost/BATCH_SIZE
print "cost during epoch " + `i` + "is ", avg_epoch_cost
I have added the complete code in this gist with slight modifications. I am training this with around 10,000 images, and the error after 488 epochs is 74.8.