I am following the D2L book and one of the exercises on the GoogLeNet chapter is to take the model from the Rethinking the Inception Architecture for Computer Vision paper so I did that and basically recreated the Inception V2 architecture based on the Inception V1 architecture presented in the D2L book, the only thing I didn't include was the grid reduction described in the paper. I expected this to improve results on my models but if anything it made them worse. I was testing the models using the D2L code presented in the GoogLeNet chapter on the fashion_mnist dataset and got a test accuracy of 0.908 using Inception V1 however on my V2 implementation from the paper I only got test acc of 0.885 and that was the highest I could get. I expected the V2 model to be better so I don't really know where I went wrong, if I made an error in my code or what but any insights into the V2 architecture or from the paper would be greatly appreciated.
This is the code for the first 3 inception blocks (from figure 5 in the paper)
class InceptionB1(nn.Module):
# `c1`--`c4` are the number of output channels for each path
def __init__(self, in_channels, c1, c2, c3, c4, **kwargs):
super(InceptionB1, self).__init__(**kwargs)
# Path 1 is a single 1 x 1 convolutional layer
self.p1_1 = nn.Conv2d(in_channels, c1, kernel_size=1)
# Path 2 is a 1 x 1 convolutional layer followed by a 3 x 3
# convolutional layer
self.p2_1 = nn.Conv2d(in_channels, c2[0], kernel_size=1)
self.p2_2 = nn.Conv2d(c2[0], c2[1], kernel_size=3, padding=1)
# Path 3 is a 1 x 1 convolutional layer followed by 2 3x3 convs (factorization)
# convolutional layer
self.p3_1 = nn.Conv2d(in_channels, c3[0], kernel_size=1)
self.p3_2 = nn.Conv2d(c3[0], c3[1], kernel_size=3, padding=1)
# Replacing path3_2 with modified path using factorization through smaller convolutions
self.p3_3 = nn.Conv2d(c3[1], c3[1], kernel_size=3, padding = 1)
# Path 4 is a 3 x 3 maximum pooling layer followed by a 1 x 1
# convolutional layer
self.p4_1 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
self.p4_2 = nn.Conv2d(in_channels, c4, kernel_size=1)
def forward(self, x):
p1 = F.relu(self.p1_1(x))
p2 = F.relu(self.p2_2(F.relu(self.p2_1(x))))
p3 = F.relu(self.p3_3(self.p3_2(F.relu(self.p3_1(x)))))
p4 = F.relu(self.p4_2(self.p4_1(x)))
# Concatenate the outputs on the channel dimension
return torch.cat((p1, p2, p3, p4), dim=1)
This is my code for the middle 5 blocks (figure 6 in the paper)
class InceptionB2(nn.Module):
# `c1`--`c4` are the number of output channels for each path
def __init__(self, in_channels, c1, c2, c3, c4, **kwargs):
super(InceptionB2, self).__init__(**kwargs)
# Path 1 is a single 1 x 1 convolutional layer
self.p1_1 = nn.Conv2d(in_channels, c1, kernel_size=1)
# Path 2 is a 1 x 1 convolutional layer followed by a 1 x 3 then 3x1 conv
# convolutional layer
self.p2_1 = nn.Conv2d(in_channels, c2[0], kernel_size=1)
self.p2_2 = nn.Conv2d(c2[0], c2[1], kernel_size=(1,3), padding=1)
self.p2_3 = nn.Conv2d(c2[1], c2[1], kernel_size=(3,1), padding=0)
# Path 3 is a 1 x 1 conv layer followed by a 1x3 conv then 3x1 then 1x3 then 3x1
# convolutional layer
self.p3_1 = nn.Conv2d(in_channels, c3[0], kernel_size=1)
self.p3_2 = nn.Conv2d(c3[0], c3[1], kernel_size=(1,3), padding=1)
self.p3_3 = nn.Conv2d(c3[1], c3[1], kernel_size=(3,1), padding=0)
self.p3_4 = nn.Conv2d(c3[1], c3[1], kernel_size=(1,3), padding=0)
self.p3_5 = nn.Conv2d(c3[1], c3[1], kernel_size=(3,1), padding=1)
# Path 4 is a 3 x 3 maximum pooling layer followed by a 1 x 1
# convolutional layer
self.p4_1 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
self.p4_2 = nn.Conv2d(in_channels, c4, kernel_size=1)
def forward(self, x):
p1 = F.relu(self.p1_1(x))
p2 = F.relu(self.p2_3(F.relu(self.p2_2(F.relu(self.p2_1(x))))))
p3 = F.relu(self.p3_5(F.relu(self.p3_4(F.relu(self.p3_3(F.relu(self.p3_2(F.relu(self.p3_1(x))))))))))
p4 = F.relu(self.p4_2(self.p4_1(x)))
# Concatenate the outputs on the channel dimension
return torch.cat((p1, p2, p3, p4), dim=1)
This is my code for the last 2 blocks (figure 7 in the paper)
class InceptionB3(nn.Module):
# `c1`--`c4` are the number of output channels for each path
def __init__(self, in_channels, c1, c2, c3, c4, **kwargs):
super(InceptionB3, self).__init__(**kwargs)
# Path 1 is a single 1 x 1 convolutional layer
self.p1_1 = nn.Conv2d(in_channels, c1, kernel_size=1)
# Path 2 is a 1 x 1 convolutional layer followed by a 1x3 and 3x1
# convolutional layer
self.p2_1 = nn.Conv2d(in_channels, c2[0], kernel_size=1)
self.p2_2_1 = nn.Conv2d(c2[0], c2[1], kernel_size=(1,3), padding=0)
self.p2_2_2 = nn.Conv2d(c2[0], c2[1], kernel_size=(3,1), padding=0)
# Path 3 is a 1 x 1 convolutional layer followed by a 3x3 then 1x3 and 3x1
# convolutional layer
self.p3_1 = nn.Conv2d(in_channels, c3[0], kernel_size=1)
self.p3_2 = nn.Conv2d(c3[0], c3[1], kernel_size=3, padding=1)
self.p3_3_1 = nn.Conv2d(c3[1], c3[1], kernel_size=(1,3), padding=0)
self.p3_3_2 = nn.Conv2d(c3[1], c3[1], kernel_size=(3,1), padding=0)
# Path 4 is a 3 x 3 maximum pooling layer followed by a 1 x 1
# convolutional layer
self.p4_1 = nn.MaxPool2d(kernel_size=3, stride=1, padding=1)
self.p4_2 = nn.Conv2d(in_channels, c4, kernel_size=1)
def forward(self, x):
p1 = F.relu(self.p1_1(x))
p2_1 = F.relu(self.p2_2_1(F.relu(self.p2_1(x))))
p2_2 = F.relu(elf.p2_2_2(F.relu(self.p2_1(x))))
p2 = torch.matmul(p2_1, p2_2)
p3_1 = F.relu(self.p3_2(F.relu(self.p3_1(x))))
p3_2_1 = F.relu(self.p3_3_1(p3_1))
p3_2_2 = F.relu(sself.p3_3_2(p3_1))
p3 = torch.matmul(p3_2_1, p3_2_2)
p4 = F.relu(self.p4_2(self.p4_1(x)))
# Concatenate the outputs on the channel dimension
return torch.cat((p1, p2, p3, p4), dim=1)
Here is the link to the D2L chapter
https://d2l.ai/chapter_convolutional-modern/googlenet.html
This is the training graph for the v1 model
This is the training graph for the v2 model
Related
I am making a simple CNN without a pooling layer and optimizer (not allowed since its a college assignment). I am using the SVHN dataset. I am using crossentropy loss but loss keeps jumping between 2.305 and 2.306 in 10 epochs. Kindly help.
class Net(nn.Module):
def __init__(self):
super(Net,self).__init__()
self.conv1 = nn.Conv2d(3, 32,kernel_size=3, padding=1)
# self.pool = nn.MaxPool2d(2, 2)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3, padding=1)
self.fc1 = nn.Sequential(
nn.Flatten(),
nn.Linear(64*32*32, 10)
)
def forward(self, x):
x = (F.relu(self.conv1(x)))
x = (F.relu(self.conv2(x)))
x = self.fc1(x)
return x
net = Net()
criterion = nn.CrossEntropyLoss()
for epoch in range(10): # loop over the dataset multiple times
net.train()
batch_loss_val=0
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
# forward + backward
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
# optimizer.step()
# print statistics
running_loss += loss.item()
if i % 50 == 0: # print every 50 mini-batches
print(f'[{epoch + 1}, {i + 1:5d}] loss: {running_loss / 50 :.3f}')
running_loss = 0.0
print('Finished Training')
You need to enable optimizer.step() to update the weights. loss.backward() is only computing the gradients.
At first I want to tell, that I am a newbie to Machine Learning. So I apologize if there is any mistake in my code.
I am training a DANN(Domain Adversarial Neural Network) on the CIFAR10 and CIFAR100 datasets and I am getting a error code
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-25-f44419d6a132> in <module>
27 D.zero_grad()
28
---> 29 Ltot.backward()
30
31 C_opt.step()
1 frames
/usr/local/lib/python3.8/dist-packages/torch/_tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
486 inputs=inputs,
487 )
--> 488 torch.autograd.backward(
489 self, gradient, retain_graph, create_graph, inputs=inputs
490 )
/usr/local/lib/python3.8/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
195 # some Python versions print out the first line of a multi-line function
196 # calls in the traceback and some print out the last line
--> 197 Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
198 tensors, grad_tensors_, retain_graph, create_graph, inputs,
199 allow_unreachable=True, accumulate_grad=True) # Calls into the C++ engine to run the backward pass
RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED when calling `cublasSgemm( handle, opa, opb, m, n, k, &alpha, a, lda, b, ldb, &beta, c, ldc)
Couldn't figure out the problem in the code. Kindly attaching the whole code for your reference.
class FeatureExtractor(nn.Module):
"""
Feature Extractor
"""
def __init__(self, in_channel=3, hidden_dims=512):
super(FeatureExtractor, self).__init__()
self.conv = nn.Sequential(
nn.Conv2d(in_channel, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2), # Add max pooling layer
nn.Conv2d(64, 128, 3, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2), # Add max pooling layer
nn.Conv2d(128, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2), # Add max pooling layer
nn.Conv2d(256, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True),
nn.MaxPool2d(2,2), # Add max pooling layer
nn.Conv2d(256, hidden_dims, 3, padding=1),
nn.BatchNorm2d(hidden_dims),
nn.ReLU(inplace=True),
nn.AdaptiveAvgPool2d((1,1)),
)
def forward(self, x):
h = self.conv(x).squeeze() # (N, hidden_dims)
return h
class Classifier(nn.Module):
"""
Classifier
"""
def __init__(self, input_size=512, num_classes=10):
super(Classifier, self).__init__()
self.layer = nn.Sequential(
nn.Linear(input_size, 256),
nn.ReLU(inplace=True),
nn.Dropout(0.5),
nn.Linear(256, num_classes),
)
def forward(self, h):
c = self.layer(h)
return c
class Discriminator(nn.Module):
"""
Simple Discriminator w/ MLP
"""
def __init__(self, input_size=512, num_classes=1):
super(Discriminator, self).__init__()
self.layer = nn.Sequential(
nn.Linear(input_size, 256),
nn.LeakyReLU(0.2),
nn.Dropout(0.5),
nn.Linear(256, 128),
nn.LeakyReLU(0.2),
nn.Linear(128, num_classes),
nn.Sigmoid(),
)
def forward(self, h):
y = self.layer(h)
return y
F = FeatureExtractor().to(DEVICE)
C = Classifier().to(DEVICE)
D = Discriminator().to(DEVICE)
transform = transforms.Compose([
transforms.Resize(28),
transforms.CenterCrop(28),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5],
std=[0.5])
])
mnist_train = datasets.CIFAR10(root='../data/', train=True, transform=transform, download=True)
mnist_test = datasets.CIFAR10(root='../data/', train=False, transform=transform, download=True)
svhn_train = datasets.CIFAR100(root='../data/', train=True,transform=transform, download=True)
svhn_test = datasets.CIFAR100(root='../data/', train=False,transform=transform, download=True)
batch_size = 64
svhn_train.data.shape
svhn_loader = DataLoader(dataset=svhn_train, batch_size=batch_size, shuffle=True, drop_last=True,num_workers=3, pin_memory=True)
mnist_loader = DataLoader(dataset=mnist_train, batch_size=batch_size, shuffle=True, drop_last=True, num_workers=3, pin_memory=True )
eval_loader = DataLoader(dataset=svhn_test, batch_size=batch_size, shuffle=False, drop_last=False,num_workers=3, pin_memory=True)
test_loader = DataLoader(dataset=mnist_test, batch_size=batch_size, shuffle=False, drop_last=False,num_workers=3, pin_memory=True)
bce = nn.BCELoss()
xe = nn.CrossEntropyLoss()
F_opt = torch.optim.Adam(F.parameters())
C_opt = torch.optim.Adam(C.parameters())
D_opt = torch.optim.Adam(D.parameters())
max_epoch = 50
step = 0
n_critic = 1 # for training more k steps about Discriminator
n_batches = len(mnist_train)//batch_size
# lamda = 0.01
D_src = torch.ones(batch_size, 1).to(DEVICE) # Discriminator Label to real
D_tgt = torch.zeros(batch_size, 1).to(DEVICE) # Discriminator Label to fake
D_labels = torch.cat([D_src, D_tgt], dim=0)
"""### Training Code
"""
def get_lambda(epoch, max_epoch):
p = epoch / max_epoch
return 2. / (1+np.exp(-10.*p)) - 1.
mnist_set = iter(mnist_loader)
def sample_mnist(step, n_batches):
global mnist_set
if step % n_batches == 0:
mnist_set = iter(mnist_loader)
return next(mnist_set)
ll_c, ll_d = [], []
acc_lst = []
torch.cuda.empty_cache()
for epoch in range(1, max_epoch+1):
for idx, (src_images, labels) in enumerate(svhn_loader):
tgt_images, _ = sample_mnist(step, n_batches)
# Training Discriminator
src, labels, tgt = src_images.to(DEVICE), labels.to(DEVICE), tgt_images.to(DEVICE)
x = torch.cat([src, tgt], dim=0)
h = F(x)
y = D(h.detach())
Ld = bce(y, D_labels)
D.zero_grad()
Ld.backward()
D_opt.step()
c = C(h[:batch_size])
y = D(h)
Lc = xe(c, labels)
Ld = bce(y, D_labels)
lamda = 0.1*get_lambda(epoch, max_epoch)
Ltot = Lc -lamda*Ld
F.zero_grad()
C.zero_grad()
D.zero_grad()
Ltot.backward()
C_opt.step()
F_opt.step()
if step % 100 == 0:
dt = datetime.datetime.now().strftime('%H:%M:%S')
print('Epoch: {}/{}, Step: {}, D Loss: {:.4f}, C Loss: {:.4f}, lambda: {:.4f} ---- {}'.format(epoch, max_epoch, step, Ld.item(), Lc.item(), lamda, dt))
ll_c.append(Lc.detach().cpu().numpy())
ll_d.append(Ld.detach().cpu().numpy())
if step % 500 == 0:
F.eval()
C.eval()
with torch.no_grad():
corrects = torch.zeros(1).to(DEVICE)
for idx, (src, labels) in enumerate(eval_loader):
src, labels = src.to(DEVICE), labels.to(DEVICE)
c = C(F(src))
_, preds = torch.max(c, 1)
corrects += (preds == labels).sum()
acc = corrects.item() / len(eval_loader.dataset)
print('***** Eval Result: {:.4f}, Step: {}'.format(acc, step))
corrects = torch.zeros(1).to(DEVICE)
for idx, (tgt, labels) in enumerate(test_loader):
tgt, labels = tgt.to(DEVICE), labels.to(DEVICE)
c = C(F(tgt))
_, preds = torch.max(c, 1)
corrects += (preds == labels).sum()
acc = corrects.item() / len(test_loader.dataset)
print('***** Test Result: {:.4f}, Step: {}'.format(acc, step))
acc_lst.append(acc)
F.train()
C.train()
step += 1
My vae class looks like this:
class Encoder(nn.Module):
def __init__(self):
super(Encoder, self).__init__()
c = capacity
self.conv1 = nn.Conv2d(in_channels=1, out_channels=c, kernel_size=4, stride=2, padding=1) # out: c x 14 x 14
self.conv2 = nn.Conv2d(in_channels=c, out_channels=c*2, kernel_size=4, stride=2, padding=1) # out: c x 7 x 7
self.fc_mu = nn.Linear(in_features=c*2*7*7, out_features=latent_dims)
self.fc_logvar = nn.Linear(in_features=c*2*7*7, out_features=latent_dims)
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.relu(self.conv2(x))
x = x.view(x.size(0), -1) # flatten batch of multi-channel feature maps to a batch of feature vectors
x_mu = self.fc_mu(x)
x_logvar = self.fc_logvar(x)
return x_mu, x_logvar
class Decoder(nn.Module):
def __init__(self):
super(Decoder, self).__init__()
c = capacity
self.fc = nn.Linear(in_features=latent_dims, out_features=c*2*7*7)
self.conv2 = nn.ConvTranspose2d(in_channels=c*2, out_channels=c, kernel_size=4, stride=2, padding=1)
self.conv1 = nn.ConvTranspose2d(in_channels=c, out_channels=1, kernel_size=4, stride=2, padding=1)
def forward(self, x):
x = self.fc(x)
x = x.view(x.size(0), capacity*2, 7, 7) # unflatten batch of feature vectors to a batch of multi-channel feature maps
x = F.relu(self.conv2(x))
x = torch.sigmoid(self.conv1(x)) # last layer before output is sigmoid, since we are using BCE as reconstruction loss
return x
class VariationalAutoencoder(nn.Module):
def __init__(self):
super(VariationalAutoencoder, self).__init__()
self.encoder = Encoder()
self.decoder = Decoder()
def forward(self, x):
latent_mu, latent_logvar = self.encoder(x)
latent = self.latent_sample(latent_mu, latent_logvar)
x_recon = self.decoder(latent)
return x_recon, latent_mu, latent_logvar
def latent_sample(self, mu, logvar):
if self.training:
# the reparameterization trick
std = logvar.mul(0.5).exp_()
eps = torch.empty_like(std).normal_()
return eps.mul(std).add_(mu)
else:
return mu
def vae_loss(recon_x, x, mu, logvar):
# recon_x is the probability of a multivariate Bernoulli distribution p.
# -log(p(x)) is then the pixel-wise binary cross-entropy.
# Averaging or not averaging the binary cross-entropy over all pixels here
# is a subtle detail with big effect on training, since it changes the weight
# we need to pick for the other loss term by several orders of magnitude.
# Not averaging is the direct implementation of the negative log likelihood,
# but averaging makes the weight of the other loss term independent of the image resolution.
recon_loss = F.binary_cross_entropy(recon_x.view(-1, 784), x.view(-1, 784), reduction='sum')
kldivergence = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
return recon_loss + variational_beta * kldivergence
I train it on MNIST dataset.
I want to sample it, or generate an array and give it to the decoder and see what the output will be.
The problem is that I don't really understand, what my z array should look like and what shape should it need.
Here is the code for sampling:
z = ...
input = torch.FloatTensor(z).to(device)
vae.eval()
output = vae.decoder(input)
plot_gallery(output.data.cpu().numpy(), 24, 24, n_row=5, n_col=5)
I have a dataset containing 1000 examples where each example has 5 features (a,b,c,d,e). I want to feed 7 examples to an LSTM so it predicts the feature (a) of the 8th day.
Reading Pytorchs documentation of nn.LSTM() I came up with the following:
input_size = 5
hidden_size = 10
num_layers = 1
output_size = 1
lstm = nn.LSTM(input_size, hidden_size, num_layers)
fc = nn.Linear(hidden_size, output_size)
out, hidden = lstm(X) # Where X's shape is ([7,1,5])
output = fc(out[-1])
output # output's shape is ([7,1])
According to the docs:
The input of the nn.LSTM is "input of shape (seq_len, batch, input_size)" with "input_size – The number of expected features in the input x",
And the output is: "output of shape (seq_len, batch, num_directions * hidden_size): tensor containing the output features (h_t) from the last layer of the LSTM, for each t."
In this case, I thought seq_len would be the sequence of 7 examples, batchis 1 and input_size is 5. So the lstm would consume each example containing 5 features refeeding the hidden layer every iteration.
What am I missing?
When I extend your code to a full example -- I also added some comments to may help -- I get the following:
import torch
import torch.nn as nn
input_size = 5
hidden_size = 10
num_layers = 1
output_size = 1
lstm = nn.LSTM(input_size, hidden_size, num_layers)
fc = nn.Linear(hidden_size, output_size)
X = [
[[1,2,3,4,5]],
[[1,2,3,4,5]],
[[1,2,3,4,5]],
[[1,2,3,4,5]],
[[1,2,3,4,5]],
[[1,2,3,4,5]],
[[1,2,3,4,5]],
]
X = torch.tensor(X, dtype=torch.float32)
print(X.shape) # (seq_len, batch_size, input_size) = (7, 1, 5)
out, hidden = lstm(X) # Where X's shape is ([7,1,5])
print(out.shape) # (seq_len, batch_size, hidden_size) = (7, 1, 10)
out = out[-1] # Get output of last step
print(out.shape) # (batch, hidden_size) = (1, 10)
out = fc(out) # Push through linear layer
print(out.shape) # (batch_size, output_size) = (1, 1)
This makes sense to me, given your batch_size = 1 and output_size = 1 (I assume, you're doing regression). I don't know where your output.shape = (7, 1) come from.
Are you sure that your X has the correct dimensions? Did you create nn.LSTM maybe with batch_first=True? There are lot of little things that can sneak in.
My conv-layer has the output shape of (64,3,3,80) where 64 is the batch size. The next layer is a dense layer of shape (3920,4096). How do I reshape the output of my conv-layer to fit with the shape of my dense layer? I am implementing in tensorflow :)
This is the layer just before the dense layer.
stride_conv = [1,1,1,1]
padding='SAME'
filter_3 = tf.Variable(initial_value=tf.random_normal([3,3,112,80]))
conv_3 = tf.nn.conv2d(conv_2,filter_3,stride_conv,padding)
Thanks!
conv3 => Reshape => FC1 (720->4096)
[64,3,3,80] => [64,720] => [64,4096]
Following code does the Conv to FC as shown above:
shape = int(np.prod(conv_3.get_shape()[1:]))
conv_3_flat = tf.reshape(conv_3, [-1, shape])
fc1w = tf.Variable(tf.truncated_normal([shape, 4096],dtype=tf.float32,stddev=1e-1), name='weights')
fc1b = tf.Variable(tf.constant(1.0, shape=[4096], dtype=tf.float32),
trainable=True, name='biases')
fc1 = tf.nn.bias_add(tf.matmul(conv_3_flat, fc1w), fc1b)
fc1 = tf.nn.relu(fc1)
Hope this helps.
Also, simple MNIST model (taken from here: https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/convolutional_network.py)
def conv_net(x, weights, biases, dropout):
# Reshape input picture
x = tf.reshape(x, shape=[-1, 28, 28, 1])
# Convolution Layer
conv1 = conv2d(x, weights['wc1'], biases['bc1'])
# Max Pooling (down-sampling)
conv1 = maxpool2d(conv1, k=2)
# Convolution Layer
conv2 = conv2d(conv1, weights['wc2'], biases['bc2'])
# Max Pooling (down-sampling)
conv2 = maxpool2d(conv2, k=2)
# Fully connected layer
# Reshape conv2 output to fit fully connected layer input
fc1 = tf.reshape(conv2, [-1, weights['wd1'].get_shape().as_list()[0]])
fc1 = tf.add(tf.matmul(fc1, weights['wd1']), biases['bd1'])
fc1 = tf.nn.relu(fc1)
# Apply Dropout
fc1 = tf.nn.dropout(fc1, dropout)
# Output, class prediction
out = tf.add(tf.matmul(fc1, weights['out']), biases['out'])
return out