So I have to implement a model from a paper, and after explaining each layer it says:
Layer normalization was used for both the input samples and for all
convolutional layers.
My implementation is as follows:
self.sinc_convolution = sinc_conv(80, 251, 16000)
self.convolution2 = nn.Conv1d(80, 60, 5)
self.convolution3 = nn.Conv1d(60, 60, 5)
self.lr = nn.LeakyReLU(0.1)
self.fc1 = nn.Linear(60, constants.embedding_size * 2)
self.fc2 = nn.Linear(constants.embedding_size * 2, constants.embedding_size)
self.input_norm = nn.LayerNorm((1, 3200))
self.layer_norm1 = nn.LayerNorm((80, 2950))
self.layer_norm2 = nn.LayerNorm((60, 2946))
self.layer_norm3 = nn.LayerNorm((60, 2942))
But I am wondering if that's alright. How can I implement it for all layers?
Related
I'm creating anti-phishing algorithm and I am facing problem of final labeling.
The possible labels are -1, 0, 1.
My neural network build is:
def create_model(self, n_hidden = 1, n_neurons = 30, learning_rate = 3e-3, input_shape = 30):
model = Sequential()
model.add(InputLayer(input_shape = input_shape))
for layer in range(n_hidden):
model.add(Dense(n_neurons, activation = "relu"))
model.add(Dense(n_neurons, activation="softmax"))
model.add(Dense(1))
optimizer = SGD(learning_rate = learning_rate)
model.compile(loss="mse", optimizer = optimizer, metrics=BinaryAccuracy())
return model
The results I am getting
looks like this:
[ 0.9413019 0.23881185 0.34624332 ... 1.0277238 -1.0253198
-1.0065029 ]
How can I make my network round the result to one of mentioned labels?
I have the following model:
class Model(nn.Module):
def __init__(self, dim_in, lambda_=.3):
super(FeatExtractorGR, self).__init__()
'''The linear transform layer'''
self.lt = nn.Linear(in_features = dim_in, out_features = 10, bias=True)
'''The encoder'''
self.feature_extractor = \
nn.Sequential(
#nn.Linear(in_features = 10, out_features = 30, bias=True),
nn.BatchNorm1d(10),
nn.ReLU(),
nn.Linear(in_features = 10, out_features = 20, bias=True),
nn.BatchNorm1d(20),
nn.ReLU(),
nn.Linear(in_features = 20, out_features = 10, bias=True),
nn.BatchNorm1d(10),
nn.ReLU()
)
def forward(self, x):
transformed = self.lt(x)
return self.feature_extractor(transformed)
I want to force the weight vectors of the linear transformation layer to be uncorrelated. I tried to include the dot products among the vectors in the cost function:
params=list(model.lt.parameters())[0]
dotprod=torch.tensordot(params, params, dims=([1],[1])).abs().fill_diagonal_(0).sum()/2
loss = other_losses + dotprod * weight
But this is not working, even with really high weight. The weight vectors from the lt layer are still highly correlated. I have also tried to remove other_losses, but no effect. What am I doing wrong?
So I trained by python/pytorch DC-GAN (deep convolutional GAN) for 30 epochs on grayscale faces, and my GAN pretty much failed. I added batch normalization and leaky relu's to my generator and discriminator (I heard those are ways to make the GAN converge), and the Adam optimizer. My GAN still only putting out random grayscale pixels (nothing even remotely related to faces.) I have no problem with the discriminator, my discriminator works very well. I then implemented weight decay of 0.01 on my discriminator to make my GAN train better (since my discriminator was doing better than my generator) but to no avail. Finally, I tried training the GAN for more epochs, 60 epochs. My GAN still generates just random pixels, sometimes outputting completely black.
The GAN training method I used worked for the MNIST dataset (but I used a way simpler GAN architecture for that.)
import torch.nn as nn
import torch.nn.functional as F
class Discriminator(nn.Module):
def __init__(self):
super().__init__()
self.conv1 = nn.Conv2d(1, 4, 3)
self.conv2 = nn.Conv2d(4, 8, 3)
self.bnorm1 = nn.BatchNorm2d(8)
self.conv3 = nn.Conv2d(8, 16, 3)
self.conv4 = nn.Conv2d(16, 32, 3)
self.bnorm2 = nn.BatchNorm2d(32)
self.conv5 = nn.Conv2d(32, 4, 3)
self.fc1 = nn.Linear(5776, 1024)
self.fc2 = nn.Linear(1024, 512)
self.fc3 = nn.Linear(512, 1)
def forward(self, x):
pred = F.leaky_relu(self.conv1(x.reshape(-1,1,48,48)))
pred = F.leaky_relu(self.bnorm1(self.conv2(pred)))
pred = F.leaky_relu(self.conv3(pred))
pred = F.leaky_relu(self.bnorm2(self.conv4(pred)))
pred = F.leaky_relu(self.conv5(pred))
pred = pred.reshape(-1, 5776)
pred = F.leaky_relu(self.fc1(pred))
pred = F.leaky_relu(self.fc2(pred))
pred = torch.sigmoid(self.fc3(pred))
return pred
class Generator(nn.Module):
def __init__(self):
super().__init__()
self.fc1 = nn.Linear(512, 1024)
self.fc2 = nn.Linear(1024, 2048)
self.fc3 = nn.Linear(2048, 5776)
self.convT1 = nn.ConvTranspose2d(4, 32, 3)
self.convT2 = nn.ConvTranspose2d(32, 16, 3)
self.bnorm1 = nn.BatchNorm2d(16)
self.convT3 = nn.ConvTranspose2d(16, 8, 3)
self.convT4 = nn.ConvTranspose2d(8, 4, 3)
self.bnorm2 = nn.BatchNorm2d(4)
self.convT5 = nn.ConvTranspose2d(4, 1, 3)
def forward(self, x):
pred = F.leaky_relu(self.fc1(x))
pred = F.leaky_relu(self.fc2(pred))
pred = F.leaky_relu(self.fc3(pred))
pred = pred.reshape(-1, 4, 38, 38)
pred = F.leaky_relu(self.convT1(pred))
pred = F.leaky_relu(self.bnorm1(self.convT2(pred)))
pred = F.leaky_relu(self.convT3(pred))
pred = F.leaky_relu(self.bnorm2(self.convT4(pred)))
pred = torch.sigmoid(self.convT5(pred))
return pred
import torch.optim as optim
discriminator = discriminator.to("cuda")
generator = generator.to("cuda")
discriminator_losses = []
generator_losses = []
for epoch in range(30):
for data,label in tensor_dataset:
data = data.to("cuda")
label = label.to("cuda")
batch_size = data.size(0)
real_labels = torch.ones(batch_size, 1).to("cuda")
fake_labels = torch.zeros(batch_size, 1).to("cuda")
noise = torch.randn(batch_size, 512).to("cuda")
D_real = discriminator(data)
D_fake = discriminator(generator(noise))
D_real_loss = F.binary_cross_entropy(D_real, real_labels)
D_fake_loss = F.binary_cross_entropy(D_fake, fake_labels)
D_loss = D_real_loss+D_fake_loss
d_optim.zero_grad()
D_loss.backward()
d_optim.step()
noise = torch.randn(batch_size, 512).to("cuda")
D_fake = discriminator(generator(noise))
G_loss = F.binary_cross_entropy(D_fake, real_labels)
g_optim.zero_grad()
G_loss.backward()
g_optim.step()
discriminator_losses.append(D_loss)
generator_losses.append(G_loss)
print(epoch)
I'm also new to Deep learning and GAN models, but this method solved similar problem for my DCGAN project. Use kernel size at least 4*4: it's my guess, but it seems that small kernels can't catch details in image, no matter how deep the network is. Other tips I found are mostly from here:(same link from above)
https://machinelearningmastery.com/how-to-train-stable-generative-adversarial-networks/
I implemented a genrative adversarial network in Keras. My training data size is about 16,000, where each image is of 32*32 size. All of my training images are the resized versions of the imageds from the imagenet dataset with regard to the object detection task. I fed the image matrix directly into the network without doing the center crop. I used the AdamOptimizer with the learning rate being 1e-4, and beta1 being 0.5 and I also set the dropout rate to be 0.1. I first trained the discrimator on 3000 real images and 3000 fake images and it achieved a 93% accuracy. Then, I trained for 500 epochs with the batch size being 32. However, my model seemed to converge in only a few epochs(<10), and the images it generated were ugly.
Plot of the Loss Function
Random Samples Generated by the Generator
I was wondering whether my training dataset is too small(compared to those in the paper of DCGAN, which are more than 300,000) or my model configuration is not correct. What's more, should I train the SGD on D for k iterations (where k is small, perhaps 1) and then training with SGD on G for one iteration as suggested by Ian Goodfellow in the original paper?(I have just tried to train them one at a time)
Below is the configuration of the generator.
g_input = Input(shape=[100])
H = Dense(1024*4*4, init='glorot_normal')(g_input)
H = BatchNormalization(mode=2)(H)
H = Activation('relu')(H)
H = Reshape( [4, 4,1024] )(H)
H = UpSampling2D(size=( 2, 2))(H)
H = Convolution2D(512, 3, 3, border_mode='same', init='glorot_uniform')(H)
H = BatchNormalization(mode=2)(H)
H = Activation('relu')(H)
H = UpSampling2D(size=( 2, 2))(H)
H = Convolution2D(256, 3, 3, border_mode='same', init='glorot_uniform')(H)
H = BatchNormalization(mode=2)(H)
H = Activation('relu')(H)
H = UpSampling2D(size=( 2, 2))(H)
H = Convolution2D(3, 3, 3, border_mode='same', init='glorot_uniform')(H)
g_V = Activation('tanh')(H)
generator = Model(g_input,g_V)
generator.compile(loss='binary_crossentropy', optimizer=opt)
generator.summary()
Below is the configuration of the discriminator：
d_input = Input(shape=shp)
H = Convolution2D(64, 5, 5, subsample=(2, 2), border_mode = 'same', init='glorot_normal')(d_input)
H = LeakyReLU(0.2)(H)
#H = Dropout(dropout_rate)(H)
H = Convolution2D(128, 5, 5, subsample=(2, 2), border_mode = 'same', init='glorot_normal')(H)
H = BatchNormalization(mode=2)(H)
H = LeakyReLU(0.2)(H)
#H = Dropout(dropout_rate)(H)
H = Flatten()(H)
H = Dense(256, init='glorot_normal')(H)
H = LeakyReLU(0.2)(H)
d_V = Dense(2,activation='softmax')(H)
discriminator = Model(d_input,d_V)
discriminator.compile(loss='categorical_crossentropy', optimizer=dopt)
discriminator.summary()
Below is the configuration of GAN as a whole:
gan_input = Input(shape=[100])
H = generator(gan_input)
gan_V = discriminator(H)
GAN = Model(gan_input, gan_V)
GAN.compile(loss='categorical_crossentropy', optimizer=opt)
GAN.summary()
I think problem is with loss function
Try
loss='categorical_crossentropy',
I suspect that your generator is trainable while you training the gan. You can verify by using generator.layers[-1].get_weights() to see if the parameters changed during training process of gan.
You should freeze discriminator before you assemble it to gan:
generator.trainnable = False
gan_input = Input(shape=[100])
H = generator(gan_input)
gan_V = discriminator(H)
GAN = Model(gan_input, gan_V)
GAN.compile(loss='categorical_crossentropy', optimizer=opt)
GAN.summary()
see this discussion:
https://github.com/fchollet/keras/issues/4674
I'm trying to train simple neural network
that consists of:
Convolution layer filter (5x5) x 8, stride 2.
Max pooling 25x25 (the image has kinda low amount of details)
Flatting output into (2x2x8) vector
Classifier with logistic regression
Altogether network has < 1000 weights.
File: nn.py
#!/bin/python
import tensorflow as tf
import create_batch
# Prepare data
batch = create_batch.batch
x = tf.reshape(batch[0], [-1,100,100,3])
y_ = batch[1]
# CONVOLUTION NETWORK
# For initialization
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.3)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.2, shape=shape)
return tf.Variable(initial)
# Convolution with stride 1
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 2, 2, 1], padding='SAME')
def max_pool_25x25(x):
return tf.nn.max_pool(x, ksize=[1, 25, 25, 1],
strides=[1, 25, 25, 1], padding='SAME')
# First layer
W_conv1 = weight_variable([5, 5, 3, 8])
b_conv1 = bias_variable([8])
x_image = tf.reshape(x, [-1,100,100,3])
# First conv1
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_25x25(h_conv1)
# Dense connection layer
# make data flat
W_fc1 = weight_variable([2 * 2 * 8, 2])
b_fc1 = bias_variable([2])
h_pool1_flat = tf.reshape(h_pool1, [-1, 2*2*8])
y_conv = tf.nn.softmax(tf.matmul(h_pool1_flat, W_fc1) + b_fc1)
#Learning
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# Session
sess = tf.Session()
sess.run(tf.initialize_all_variables())
# Start input enqueue threads.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(200):
if i%10 == 0:
train_accuracy = accuracy.eval(session=sess)
print("step %d, training accuracy %g"%(i, train_accuracy))
train_step.run(session=sess)
File: create_batch.py
#!/bin/python
import tensorflow as tf
PATH1 = "../dane/trening/NK/"
PATH2 = "../dane/trening/K/"
def create_labeled_image_list():
filenames = [(PATH1 + "nk_%d.png" % i) for i in range(300)]
labels = [[1,0] for i in range(300)]
filenames += [(PATH2 + "kulki_%d.png" % i) for i in range(300)]
labels += [[0,1] for i in range(300)]
return filenames, labels
def read_images_from_disk(input_queue):
label = input_queue[1]
file_contents = tf.read_file(input_queue[0])
example = tf.image.decode_png(file_contents, channels=3)
example.set_shape([100, 100, 3])
example = tf.to_float(example)
print ("READ, label:")
print(label)
return example, label
# Start
image_list, label_list = create_labeled_image_list()
# Create appropriate tensors for naming
images = tf.convert_to_tensor(image_list, dtype=tf.string)
labels = tf.convert_to_tensor(label_list, dtype=tf.float32)
input_queue = tf.train.slice_input_producer([images, labels],
shuffle=True)
image, label = read_images_from_disk(input_queue)
batch = tf.train.batch([image, label], batch_size=600)
I'm feeding 100x100 images i have two classess 300 images each.
Basically randomly initialzied network at step 0 has better accuracy than trained one.
Network stops learning after it reaches 0.5 accuracy (basically coin flip). Images contain blue blooby thing (class 1) or grass (class 2).
I'm traning network using whole imageset at once (600 images), the loss function is cross entropy.
What I'm doing wrong?
OK, I've find a fix there were two errors, now the network is learning.
Images were RGBA despite the fact I declared them as RGB in tf
I did not perform normalization of Images to [-1,1] float32.
In tensorflow it should be done with something like this:
# i use "im" for image
tf.image.convert_image_dtype(im, dtype=float32)
im = tf.sub(im, -0.5)
im = tf.mul(im, 2.0)
To all newbies to ML - prepare your data with caution!
Thanks.