Spare autoencoders in tensorflow - machine-learning

Question

Is there an example of sparse autoencoders in tensorflow? I was to able to run and understand the normal one from here https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/autoencoder.py
For sparse, do I just need to modify the cost function?
from __future__ import division, print_function, absolute_import
import scipy.fftpack
import pdb, random
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from read_audio import read_audio
start,end= 3050,5723
#start=end=None
audio_data=read_audio("LDC93S1",start,end)
def overlapping_chunks(l, sub_array_size, overlap_size):
return [l[i:i+sub_array_size] for i in range(0, len(l)-overlap_size, overlap_size)]
def conv_frq_domain(signal):
fft_abs=abs(scipy.fft(signal))
fft_sorted=np.sort(fft_abs)[::-1]
top_100=fft_sorted[:100]
return top_100
sample_len=100
samples=overlapping_chunks(audio_data,sample_len,50)
freq_samples=[]
for sample in samples:
freq_samples.append(conv_frq_domain(sample))
examples=samples
print("Number of samples", str(len(examples)))
#pdb.set_trace()
# Parameters
learning_rate = 0.001
training_epochs = 2000
batch_size = 2
display_step = 100
# Network Parameters
n_hidden_1 = 1000 # 1st layer num features
n_hidden_2 = 650 # 2nd layer num features
n_input = sample_len
# tf Graph input (only pictures)
X = tf.placeholder("float", [None, n_input])
weights = {
'encoder_h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),
'encoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),
'decoder_h1': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_1])),
'decoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_input])),
}
biases = {
'encoder_b1': tf.Variable(tf.random_normal([n_hidden_1])),
'encoder_b2': tf.Variable(tf.random_normal([n_hidden_2])),
'decoder_b1': tf.Variable(tf.random_normal([n_hidden_1])),
'decoder_b2': tf.Variable(tf.random_normal([n_input])),
}
# Building the encoder
def encoder(x):
# Encoder Hidden layer with sigmoid activation #1
layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['encoder_h1']),
biases['encoder_b1']))
# Decoder Hidden layer with sigmoid activation #2
layer_2 = tf.nn.sigmoid(tf.add(tf.matmul(layer_1, weights['encoder_h2']),
biases['encoder_b2']))
return layer_1,layer_2
# Building the decoder
def decoder(x):
# Encoder Hidden layer with sigmoid activation #1
layer_1 = tf.nn.sigmoid(tf.add(tf.matmul(x, weights['decoder_h1']),
biases['decoder_b1']))
# Decoder Hidden layer with sigmoid activation #2
layer_2 = tf.add(tf.matmul(layer_1, weights['decoder_h2']),
biases['decoder_b2'])
return layer_2
def kl_divergence(p_1, p_hat):
num_len=p_1.get_shape()[0]
term1 = p_1 * tf.log(p_1)
term2 = p_1 * tf.log(p_hat)
term3 = tf.sub(tf.ones(num_len),p_1) * tf.log(tf.sub(tf.ones(num_len),p_1))
term4 = tf.sub(tf.ones(num_len),p_1) * tf.log(tf.sub(tf.ones(num_len) ,p_hat))
return tf.sub(tf.add(term1,term3),tf.add(term2,term4))
def sparsity_penalty(hidden_layer_acts, sparsity_level=0.05, sparse_reg=1e-3, batch_size=-1):
# = T.extra_ops.repeat(sparsity_level, self.nhid)
sparsity_level_vec=tf.ones(hidden_layer_acts.get_shape()[1])*sparsity_level
sparsity_penalty = 0
avg_act = Mean = tf.reduce_mean(hidden_layer_acts,1)
kl_div = kl_divergence(sparsity_level_vec, avg_act)
sparsity_penalty = sparse_reg * tf.reduce_sum(kl_div,0)
return sparsity_penalty
# Construct model
encoder_op1, encoder_op2 = encoder(X)
decoder_op = decoder(encoder_op2)
# Prediction
y_pred = decoder_op
# Targets (Labels) are the input data.
y_true = X
# Define loss and optimizer, minimize the squared error
cost = tf.reduce_mean(tf.pow(y_true - y_pred, 2)+sparsity_penalty(encoder_op2))
optimizer = tf.train.RMSPropOptimizer(learning_rate).minimize(cost)
# Initializing the variables
init = tf.initialize_all_variables()
# Launch the graph
train_data=examples
with tf.Session() as sess:
sess.run(init)
total_batch = int(len(examples)/batch_size)
# Training cycle
for epoch in range(training_epochs):
#random.shuffle(train_data)
# Loop over all batches
for i in range(total_batch):
batch_xs=train_data[i:i+2]
# Run optimization op (backprop) and cost op (to get loss value)
_, c = sess.run([optimizer, cost], feed_dict={X: batch_xs})
if epoch ==2500:
pdb.set_trace()
encode_decode = sess.run(y_pred, feed_dict={X: batch_xs})
# Display logs per epoch step
if epoch % display_step == 0:
print("Epoch:", '%04d' % (epoch+1),"cost=", "{:.9f}".format(c))
print("Optimization Finished!")