How to train a CNN properly to predict well on unseen data? - machine-learning

Question

I have been developing a program for my school project which recognizes numbers. For that I have used Python, Keras and the MNIST Dataset. This is the code I used to train it:
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Convolution2D, MaxPooling2D, Activation, AveragePooling2D
from keras import backend as K
import matplotlib.pyplot as plt
import matplotlib
batch_size = 32
num_classes = 10
epochs = 10
img_rows, img_cols = 28, 28
(x_train, y_train), (x_test, y_test) = mnist.load_data()
if K.image_data_format() == 'channels_first':
x_train = x_train.reshape(x_train.shape[0], 1, img_rows, img_cols)
x_test = x_test.reshape(x_test.shape[0], 1, img_rows, img_cols)
input_shape = (1, img_rows, img_cols)
else:
x_train = x_train.reshape(x_train.shape[0], img_rows, img_cols, 1)
x_test = x_test.reshape(x_test.shape[0], img_rows, img_cols, 1)
input_shape = (img_rows, img_cols, 1)
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Convolution2D(6, (5, 5), input_shape=input_shape))
model.add(Activation('sigmoid'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(12, (5, 5)))
model.add(Activation('sigmoid'))
model.add(AveragePooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(192))
model.add(Dense(10))
model.add(Activation('sigmoid'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
hist = model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
model.save('model3.h5')
train_loss = hist.history['loss']
val_loss = hist.history['val_loss']
train_acc = hist.history['acc']
val_acc = hist.history['val_acc']
xc = range(epochs)
plt.figure(1,figsize=(7,5))
plt.plot(xc,train_loss)
plt.plot(xc,val_loss)
plt.xlabel('num of Epochs')
plt.ylabel('loss')
plt.title('train_loss vs val_loss')
plt.grid(True)
plt.legend(['train','val'])
print(plt.style.available) # use bmh, classic,ggplot for big pictures
plt.style.use(['classic'])
plt.figure(2,figsize=(7,5))
plt.plot(xc,train_acc)
plt.plot(xc,val_acc)
plt.xlabel('num of Epochs')
plt.ylabel('accuracy')
plt.title('train_acc vs val_acc')
plt.grid(True)
plt.legend(['train','val'],loc=4)
#print plt.style.available # use bmh, classic,ggplot for big pictures
plt.style.use(['classic'])
plt.show()
I saved the model under the name model3.h5. However, in another program I wrote, I was trying to predict with the model I saved the numbers I entered in Paint. I had 10 pictures (0-9) and while predicting the model predicted that all numbers are number 8, which is of course wrong.
However, during training, accuracy was close to 98.5% and the loss was less than 0.1. Am I doing something wrong?
Here is the code that I run for predicting it on unseen data. It resizes the picture to 28 columns and 28 rows so it can run on my CNN.
This is my first project on Convolutional Neural Networks and I don't know "some extra techniques" that could help me do better on the unseen data.
I tried some different architectures as well (doing convolutional layers with max pooling and relu activation functions, then adding full connected layers) but the result was still the same. I also tried to set it to 100 or 200 epochs, still no use...
import os, cv2
import numpy as np
import matplotlib.pyplot as plt
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from keras import backend as K
from keras.models import load_model
K.set_image_dim_ordering('tf')
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD,RMSprop,adam
PATH = os.getcwd()
data_path = PATH + '\myNumbers'
data_dir_list = os.listdir(data_path) #direktoriji unutra
img_data = []
for file in data_dir_list:
test_image = cv2.imread(data_path + "\\" + file)
test_image = cv2.cvtColor(test_image, cv2.COLOR_RGB2GRAY)
test_image = cv2.resize(test_image,(28,28))
test_image = np.array(test_image)
test_image = test_image.astype('float32')
test_image /= 255
print (test_image.shape)
test_image= np.expand_dims(test_image, axis=3)
test_image= np.expand_dims(test_image, axis=0)
print (test_image.shape)
img_data.append(test_image)
model = load_model("model3.h5")
for img in img_data:
print(model.predict(img))
print(model.predict_classes(img))