Wrong predictions with own MNIST-like image data in trained CNN model - machine-learning

Question

I have made a CNN model to predict number, trained by MNIST data. Using keras wrapper for tensorflow. I am having trouble in predicting my own input data. I have trained the model with epochs =100 and tested model with test set of MNIST which is working fine with accuracy of 97% approx. I have saved this model as 'my_model_conv2d.h5'.
1st Code:
# importing modules
import numpy as np
import keras
import matplotlib.pyplot as plt
from keras.optimizers import SGD
from keras.models import Sequential
from keras.layers import Dense,Activation, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.datasets import mnist
from keras.utils import np_utils
plt.rcParams['figure.figsize'] = (7,7)
#reading MNIST data
(X_train,y_train),(X_test,y_test)=mnist.load_data('mnist.npz')
print X_train.shape, " train datad shape"
print y_train.shape, " labels shape"
#copying data for plotting
test=X_test.copy()
#reshaping data type according to the tensorflow param.
X_train=X_train.reshape(X_train.shape[0],X_train.shape[1],X_train.shape[2],1)
X_test=X_test.reshape(X_test.shape[0],X_test.shape[1],X_test.shape[2],1)
input_shape=(X_train.shape[1],X_train.shape[2],1)
X_train=X_train.astype('float32')
X_test=X_test.astype('float32')
X_train/=255
X_test/=255
# one_hot vector
Y_train=np_utils.to_categorical(y_train,10)
Y_test=np_utils.to_categorical(y_test,10)
print X_train.shape, " train datad shape"
print y_train.shape, " labels shape"
#building CNN model
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(10))
model.add(Activation('softmax'))
#optimizer type initialization
sgd=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#compiling model
model.compile(loss='categorical_crossentropy', optimizer='sgd')
#training model
model.fit(X_train,Y_train, batch_size=128,epochs=100, verbose=1,validation_data=(X_test,Y_test))
#evaluating
score=model.evaluate(X_test,Y_test,verbose=1)
print score, "score"
#predicting classes using MNIST test data
predicted_classes=model.predict_classes(X_test)
print predicted_classes.shape, "predicted_classes.shape"
correct_indices = np.nonzero(predicted_classes == y_test)[0]
incorrect_indices = np.nonzero(predicted_classes != y_test)[0]
#saving model for future use
model.save('my_model_conv2d.h5')
print len(correct_indices), "no . of correct samples"
print len(incorrect_indices), "no . of incorrect samples"
print str((len(incorrect_indices)/float(len(y_test)))*100)+'%', "error percentage"
#plotting graph of predicted test data
plt.figure()
for i, correct in enumerate(correct_indices[:9]):
plt.subplot(3,3,i+1)
plt.imshow(test[correct], cmap='gray', interpolation='none')
plt.title("Predicted {}, Class {}".format(predicted_classes[correct], y_test[correct]))
plt.show()
plt.figure()
for i, incorrect in enumerate(incorrect_indices[:9]):
plt.subplot(3,3,i+1)
plt.imshow(test[incorrect], cmap='gray', interpolation='none')
plt.title("Predicted {}, Class {}".format(predicted_classes[incorrect], y_test[incorrect]))
plt.show()
I have made second python code which opens the model 'my_model_conv2d.h5'. In this code, I have also made an interactive window to draw a number, which is used as an input image for the prediction. I have taken care of background and font color of image and also the size (28,28), approximately it's similar to MNIST data.
2nd Code:
import cv2
import numpy as np
from keras.models import Sequential,load_model
drawing=False
mode=True
#function for mouse events
def interactive_drawing(event,x,y,flags,param):
global ix,iy,drawing, mode
if event==cv2.EVENT_LBUTTONDOWN:
drawing=True
ix,iy=x,y
print "EVENT_LBUTTONDOWN"
elif event==cv2.EVENT_MOUSEMOVE:
if drawing==True:
print "EVENT_MOUSEMOVE"
if mode==True:
print "EVENT_MOUSEMOVE"
cv2.line(img,(ix,iy),(x,y),1,1)
ix=x
iy=y
elif event==cv2.EVENT_LBUTTONUP:
drawing=False
print "EVENT_LBUTTONUP"
if mode==True:
print "EVENT_LBUTTONUP"
cv2.line(img,(ix,iy),(x,y),1,1)
ix=x
iy=y
return x,y
#function for predicting number
def cnn(img):
im=img.copy()
#loading cnn model
model = load_model('my_model_conv2d.h5')
img=img.reshape(1,28,28,1).astype('float32')
#prediction of class using drawn image as an input
predicted_class=model.predict_classes(img)
print predicted_class, "class"
cv2.imwrite('actual=9:predicted='+str(predicted_class)+'.jpg',im*255)
#image same as mnist image
img = np.zeros((28,28), 'float32')
cv2.namedWindow('Drawing_window')
cv2.setMouseCallback('Drawing_window',interactive_drawing)
while(1):
cv2.imshow('Drawing_window',img*255)
k=cv2.waitKey(1)&0xFF
if k==27:
break
cv2.destroyAllWindows()
#calling cnn function for prediction
cnn(img)
Most of the predictions are wrong with my input data. Also, I have made fully connected layer before implementing CNN, but the result was same. So, I tried CNN but the problem is same. I have checked the test data of MNIST in the second code which is working fine. You can check the result here https://drive.google.com/open?id=1E26zinOLrMw7XKhsHd9vnSYoHXzadiJ3 . Please check the image name, name of the image will suggest actual and predicted value.
Where I am doing wrong please suggest.

Answer 1

You need to use the exact same preprocessing applied to the images during training to images during inference.
This is why your program normalizes the train and test data in the same way.
X_train/=255
X_test/=255
This is also true to any other preprocessing you might want to do such as PCA or normalization by z-score.
So make sure that, in your case, your input image is in the range (0,1) by dividing it by 255 (if a range of 0 to 255 is to be expected from that opencv call)
Edit:
I just went ahead and trained the model and tried your program. Indeed it seems to make alot of mistakes (much more that what's to be expected since it got 1% validation error).
My guess is that since mnist is already somewhat preprocessed,
The digits have been size-normalized and centered in a fixed-size
image.
your model might expect your test images to be preprocessed as well.
I would recommend then using some mnist variant with more noise/rotation that might mitigate the problem. Eg: mnist-rot,mnist-back-rand.