Deep Learning Model training infinitely - machine-learning

Question

I am training a deep learning model for meme image classification.
I am trying to cascade VGG model and LSTM text classification, model trains infinitely and never stops after mentioned epochs. I tried all possible ways to terminate using early stopping, still it runs infinitely.
My data folder has 2 folders one with meme image and another one with tag text files
Below is the code.
label_df_clean = t
num_of_samples = label_df_clean.shape[0]
num_of_samples
# ## Glove
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
maxlen = 100
training_samples = num_of_samples
tag_vocabulary_size = 10000
max_words = tag_vocabulary_size
glove_dir = 'glove/glove.6B/'
embeddings_index = {}
f = open(os.path.join(glove_dir, 'glove.6B.100d.txt'), encoding = "UTF-8")
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))
tokenizer = Tokenizer(num_words=max_words)
texts = []
for tag_list in label_df_clean['word_tags']:
texts.append(' '.join(tag_list))
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
word_index = tokenizer.word_index
print('Found {} unique tokens'.format(len(word_index)))
tag_data = pad_sequences(sequences, maxlen=maxlen)
tag_data.shape
tag_data
embedding_dim = 100
embedding_matrix = np.zeros((max_words, embedding_dim))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if i < max_words:
if embedding_vector is not None:
# Words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
embedding_matrix:
tag_data.shape
tag_input = Input(shape=(None,), dtype='int32', name='tag')
embedded_tag = layers.Embedding(max_words, embedding_dim)(tag_input)
encoded_tag = layers.LSTM(512)(embedded_tag)
# ## CONV2D
from keras.applications import VGG16
image_input = Input(shape=(150, 150, 3), name='image')
vgg16 = VGG16(weights='imagenet',
include_top=False,
input_shape=(150, 150, 3))(image_input)
x = layers.Flatten()(vgg16)
x = layers.Dense(256, activation='relu')(x)
import tensorflow
concatenated = layers.concatenate([x, encoded_tag], axis=-1)
output = layers.Dense(1, activation='sigmoid')(concatenated)
model = Model([image_input, tag_input], output)
# model.layers[1].trainable = False # freeze VGG16 convolutional base
model.layers[4].set_weights([embedding_matrix])
# model.layers[4].trainable = False # freeze GloVe word embedding
class new_callback(tensorflow.keras.callbacks.Callback):
def epoch_end(self, epoch, logs={}):
if(logs.get('accuracy')> 0.65): # select the accuracy
print("\n !!! 65% accuracy, no further training !!!")
self.model.stop_training = True
callbacks = new_callback()
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
model.summary()
# # model.layers[1].trainable = False # freeze VGG16 convolutional base
# model.layers[4].set_weights([embedding_matrix])
# model.layers[4].trainable = False # freeze GloVe word embedding
label_df_clean.head()
dim = (150, 150)
X_image_train = []
X_tag_train = tag_data
y_train = label_df_clean["label"]
for fname in listdir(small_image_path):
fpath = os.path.join(small_image_path, fname)
im = cv2.imread(fpath)
im_resized = cv2.resize(im, dim, interpolation = cv2.INTER_AREA)
X_image_train.append(im_resized)
# y_train.append(1)
# # add wrong tag samples
# num_negative_samples = len(y_train)
# for i in range(num_negative_samples):
# image = X_image_train[i]
# X_image_train.append(image)
# j = (i + 1) % num_negative_samples # get a different tag
# tag = X_tag_train[j]
# X_tag_train = np.append(X_tag_train, tag)
# y_train
# from sklearn import preprocessing
# le = preprocessing.LabelEncoder()
# le.fit(y_train)
# y_train = le.transform(y_train)
X_image_train = np.array(X_image_train)
X_tag_train = np.array(tag_data)
y_train = np.array(y_train)
perm = np.arange(y_train.shape[0])
np.random.shuffle(perm)
X_image_train = X_image_train[perm]
X_tag_train = X_tag_train[perm]
y_train = y_train[perm]
X_image_train.shape, X_tag_train.shape, y_train.shape
X_image_train
X_tag_train
y_train
model.fit([X_image_train, X_tag_train],y_train, batch_size = 64, epochs = 10, callbacks =[callbacks])')
Model continuously trains and never stops. Note code is from Jupyter notebook, please ignore indentation and print issues.
Please do let me know if u find logic issues in model train.
Note: I have not used label encoder since my train is binary data.
I currently use Keras 2.3.1 and TensorFlow 2.1.0.