Maybe Overfitting trying to implement rnn with keras - machine-learning

Question

I am trying to create a RNN for sentiment classification (-1(neg) or 1(pos)).My dataset file looks like this:
text,polarity
"this is a line",1
"this is asecond line",-1
.....
etc.
with 2603 rows like these.
This is what i got so far and it reaches maximum accuracy very fast, so i assume it is wrong. What am i doing wrong though?
Code:
from __future__ import print_function
from keras.preprocessing import sequence
import tensorflow as tf
from keras.models import Sequential
from keras.layers import Dense, Embedding, Activation, Dropout
from keras.preprocessing.text import Tokenizer
from keras.layers import LSTM
from keras.datasets import imdb
import pandas as pd
import string
import nltk
from sklearn.model_selection import train_test_split
from collections import Counter
batch_size = 32
num_words=2000
epochs=15
print('Loading data...')
data = pd.read_csv("yolo.csv",header=0,encoding = 'UTF-8')
X = data['text']
Y = data['polarity']
x_train,x_test,y_train,y_test = train_test_split(X,Y,test_size=0.2) #split train/test data
tokenizer = Tokenizer(num_words=num_words, lower=False, split=' ')
tokenizer.fit_on_texts(x_train)
x_train= tokenizer.texts_to_sequences(x_train)
x_test= tokenizer.texts_to_sequences(x_test)
x_train= tokenizer.sequences_to_matrix(x_train, mode='binary')
x_test= tokenizer.sequences_to_matrix(x_test, mode='binary')
y_train = tf.keras.utils.to_categorical(y_train,2)
y_test = tf.keras.utils.to_categorical(y_test,2)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Build model...')
model = Sequential()
model.add(Dense(512, input_shape=(num_words, )))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print (model.metrics_names)
history= model.fit (x_train,y_train, batch_size=batch_size, epochs=epochs, verbose=1)
score= model.evaluate(x_test, y_test, batch_size=batch_size, verbose=1)
print ('Test loss: ()'.format(score[0]))
print ('Test accuracy:{}'.format(score[1]))