Implement Logistc Regression with L2 regularization Using SGD: without using sklearn - machine-learning

Question

import math
from math import log10
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn import linear_model
from sklearn.model_selection import train_test_split
def sigmoid(w,x,b):
return(1/(1+math.exp(-(np.dot(x,w)+b))))
def l2_regularizer(w):
l2_reg_sum=0.0
for i in range(len(w)):
l2_reg_sum+=(w[i]**2)
return l2_reg_sum
def compute_log_loss(X_train,y_train,w,b,alpha):
loss=0.0
X_train=np.clip(X_train, alpha, 1-alpha)
for i in range(N):
loss+= ((y_train[i]*log10(sigmoid(w,X_train[i],b)))+((1-y_train[i])*log10(1-sigmoid(w,X_train[i],b))))
#loss =-1*np.mean(actual*np.log(predicted)+(1-actual))*np.log(1-predicted)
#loss=-1*np.mean(y_train*np.log(sigmoid(w,X_proba,b))+(1-y_train))*np.log(1-sigmoid(w,X_proba,b))
loss=((-1/N)*loss)
return loss
X, y = make_classification(n_samples=50000, n_features=15, n_informative=10, n_redundant=5,
n_classes=2, weights=[0.7], class_sep=0.7, random_state=15)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=15)
w = np.zeros_like(X_train[0])
b = 0
eta0 = 0.0001
alpha = 0.0001
N = len(X_train)
n_epochs = 3
W=[]
B=[]
W.append(w)
B.append(b)
loss_list=[]
log_loss_train=0.0
log_loss_train=compute_log_loss(X_train,y_train,w,b,alpha)
loss_list.append(log_loss_train)
print(loss_list)
for epoch in range(1,n_epochs):
grad_loss=0.0
grad_intercept=0.0
for i in range(N):
first_term_grad_loss=((1-((alpha*eta0)/N))*W[epoch-1])
second_term_grad_loss=((eta0*X_train[i])*(y_train[i]-sigmoid(W[epoch-1],X_train[i],B[epoch-1])))
grad_loss+=(first_term_grad_loss+second_term_grad_loss)
first_term_grad_intercept=B[epoch-1]
second_term_grad_intercept=(eta0*(y_train[i]-sigmoid(W[epoch-1],X_train[i],B[epoch-1])))
grad_intercept+=(first_term_grad_intercept+second_term_grad_intercept)
B.append(grad_intercept)
W.append(grad_loss)
log_loss_train=0.0
log_loss_train=compute_log_loss(X_train,y_train,W[epoch],B[epoch],alpha)
loss_list.append(log_loss_train)
print(loss_list)
I am getting math range error while calculating the Sigmoid and i am not able to understand how to handle this.sigmoid calculation throwing error because of may be some large calculation.
File "C:\Users\SUMO.spyder-py3-dev\temp.py", line 12, in sigmoid return(1/(1+math.exp(-(np.dot(x,w)+b)))) OverflowError: math range error.

Answer 1

First, you need to identify your hypothesis is positive or negative. Then handle problems separately for positive and negative hypotheses like below.
def sigmoid(w,x,b):
hypothesis = np.dot(x,w)+b
if hypothesis < 0:
return (1 - 1/(1+math.exp(hypothesis)))
return (1/(1+math.exp(-hypothesis)))

Answer 2

Try to use np.exp() instead of math.exp(-(np.dot(x,w)+b)) because math.exp works on scalar values and np.exp() works on np arrays.