I'm interested in minimizing the trace of the covariance matrix associated with a Gaussian process in two dimensions. That is, I want to minimize tr(Σ) where Σ is given by:
Sigma
and K() is the kernel function with design points X and query points X*
As a minimum working example, I have tried the below implementation. This is clearly not DCP compliant and I have a strong feeling there is a better way to implement this such that it would be DCP compliant; however, I am somewhat of a novice to cvxpy and so would appreciate any suggestions.
import cvxpy as cp
import numpy as np
from itertools import product
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF, ConstantKernel as C
def cp_kernel(X, Y):
return cp.exp(cp.norm2(X-Y))
T = 10 # number of X's
u_max = 1
Xq = np.mgrid[0:1:0.1, 0:1:0.1].reshape(2,-1).T
x = cp.Variable((T,2))
u = cp.Variable((T,2))
nu = cp.Variable((T,1))
A = np.matrix('1 0; 0 1')
B = np.matrix('1 0; 0 1')
kernel = RBF()
KXqXq = kernel(Xq)
obj = cp.trace(KXqXq - cp.matrix_frac(cp.bmat([[cp_kernel(x[i], Xq[j]) for j in range(len(Xq))] for i in range(T)]), cp.bmat([[cp_kernel(x[i], x[j]) for j in range(T)] for i in range(T)]) + cp.diag(nu)))
cons = [0 <= x, x <= 1]
cons += [0 <= u, u <= u_max]
cons += [0 <= nu]
cons += [cp.sum(nu) == 1]
for t in range(T-1):
cons += [x[t+1] == A # x[t] + B # u[t]]
cp.Problem(cp.Minimize(obj), cons).solve()
Related
Does anyone know why cvxpy throws an infeasible error when I change the number of samples in constrained OLS? I'm attaching code to re-create my issue. n=100000 is fine, but n=400000 fails.
import cvxpy as cp
import numpy as np
class constrained_ols:
def __init__ (self, xdim=6):
self.xdim = xdim
return None
def fit(self, x, y):
import cvxpy as cp
w = cp.Variable(self.xdim)
i = cp.Variable()
quad_prog = cp.Minimize(cp.sum_squares(y-(x#w+i)))
cons = [w>=0, cp.sum(w)<=1.02, cp.sum(w)>=.98]
problem = cp.Problem(quad_prog, cons)
problem.solve()
print(problem.status)
self.coef_ = w.value
self.intercept_ = i.value
def predict(self, x):
return x # self.coef_
n = 100000
x = np.random.normal(0,1,(n,10))
y = np.random.normal(0,1,n)
model=constrained_ols(xdim=10)
model.fit(x,y)
model.coef_,model.intercept_
I was expecting to get a vector of 10 coefficients and an intercept but instead I got none values.
I'm trying to Implement linear regression in python using the following gradient decent formulas (Notice that these formulas are after partial derive)
slope
y_intercept
but the code keeps giving me wearied results ,I think (I'm not sure) that the error is in the gradient_descent function
import numpy as np
class LinearRegression:
def __init__(self , x:np.ndarray ,y:np.ndarray):
self.x = x
self.m = len(x)
self.y = y
def calculate_predictions(self ,slope:int , y_intercept:int) -> np.ndarray: # Calculate y hat.
predictions = []
for x in self.x:
predictions.append(slope * x + y_intercept)
return predictions
def calculate_error_cost(self , y_hat:np.ndarray) -> int:
error_valuse = []
for i in range(self.m):
error_valuse.append((y_hat[i] - self.y[i] )** 2)
error = (1/(2*self.m)) * sum(error_valuse)
return error
def gradient_descent(self):
costs = []
# initialization values
temp_w = 0
temp_b = 0
a = 0.001 # Learning rate
while True:
y_hat = self.calculate_predictions(slope=temp_w , y_intercept= temp_b)
sum_w = 0
sum_b = 0
for i in range(len(self.x)):
sum_w += (y_hat[i] - self.y[i] ) * self.x[i]
sum_b += (y_hat[i] - self.y[i] )
w = temp_w - a * ((1/self.m) *sum_w)
b = temp_b - a * ((1/self.m) *sum_b)
temp_w = w
temp_b = b
costs.append(self.calculate_error_cost(y_hat))
try:
if costs[-1] > costs[-2]: # If global minimum reached
return [w,b]
except IndexError:
pass
I Used this dataset:-
https://www.kaggle.com/datasets/tanuprabhu/linear-regression-dataset?resource=download
after downloading it like this:
import pandas
p = pandas.read_csv('linear_regression_dataset.csv')
l = LinearRegression(x= p['X'] , y= p['Y'])
print(l.gradient_descent())
But It's giving me [-568.1905905426412, -2.833321633515304] Which is decently not accurate.
I want to implement the algorithm not using external modules like scikit-learn for learning purposes.
I tested the calculate_error_cost function and it worked as expected and I don't think that there is an error in the calculate_predictions function
One small problem you have is that you are returning the last values of w and b, when you should be returning the second-to-last parameters (because they yield a lower cost). This should not really matter that much... unless your learning rate is too high and you are immediately getting a higher value for the cost function on the second iteration. This I believe is your real problem, judging from the dataset you shared.
The algorithm does work on the dataset, but you need to change the learning rate. I ran it in the example below and it gave the result shown in the image. One caveat is that I added a limit to the iterations to avoid the algorithm from taking too long (and only marginally improving the result).
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
class LinearRegression:
def __init__(self , x:np.ndarray ,y:np.ndarray):
self.x = x
self.m = len(x)
self.y = y
def calculate_predictions(self ,slope:int , y_intercept:int) -> np.ndarray: # Calculate y hat.
predictions = []
for x in self.x:
predictions.append(slope * x + y_intercept)
return predictions
def calculate_error_cost(self , y_hat:np.ndarray) -> int:
error_valuse = []
for i in range(self.m):
error_valuse.append((y_hat[i] - self.y[i] )** 2)
error = (1/(2*self.m)) * sum(error_valuse)
return error
def gradient_descent(self):
costs = []
# initialization values
temp_w = 0
temp_b = 0
iteration = 0
a = 0.00001 # Learning rate
while iteration < 1000:
y_hat = self.calculate_predictions(slope=temp_w , y_intercept= temp_b)
sum_w = 0
sum_b = 0
for i in range(len(self.x)):
sum_w += (y_hat[i] - self.y[i] ) * self.x[i]
sum_b += (y_hat[i] - self.y[i] )
w = temp_w - a * ((1/self.m) *sum_w)
b = temp_b - a * ((1/self.m) *sum_b)
costs.append(self.calculate_error_cost(y_hat))
try:
if costs[-1] > costs[-2]: # If global minimum reached
print(costs)
return [temp_w,temp_b]
except IndexError:
pass
temp_w = w
temp_b = b
iteration += 1
print(iteration)
return [temp_w,temp_b]
p = pd.read_csv('linear_regression_dataset.csv')
x_data = p['X']
y_data = p['Y']
lin_reg = LinearRegression(x_data, y_data)
y_hat = lin_reg.calculate_predictions(*lin_reg.gradient_descent())
fig = plt.figure()
plt.plot(x_data, y_data, 'r.', label='Data')
plt.plot(x_data, y_hat, 'b-', label='Linear Regression')
plt.xlabel('x')
plt.ylabel('y')
plt.legend()
plt.show()
I'm attempting to run CVXPY to solve a kernelized lasso regression.
When the number of predictors goes up (my goal is to have 3000 of them), it crashes with either a "Killed" error or a "bad alloc" error.
import cvxpy as cp
import numpy as np
import scipy
np.random.seed(0)
NUM_PREDICTORS = 500
NUM_SAMPLES = 1
l1 = 10
x = np.random.randn(NUM_SAMPLES, NUM_PREDICTORS)
y = np.random.randn(NUM_SAMPLES, 1)
xx = x.T # x
yx = y.T # x
xx_sqrt = scipy.linalg.sqrtm(xx)
b = cp.Variable(yx.T.shape)
u = cp.sum_squares(xx_sqrt # b) - cp.sum(2 * yx # b) + l1 * cp.norm(b, 1)
obj = cp.Minimize(u)
prob = cp.Problem(obj)
prob.solve()
print('done')
I tried trapezoidal collocation using to solve a double integrator control problem. I am doing in CVXPY. This is a two point boundary value problem. After solving there is a jump in control beyond limits to meet the end condition. Could not find reason. I have attached the code below. Thanks in advance
import numpy as np
from cvxpy import *
import matplotlib.pyplot as plt
np.random.seed(1)
n= 2
m = 1
T = 50
alpha = 0.02
beta = 5
A = np.zeros((2,2))
A[0,1] = 1
B= np.zeros((2,1))
B[1,0] = 1
dt = 0.02
x_0 = np.zeros(2)
x = Variable((2,T+1))
u = Variable((1,T+1))
cost = 0
constr =[]
for t in range(T):
cost += sum_squares(u[:,t])
constr += [x[0,t+1] == x[0,t] + 0.5*dt*(x[1,t+1]+x[1,t]) ]
constr += [x[1,t+1] == x[1,t] + 0.5*dt*(u[0,t+1]+u[0,t]) ]
constr +=[x[0,T]==1,x[1,T]==0]
constr +=[x[:,0]==x_0 ]
problem = Problem(Minimize(cost),constr)
#problem.solve()
#problem.solve(solver='SCS',eps=1e-5,max_iters=1000,verbose=True)
problem.solve(verbose=True, solver='ECOS')
f = plt.figure()
ax = f.add_subplot(411)
plt.plot(u[0,:].value)
plt.subplot(4,1,3)
x1 = x[0,:].value
plt.plot(x1)
plt.subplot(4,1,4)
x2 = x[1,:].value
plt.plot(x2)
plt.show()
states and control plot
I have a convex optimization problem I am trying to solve with cvxpy. Given a 1 x n row vector y and an m x n matrix C, I want to find a scalar b and a 1 x m row vector a such that the sum of squares of y - (aC + b(aC # aC)) is as small as possible (the # denotes element wise multiplication). In addition, all entires in a must be nonnegative and sum to 1 and -100 <= b <= 100. Below is my attempt to solve this using cvxpy.
import numpy as np
import cvxpy as cvx
def find_a(y, C, b_min=-100, b_max=100):
b = cvx.Variable()
a = cvx.Variable( (1,C.shape[0]) )
aC = a * C # this should be matrix multiplication
x = (aC + cvx.multiply(b, cvx.square(aC)))
objective = cvx.Minimize ( cvx.sum_squares(y - x) )
constraints = [0. <= a,
a <= 1.,
b_min <= b,
b <= b_max,
cvx.sum(a) == 1.]
prob = cvx.Problem(objective, constraints)
result = prob.solve()
print a.value
print result
y = np.asarray([[0.10394265, 0.25867508, 0.31258457, 0.36452763, 0.36608997]])
C = np.asarray([
[0., 0.00169811, 0.01679245, 0.04075472, 0.03773585],
[0., 0.00892802, 0.03154158, 0.06091544, 0.07315024],
[0., 0.00962264, 0.03245283, 0.06245283, 0.07283019],
[0.04396226, 0.05245283, 0.12245283, 0.18358491, 0.23886792]])
find_a(y, C)
I keep getting a DCPError: Problem does not follow DCP rules. error when I try to solve for a. I am thinking that either my function is not really convex, or I do not understand how to construct the proper cvxpy Problem. Any help would be greatly appreciated.