Trying to implement the reaserch paper:
https://ieeexplore.ieee.org/document/9479786/
Training a Monotone Network with architechture:
class Model(nn.Module):
def __init__(self, q, s):
self.layer_s_list = [nn.Linear(5, s) for _ in range(q)]
self.inv_w, self.inv_b = self.get_layer_weights()
def forward(self, x):
# print(inv_w[0].shape, inv_b[0].shape)
output_lst = []
for layer in self.layer_s_list:
v, id = torch.max(layer(x), 1)
output_lst.append(v.detach().numpy())
output_lst = np.array(output_lst)
output_lst = torch.from_numpy(output_lst)
out, _ = torch.min(output_lst, 0)
allo_out = F.softmax(out)
pay_out = nn.ReLU(inplace = True)(out)
inv_out_lst = []
for q_idx in range(len(self.inv_w)):
# print(inv_w[q_idx].shape, pay_out.shape, inv_b[q_idx].shape)
y, _ = torch.min(torch.linalg.pinv(self.inv_w[q_idx]) * (pay_out - self.inv_b[q_idx]), 0)
inv_out_lst.append(y.detach().numpy())
final_out = np.array(inv_out_lst)
final_out = torch.from_numpy(final_out)
final_out, _ = torch.max(final_out, 1)
return final_out, allo_out
def get_layer_weights(self):
weights_lst = []
bias_lst = []
for layer in self.layer_s_list:
weights_lst.append(layer.state_dict()['weight'])
bias_lst.append(layer.state_dict()['bias'])
return weights_lst, bias_lst
When I initialise the network and run for random inputs:
q = 5
s = 10
x = torch.rand((10, 5), requires_grad = True)
net = Model(q, s)
y, z = net(x)`
It gives the following error:
AttributeError Traceback (most recent call last)
<ipython-input-3-aac6d239df1f> in <module>
1 x = torch.rand((10, 5), requires_grad = True)
2 net = Model(5, 10)
----> 3 y = net(x)
1 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in __getattr__(self, name)
1206 return modules[name]
1207 raise AttributeError("'{}' object has no attribute '{}'".format(
-> 1208 type(self).__name__, name))
1209
1210 def __setattr__(self, name: str, value: Union[Tensor, 'Module']) -> None:
AttributeError: 'Model' object has no attribute '_backward_hooks'
Please help me understand what this error is and how to fix it.
You forgot to initialize your model with __init__ method of its parent class nn.Module. A simple solution is to add one line in your Model.__init__:
class Model(nn.Module):
def __init__(self, q, s):
super(Model, self).__init__()
self.layer_s_list = [nn.Linear(5, s) for _ in range(q)]
self.inv_w, self.inv_b = self.get_layer_weights()
Related
I have coded a PPO algorithm from scratch and tried testing it on Pendulum-v2. For some reason, the algorithm doesn't learn anything. I wrote this code to understand the mechanics of how PPO works and therefore have kept everything simple. I also referred popular code bases to ensure that I don't deviate too much from the norm.
Here are my actor and critic networks -
class Actor(nn.Module):
def __init__(self, state_size, action_size):
super(Actor, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.linear_relu_stack = nn.Sequential(
nn.Linear(state_size, 300),
nn.ReLU(),
nn.Linear(300, 128),
nn.ReLU(),
nn.Linear(128, 128),
nn.ReLU(),
nn.Linear(128, action_size),
nn.Softmax()
)
def forward(self,x):
x = self.linear_relu_stack(x)
return x
class Critic(nn.Module):
def __init__(self, state_size, action_size):
super(Critic, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.linear_stack = nn.Sequential(
nn.Linear(state_size, 300),
nn.ReLU(),
nn.Linear(300, 128),
nn.ReLU(),
nn.Linear(128, 128),
nn.ReLU(),
nn.Linear(128, 1)
)
def forward(self, x):
x = self.linear_stack(x)
return x
Here is the rollout phase where we collect data -
def rollout():
for i in range(ppo_batch): # 100 episodes should be good?
print("Rollout process, i = ", i)
obs = torch.tensor(env.reset(), dtype=torch.float32).unsqueeze(0)
tot_rewards = 0
transitions = []
iter = 0
done = False
while not done:
act_probs = torch.distributions.Categorical(actor(obs.to(device)))
action = act_probs.sample()
action = action.cpu().detach().numpy()
next_state, reward, done, info = env.step(action)
action = torch.tensor(action, dtype=torch.float32).to(device)
tot_rewards += np.power(gamma, iter) * reward
iter += 1
transitions.append((obs, action, act_probs.log_prob(action), tot_rewards))
obs = torch.tensor(next_state, dtype=torch.float32).unsqueeze(0)
print("Discounted Reward = ", tot_rewards)
batch_obs = torch.Tensor([s.numpy() for (s, a, a_p, r) in transitions]).to(device)
# print("batch_obs shape = ", np.array(batch_obs).shape)
batch_act = torch.Tensor([a for (s, a, a_p, r) in transitions]).to(device)
batch_log_probs = torch.Tensor([a_p for (s, a, a_p, r) in transitions]).to(device)
batch_rtgs = torch.Tensor([r for (s, a, a_p, r) in transitions]).flip(dims = (0,)).to(device)
return batch_obs, batch_act, batch_log_probs, batch_rtgs
This is where the learning happens -
for i in range(episodes):
batch_obs, batch_act, batch_log_probs, batch_rtgs = rollout()
value = critic(batch_obs)
batch_rtgs = batch_rtgs
# todo Why are we detaching value
A_k = batch_rtgs - value.squeeze().detach()
for _ in range(training_iters):
value = critic(batch_obs).squeeze()
act_probs = torch.distributions.Categorical(actor(batch_obs))
action = act_probs.sample()
log_probs = act_probs.log_prob(action).squeeze()
ratios = torch.exp(log_probs - batch_log_probs)
surr1 = ratios*A_k
surr2 = torch.clamp(ratios, 1 - clip, 1 + clip)*A_k
actor_loss = -torch.min(surr1, surr2).mean()
critic_loss = (value - batch_rtgs).pow(2).mean()
#todo No idea why we are doing retain_graph = True
policy_opt.zero_grad()
actor_loss.backward(retain_graph=True)
policy_opt.step()
value_opt.zero_grad()
critic_loss.backward(retain_graph=True)
value_opt.step()
What I've already done -
A major bug I found was that the actor and critic loss didn't have dimension 1, before taking the mean. Therefore the loss function was an n-dimensional matrix before being averaged which was obviously wrong. I believe I have fixed it now.
Here's the entire code -
#Modified this code - https://github.com/DeepReinforcementLearning/DeepReinforcementLearningInAction/blob/master/Chapter%204/Ch4_book.ipynb
#Also, modified this code - https://github.com/higgsfield/RL-Adventure-2/blob/master/1.actor-critic.ipynb
# Also, modified this code - https://github.com/ericyangyu/PPO-for-Beginners/blob/9abd435771aa84764d8d0d1f737fa39118b74019/ppo.py#L151
import numpy as np
import gym
import torch
from torch import nn
import matplotlib.pyplot as plt
env = gym.make('Pendulum-v1')
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
learning_rate = 0.0001
episodes = 10000
gamma = 0.99
clip = 0.2
#No idea whether these hyperparameters are good
ppo_batch = 30
training_iters = 5
dim_action = env.action_space.shape[0]
class Actor(nn.Module):
def __init__(self, state_size, action_size):
super(Actor, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.linear_relu_stack = nn.Sequential(
nn.Linear(state_size, 300),
nn.ReLU(),
nn.Linear(300, 128),
nn.ReLU(),
nn.Linear(128, 128),
nn.ReLU(),
nn.Linear(128, action_size),
nn.Softmax()
)
def forward(self,x):
x = self.linear_relu_stack(x)
return x
class Critic(nn.Module):
def __init__(self, state_size, action_size):
super(Critic, self).__init__()
self.state_size = state_size
self.action_size = action_size
self.linear_stack = nn.Sequential(
nn.Linear(state_size, 300),
nn.ReLU(),
nn.Linear(300, 128),
nn.ReLU(),
nn.Linear(128, 128),
nn.ReLU(),
nn.Linear(128, 1)
)
def forward(self, x):
x = self.linear_stack(x)
return x
def rollout():
for i in range(ppo_batch): # 100 episodes should be good?
print("Rollout process, i = ", i)
obs = torch.tensor(env.reset(), dtype=torch.float32).unsqueeze(0)
tot_rewards = 0
transitions = []
iter = 0
done = False
while not done:
act_probs = torch.distributions.Categorical(actor(obs.to(device)))
action = act_probs.sample()
action = action.cpu().detach().numpy()
next_state, reward, done, info = env.step(action)
action = torch.tensor(action, dtype=torch.float32).to(device)
tot_rewards += np.power(gamma, iter) * reward
iter += 1
transitions.append((obs, action, act_probs.log_prob(action), tot_rewards))
obs = torch.tensor(next_state, dtype=torch.float32).unsqueeze(0)
print("Discounted Reward = ", tot_rewards)
batch_obs = torch.Tensor([s.numpy() for (s, a, a_p, r) in transitions]).to(device)
# print("batch_obs shape = ", np.array(batch_obs).shape)
batch_act = torch.Tensor([a for (s, a, a_p, r) in transitions]).to(device)
batch_log_probs = torch.Tensor([a_p for (s, a, a_p, r) in transitions]).to(device)
batch_rtgs = torch.Tensor([r for (s, a, a_p, r) in transitions]).flip(dims = (0,)).to(device)
return batch_obs, batch_act, batch_log_probs, batch_rtgs
actor = Actor(env.observation_space.shape[0], dim_action).to(device)
critic = Critic(env.observation_space.shape[0], dim_action).to(device)
policy_opt = torch.optim.Adam(params = actor.parameters(), lr = learning_rate)
value_opt = torch.optim.Adam(params = critic.parameters(), lr = learning_rate)
score = []
for i in range(episodes):
batch_obs, batch_act, batch_log_probs, batch_rtgs = rollout()
value = critic(batch_obs)
batch_rtgs = batch_rtgs
# todo Why are we detaching value
A_k = batch_rtgs - value.squeeze().detach()
for _ in range(training_iters):
value = critic(batch_obs).squeeze()
act_probs = torch.distributions.Categorical(actor(batch_obs))
action = act_probs.sample()
log_probs = act_probs.log_prob(action).squeeze()
ratios = torch.exp(log_probs - batch_log_probs)
surr1 = ratios*A_k
surr2 = torch.clamp(ratios, 1 - clip, 1 + clip)*A_k
actor_loss = -torch.min(surr1, surr2).mean()
critic_loss = (value - batch_rtgs).pow(2).mean()
#todo No idea why we are doing retain_graph = True
policy_opt.zero_grad()
actor_loss.backward(retain_graph=True)
policy_opt.step()
value_opt.zero_grad()
critic_loss.backward(retain_graph=True)
value_opt.step()
I'm trying to implement a model using lstm to generate sonnects in pytorch. When I tested the dataLoader, It took a lot of time and could not return data as expected. I review it and don't know where error locates. Please help me, here is the code(more detail in(https://paste.ubuntu.com/p/vspS3msNVW/))
(10th cell)
class SonnetDataset(Dataset):
def __init__(self, sonnet_in_ids: list, vocab: list, max_seq_length: int):
super().__init__()
self.data = sonnet_in_ids
self.vocab = vocab
self.vocab_size = len(vocab)
self.pad_id = self.vocab.index('<PAD>')
self.start_id = self.vocab.index('<START>')
self.end_id = self.vocab.index('<END>')
self.max_seq_length = max_seq_length + 2
print('init successfully')
def __len__(self):
return len(self.data)
def __getitem__(self, index) -> torch.LongTensor:
print('get item')
x = self.data[index]
x = [self.start_id] + x + [self.end_id]
# padding
x += [self.pad_id] * (self.max_seq_length - len(x))
x = torch.LongTensor(x)
print(x)
return x
batch_size = 4
train_set = SonnetDataset(sonnet_in_ids=sonnets_in_ids, vocab=vocab, max_seq_length=max_length)
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=4, drop_last=True)
next(iter(train_loader))
Could not get data
**I am using 100 tiramisu code and I am getting this error. I know it is probably because of version changes in Keras but not sure how to fix it.
I have changed the old merger method to keras.layer.concatenate but it still gives the same error.**
def relu(x): return Activation('relu')(x)
def dropout(x, p): return Dropout(p)(x) if p else x
def bn(x): return BatchNormalization(mode=2, axis=-1)(x)
def relu_bn(x): return relu(bn(x))
def concat(xs): return keras.layers.Concatenate(xs, mode='concat', concat_axis=-1)
def conv(x, nf, sz, wd, p, stride=1):
# x = Convolution2D(nf, sz, sz, init='he_uniform', border_mode='same',
# subsample=(stride,stride), W_regularizer=regularizers.l1_l2(wd))(x)
x = Convolution2D(nf, (sz, sz), padding='same',
strides=(stride,stride), kernel_regularizer=regularizers.l1_l2(wd))(x)
return dropout(x, p)
def down_path(x, nb_layers, growth_rate, p, wd):
skips = []
for i,n in enumerate(nb_layers):
x,added = dense_block(n,x,growth_rate,p,wd)
skips.append(x)
x = transition_dn(x, p=p, wd=wd)
return skips, added
def transition_up(added, wd=0):
x = concat(added)
_,r,c,ch = x.get_shape().as_list()
W_regularizer=l2(wd))(x)
return Deconvolution2D(ch, (3, 3), (None,r*2,c*2,ch),
padding='same', stride=(2,2), kernel_regularizer=l2(wd))(x)
def up_path(added, skips, nb_layers, growth_rate, p, wd):
for i,n in enumerate(nb_layers):
x = transition_up(added, wd)
x = concat([x,skips[i]])
x,added = dense_block(n,x,growth_rate,p,wd)
return x
def reverse(a): return list(reversed(a))
def create_tiramisu(nb_classes, img_input, nb_dense_block=6,
growth_rate=16, nb_filter=48, nb_layers_per_block=5, p=None, wd=0):
if type(nb_layers_per_block) is list or type(nb_layers_per_block) is tuple:
nb_layers = list(nb_layers_per_block)
else: nb_layers = [nb_layers_per_block] * nb_dense_block
x = conv(img_input, nb_filter, 3, wd, 0)
skips,added = down_path(x, nb_layers, growth_rate, p, wd)
x = up_path(added, reverse(skips[:-1]), reverse(nb_layers[:-1]), growth_rate, p, wd)
x = conv(x, nb_classes, 1, wd, 0)
_,r,c,f = x.get_shape().as_list()
x = Reshape((-1, nb_classes))(x)
return Activation('softmax')(x)
input_shape = (224,224,3)
img_input = Input(shape=input_shape)
x = create_tiramisu(32, img_input, nb_layers_per_block=[4,5,7,10,12,15], p=0.2, wd=1e-4)
The error I am getting is:
TypeError Traceback (most recent call last)
<ipython-input-80-acecdf7dd0b2> in <module>()
1 input_shape = (224,224,3)
2 img_input = Input(shape=input_shape)
----> 3 x = create_tiramisu(32, img_input, nb_layers_per_block=[4,5,7,10,12,15], p=0.2, wd=1e-4)
10 frames
/usr/local/lib/python3.7/dist-packages/keras/utils/generic_utils.py in validate_kwargs(kwargs, allowed_kwargs, error_message)
1172 for kwarg in kwargs:
1173 if kwarg not in allowed_kwargs:
-> 1174 raise TypeError(error_message, kwarg)
1175
1176
TypeError: ('Keyword argument not understood:', 'mode')
I have tried to change a few arguments which changed from Keras version but still give the wrong answer.
I want to perform PCA on a list of dataframes, such as cna. My code raised ValueError: Found array with 0 feature(s) (shape=(54, 0)) while a minimum of 1 is required.
class StringConverter(dict):
def __contains__(self, item):
return True
def __getitem__(self, item):
return str
def get(self, default=None):
return str
def get_pathways():
pathways = []
with open(dir + "hsa.txt", newline='') as csvfile:
freader = csv.reader(csvfile, delimiter='\t')
for row in freader:
pathways.append(row)
return pathways
class DataProcessing:
def __init__(self, data, header=0):
self.df = pd.read_csv(data, sep="\t", header=header)
def split_data(self):
X = self.df.iloc[:, :-1]
y = self.df.iloc[:, -1]
return X, y
def pca(self):
pca = PCA()
if np.any(np.isnan(self.df)):
pass
elif np.all(np.isfinite(self.df)):
pass
else:
pca.fit(self.df.iloc[1:, 3:])
self.pca_components = pca.components_
return self.pca_components
def main():
cna = DataProcessing(dir + "data_linear_cna.txt")
patients_source = DataProcessing(dir + "data_clinical_patient_reduced.txt", 0)
patients = {}
# get list of allowed patients
for index, row in patients_source.df.iterrows():
if row.OS_MONTHS != '[Not Available]':
if float(row.OS_MONTHS) > 2 * 12:
patients[row.PATIENT_ID] = 1
pathways = get_pathways()
#### Process the CNA data
q = 5
C = []
G = []
M = []
# find common samples
n = cna.df.shape[1]
common_samples = {}
cna_sample_index = {}
for i in range(2, n):
sample_name = list(cna.df)[i][0:-3]
if sample_name in patients:
common_samples[sample_name] = 1
cna_sample_index[sample_name] = i
ordered_common_samples = list(common_samples.keys())
ordered_common_samples.sort()
process_cna = True
if process_cna:
C = []
for i, p in enumerate(pathways):
allowed_symbols = {}
first = True
for s in p:
if first:
first = False
else:
if s != 'NA':
allowed_symbols[s] = 1
# going through each sample
B = []
allowed_columns = []
for s in ordered_common_samples:
B.append([])
allowed_columns.append(cna_sample_index[s])
Bi = 0
for index, row in cna.df.iterrows():
if row[0].upper() in allowed_symbols:
Bi2 = Bi
for c in allowed_columns:
B[Bi2].append(cna.df.iloc[index, c])
Bi2 = Bi2 + 1
pca_B = PCA()
pca_B.fit(B)
C.append(pca_B.transform(B)[:, 0:q])
cna.pca()
main()
Traceback:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-59-d61a65bf4d9e> in <module>()
248
249
--> 250 main()
4 frames
/usr/local/lib/python3.7/dist-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, accept_large_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, estimator)
815 "Found array with %d feature(s) (shape=%s) while"
816 " a minimum of %d is required%s."
--> 817 % (n_features, array.shape, ensure_min_features, context)
818 )
819
ValueError: Found array with 0 feature(s) (shape=(54, 0)) while a minimum of 1 is required.
I get an error saying that the input should be of type Tensor, not tuple. I do not know how to work around this problem, as I am already implementing the return_dict=False method as stated in the migration plan.
My model is as follows:
class XLNetClassifier(torch.nn.Module):
def __init__(self, dropout_rate=0.1):
super(XLNetClassifier, self).__init__()
self.XLNet = XLNetModel.from_pretrained('xlnet-base-cased', return_dict=False)
self.d1 = torch.nn.Dropout(dropout_rate)
self.l1 = torch.nn.Linear(768, 64)
self.bn1 = torch.nn.LayerNorm(64)
self.d2 = torch.nn.Dropout(dropout_rate)
self.l2 = torch.nn.Linear(64, 3)
def forward(self, input_ids, attention_mask):
x = self.XLNet(input_ids=input_ids, attention_masks = attention_mask)
x = self.d1(x)
x = self.l1(x)
x = self.bn1(x)
x = torch.nn.Tanh()(x)
x = self.d2(x)
x = self.l2(x)
return x
The error occurs when calling the dropout.
The XLNetModel returns two output values:
last_hidden_state
mems
That means you get a tuple and not a single tensor as the error message says. Your class definition should therefore be:
from transformers import XLNetModel, XLNetTokenizerFast
import torch
class XLNetClassifier(torch.nn.Module):
def __init__(self, dropout_rate=0.1):
super(XLNetClassifier, self).__init__()
self.XLNet = XLNetModel.from_pretrained('xlnet-base-cased', return_dict=False)
self.d1 = torch.nn.Dropout(dropout_rate)
self.l1 = torch.nn.Linear(768, 64)
self.bn1 = torch.nn.LayerNorm(64)
self.d2 = torch.nn.Dropout(dropout_rate)
self.l2 = torch.nn.Linear(64, 3)
def forward(self, input_ids, attention_mask):
x = self.XLNet(input_ids=input_ids, attention_masks = attention_mask)
x = self.d1(x[0])
x = self.l1(x)
x = self.bn1(x)
x = torch.nn.Tanh()(x)
x = self.d2(x)
x = self.l2(x)
return x
tokenizer = XLNetTokenizerFast.from_pretrained('xlnet-base-cased')
model = XLNetClassifier()
inputs = tokenizer("Hello, my dog is cute", return_tensors="pt", return_token_type_ids=False)
outputs = model(**inputs)
or even better without return_dict=False
class XLNetClassifier(torch.nn.Module):
def __init__(self, dropout_rate=0.1):
super(XLNetClassifier, self).__init__()
self.XLNet = XLNetModel.from_pretrained('xlnet-base-cased')
self.d1 = torch.nn.Dropout(dropout_rate)
self.l1 = torch.nn.Linear(768, 64)
self.bn1 = torch.nn.LayerNorm(64)
self.d2 = torch.nn.Dropout(dropout_rate)
self.l2 = torch.nn.Linear(64, 3)
def forward(self, input_ids, attention_mask):
x = self.XLNet(input_ids=input_ids, attention_masks = attention_mask)
x = self.d1(x.last_hidden_state)
x = self.l1(x)
x = self.bn1(x)
x = torch.nn.Tanh()(x)
x = self.d2(x)
x = self.l2(x)
return x