improved stability

rl_agent/policy.py

@@ -8,19 +8,19 @@ class Policy(nn.Module):
         
         super(Policy, self).__init__()
 
-        self.layer1 = nn.Linear(input_channels, 2 * input_channels)
+        self.layer1 = nn.Linear(input_channels, 1)
         self.tanh1   = nn.Tanh()
-        self.layer2 = nn.Linear(2 * input_channels, 1)
-        self.tanh2 = nn.Tanh()
+        # self.layer2 = nn.Linear(2 * input_channels, 1)
+        # self.tanh2 = nn.Tanh()
 
     def forward(self, state):
 
         hidden = self.layer1(state)
         hidden = self.tanh1(hidden)
-        hidden = self.layer2(hidden)
-        action = self.tanh2(hidden)
+        # hidden = self.layer2(hidden)
+        # action = self.tanh2(hidden)
 
-        return action
+        return hidden
 
 
 

rl_agent/test_env.py

@@ -7,6 +7,8 @@ import numpy as np
 import torch
 from collections import OrderedDict
 
+import matplotlib.pyplot as plt
+
 if __name__ == "__main__":
 
     data = pd.read_csv('./data/EURUSD_Candlestick_1_M_BID_01.01.2021-04.02.2023.csv')
@@ -19,10 +21,10 @@ if __name__ == "__main__":
     test = data[date_split:]
 
     initial_money = 10.0
-    
-    learning_rate = 0.01
+
+    learning_rate = 0.001
     first_momentum = 0.0
-    second_momentum = 0.0
+    second_momentum = 0.0001
     transaction_cost = 0.0001
     adaptation_rate = 0.01
     state_size = 15
@@ -47,7 +49,50 @@ if __name__ == "__main__":
     for name, param in agent.named_parameters():
         model_gradients_history.update({name: torch.zeros_like(param)})
 
-    for _ in range(state_size, 1440):
+
+    p = []
+
+
+    for _ in range(state_size, 28800):
+        observation = torch.as_tensor(observation).float()
+        action = agent(observation)
+        observation, reward, _ = env.step(action.data.to("cpu").numpy())
+
+
+        
+        
+        action.backward()
+
+        for name, param in agent.named_parameters():
+        
+            grad_n = param.grad
+            param = param + optimizer.step(grad_n, reward, observation[-1], model_gradients_history[name])
+            checkpoint[name] = param
+            model_gradients_history.update({name: grad_n})
+        
+        p.append(env.profits)
+        optimizer.after_step(reward)
+        agent.load_state_dict(checkpoint)
+
+    ###########
+    ###########
+
+    history = []
+    for i in range(1, state_size):
+        c = test.iloc[i, :]['Close'] - test.iloc[i-1, :]['Close']
+        history.append(c)
+
+    env = Environment(test, history=history, state_size=state_size)
+    observation = env.reset()
+    
+
+    model_gradients_history = dict()
+    checkpoint = OrderedDict()
+
+    for name, param in agent.named_parameters():
+        model_gradients_history.update({name: torch.zeros_like(param)})
+
+    for _ in range(state_size, 14400):
         observation = torch.as_tensor(observation).float()
         action = agent(observation)
         observation, reward, _ = env.step(action.data.numpy())
@@ -64,6 +109,7 @@ if __name__ == "__main__":
             checkpoint[name] = param
             model_gradients_history.update({name: grad_n})
 
+        optimizer.after_step(reward)
         agent.load_state_dict(checkpoint)
 
     print(env.profits)

rl_agent/utils.py

@@ -15,7 +15,7 @@ class myOptimizer():
         numerator = self.mu_square - (self.mu * reward)
         denominator = np.sqrt((self.mu_square - (self.mu ** 2)) ** 3)
 
-        gradient = numerator / (denominator + 1e-8)
+        gradient = numerator / denominator
 
         current_grad = (-1.0 * self.transaction_cost * grad_n)