diff options
Diffstat (limited to 'rl')
| -rw-r--r-- | rl/gym_demo/carl_pole.py | 60 | ||||
| -rw-r--r-- | rl/gym_demo/lunar/dqn.py | 112 | ||||
| -rw-r--r-- | rl/gym_demo/lunar/main.py | 35 | ||||
| -rw-r--r-- | rl/gym_demo/lunar/utils.py | 36 | ||||
| -rw-r--r-- | rl/gym_demo/taxi.py | 0 |
5 files changed, 207 insertions, 36 deletions
diff --git a/rl/gym_demo/carl_pole.py b/rl/gym_demo/carl_pole.py index 146c337..dd22d1d 100644 --- a/rl/gym_demo/carl_pole.py +++ b/rl/gym_demo/carl_pole.py @@ -2,46 +2,34 @@ import gym import numpy as np -class BespokeAgent: - def __init__(self, env): - pass +env_name = 'CartPole-v1' - def decide(self, observation): - position, velocity = observation - lb = min(-0.09*(position + 0.25) ** 2 + 0.03, 0.3*(position + 0.9)**4 - 0.008) - ub = -0.07*(position + 0.38) ** 2 + 0.07 - if lb < velocity < ub: - action = 2 - else: - action = 0 - # print('observation: {}, lb: {}, ub: {} => action: {}'.format(observation, lb, ub, action)) - return action +env = gym.make(env_name) - def learn(self, *argg): - pass +class Agent: + def __init__(self, env): + self.action_size = env.action_space.n -def play(i, agent, env, render=True, train=False): - episode_reward = 0 - observation = env.reset() - while True: - if render: - env.render() - action = agent.decide(observation) - next_observation, reward, done, _ = env.step(action) - episode_reward += reward - if train: - agent.learn(observation, action, reward, done) - if done: - env.close() - break - observation = next_observation - print(i, episode_reward) - return i, episode_reward + def action_policy(self, observation): + pos, vel, angle, _ = observation + if angle < 0: + return 0 + return 1 if __name__ == '__main__': - env = gym.make('MountainCar-v0') - agent = BespokeAgent(env) - rewards = [play(i, agent, env) for i in range(100)] - print(rewards) + + observation = env.reset() + agent = Agent(env) + reward_history = [] + for _ in range(100): + # env.render() + # action = agent.action_policy(observation) + action = env.action_space.sample() + observation, reward, done, info = env.step(action) + reward_history.append(reward) + if done: + # env.env.close() + env.reset() + print(reward_history, np.mean(reward_history)) diff --git a/rl/gym_demo/lunar/dqn.py b/rl/gym_demo/lunar/dqn.py new file mode 100644 index 0000000..a757a92 --- /dev/null +++ b/rl/gym_demo/lunar/dqn.py @@ -0,0 +1,112 @@ +import torch as T +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import numpy as np + + +class DeepQNetwork(nn.Module): + def __init__(self, lr, input_dims, fc1_dims, fc2_dims, + n_actions): + super(DeepQNetwork, self).__init__() + self.input_dims = input_dims + self.fc1_dims = fc1_dims + self.fc2_dims = fc2_dims + self.n_actions = n_actions + self.fc1 = nn.Linear(*self.input_dims, self.fc1_dims) + self.fc2 = nn.Linear(self.fc1_dims, self.fc2_dims) + self.fc3 = nn.Linear(self.fc2_dims, self.n_actions) + + self.optimizer = optim.Adam(self.parameters(), lr=lr) + self.loss = nn.MSELoss() + self.device = T.device('cuda:0' if T.cuda.is_available() else 'cpu') + self.to(self.device) + + def forward(self, state): + x = F.relu(self.fc1(state)) + x = F.relu(self.fc2(x)) + actions = self.fc3(x) + + return actions + + +class Agent: + def __init__(self, gamma, epsilon, lr, input_dims, batch_size, n_actions, + max_mem_size=100000, eps_end=0.05, eps_dec=5e-4): + self.gamma = gamma + self.epsilon = epsilon + self.eps_min = eps_end + self.eps_dec = eps_dec + self.lr = lr + self.action_space = [i for i in range(n_actions)] + self.mem_size = max_mem_size + self.batch_size = batch_size + self.mem_cntr = 0 + self.iter_cntr = 0 + self.replace_target = 100 + + self.Q_eval = DeepQNetwork(lr, n_actions=n_actions, + input_dims=input_dims, + fc1_dims=256, fc2_dims=256) + self.state_memory = np.zeros((self.mem_size, *input_dims), + dtype=np.float32) + self.new_state_memory = np.zeros((self.mem_size, *input_dims), + dtype=np.float32) + self.action_memory = np.zeros(self.mem_size, dtype=np.int32) + self.reward_memory = np.zeros(self.mem_size, dtype=np.float32) + self.terminal_memory = np.zeros(self.mem_size, dtype=np.bool) + + def store_transition(self, state, action, reward, state_, terminal): + index = self.mem_cntr % self.mem_size + self.state_memory[index] = state + self.new_state_memory[index] = state_ + self.reward_memory[index] = reward + self.action_memory[index] = action + self.terminal_memory[index] = terminal + + self.mem_cntr += 1 + + def choose_action(self, observation): + if np.random.random() > self.epsilon: + state = T.tensor([observation]).to(self.Q_eval.device) + actions = self.Q_eval.forward(state) + action = T.argmax(actions).item() + else: + action = np.random.choice(self.action_space) + + return action + + def learn(self): + if self.mem_cntr < self.batch_size: + return + + self.Q_eval.optimizer.zero_grad() + + max_mem = min(self.mem_cntr, self.mem_size) + + batch = np.random.choice(max_mem, self.batch_size, replace=False) + batch_index = np.arange(self.batch_size, dtype=np.int32) + + state_batch = T.tensor(self.state_memory[batch]).to(self.Q_eval.device) + new_state_batch = T.tensor( + self.new_state_memory[batch]).to(self.Q_eval.device) + action_batch = self.action_memory[batch] + reward_batch = T.tensor( + self.reward_memory[batch]).to(self.Q_eval.device) + terminal_batch = T.tensor( + self.terminal_memory[batch]).to(self.Q_eval.device) + + q_eval = self.Q_eval.forward(state_batch)[batch_index, action_batch] + q_next = self.Q_eval.forward(new_state_batch) + q_next[terminal_batch] = 0.0 + + q_target = reward_batch + self.gamma*T.max(q_next, dim=1)[0] + + loss = self.Q_eval.loss(q_target, q_eval).to(self.Q_eval.device) + loss.backward() + self.Q_eval.optimizer.step() + + self.iter_cntr += 1 + self.epsilon = self.epsilon - self.eps_dec \ + if self.epsilon > self.eps_min else self.eps_min + diff --git a/rl/gym_demo/lunar/main.py b/rl/gym_demo/lunar/main.py new file mode 100644 index 0000000..b718b71 --- /dev/null +++ b/rl/gym_demo/lunar/main.py @@ -0,0 +1,35 @@ +import gym +from dqn import Agent +from utils import plotLearning +import numpy as np + +if __name__ == '__main__': + env = gym.make('LunarLander-v2') + agent = Agent(gamma=0.99, epsilon=1.0, batch_size=64, n_actions=4, eps_end=0.01, + input_dims=[8], lr=0.001) + scores, eps_history = [], [] + n_games = 500 + + for i in range(n_games): + score = 0 + done = False + observation = env.reset() + while not done: + action = agent.choose_action(observation) + observation_, reward, done, info = env.step(action) + score += reward + agent.store_transition(observation, action, reward, + observation_, done) + agent.learn() + observation = observation_ + scores.append(score) + eps_history.append(agent.epsilon) + + avg_score = np.mean(scores[-100:]) + + print('episode ', i, 'score %.2f' % score, + 'average score %.2f' % avg_score, + 'epsilon %.2f' % agent.epsilon) + x = [i + 1 for i in range(n_games)] + filename = 'lunar_lander.png' + plotLearning(x, scores, eps_history, filename) diff --git a/rl/gym_demo/lunar/utils.py b/rl/gym_demo/lunar/utils.py new file mode 100644 index 0000000..e881c76 --- /dev/null +++ b/rl/gym_demo/lunar/utils.py @@ -0,0 +1,36 @@ +import matplotlib.pyplot as plt +import numpy as np +import gym + +def plotLearning(x, scores, epsilons, filename, lines=None): + fig=plt.figure() + ax=fig.add_subplot(111, label="1") + ax2=fig.add_subplot(111, label="2", frame_on=False) + + ax.plot(x, epsilons, color="C0") + ax.set_xlabel("Game", color="C0") + ax.set_ylabel("Epsilon", color="C0") + ax.tick_params(axis='x', colors="C0") + ax.tick_params(axis='y', colors="C0") + + N = len(scores) + running_avg = np.empty(N) + for t in range(N): + running_avg[t] = np.mean(scores[max(0, t-20):(t+1)]) + + ax2.scatter(x, running_avg, color="C1") + #ax2.xaxis.tick_top() + ax2.axes.get_xaxis().set_visible(False) + ax2.yaxis.tick_right() + #ax2.set_xlabel('x label 2', color="C1") + ax2.set_ylabel('Score', color="C1") + #ax2.xaxis.set_label_position('top') + ax2.yaxis.set_label_position('right') + #ax2.tick_params(axis='x', colors="C1") + ax2.tick_params(axis='y', colors="C1") + + if lines is not None: + for line in lines: + plt.axvline(x=line) + + plt.savefig(filename)
\ No newline at end of file diff --git a/rl/gym_demo/taxi.py b/rl/gym_demo/taxi.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/rl/gym_demo/taxi.py |