diff --git a/spinup/algos/pytorch/vpg/core.py b/spinup/algos/pytorch/vpg/core.py index 84e9f9889..1710eb36a 100644 --- a/spinup/algos/pytorch/vpg/core.py +++ b/spinup/algos/pytorch/vpg/core.py @@ -1,6 +1,6 @@ import numpy as np import scipy.signal -from gym.spaces import Box, Discrete +from gymnasium.spaces import Box, Discrete import torch import torch.nn as nn diff --git a/spinup/algos/pytorch/vpg/vpg.py b/spinup/algos/pytorch/vpg/vpg.py index 4639b56ae..4d918340f 100644 --- a/spinup/algos/pytorch/vpg/vpg.py +++ b/spinup/algos/pytorch/vpg/vpg.py @@ -1,7 +1,7 @@ import numpy as np import torch from torch.optim import Adam -import gym +import gymnasium as gym import time import spinup.algos.pytorch.vpg.core as core from spinup.utils.logx import EpochLogger @@ -15,8 +15,10 @@ class VPGBuffer: with the environment, and using Generalized Advantage Estimation (GAE-Lambda) for calculating the advantages of state-action pairs. """ - - def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95): + # gamme: the dicount paramter that is used for advantage estimation + # lambda: again the control parameter used in Generalized Advantage Estimation + # cite Schulman et.al, 2016. + def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95): self.obs_buf = np.zeros(core.combined_shape(size, obs_dim), dtype=np.float32) self.act_buf = np.zeros(core.combined_shape(size, act_dim), dtype=np.float32) self.adv_buf = np.zeros(size, dtype=np.float32) @@ -156,14 +158,15 @@ def vpg(env_fn, actor_critic=core.MLPActorCritic, ac_kwargs=dict(), seed=0, number of policy updates) to perform. gamma (float): Discount factor. (Always between 0 and 1.) - + *The next 2 params are very important in the sense that it shows how 2 models are trained simultaneosuly + *one for training the policy and another for obtaining the value function. pi_lr (float): Learning rate for policy optimizer. vf_lr (float): Learning rate for value function optimizer. train_v_iters (int): Number of gradient descent steps to take on value function per epoch. - + *GAE: Generalized Advantage Estimation lam (float): Lambda for GAE-Lambda. (Always between 0 and 1, close to 1.) @@ -329,12 +332,12 @@ def update(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--env', type=str, default='HalfCheetah-v2') - parser.add_argument('--hid', type=int, default=64) - parser.add_argument('--l', type=int, default=2) - parser.add_argument('--gamma', type=float, default=0.99) + parser.add_argument('--hid', type=int, default=64) # This is essentially the hidden dimensions of the intermediate layer + parser.add_argument('--l', type=int, default=2) # This is the number of hidden layers + parser.add_argument('--gamma', type=float, default=0.99) #The discount factor parser.add_argument('--seed', '-s', type=int, default=0) parser.add_argument('--cpu', type=int, default=4) - parser.add_argument('--steps', type=int, default=4000) + parser.add_argument('--steps', type=int, default=4000) # This can be thought like the ones that are used to determine the episode length, the actual length can <= the one specified here parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--exp_name', type=str, default='vpg') args = parser.parse_args() diff --git a/spinup/examples/pytorch/pg_math/1_simple_pg.py b/spinup/examples/pytorch/pg_math/1_simple_pg.py index af4bbbd34..4763537c1 100644 --- a/spinup/examples/pytorch/pg_math/1_simple_pg.py +++ b/spinup/examples/pytorch/pg_math/1_simple_pg.py @@ -3,9 +3,13 @@ from torch.distributions.categorical import Categorical from torch.optim import Adam import numpy as np -import gym -from gym.spaces import Discrete, Box +import gymnasium as gym +from gymnasium.spaces import Discrete, Box +''' +Below is defined the network which is essentially the policy for the agent, it is a neural network which takes into +account the observation dimensions. +''' def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity): # Build a feedforward neural network. layers = [] @@ -14,25 +18,29 @@ def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity): layers += [nn.Linear(sizes[j], sizes[j+1]), act()] return nn.Sequential(*layers) -def train(env_name='CartPole-v0', hidden_sizes=[32], lr=1e-2, +# The env-name is crucial and tells the code the pre-defined environment that will be used for testing the algorithm +def train(env_name='CartPole-v1', hidden_sizes=[32], lr=1e-2, epochs=50, batch_size=5000, render=False): # make environment, check spaces, get obs / act dims env = gym.make(env_name) + # This right here is demarcating the continuous space represented by the box and discrete ones represented by Discrete assert isinstance(env.observation_space, Box), \ "This example only works for envs with continuous state spaces." assert isinstance(env.action_space, Discrete), \ "This example only works for envs with discrete action spaces." - + # the observation space dimension is the input and the action space is the output. For language this can be a discrete vocabulary + # this leads to the intuition of designing rewards based on tokens rather than the whole sentences which by itself is also an action space. + #n_acts defines the action space of the environment left or right obs_dim = env.observation_space.shape[0] n_acts = env.action_space.n - # make core of policy network logits_net = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts]) # make function to compute action distribution def get_policy(obs): logits = logits_net(obs) + #This is a function which returns the action by sampling from the logits by modeling it as a Categorical distribution return Categorical(logits=logits) # make action selection function (outputs int actions, sampled from policy) @@ -44,7 +52,7 @@ def compute_loss(obs, act, weights): logp = get_policy(obs).log_prob(act) return -(logp * weights).mean() - # make optimizer + # make optimizer, this will be used for training the policy optimizer = Adam(logits_net.parameters(), lr=lr) # for training policy @@ -55,54 +63,48 @@ def train_one_epoch(): batch_weights = [] # for R(tau) weighting in policy gradient batch_rets = [] # for measuring episode returns batch_lens = [] # for measuring episode lengths - + ep_rews = [] # list for rewards accrued throughout ep # reset episode-specific variables - obs = env.reset() # first obs comes from starting distribution - done = False # signal from environment that episode is over - ep_rews = [] # list for rewards accrued throughout ep - + obs, info = env.reset() # first obs comes from starting distribution + done = False # signal from environment that episode is over + # render first episode of each epoch finished_rendering_this_epoch = False - # collect experience by acting in the environment with current policy + # collect experience by acting in the environment with current policy, this thing is called an episode while True: - # rendering if (not finished_rendering_this_epoch) and render: env.render() # save obs - batch_obs.append(obs.copy()) + batch_obs.append(obs.copy()) # Copying the observations array of the tuple # act in the environment act = get_action(torch.as_tensor(obs, dtype=torch.float32)) - obs, rew, done, _ = env.step(act) - + #Every step of the action gives the rewards from the environment + obs, reward, done, _, _ = env.step(act) # save action, reward batch_acts.append(act) - ep_rews.append(rew) - + ep_rews.append(reward) if done: # if episode is over, record info about episode ep_ret, ep_len = sum(ep_rews), len(ep_rews) batch_rets.append(ep_ret) batch_lens.append(ep_len) - + print("Episode Length", ep_len) # the weight for each logprob(a|s) is R(tau) - batch_weights += [ep_ret] * ep_len + batch_weights+= [ep_ret] * ep_len # reset episode-specific variables - obs, done, ep_rews = env.reset(), False, [] - + obs, info = env.reset() + done, ep_rews = False, [] # won't render again this epoch finished_rendering_this_epoch = True - # end experience loop if we have enough of it if len(batch_obs) > batch_size: break - # take a single policy gradient update step - optimizer.zero_grad() batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32), act=torch.as_tensor(batch_acts, dtype=torch.int32), weights=torch.as_tensor(batch_weights, dtype=torch.float32) @@ -120,7 +122,7 @@ def train_one_epoch(): if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() - parser.add_argument('--env_name', '--env', type=str, default='CartPole-v0') + parser.add_argument('--env_name', '--env', type=str, default='CartPole-v1') parser.add_argument('--render', action='store_true') parser.add_argument('--lr', type=float, default=1e-2) args = parser.parse_args() diff --git a/spinup/examples/pytorch/pg_math/2_rtg_pg.py b/spinup/examples/pytorch/pg_math/2_rtg_pg.py index 5fc5dcfd5..7cdd9f60b 100644 --- a/spinup/examples/pytorch/pg_math/2_rtg_pg.py +++ b/spinup/examples/pytorch/pg_math/2_rtg_pg.py @@ -3,8 +3,8 @@ from torch.distributions.categorical import Categorical from torch.optim import Adam import numpy as np -import gym -from gym.spaces import Discrete, Box +import gymnasium as gym +from gymnasium.spaces import Discrete, Box def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity): # Build a feedforward neural network. @@ -21,7 +21,7 @@ def reward_to_go(rews): rtgs[i] = rews[i] + (rtgs[i+1] if i+1 < n else 0) return rtgs -def train(env_name='CartPole-v0', hidden_sizes=[32], lr=1e-2, +def train(env_name='CartPole-v1', hidden_sizes=[32], lr=1e-2, epochs=50, batch_size=5000, render=False): # make environment, check spaces, get obs / act dims @@ -64,7 +64,7 @@ def train_one_epoch(): batch_lens = [] # for measuring episode lengths # reset episode-specific variables - obs = env.reset() # first obs comes from starting distribution + obs, info = env.reset() # first obs comes from starting distribution done = False # signal from environment that episode is over ep_rews = [] # list for rewards accrued throughout ep @@ -83,11 +83,11 @@ def train_one_epoch(): # act in the environment act = get_action(torch.as_tensor(obs, dtype=torch.float32)) - obs, rew, done, _ = env.step(act) + obs, reward, done, _, _ = env.step(act) # save action, reward batch_acts.append(act) - ep_rews.append(rew) + ep_rews.append(reward) if done: # if episode is over, record info about episode @@ -99,8 +99,8 @@ def train_one_epoch(): batch_weights += list(reward_to_go(ep_rews)) # reset episode-specific variables - obs, done, ep_rews = env.reset(), False, [] - + obs, _ = env.reset() + done, ep_rews = False, [] # won't render again this epoch finished_rendering_this_epoch = True @@ -127,7 +127,7 @@ def train_one_epoch(): if __name__ == '__main__': import argparse parser = argparse.ArgumentParser() - parser.add_argument('--env_name', '--env', type=str, default='CartPole-v0') + parser.add_argument('--env_name', '--env', type=str, default='CartPole-v1') parser.add_argument('--render', action='store_true') parser.add_argument('--lr', type=float, default=1e-2) args = parser.parse_args()