Updated the scripts to make it compatible with the library

spinup/algos/pytorch/vpg/core.py

@@ -1,6 +1,6 @@
 import numpy as np
 import scipy.signal
-from gym.spaces import Box, Discrete
+from gymnasium.spaces import Box, Discrete
 
 import torch
 import torch.nn as nn

spinup/algos/pytorch/vpg/vpg.py

@@ -1,7 +1,7 @@
 import numpy as np
 import torch
 from torch.optim import Adam
-import gym
+import gymnasium as gym
 import time
 import spinup.algos.pytorch.vpg.core as core
 from spinup.utils.logx import EpochLogger
@@ -15,8 +15,10 @@ class VPGBuffer:
     with the environment, and using Generalized Advantage Estimation (GAE-Lambda)
     for calculating the advantages of state-action pairs.
     """
-
-    def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):
+    # gamme: the dicount paramter that is used for advantage estimation
+    # lambda: again the control parameter used in Generalized Advantage Estimation
+    # cite Schulman et.al, 2016.
+    def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95): 
         self.obs_buf = np.zeros(core.combined_shape(size, obs_dim), dtype=np.float32)
         self.act_buf = np.zeros(core.combined_shape(size, act_dim), dtype=np.float32)
         self.adv_buf = np.zeros(size, dtype=np.float32)
@@ -156,14 +158,15 @@ def vpg(env_fn, actor_critic=core.MLPActorCritic, ac_kwargs=dict(),  seed=0,
             number of policy updates) to perform.
 
         gamma (float): Discount factor. (Always between 0 and 1.)
-
+        *The next 2 params are very important in the sense that it shows how 2 models are trained simultaneosuly
+        *one for training the policy and another for obtaining the value function.
         pi_lr (float): Learning rate for policy optimizer.
 
         vf_lr (float): Learning rate for value function optimizer.
 
         train_v_iters (int): Number of gradient descent steps to take on 
             value function per epoch.
-
+        *GAE: Generalized Advantage Estimation
         lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,
             close to 1.)
 
@@ -329,12 +332,12 @@ def update():
     import argparse
     parser = argparse.ArgumentParser()
     parser.add_argument('--env', type=str, default='HalfCheetah-v2')
-    parser.add_argument('--hid', type=int, default=64)
-    parser.add_argument('--l', type=int, default=2)
-    parser.add_argument('--gamma', type=float, default=0.99)
+    parser.add_argument('--hid', type=int, default=64) # This is essentially the hidden dimensions of the intermediate layer
+    parser.add_argument('--l', type=int, default=2) # This is the number of hidden layers
+    parser.add_argument('--gamma', type=float, default=0.99) #The discount factor
     parser.add_argument('--seed', '-s', type=int, default=0)
     parser.add_argument('--cpu', type=int, default=4)
-    parser.add_argument('--steps', type=int, default=4000)
+    parser.add_argument('--steps', type=int, default=4000) # This can be thought like the ones that are used to determine the episode length, the actual length can <= the one specified here
     parser.add_argument('--epochs', type=int, default=50)
     parser.add_argument('--exp_name', type=str, default='vpg')
     args = parser.parse_args()

spinup/examples/pytorch/pg_math/1_simple_pg.py

@@ -3,9 +3,13 @@
 from torch.distributions.categorical import Categorical
 from torch.optim import Adam
 import numpy as np
-import gym
-from gym.spaces import Discrete, Box
+import gymnasium as gym
+from gymnasium.spaces import Discrete, Box
 
+'''
+Below is defined the network which is essentially the policy for the agent, it is a neural network which takes into 
+account the observation dimensions.
+'''
 def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity):
     # Build a feedforward neural network.
     layers = []
@@ -14,25 +18,29 @@ def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity):
         layers += [nn.Linear(sizes[j], sizes[j+1]), act()]
     return nn.Sequential(*layers)
 
-def train(env_name='CartPole-v0', hidden_sizes=[32], lr=1e-2, 
+# The env-name is crucial and tells the code the pre-defined environment that will be used for testing the algorithm
+def train(env_name='CartPole-v1', hidden_sizes=[32], lr=1e-2, 
           epochs=50, batch_size=5000, render=False):
 
     # make environment, check spaces, get obs / act dims
     env = gym.make(env_name)
+    # This right here is demarcating the continuous space represented by the box and discrete ones represented by Discrete
     assert isinstance(env.observation_space, Box), \
         "This example only works for envs with continuous state spaces."
     assert isinstance(env.action_space, Discrete), \
         "This example only works for envs with discrete action spaces."
-
+    # the observation space dimension is the input and the action space is the output. For language this can be a discrete vocabulary
+    # this leads to the intuition of designing rewards based on tokens rather than the whole sentences which by itself is also an action space.
+    #n_acts defines the action space of the environment left or right
     obs_dim = env.observation_space.shape[0]
     n_acts = env.action_space.n
-
     # make core of policy network
     logits_net = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts])
 
     # make function to compute action distribution
     def get_policy(obs):
         logits = logits_net(obs)
+        #This is a function which returns the action by sampling from the logits by modeling it as a Categorical distribution
         return Categorical(logits=logits)
 
     # make action selection function (outputs int actions, sampled from policy)
@@ -44,7 +52,7 @@ def compute_loss(obs, act, weights):
         logp = get_policy(obs).log_prob(act)
         return -(logp * weights).mean()
 
-    # make optimizer
+    # make optimizer, this will be used for training the policy
     optimizer = Adam(logits_net.parameters(), lr=lr)
 
     # for training policy
@@ -55,54 +63,48 @@ def train_one_epoch():
         batch_weights = []      # for R(tau) weighting in policy gradient
         batch_rets = []         # for measuring episode returns
         batch_lens = []         # for measuring episode lengths
-
+        ep_rews = []                # list for rewards accrued throughout ep
         # reset episode-specific variables
-        obs = env.reset()       # first obs comes from starting distribution
-        done = False            # signal from environment that episode is over
-        ep_rews = []            # list for rewards accrued throughout ep
-
+        obs, info = env.reset()     # first obs comes from starting distribution
+        done = False                # signal from environment that episode is over
+
         # render first episode of each epoch
         finished_rendering_this_epoch = False
 
-        # collect experience by acting in the environment with current policy
+        # collect experience by acting in the environment with current policy, this thing is called an episode
         while True:
-
             # rendering
             if (not finished_rendering_this_epoch) and render:
                 env.render()
 
             # save obs
-            batch_obs.append(obs.copy())
+            batch_obs.append(obs.copy()) # Copying the observations array of the tuple
 
             # act in the environment
             act = get_action(torch.as_tensor(obs, dtype=torch.float32))
-            obs, rew, done, _ = env.step(act)
-
+            #Every step of the action gives the rewards from the environment
+            obs, reward, done, _, _ = env.step(act)
             # save action, reward
             batch_acts.append(act)
-            ep_rews.append(rew)
-
+            ep_rews.append(reward)
             if done:
                 # if episode is over, record info about episode
                 ep_ret, ep_len = sum(ep_rews), len(ep_rews)
                 batch_rets.append(ep_ret)
                 batch_lens.append(ep_len)
-
+                print("Episode Length", ep_len)
                 # the weight for each logprob(a|s) is R(tau)
-                batch_weights += [ep_ret] * ep_len
+                batch_weights+= [ep_ret] * ep_len
 
                 # reset episode-specific variables
-                obs, done, ep_rews = env.reset(), False, []
-
+                obs, info = env.reset()
+                done, ep_rews = False, []
                 # won't render again this epoch
                 finished_rendering_this_epoch = True
-
                 # end experience loop if we have enough of it
                 if len(batch_obs) > batch_size:
                     break
-
         # take a single policy gradient update step
-        optimizer.zero_grad()
         batch_loss = compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),
                                   act=torch.as_tensor(batch_acts, dtype=torch.int32),
                                   weights=torch.as_tensor(batch_weights, dtype=torch.float32)
@@ -120,7 +122,7 @@ def train_one_epoch():
 if __name__ == '__main__':
     import argparse
     parser = argparse.ArgumentParser()
-    parser.add_argument('--env_name', '--env', type=str, default='CartPole-v0')
+    parser.add_argument('--env_name', '--env', type=str, default='CartPole-v1')
     parser.add_argument('--render', action='store_true')
     parser.add_argument('--lr', type=float, default=1e-2)
     args = parser.parse_args()

spinup/examples/pytorch/pg_math/2_rtg_pg.py

@@ -3,8 +3,8 @@
 from torch.distributions.categorical import Categorical
 from torch.optim import Adam
 import numpy as np
-import gym
-from gym.spaces import Discrete, Box
+import gymnasium as gym
+from gymnasium.spaces import Discrete, Box
 
 def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity):
     # Build a feedforward neural network.
@@ -21,7 +21,7 @@ def reward_to_go(rews):
         rtgs[i] = rews[i] + (rtgs[i+1] if i+1 < n else 0)
     return rtgs
 
-def train(env_name='CartPole-v0', hidden_sizes=[32], lr=1e-2, 
+def train(env_name='CartPole-v1', hidden_sizes=[32], lr=1e-2, 
           epochs=50, batch_size=5000, render=False):
 
     # make environment, check spaces, get obs / act dims
@@ -64,7 +64,7 @@ def train_one_epoch():
         batch_lens = []         # for measuring episode lengths
 
         # reset episode-specific variables
-        obs = env.reset()       # first obs comes from starting distribution
+        obs, info = env.reset()       # first obs comes from starting distribution
         done = False            # signal from environment that episode is over
         ep_rews = []            # list for rewards accrued throughout ep
 
@@ -83,11 +83,11 @@ def train_one_epoch():
 
             # act in the environment
             act = get_action(torch.as_tensor(obs, dtype=torch.float32))
-            obs, rew, done, _ = env.step(act)
+            obs, reward, done, _, _ = env.step(act)
 
             # save action, reward
             batch_acts.append(act)
-            ep_rews.append(rew)
+            ep_rews.append(reward)
 
             if done:
                 # if episode is over, record info about episode
@@ -99,8 +99,8 @@ def train_one_epoch():
                 batch_weights += list(reward_to_go(ep_rews))
 
                 # reset episode-specific variables
-                obs, done, ep_rews = env.reset(), False, []
-
+                obs, _   = env.reset()
+                done, ep_rews = False, []
                 # won't render again this epoch
                 finished_rendering_this_epoch = True
 
@@ -127,7 +127,7 @@ def train_one_epoch():
 if __name__ == '__main__':
     import argparse
     parser = argparse.ArgumentParser()
-    parser.add_argument('--env_name', '--env', type=str, default='CartPole-v0')
+    parser.add_argument('--env_name', '--env', type=str, default='CartPole-v1')
     parser.add_argument('--render', action='store_true')
     parser.add_argument('--lr', type=float, default=1e-2)
     args = parser.parse_args()