\documentclass[12pt]{article}
\usepackage{fullpage,enumitem,amsmath,amssymb,graphicx,url,hyperref}
\usepackage{sectsty}

\hypersetup{
    colorlinks=true,
    linkcolor=blue,
}

\sectionfont{\fontsize{15}{20}\selectfont}

\DeclareMathOperator*{\argmax}{arg\,max}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator{\E}{\mathbb{E}}
\usepackage{tgpagella}
\usepackage{tcolorbox}
\usepackage{forest}
\usepackage{booktabs}

\begin{document}

\begin{center}
{\Large \textbf{CS224R Spring 2025 Homework 2 \\ Online Reinforcement Learning}}
\\ {\large Due 5/2/2025}

\begin{tabular}{rl}
SUNet ID: & $\hspace{6cm}$\\
Name: & \\
Collaborators: & 
\end{tabular}
\end{center}

\noindent By turning in this assignment, I agree by the Stanford honor code and declare
that all of this is my own work.

\newpage
\section*{Problem 1: Impact of Reward Functions}
\begin{enumerate}
\item Design a reward function for the Quadruped task such that the agent walks in a clockwise circle (watch \texttt{mujoco\_mpc/videos/part1.avi} for an example of the desired behavior). The structure of the reward function for the Quadruped task is:
\begin{align*}
    r_t = -w_0 \cdot r_{t, \text{height}} - w_1 \cdot r_{t, \text{pos}}(w_2, w_3) + c
\end{align*}
where
\begin{itemize}
    \item $r_{t, \text{height}}$ is the absolute difference between the agent's torso height over its feet and the target height of $1$,
    \item $r_{t, \text{pos}}(w_2, w_3)$ is the $\ell^2$ norm of the difference between the agent's torso position and the goal position. The goal moves at each time-step according to the desired \emph{walk speed} $w_2$ and \emph{walk direction} $w_3$, and
    \item $c$ consists of other reward terms for balance, effort, and posture.
\end{itemize}

You will design the reward function by choosing values for $w_0, w_1, w_2,$ and $w_3$, which can be any real number. Here is how you can run the Quadruped task with $w_0 = w_1 = w_2 = w_3 = 0$:
\begin{tcolorbox}[width=\linewidth, sharp corners=all, colback=white!95!black]
\begin{verbatim}
./build/bin/mjpc --task="Quadruped Flat" --steps=100 \
    --horizon=0.35 --w0=0.0 --w1=0.0 --w2=0.0 --w3=0.0
\end{verbatim}
\end{tcolorbox}

\vspace{0.2cm}

The program will run the simulator for the specified number of time-steps and save a video in the \texttt{mujoco\_mpc/videos/} directory.
\\
\\
Fill in Table~\ref{table:Problem_1} with the parameters of your reward function.
\begin{table}[h]
\centering
\begin{tabular}{cc}
    \toprule
    Parameter & Value \\
    \midrule
    $w_0$ & \textbf{\color{red}Your answer here} \\
    $w_1$ & \textbf{\color{red}Your answer here} \\
    $w_2$ & \textbf{\color{red}Your answer here} \\
    $w_3$ & \textbf{\color{red}Your answer here} \\
    \bottomrule
\end{tabular}
\label{table:Problem_1}
\caption{Parameters of your reward function.}
\end{table}

\item In this next part, you will see how different reward functions impact the agent's behavior in the In-Hand Manipulation task. The structure of the reward function for the Hand task is:
\begin{align*}
    r_t = -w_0 \cdot r_{t, \text{cube pos}} - w_1 \cdot r_{t, \text{cube ori}} - w_2 \cdot r_{t, \text{cube vel}} - w_3 \cdot r_{t, \text{actuator}}
\end{align*}
where
\begin{itemize}
    \item $r_{t, \text{cube pos}}$ is the $\ell^2$ norm of the difference between the cube's position and the hand palm position,
    \item $r_{t, \text{cube ori}}$ is the $\ell^2$ norm of the difference between the cube's orientation and the goal orientation,
    \item $r_{t, \text{cube vel}}$ is the $\ell^2$ norm of the cube's linear velocity, and
    \item $r_{t, \text{actuator}}$ is the $\ell^2$ norm of the control inputs (i.e., the forces applied that cause the robot to move).
\end{itemize}
For each part below, you will watch the videos located in \texttt{mujoco\_mpc/videos}. Then, in 1-2 sentences, describe the agent's behavior and why the reward function leads to this behavior.

\begin{enumerate}[itemsep=20pt]
    \item Watch \texttt{mujoco\_mpc/videos/part2a.avi}, which was generated with the parameters $w_0 = 20$, $w_1 = 3$, $w_2 = 10$, and $w_3 = 0.1$:
\begin{tcolorbox}[width=\linewidth, sharp corners=all, colback=white!95!black]
\begin{verbatim}
./build/bin/mjpc --task="Hand" --steps=100 \
    --horizon=2.5 --w0=20.0 --w1=3.0 --w2=10.0 --w3=0.1
\end{verbatim}
\end{tcolorbox}

\textbf{\color{red}Describe the agent's behavior and why the reward function leads to this behavior in 1-2 sentences.}

    \item Watch \texttt{mujoco\_mpc/videos/part2b.avi}, which was generated with the parameters $w_0 = 20$, $w_1 = 3$, $w_2 = 10$, and $w_3 = 1$:
\begin{tcolorbox}[width=\linewidth, sharp corners=all, colback=white!95!black]
\begin{verbatim}
./build/bin/mjpc --task="Hand" --steps=100 \
    --horizon=0.25 --w0=20.0 --w1=3.0 --w2=10.0 --w3=1.0
\end{verbatim}
\end{tcolorbox}
\textbf{\color{red}Describe the agent's behavior and why the reward function leads to this behavior in 1-2 sentences.}

    \item Watch \texttt{mujoco\_mpc/videos/part2c.avi}, which was generated with the parameters $w_0 = 0$, $w_1 = 0$, $w_2 = 0$, and $w_3 = 1$:
\begin{tcolorbox}[width=\linewidth, sharp corners=all, colback=white!95!black]
\begin{verbatim}
./build/bin/mjpc --task="Hand" --steps=100 \
    --horizon=2.5 --w0=0.0 --w1=0.0 --w2=0.0 --w3=1.0
\end{verbatim}
\end{tcolorbox}
\textbf{Hint:} An episode terminates when the cube falls out of the hand.\\
\\
\textbf{\color{red}Describe the agent's behavior and why the reward function leads to this behavior in 1-2 sentences.}
    
\end{enumerate}

\end{enumerate}

\newpage

\section*{Problem 2: }
\begin{enumerate}
\item Optimizing behavior in environments with sparse rewards is difficult due to limited reward supervision. To alleviate this, we provide the agent with 20 successful demonstrations, which we will use to pre-train with behavior cloning. In \textbf{ac.py} complete the \textbf{bc} function of the \textbf{ACAgent} class to train the policy using supervised behavior cloning. That is, given state-action pairs $o_t, a_t$ optimize the loss
$$\mathcal{L}_{\pi_{\theta}, f_{\theta}}(o_t, a_t) = -\log \pi_{\theta}(a_t|o_t)$$
We will also use this to update the actor during later RL training, to improve stability.

\item In the second part, we will try to improve the performance of the policy with additional fine-tuning with reinforcement learning. Reinforcement learning allows the robot to improve using its own experience, without needing additional demonstrations. Your implementation will be in the \textbf{ACAgent} class.

\begin{itemize}

    \item We begin by implementing the \textbf{update\_critic} method using the Bellman objective. Consider transitions $(o_t, a_t, r_t, o_{t+1}, \gamma)$ and implement the following steps:
    \begin{enumerate}
        \item Sample next state actions from the policy $a_{t+1}'\sim \pi_{\theta}(o_{t+1})$.
        \item Compute the Bellman targets $$y = r_t + \gamma\min\{\bar{Q}_{\theta^i}(o_{t+1}, a_{t+1}'), \bar{Q}_{\theta^j}(o_{t+1}, a_{t+1}')\}$$

        where $\bar{Q}_{\theta^i}$ and $\bar{Q}_{\theta^j}$ are two randomly sampled target critics. We take the minimum over two Q-values to mitigate critic overestimation errors.
        \item Compute the loss:
            
            $$\mathcal{L}_{Q_{\theta}, f_{\theta}} = \sum_{i=1}^N (Q_{\theta^i}(o_{t}, a_t)-\mathbf{sg}(y))^2$$
        where $\mathbf{sg}$ stands for the stop gradient operator.
        \item Take a gradient step with respect to the critic parameters.
        \item Update the target critic parameters using exponential moving average.
        $$\bar{Q}_{\theta^i} = (1-\tau)\bar{Q}_{\theta^i} + \tau Q_{\theta^i}$$
        By using an exponential moving average, our target critic parameters are updated more slowly than those of the main critic.
    
    \end{enumerate}
    
    \item Next, we will improve the policy in the \textbf{update\_actor} method. Sample an action from the actor $a_t'\sim\pi_{\theta}(o_t)$ and compute the objective that optimizes the actor to maximize the Q-value estimates from the critics:

    $$\mathcal{L}_{\pi_{\theta}} = -\frac{1}{N}\sum_{i=1}^NQ_{\theta^i}(o_t, a_t')$$

    Take a gradient step on this objective with respect to the policy only.

\item Once you are done, run the RL fine-tuning with

\begin{tcolorbox}[width=\linewidth, sharp corners=all, colback=white!95!black]
\begin{verbatim}
python train.py agent.num_critics=2 utd=1
\end{verbatim}
\end{tcolorbox}

\textbf{\color{red}Attach the plot ``eval/episode\_success" for this run from Tensorboard. You should achieve a success rate of at least 90\% before 100K environment steps.}

\item In the final part, we will explore some optimization parameter choices. The update-to-data (UTD) ratio stands for the number of critic gradient steps we do, with respect to each environment step. So far we have used only 2 critics and UTD of 1. Repeat the previous part with:

\begin{tcolorbox}[width=\linewidth, sharp corners=all, colback=white!95!black]
\begin{verbatim}
python train.py agent.num_critics=10 utd=5
\end{verbatim}
\end{tcolorbox}
Here, we perform critic gradient steps 5x more often. However, too many gradient steps can make learning unstable by increasing error in the critic Q-values.

\textbf{\color{red}Attach the plot ``eval/episode\_success" for this run and compare it to the previous question. Provide an explanation of why we observe these effects.}

\end{itemize}

\end{enumerate}

\end{document}
