Chapter 6.2 Random Walk

7.2 μs

xxxxxxxxxx
 
begin
    using ReinforcementLearning
    using Flux
    using Statistics
    using Plots
end

49.9 s

In this section, we'll use the RandomWalk1D env provided in ReinforcementLearning.

7.3 μs

env

# RandomWalk1D

## Traits

| Trait Type        |                                          Value |
|:----------------- | ----------------------------------------------:|
| NumAgentStyle     |        ReinforcementLearningBase.SingleAgent() |
| DynamicStyle      |         ReinforcementLearningBase.Sequential() |
| InformationStyle  | ReinforcementLearningBase.PerfectInformation() |
| ChanceStyle       |      ReinforcementLearningBase.Deterministic() |
| RewardStyle       |     ReinforcementLearningBase.TerminalReward() |
| UtilityStyle      |         ReinforcementLearningBase.GeneralSum() |
| ActionStyle       |   ReinforcementLearningBase.MinimalActionSet() |
| StateStyle        | ReinforcementLearningBase.Observation{Int64}() |
| DefaultStateStyle | ReinforcementLearningBase.Observation{Int64}() |

## Is Environment Terminated?

No

## State Space

`Base.OneTo(7)`

## Action Space

`Base.OneTo(2)`

## Current State

```
4
```

xxxxxxxxxx
 
env = RandomWalk1D(;rewards=0.0=>1.0)

26.5 ms

xxxxxxxxxx
 
NS, NA = length(state_space(env)), length(action_space(env))

512 ns

As is explained in the book, the true values of state A to E are:

8.1 μs

true_values

Float641

0.166667

0.333333

0.5

0.666667

0.833333

xxxxxxxxxx
 
true_values = [i/6 for i in 1:5]

1.6 μs

To estimate the state values, we'll use the VBasedPolicy with a random action generator.

5.1 μs

create_TD_agent (generic function with 1 method)

xxxxxxxxxx
 
create_TD_agent(α) = Agent(
    policy=VBasedPolicy(
        learner = TDLearner(
            approximator=TabularApproximator(fill(0.5, NS), Descent(α)),
            method=:SRS,
            γ=1.0,
            n=0,
        ),
        mapping = (env, V) -> rand(1:NA)
    ),
    trajectory=VectorSARTTrajectory()
)

50.7 μs

xxxxxxxxxx
 
begin
    p_6_2_left = plot(;legend=:bottomright)
    for i in [1, 10, 100]
        agent = create_TD_agent(0.1)
        run(agent, env, StopAfterEpisode(i))
        plot!(
            p_6_2_left,
            agent.policy.learner.approximator.table[2:end - 1],
            label="episode = $i"
        )
    end
    plot!(p_6_2_left, true_values, label="true value")
    p_6_2_left
end

4.7 s

To calculate the RMS error, we need to define such a hook first.

4.1 μs

RecordRMS

xxxxxxxxxx
 
Base.@kwdef struct RecordRMS <: AbstractHook
    rms::Vector{Float64} = []
end

5.8 ms

xxxxxxxxxx
 
(f::RecordRMS)(::PostEpisodeStage, agent, env) = push!(
    f.rms,
    sqrt(mean((agent.policy.learner.approximator.table[2:end - 1] - true_values).^2))
)

145 μs

Now let's take a look at the performance of TDLearner under different α.

4.4 μs

xxxxxxxxxx
 
begin 
    p_6_2_right = plot()
​
    for α in [0.05, 0.1, 0.15]
        rms = []
        for _ in 1:100
            agent = create_TD_agent(α)
            hook = RecordRMS()
            run(agent, env, StopAfterEpisode(100),hook)
            push!(rms, hook.rms)
        end
        plot!(p_6_2_right, mean(rms), label ="TD alpha=$α", linestyle=:dashdot)
    end
    p_6_2_right
end

2.0 s

Then we can compare the differences between TDLearner and MonteCarloLearner.

4.6 μs

create_MC_agent (generic function with 1 method)

xxxxxxxxxx
 
create_MC_agent(α) = Agent(
    policy=VBasedPolicy(
        learner=MonteCarloLearner(
            approximator=TabularApproximator(fill(0.5, NS), Descent(α)),
            kind=EVERY_VISIT
        ),
        mapping = (env, V) -> rand(1:NA)
    ),
    trajectory=VectorSARTTrajectory()
)

48.7 μs

xxxxxxxxxx
 
for α in [0.01, 0.02, 0.03, 0.04]
    rms = []
    for _ in 1:100
        agent = create_MC_agent(α)
        hook = RecordRMS()
        run(agent, env, StopAfterEpisode(100),hook)
        push!(rms, hook.rms)
    end
    plot!(p_6_2_right, mean(rms), label ="MC alpha=$α")
end

299 ms

xxxxxxxxxx
 
p_6_2_right

85.0 ns

xxxxxxxxxx
 
begin
    fig_6_2 = plot()
​
    rms = []
    for _ in 1:100
        agent = create_TD_agent(0.1)
        hook = RecordRMS()
        run(agent, env, StopAfterEpisode(100),hook)
        push!(rms, hook.rms)
    end
    plot!(fig_6_2, mean(rms), label ="TD alpha=0.1", linestyle=:dashdot)
​
​
    rms = []
    for _ in 1:100
        agent = create_MC_agent(0.1)
        hook = RecordRMS()
        run(agent, env, StopAfterEpisode(100),hook)
        push!(rms, hook.rms)
    end
    plot!(fig_6_2, mean(rms), label ="MC alpha=0.1")
​
    fig_6_2
end

578 ms

Warning

Some of you might have noticed that the above figure is not the same with the one on the book of Figure 6.2. Actually we are not doing **BATCH TRAINING** here, because we're emptying the `trajectory` at the end of each episode. We leave it as an exercise for readers to practice developing new customized algorithms with `ReinforcementLearning.jl`. 😉

2.5 μs