Chapter 7.2 n-step Sarsa

5.6 μs

xxxxxxxxxx
 
begin
    using ReinforcementLearning
    using Statistics
    using Flux
    using Plots
end

63.3 s

env

# RandomWalk1D

## Traits

| Trait Type        |                                          Value |
|:----------------- | ----------------------------------------------:|
| NumAgentStyle     |        ReinforcementLearningBase.SingleAgent() |
| DynamicStyle      |         ReinforcementLearningBase.Sequential() |
| InformationStyle  | ReinforcementLearningBase.PerfectInformation() |
| ChanceStyle       |      ReinforcementLearningBase.Deterministic() |
| RewardStyle       |     ReinforcementLearningBase.TerminalReward() |
| UtilityStyle      |         ReinforcementLearningBase.GeneralSum() |
| ActionStyle       |   ReinforcementLearningBase.MinimalActionSet() |
| StateStyle        | ReinforcementLearningBase.Observation{Int64}() |
| DefaultStateStyle | ReinforcementLearningBase.Observation{Int64}() |

## Is Environment Terminated?

No

## State Space

`Base.OneTo(21)`

## Action Space

`Base.OneTo(2)`

## Current State

```
11
```

xxxxxxxxxx
 
env = RandomWalk1D(N=21)

18.5 ms

xxxxxxxxxx
 
ns, na = length(state_space(env)), length(action_space(env))

395 ns

true_values

-1.0:0.1:1.0

xxxxxxxxxx
 
true_values = -1:0.1:1

2.8 μs

Again, we first define a hook to calculate RMS

3.7 μs

xxxxxxxxxx
 
struct RecordRMS <: AbstractHook
    rms::Vector{Float64}
    RecordRMS() = new([])
end

1.6 ms

xxxxxxxxxx
 
(f::RecordRMS)(::PostEpisodeStage, agent, env) = push!(f.rms, sqrt(mean((agent.policy.learner.approximator.table[2:end-1] - true_values[2:end-1]).^2)))

151 μs

run_once (generic function with 1 method)

xxxxxxxxxx
 
function run_once(α, n)
    env = RandomWalk1D(N=21)
    agent = Agent(
        policy=VBasedPolicy(
            learner=TDLearner(
                approximator=TabularVApproximator(;n_state=ns, opt=Descent(α)), 
                method=:SRS,
                n=n
                ),
            mapping= (env, V) -> rand(1:na)
            ),
        trajectory=VectorSARTTrajectory()
    )
    hook = RecordRMS()
    run(agent, env, StopAfterEpisode(10; is_show_progress=false), hook)
    mean(hook.rms)
end

53.9 μs

xxxxxxxxxx
 
begin
    A = 0.:0.05:1.0
    p = plot()
    for n in [2^i for i in 0:9]
        avg_rms = Float64[]
        for α in A
            rms = []
            for _ in 1:100
                push!(rms, run_once(α, n))
            end
            push!(avg_rms, mean(rms))
        end
        plot!(p, A, avg_rms, label="n = $n")
    end
​
    ylims!(p, 0.25, 0.55)
    p
end

105 s