Chapter 6.7 Maximization Bias and Double Learning

In example 6.7, authors introduced a MDP problem to compare the different performance of Q-Learning and Double-Q-Learning. This environment is kind of special compared to the environments we have seen before. In the first step, only LEFT and RIGHT are allowed. In the second step, if the LEFT is chosen previously, then we have 10 valid actions. We call this kind of environment is of FULL_ACTION_SET.

16.1 μs

xxxxxxxxxx
 
begin
    using ReinforcementLearning
    using Statistics
    using Flux
    using Plots
end

50.7 s

MaximizationBiasEnv

xxxxxxxxxx
 
"""
states:
1:A
2:B
3:terminal
actions:
1: left
2: right
"""
Base.@kwdef mutable struct MaximizationBiasEnv <: AbstractEnv
    position::Int = 1
    reward::Float64 = 0.0
end

5.2 ms

xxxxxxxxxx
 
RLBase.state_space(env::MaximizationBiasEnv) = Base.OneTo(3)

33.5 μs

xxxxxxxxxx
 
RLBase.action_space(env::MaximizationBiasEnv) = Base.OneTo(10)

21.3 μs

xxxxxxxxxx
 
RLBase.ActionStyle(env::MaximizationBiasEnv) = FULL_ACTION_SET

18.2 μs

xxxxxxxxxx
 
const LEFT = 1

2.5 μs

xxxxxxxxxx
 
const RIGHT = 2

1.1 μs

xxxxxxxxxx
 
function RLBase.legal_action_space(env::MaximizationBiasEnv)
    if env.position == 1
        (LEFT, RIGHT)
    else
        Base.OneTo(10)
    end
end

26.5 μs

xxxxxxxxxx
 
function RLBase.legal_action_space_mask(env::MaximizationBiasEnv)
    m = fill(false, 10)
    if env.position == 1
        m[LEFT] = true
        m[RIGHT] = true
    else
        m .= true
    end
    m
end

28.0 μs

xxxxxxxxxx
 
function (env::MaximizationBiasEnv)(a::Int)
    if env.position == 1
        if a == 1
            env.position = 2
            env.reward = 0.0
        else
            env.position = 3
            env.reward = 0.0
        end
    elseif env.position == 2
        env.position = 3
        env.reward = randn() - 0.1
    end
    nothing
end

151 μs

xxxxxxxxxx
 
function RLBase.reset!(env::MaximizationBiasEnv)
    env.position = 1
    env.reward = 0.0
    nothing
end

20.4 μs

xxxxxxxxxx
 
RLBase.reward(env::MaximizationBiasEnv) = env.reward

18.3 μs

xxxxxxxxxx
 
RLBase.is_terminated(env::MaximizationBiasEnv) = env.position == 3

22.4 μs

xxxxxxxxxx
 
RLBase.state(env::MaximizationBiasEnv) = env.position

20.3 μs

Now the environment is well defined.

7.0 μs

world

# MaximizationBiasEnv

## Traits

| Trait Type        |                                            Value |
|:----------------- | ------------------------------------------------:|
| NumAgentStyle     |          ReinforcementLearningBase.SingleAgent() |
| DynamicStyle      |           ReinforcementLearningBase.Sequential() |
| InformationStyle  | ReinforcementLearningBase.ImperfectInformation() |
| ChanceStyle       |           ReinforcementLearningBase.Stochastic() |
| RewardStyle       |           ReinforcementLearningBase.StepReward() |
| UtilityStyle      |           ReinforcementLearningBase.GeneralSum() |
| ActionStyle       |        ReinforcementLearningBase.FullActionSet() |
| StateStyle        |     ReinforcementLearningBase.Observation{Any}() |
| DefaultStateStyle |     ReinforcementLearningBase.Observation{Any}() |

## Is Environment Terminated?

No

## State Space

`Base.OneTo(3)`

## Action Space

`Base.OneTo(10)`

## Current State

```
1
```

xxxxxxxxxx
 
world = MaximizationBiasEnv()

2.1 ms

xxxxxxxxxx
 
NS, NA = length(state_space(world)), length(action_space(world))

94.0 ns

To calculate the percentage of chosing LEFT action in the first step, we'll create customized hook here:

5.7 μs

CountOfLeft

xxxxxxxxxx
 
Base.@kwdef mutable struct CountOfLeft <: AbstractHook
    counts::Vector{Bool} = []
end

5.3 ms

xxxxxxxxxx
 
function (f::CountOfLeft)(::PreActStage, agent, env, action)
    if state(env) == 1
        push!(f.counts, action == LEFT)
    end
end

127 μs

Next we create two agent factories, Q-Learning and Double-Q-Learning.

5.3 μs

create_double_Q_agent (generic function with 1 method)

xxxxxxxxxx
 
create_double_Q_agent() = Agent(
    policy=QBasedPolicy(
        learner=DoubleLearner(
            L1=TDLearner(
                approximator=TabularQApproximator(
                    n_state=NS,
                    n_action=NA,
                    opt=Descent(0.1),
                ),
                method=:SARS,
                γ=1.,
                n=0,
            ),
            L2=TDLearner(
                approximator=TabularQApproximator(;
                    n_state=NS,
                    n_action=NA,
                    opt=Descent(0.1),
                ),
                method=:SARS,
                γ=1.,
                n=0,
            ),
            ),
        explorer=EpsilonGreedyExplorer(0.1;is_break_tie=true)
    ),
    trajectory=VectorSARTTrajectory()
)

50.5 μs

create_Q_agent (generic function with 1 method)

xxxxxxxxxx
 
create_Q_agent() = Agent(
    policy=QBasedPolicy(
        learner=TDLearner(
            approximator=TabularQApproximator(;n_state=NS, n_action=NA, opt=Descent(0.1)),
            method=:SARS,
            γ=1.0,
            n=0
            ),
        explorer=EpsilonGreedyExplorer(0.1;is_break_tie=true)
        ),
    trajectory=VectorSARTTrajectory()
)

38.6 μs

xxxxxxxxxx
 
begin
    DQ_stats = []
    for _ in 1:1000
        hook = CountOfLeft()
        run(create_double_Q_agent(), world, StopAfterEpisode(300),hook)
        push!(DQ_stats, hook.counts)
    end
    plot(mean(DQ_stats), legend=:topright, label="double q")
    
    Q_stats = []
    for _ in 1:1000
        hook = CountOfLeft()
        run(create_Q_agent(), world, StopAfterEpisode(300),hook)
        push!(Q_stats, hook.counts)
    end
    plot!(mean(Q_stats), legend=:topright, label="q")
    hline!([0.05], linestyle=:dash, label="optimal")
end

11.8 s