begin

    using ReinforcementLearning

    using Flux

    using Statistics

    using Plots

    using Distributions

end

begin

    const pₕ = 0.4

    const WinCapital = 100

​

    decode_state(s::Int) = s - 1

    encode_state(s::Int) = s + 1

​

    function nextstep(s::Int, a::Int)

        s = decode_state(s)

        a = min(s, a)

        if s == WinCapital || s==0

            [(0., false,encode_state(s))=>1.0]

        else

            [

                ((s+a >= WinCapital ? 1.0 : 0.), false, encode_state(min(s+a, WinCapital))) => pₕ,

                (0., false, encode_state(max(s-a, 0))) => 1-pₕ

            ]

        end

    end

​

    struct GamblerProblemEnvModel <: AbstractEnvironmentModel

        cache

    end

    function GamblerProblemEnvModel()

        GamblerProblemEnvModel(

            Dict(

                (s,a) => nextstep(s,a)

                for s in 1:(WinCapital+1) for a in 1:WinCapital

            )

        )

    end

    RLBase.state_space(m::GamblerProblemEnvModel) = Base.OneTo(WinCapital+1)

    RLBase.action_space(m::GamblerProblemEnvModel) = Base.OneTo(WinCapital)

    (m::GamblerProblemEnvModel)(s, a) = m.cache[(s,a)]

end

V = TabularVApproximator(;n_state=1+WinCapital,opt=Descent(1))

RLZoo.value_iteration!(V=V, model=GamblerProblemEnvModel(), γ=1.0, max_iter=1000)

plot(V.table[2:end-1])