强化学习-笔记

import gym
from gym import envs
env_specs = envs.registry.all()   # 查看库中都注册了哪些环境
# for e in env_specs:
#     print(e)

env = gym.make("CartPole-v1")  # 取出环境平衡车-v1
env.reset()                    # 初始化环境对象env, 返回智能体的初始观测值：array([-0.00667078, -0.04023064, -0.01557324,  0.04904377])
# env.observation_space          # 环境的观察空间：Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32)
# env.action_space               # 环境的动作空间：Discrete(2)
# 环境初始化后即可使用，核心：使用对象的step()方法
action = env.action_space.sample()  # step参数需要取自动作空间，可以使用sample从动作空间随机取一个动作;每次调用会使得环境前进一步，需要放在循环里完成整个回合。
# env.render()                   # 显示当前环境
# env.close()                    # 关闭当前环境

# 如果要得到更好的结果，不能在每一次都采取随机，还是要知道什么动作对环境做了什么会更好。要拿到step函数返回的东西，其会返回四个值：
# observation(object): 一个特定环境的对象，代表智能体对环境的观察，观测值，与env.reset()意义相同
# reward(float): 前一个动作的奖励量
# done(boolean): 是否需要再次重置环境
# info(dict): 对调试有用的诊断信息
env.observation_space.high

import gym
env = gym.make("CartPole-v1")
for i_episode in range(20):
    observation = env.reset()
    for t in range(100):
        env.render()
        action = env.action_space.sample()
        observation, reward, done, info = env.step(action)
        print(observation, reward, done, info)
        if done:
            print(f"Episode finished after {t+1} timesteps")
            break
env.close()

import gym
env = gym.make("MountainCar-v0")
print(f"观测空间={env.observation_space}")
print(f"动作空间={env.action_space}")
print(f"观测范围={env.observation_space.low}~{env.observation_space.high}")
print(f"动作数={env.action_space.n}")

class BespokeAgent:
    def __init__(self, env):
        pass
    # 决策
    def decide(self, observation):
        position, velocity = observation
        lb = min(-0.09 * (position + 0.25) ** 2 + 0.03, 0.3*(position + 0.9) **4 - 0.008)
        ub = -0.07 * (position + 0.38) **2 + 0.06
        if lb < velocity < ub:
            action = 2
        else:
            action = 0
        return action
    # 学习
    def learn(self, *args):
        pass
agent = BespokeAgent(env)

def play_montecarlo(env, agent, render=True, train=False):
    episode_reward = 0.
    observation = env.reset()
    while True:
        if render:
            env.render()    # 显示图形界面
        action = agent.decide(observation)     # 智能体决定采取的动作
        next_observation, reward, done, _ = env.step(action)     # 执行动作
        episode_reward += reward            # 收集回合奖励
        if train:
            agent.learn(observation, action, reward, done) # 学习
        if done:
            break
        observation = next_observation
    return episode_reward      # 返回回合总奖励
# env.seed(0)
# 执行一次
episode_reward = play_montecarlo(env, agent, train=True)
print(f"回合奖励:{episode_reward}")
env.close()

# 连续交互100回合奖励
import numpy as np
episode_rewards = [play_montecarlo(env, agent, train=True) for _ in range(10)]
print(f"平均回合奖励={np.mean(episode_rewards)}")
env.close()

离散markov决策

import gym
env = gym.make("CliffWalking-v0")
print(f"观测空间={env.observation_space}")
print(f"动作空间={env.action_space}")
print(f"状态数量={env.nS} 动作数量={env.nA}")
print(f"地图大小={env.shape}")

import numpy as np
def play_once(env, policy):
    total_reward = 0
    state = env.reset()
    loc = np.unravel_index(state, env.shape)
    print(f"状态={state}, 位置={loc}")
    while True:
        action = np.random.choice(env.nA, p=policy[state])  # 0-4随机采样，以特定概率分布采样，即每次拿出最优动作
        # 执行step，环境做出反馈
        next_state, reward, done, _ = env.step(action)
        print(f"状态：{state} 位置：{loc} 奖励：{reward}")
        total_reward += reward
        if done:
            break
        state = next_state
    return total_reward
    
actions = np.ones(env.shape, dtype=int)
actions[-1, :] = 0
actions[:, -1] = 2
optimal_policy = np.eye(4)[actions.reshape(-1)]    # 生成最优动作向量
total_rewards = play_once(env, optimal_policy)
print(f"总奖励：{total_rewards}")

def evaluate_bellman(env, policy, gamma=1.):
    a, b = np.eye(env.nS), np.zeros((env.nS))     # 分别表示状态转移和状态价值
    for state in range(env.nS-1):
        for action in range(env.nA):
            pi = policy[state][action]
            # print(f"{state} {action}: pi: {pi}")    # 表示每一状态下各个动作产生的概率， env.P 存储环境的动力
            for p, next_state, reward, done in env.P[state][action]:
                a[state, next_state] -= (pi * gamma)
                b[state] += (pi * reward * p)
    # 求解线性方程组,得到状态价值
    v = np.linalg.solve(a,b)
    print(v)
    q = np.zeros((env.nS, env.nA))
    # 利用状态价值再求解动作价值
    for state in range(env.nS-1):
        for action in range(env.nA):
            for p, next_state, reward, done, in env.P[state][action]:
                q[state][action] += ((reward + gamma*v[next_state])*p)
    return v, q
policy = np.random.uniform(size=(env.nS, env.nA))
policy = policy / np.sum(policy, axis=1)[:, np.newaxis]   # 外层新加一维度
state_values, action_values = evaluate_bellman(env, policy)
print(f"状态价值={state_values}\n动作价值={action_values}")

# 评估最优策略
optimal_state_values, optimal_action_values = evaluate_bellman(env, optimal_policy)
print(f"最优状态价值={optimal_state_values}\n最优动作价值={optimal_action_values}")

###########################################################################