import gym from gym import envs env_specs = envs.registry.all() # 查看库中都注册了哪些环境 # for e in env_specs: # print(e) env = gym.make("CartPole-v1") # 取出环境平衡车-v1 env.reset() # 初始化环境对象env, 返回智能体的初始观测值:array([-0.00667078, -0.04023064, -0.01557324, 0.04904377]) # env.observation_space # 环境的观察空间:Box(-3.4028234663852886e+38, 3.4028234663852886e+38, (4,), float32) # env.action_space # 环境的动作空间:Discrete(2) # 环境初始化后即可使用,核心:使用对象的step()方法 action = env.action_space.sample() # step参数需要取自动作空间,可以使用sample从动作空间随机取一个动作;每次调用会使得环境前进一步,需要放在循环里完成整个回合。 # env.render() # 显示当前环境 # env.close() # 关闭当前环境 # 如果要得到更好的结果,不能在每一次都采取随机,还是要知道什么动作对环境做了什么会更好。要拿到step函数返回的东西,其会返回四个值: # observation(object): 一个特定环境的对象,代表智能体对环境的观察,观测值,与env.reset()意义相同 # reward(float): 前一个动作的奖励量 # done(boolean): 是否需要再次重置环境 # info(dict): 对调试有用的诊断信息 env.observation_space.high
import gym env = gym.make("CartPole-v1") for i_episode in range(20): observation = env.reset() for t in range(100): env.render() action = env.action_space.sample() observation, reward, done, info = env.step(action) print(observation, reward, done, info) if done: print(f"Episode finished after {t+1} timesteps") break env.close()
import gym env = gym.make("MountainCar-v0") print(f"观测空间={env.observation_space}") print(f"动作空间={env.action_space}") print(f"观测范围={env.observation_space.low}~{env.observation_space.high}") print(f"动作数={env.action_space.n}")
class BespokeAgent: def __init__(self, env): pass # 决策 def decide(self, observation): position, velocity = observation lb = min(-0.09 * (position + 0.25) ** 2 + 0.03, 0.3*(position + 0.9) **4 - 0.008) ub = -0.07 * (position + 0.38) **2 + 0.06 if lb < velocity < ub: action = 2 else: action = 0 return action # 学习 def learn(self, *args): pass agent = BespokeAgent(env)
def play_montecarlo(env, agent, render=True, train=False): episode_reward = 0. observation = env.reset() while True: if render: env.render() # 显示图形界面 action = agent.decide(observation) # 智能体决定采取的动作 next_observation, reward, done, _ = env.step(action) # 执行动作 episode_reward += reward # 收集回合奖励 if train: agent.learn(observation, action, reward, done) # 学习 if done: break observation = next_observation return episode_reward # 返回回合总奖励 # env.seed(0) # 执行一次 episode_reward = play_montecarlo(env, agent, train=True) print(f"回合奖励:{episode_reward}") env.close()
# 连续交互100回合奖励 import numpy as np episode_rewards = [play_montecarlo(env, agent, train=True) for _ in range(10)] print(f"平均回合奖励={np.mean(episode_rewards)}") env.close()
离散markov决策
import gym env = gym.make("CliffWalking-v0") print(f"观测空间={env.observation_space}") print(f"动作空间={env.action_space}") print(f"状态数量={env.nS} 动作数量={env.nA}") print(f"地图大小={env.shape}") import numpy as np def play_once(env, policy): total_reward = 0 state = env.reset() loc = np.unravel_index(state, env.shape) print(f"状态={state}, 位置={loc}") while True: action = np.random.choice(env.nA, p=policy[state]) # 0-4随机采样,以特定概率分布采样,即每次拿出最优动作 # 执行step,环境做出反馈 next_state, reward, done, _ = env.step(action) print(f"状态:{state} 位置:{loc} 奖励:{reward}") total_reward += reward if done: break state = next_state return total_reward actions = np.ones(env.shape, dtype=int) actions[-1, :] = 0 actions[:, -1] = 2 optimal_policy = np.eye(4)[actions.reshape(-1)] # 生成最优动作向量 total_rewards = play_once(env, optimal_policy) print(f"总奖励:{total_rewards}") def evaluate_bellman(env, policy, gamma=1.): a, b = np.eye(env.nS), np.zeros((env.nS)) # 分别表示状态转移和状态价值 for state in range(env.nS-1): for action in range(env.nA): pi = policy[state][action] # print(f"{state} {action}: pi: {pi}") # 表示每一状态下各个动作产生的概率, env.P 存储环境的动力 for p, next_state, reward, done in env.P[state][action]: a[state, next_state] -= (pi * gamma) b[state] += (pi * reward * p) # 求解线性方程组,得到状态价值 v = np.linalg.solve(a,b) print(v) q = np.zeros((env.nS, env.nA)) # 利用状态价值再求解动作价值 for state in range(env.nS-1): for action in range(env.nA): for p, next_state, reward, done, in env.P[state][action]: q[state][action] += ((reward + gamma*v[next_state])*p) return v, q policy = np.random.uniform(size=(env.nS, env.nA)) policy = policy / np.sum(policy, axis=1)[:, np.newaxis] # 外层新加一维度 state_values, action_values = evaluate_bellman(env, policy) print(f"状态价值={state_values}\n动作价值={action_values}") # 评估最优策略 optimal_state_values, optimal_action_values = evaluate_bellman(env, optimal_policy) print(f"最优状态价值={optimal_state_values}\n最优动作价值={optimal_action_values}")
###########################################################################
标签:observation,笔记,学习,print,state,env,action,强化,reward From: https://www.cnblogs.com/demo-deng/p/16803595.html