纯强化学习推理模型：前沿技术领域的未来之星

    import numpy as np
    
    class QLearningAgent:
    def __init__(self, state_size, action_size, learning_rate=0.1, 
                 discount_factor=0.95, exploration_rate=1.0, 
                 exploration_decay=0.995):
        self.state_size = state_size
        self.action_size = action_size
        self.learning_rate = learning_rate
        self.discount_factor = discount_factor
        self.exploration_rate = exploration_rate
        self.exploration_decay = exploration_decay
        self.q_table = np.zeros((state_size, action_size))
    
    def act(self, state):
        if np.random.rand() < self.exploration_rate:
            return np.random.choice(self.action_size)  # 探索
        return np.argmax(self.q_table[state])  # 利用
    
    def learn(self, state, action, reward, next_state, done):
        current_q = self.q_table[state, action]
        max_next_q = np.max(self.q_table[next_state]) if not done else 0
        new_q = current_q + self.learning_rate * (reward + 
                        self.discount_factor * max_next_q - current_q)
        self.q_table[state, action] = new_q
        
        # 衰减探索率
        self.exploration_rate *= self.exploration_decay
    
    
    python
    
    
![](https://ad.itadn.com/c/weblog/blog-img/images/2025-08-17/aoKcSrn30NCwmAZ5LYQFhGOjq76W.png)

    import torch
    import torch.nn as nn
    import torch.optim as optim
    import torch.nn.functional as F
    
    class PolicyNetwork(nn.Module):
    def __init__(self, state_size, action_size, hidden_size=32):
        super(PolicyNetwork, self).__init__()
        self.fc1 = nn.Linear(state_size, hidden_size)
        self.fc2 = nn.Linear(hidden_size, action_size)
    
    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.softmax(self.fc2(x), dim=-1)
        return x
    
    def reinforce(env, policy, optimizer, num_episodes=1000, gamma=0.99):
    for episode in range(num_episodes):
        state = env.reset()
        rewards = []
        log_probs = []
        
        # 生成轨迹
        done = False
        while not done:
            state_tensor = torch.FloatTensor(state).unsqueeze(0)
            action_probs = policy(state_tensor)
            action = torch.multinomial(action_probs, 1).item()
            
            next_state, reward, done, _ = env.step(action)
            
            rewards.append(reward)
            log_probs.append(torch.log(action_probs.squeeze(0)[action]))
            state = next_state
        
        # 计算回报和损失
        returns = []
        R = 0
        for r in reversed(rewards):
            R = r + gamma * R
            returns.insert(0, R)
        
        returns = torch.FloatTensor(returns)
        returns = (returns - returns.mean()) / (returns.std() + 1e-9)
        
        policy_loss = []
        for log_prob, R in zip(log_probs, returns):
            policy_loss.append(-log_prob * R)
        
        optimizer.zero_grad()
        loss = torch.stack(policy_loss).sum()
        loss.backward()
        optimizer.step()
    
    
    python
    
    
![](https://ad.itadn.com/c/weblog/blog-img/images/2025-08-17/oeWlrRTw4ULNyu03bd1k92j6mXVg.png)

    # 创建conda环境
    conda create -n pure_rl python=3.8
    conda activate pure_rl
    
    # 安装核心库
    pip install numpy matplotlib torch gymnasium
    
    # 可选：安装GPU支持的PyTorch
    pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113
    
    
    bash

    import random
    import numpy as np
    import torch
    import torch.nn as nn
    import torch.optim as optim
    from collections import deque
    import gymnasium as gym
    
    class DQN(nn.Module):
    def __init__(self, state_size, action_size):
        super(DQN, self).__init__()
        self.fc1 = nn.Linear(state_size, 24)
        self.fc2 = nn.Linear(24, 24)
        self.fc3 = nn.Linear(24, action_size)
    
    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = torch.relu(self.fc2(x))
        return self.fc3(x)
    
    class DQNAgent:
    def __init__(self, state_size, action_size):
        self.state_size = state_size
        self.action_size = action_size
        self.memory = deque(maxlen=10000)
        self.gamma = 0.95    # 折扣因子
        self.epsilon = 1.0   # 探索率
        self.epsilon_min = 0.01
        self.epsilon_decay = 0.995
        self.learning_rate = 0.001
        self.model = DQN(state_size, action_size)
        self.optimizer = optim.Adam(self.model.parameters(), lr=self.learning_rate)
    
    def remember(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))
    
    def act(self, state):
        if np.random.rand() <= self.epsilon:
            return random.randrange(self.action_size)
        state = torch.FloatTensor(state).unsqueeze(0)
        act_values = self.model(state)
        return torch.argmax(act_values).item()
    
    def replay(self, batch_size):
        if len(self.memory) < batch_size:
            return
        
        minibatch = random.sample(self.memory, batch_size)
        states = torch.FloatTensor(np.array([t[0] for t in minibatch]))
        actions = torch.LongTensor(np.array([t[1] for t in minibatch]))
        rewards = torch.FloatTensor(np.array([t[2] for t in minibatch]))
        next_states = torch.FloatTensor(np.array([t[3] for t in minibatch]))
        dones = torch.FloatTensor(np.array([t[4] for t in minibatch]))
        
        current_q = self.model(states).gather(1, actions.unsqueeze(1))
        next_q = self.model(next_states).max(1)[0].detach()
        target = rewards + (1 - dones) * self.gamma * next_q
        
        loss = nn.MSELoss()(current_q.squeeze(), target)
        self.optimizer.zero_grad()
        loss.backward()
        self.optimizer.step()
        
        if self.epsilon > self.epsilon_min:
            self.epsilon *= self.epsilon_decay
    
    def train_dqn(env, agent, episodes=1000, batch_size=32):
    scores = []
    for e in range(episodes):
        state, _ = env.reset()
        total_reward = 0
        done = False
        while not done:
            action = agent.act(state)
            next_state, reward, done, _, _ = env.step(action)
            agent.remember(state, action, reward, next_state, done)
            state = next_state
            total_reward += reward
            agent.replay(batch_size)
        
        scores.append(total_reward)
        if e % 50 == 0:
            print(f"Episode: {e}, Score: {total_reward}, Epsilon: {agent.epsilon:.2f}")
    
    return scores
    
    # 初始化环境和智能体
    env = gym.make('CartPole-v1')
    state_size = env.observation_space.shape[0]
    action_size = env.action_space.n
    agent = DQNAgent(state_size, action_size)
    
    # 训练智能体
    scores = train_dqn(env, agent, episodes=500)
    
    # 可视化训练结果
    import matplotlib.pyplot as plt
    plt.plot(scores)
    plt.xlabel('Episode')
    plt.ylabel('Score')
    plt.title('DQN Training Performance on CartPole')
    plt.show()
    
    
    python
    
    
![](https://ad.itadn.com/c/weblog/blog-img/images/2025-08-17/bFQEB4luOtqaSIh9y3nWP0Lz1pfR.png)

 * 三层全连接网络，输入层大小与状态空间匹配
 * 输出层大小与动作空间匹配，输出每个动作的Q值
 * 使用ReLU激活函数引入非线性

 * 经验回放缓冲区：存储转移样本(state,action,reward,next_state,done)
 * ε-贪婪策略：平衡探索与利用
 * 训练方法：从经验回放中采样小批量进行训练

 * 每个episode中，智能体与环境交互收集经验
 * 使用经验回放更新网络参数
 * 逐渐降低探索率ε，使策略趋于稳定

 * γ=0.95：考虑较长期的回报
 * ε从1.0衰减到0.01：初期侧重探索，后期侧重利用
 * 学习率0.001：适中的参数更新步长