【人工智能】Python与强化学习：从零实现多臂老虎机（Multi-Armed Bandit）问题

    import numpy as np
    
    class MultiArmedBandit:
    """
    多臂老虎机模拟环境
    """
    def __init__(self, arms):
        """
        初始化
        :param arms: 每个老虎机的中奖概率列表
        """
        self.arms = arms
        self.k = len(arms)
    
    def pull(self, arm):
        """
        拉动指定老虎机的手柄
        :param arm: 选择的老虎机编号（0 <= arm < k）
        :return: 奖励（0 或 1）
        """
        if arm < 0 or arm >= self.k:
            raise ValueError("无效的老虎机编号")
        return 1 if np.random.rand() < self.arms[arm] else 0

    # 定义每个老虎机的中奖概率
    arm_probabilities = [0.1, 0.5, 0.8]
    
    # 创建老虎机环境
    env = MultiArmedBandit(arm_probabilities)
    
    # 测试拉动手柄
    for i in range(len(arm_probabilities)):
    reward = env.pull(i)
    print(f"拉动第{i}个老虎机，奖励: {reward}")

    class EpsilonGreedy:
    """
    ε-贪婪算法实现
    """
    def __init__(self, k, epsilon=0.1):
        """
        初始化
        :param k: 老虎机数量
        :param epsilon: 探索概率
        """
        self.k = k
        self.epsilon = epsilon
        self.counts = np.zeros(k)  # 每个老虎机的选择次数
        self.values = np.zeros(k)  # 每个老虎机的平均奖励
    
    def select_arm(self):
        """
        选择一个老虎机
        :return: 选择的老虎机编号
        """
        if np.random.rand() < self.epsilon:
            return np.random.randint(self.k)  # 探索
        return np.argmax(self.values)  # 利用
    
    def update(self, arm, reward):
        """
        更新指定老虎机的奖励信息
        :param arm: 老虎机编号
        :param reward: 奖励值
        """
        self.counts[arm] += 1
        n = self.counts[arm]
        self.values[arm] += (reward - self.values[arm]) / n

    # 参数设置
    episodes = 1000
    env = MultiArmedBandit([0.2, 0.5, 0.75])
    agent = EpsilonGreedy(k=3, epsilon=0.1)
    
    # 训练
    rewards = []
    for _ in range(episodes):
    arm = agent.select_arm()
    reward = env.pull(arm)
    agent.update(arm, reward)
    rewards.append(reward)
    
    print(f"累计奖励: {sum(rewards)}")

    class UCB:
    """
    上置信界算法实现
    """
    def __init__(self, k):
        self.k = k
        self.counts = np.zeros(k)
        self.values = np.zeros(k)
    
    def select_arm(self, t):
        ucb_values = self.values + np.sqrt(2 * np.log(t + 1) / (self.counts + 1e-5))
        return np.argmax(ucb_values)
    
    def update(self, arm, reward):
        self.counts[arm] += 1
        n = self.counts[arm]
        self.values[arm] += (reward - self.values[arm]) / n

    agent = UCB(k=3)
    
    rewards = []
    for t in range(episodes):
    arm = agent.select_arm(t)
    reward = env.pull(arm)
    agent.update(arm, reward)
    rewards.append(reward)
    
    print(f"UCB累计奖励: {sum(rewards)}")

    class ThompsonSampling:
    def __init__(self, k):
        self.k = k
        self.alpha = np.ones(k) 
    
     # 胜利次数
        self.beta = np.ones(k)   # 失败次数
    
    def select_arm(self):
        sampled_theta = np.random.beta(self.alpha, self.beta)
        return np.argmax(sampled_theta)
    
    def update(self, arm, reward):
        if reward == 1:
            self.alpha[arm] += 1
        else:
            self.beta[arm] += 1

    agent = ThompsonSampling(k=3)
    
    rewards = []
    for _ in range(episodes):
    arm = agent.select_arm()
    reward = env.pull(arm)
    agent.update(arm, reward)
    rewards.append(reward)
    
    print(f"汤普森采样累计奖励: {sum(rewards)}")

    import matplotlib.pyplot as plt
    
    plt.plot(rewards_epsilon, label="Epsilon-Greedy")
    plt.plot(rewards_ucb, label="UCB")
    plt.plot(rewards_ts, label="Thompson Sampling")
    plt.legend()
    plt.show()