【深度学习】新的深度学习优化器探索（协同优化）

    文章目录
    1 RAdam VS Adam
    2 自适应优化
    3 LookAhead
    	3.1 “侵入式”优化器
    	3.2 LookAhead 中的参数：
    4 RAdam 加 LookAhead 的一个实现：Ranger
    
    
      
      
      
      
      
      
      
    
    代码解读

    x += - learning_rate * dx
    
    
      
    
    代码解读

    v = mu * v - learning_rate * dx # 与速度融合
    x += v # 与位置融合
    
    
      
      
    
    代码解读

    1、备份模型现有的权重θ；
    
    2、从θ出发，用指定优化器更新k步，得到新权重θ̃ ；
    
    3、更新模型权重为θ←θ+α(θ̃ −θ)。
    
    
      
      
      
      
      
    
    代码解读

    from keras.optimizers import Optimizer
    from keras import backend as K
    
    
    class InjectOptimizer(Optimizer):
    """定义注入式优化器的基类
        需要传入模型，直接修改模型的训练函数，而不按常规流程使用优化器，所以称为“侵入式”
        其实下面的大部分代码，都是直接抄自keras的源码：
        https://github.com/keras-team/keras/blob/master/keras/engine/training.py#L497
        也就是keras中的_make_train_function函数。
    """
    def get_updates(self, loss, params):
        return []
    def get_grouped_updates(self, loss, params):
        raise NotImplementedError
    def inject(self, model):
        """传入模型做注入
        """
        if not hasattr(model, 'train_function'):
            raise RuntimeError('You must compile your model before using it.')
        model._check_trainable_weights_consistency()
        if model.train_function is None:
            inputs = (model._feed_inputs +
                      model._feed_targets +
                      model._feed_sample_weights)
            if model._uses_dynamic_learning_phase():
                inputs += [K.learning_phase()]
            with K.name_scope('training'):
                train_functions = []
                with K.name_scope(model.optimizer.__class__.__name__):
                    grouped_training_updates = self.get_grouped_updates(
                        params=model._collected_trainable_weights,
                        loss=model.total_loss)
                for i, updates in enumerate(grouped_training_updates[1:]):
                    f = K.function(
                        inputs,
                        [model.total_loss],
                        updates=updates,
                        name='train_function_%s' % (i + 1),
                        **model._function_kwargs)
                    train_functions.append(f)
                # Gets loss and metrics. Updates weights at each call.
                first_updates = (model.updates +
                                 grouped_training_updates[0],
                                 model.metrics_updates)
                first_train = K.function(
                    inputs,
                    [model.total_loss] + model.metrics_tensors,
                    updates=first_updates,
                    name='train_function',
                    **model._function_kwargs)
                def F(inputs):
                    R = first_train(inputs)
                    for f in train_functions:
                        f(inputs)
                    return R
                model.train_function = F
    
    
    class HeunOptimizer(InjectOptimizer):
    """Heun优化器
    ( https://en.wikipedia.org/wiki/Heun%27s_method )
    """
    def __init__(self, lr, **kwargs):
        super(HeunOptimizer, self).__init__(**kwargs)
        with K.name_scope(self.__class__.__name__):
            self.lr = K.variable(lr, name='lr')
    def get_updates_1(self, loss, params, cache_grads):
        updates = []
        grads = self.get_gradients(loss, params)
        for p, g, cg in zip(params, grads, cache_grads):
            updates.append(K.update(cg, g))
            updates.append(K.update(p, p - self.lr * g))
        return updates
    def get_updates_2(self, loss, params, cache_grads):
        updates = []
        grads = self.get_gradients(loss, params)
        for p, g, cg in zip(params, grads, cache_grads):
            updates.append(K.update(p, p - 0.5 * self.lr * (g - cg)))
        return updates
    def get_grouped_updates(self, loss, params):
        cache_grads = [K.zeros(K.int_shape(p)) for p in params]
        return [
            self.get_updates_1(loss, params, cache_grads),
            self.get_updates_2(loss, params, cache_grads)
        ]
    
    
    
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
      
    
    代码解读

    opt = HeunOptimizer(0.1)
    model.compile(loss='mse', optimizer=opt)
    opt.inject(model) # 必须执行这步
    
    model.fit(x_train, y_train, epochs=100, batch_size=32)
    
    
      
      
      
      
      
    
    代码解读

    model.compile(optimizer=Adam(1e-3), loss='mse') # 用你想用的优化器
    lookahead = Lookahead(k=5, alpha=0.5) # 初始化Lookahead
    lookahead.inject(model) # 插入到模型中
    
    
      
      
      
    
    代码解读