Pytorch做训练的时候,可以调整训练学习率。
通过调整合适的学习率曲线可以提高模型训练效率和优化模型性能。
各种学习率曲线示例代码如下:
import torch import torch.optim as optim import torch.nn as nn import matplotlib.pyplot as plt import numpy as np if __name__ == '__main__': lr_init = 0.5 #初始学习率 parameter = [nn.Parameter(torch.tensor([1, 2, 3], dtype=torch.float32))] optimizer = optim.SGD(parameter, lr=lr_init) scheduler_list=[] #每迭代step_size次,学习率乘以gamma scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.5) scheduler_list.append(scheduler) #在迭代到millestones次时,学习率乘以gamma scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[20,40,80], gamma=0.5) scheduler_list.append(scheduler) #每次学习率是上一次的gamma倍 scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9) scheduler_list.append(scheduler) #前total_inters次迭代,学习率从lr_init*(start_factor~end_factor)线性下降,total_iters次之后稳定在end_factor scheduler = torch.optim.lr_scheduler.LinearLR(optimizer,start_factor=1,end_factor=0.01,total_iters=50) scheduler_list.append(scheduler) #学习率在base_lr~max_lr之间循环,上升step_size_up个周期,下降step_size_down个周期 scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer,base_lr=0.01,max_lr=0.1,step_size_up=10,step_size_down=30) scheduler_list.append(scheduler) #学习率为cos曲线,T_max为半个周期,最小为eta_min,最大为lr_init scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=20,eta_min=0.1) scheduler_list.append(scheduler) #cos退火学习率,第一个周期为T_0,后面每一个周期为前一个的T_mult倍,最小值为eta_min,最大值为lr_init scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=2, eta_min=0.01) scheduler_list.append(scheduler) #学习率先上升后下降,pct_start学习率上升部分占比,最大学习率=max_lr,初始学习率=max_lr/div_factor,最终学习率=初始学习率/final_div_factor scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer,max_lr=0.1,pct_start=0.2,total_steps=100,div_factor=10,final_div_factor=5) scheduler_list.append(scheduler) #total_iters次以内学习率为lr_init,total_iters之后学习率为lr_init/factor scheduler = torch.optim.lr_scheduler.ConstantLR(optimizer,factor=0.5,total_iters=50) scheduler_list.append(scheduler) #多个学习率组合,将学习率在milestones次循环处分割为两端 scheduler = torch.optim.lr_scheduler.SequentialLR(optimizer,schedulers=[ torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.9), torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.5)], milestones=[50]) scheduler_list.append(scheduler) #同样多种学习率组合,可以给出连续学习率 scheduler = torch.optim.lr_scheduler.ChainedScheduler([ torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99), torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.5)]) scheduler_list.append(scheduler) #自定义lambda函数设定学习率,这里是lr_init * 1.0/(step+1) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda step: 1.0 /(step+1)) scheduler_list.append(scheduler) #自定义lambad函数设定学习率,这里是lr[t] = 0.95*lr[t-1] scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer,lr_lambda=lambda epoch:0.95) scheduler_list.append(scheduler) #当指标度量停止改进时,ReduceLROnPlateau会降低学习率,scheduler.step中需要设置loss参数 # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,mode='min',factor=0.5,patience=5,threshold=1e-4,threshold_mode='abs',cooldown=0,min_lr=0.001,eps=1e-8) # scheduler_list.append(scheduler) learning_rates = [] for sch in scheduler_list: rates=[] optimizer.param_groups[0]['lr'] = lr_init for _ in range(100): optimizer.step() sch.step() rates.append(sch.get_last_lr()[0]) learning_rates.append(rates) numpy_rates = np.array(learning_rates) for i in range(numpy_rates.shape[0]): plt.subplot(4,4,i+1) plt.plot(numpy_rates[i,:]) plt.show()
各种学习率曲线如下: