edit

Author: MorvanZhou

pytorchTUT/306_optimizer.py

@@ -0,0 +1,84 @@
+"""
+Know more, visit 莫烦Python: https://morvanzhou.github.io/tutorials/
+My Youtube Channel: https://www.youtube.com/user/MorvanZhou
+
+Dependencies:
+torch: 0.1.11
+"""
+import torch
+import torch.utils.data as Data
+import torch.nn.functional as F
+from torch.autograd import Variable
+import matplotlib.pyplot as plt
+
+torch.manual_seed(1)    # reproducible
+
+LR = 0.01
+BATCH_SIZE = 32
+EPOCH = 12
+
+# fake dataset
+x = torch.unsqueeze(torch.linspace(-1, 1, 1000), dim=1)
+y = x.pow(2) + 0.1*torch.normal(torch.zeros(*x.size()))
+
+# plot dataset
+plt.scatter(x.numpy(), y.numpy())
+plt.show()
+
+# put dateset into torch dataset
+torch_dataset = Data.TensorDataset(data_tensor=x, target_tensor=y)
+loader = Data.DataLoader(dataset=torch_dataset, batch_size=BATCH_SIZE, shuffle=True, num_workers=2,)
+
+
+# default network
+class Net(torch.nn.Module):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.hidden = torch.nn.Linear(1, 20)   # hidden layer
+        self.predict = torch.nn.Linear(20, 1)   # output layer
+
+    def forward(self, x):
+        x = F.relu(self.hidden(x))      # activation function for hidden layer
+        x = self.predict(x)             # linear output
+        return x
+
+# different nets
+net_SGD         = Net()
+net_Momentum    = Net()
+net_RMSprop     = Net()
+net_Adam        = Net()
+nets = [net_SGD, net_Momentum, net_RMSprop, net_Adam]
+
+# different optimizers
+opt_SGD         = torch.optim.SGD(net_SGD.parameters(), lr=LR)
+opt_Momentum    = torch.optim.SGD(net_Momentum.parameters(), lr=LR, momentum=0.8)
+opt_RMSprop     = torch.optim.RMSprop(net_RMSprop.parameters(), lr=LR, alpha=0.9)
+opt_Adam        = torch.optim.Adam(net_Adam.parameters(), lr=LR, betas=(0.9, 0.99))
+optimizers = [opt_SGD, opt_Momentum, opt_RMSprop, opt_Adam]
+
+loss_func = torch.nn.MSELoss()
+losses_his = [[], [], [], []]   # record loss
+
+# training
+for epoch in range(EPOCH):
+    print('Epoch: ', epoch)
+    for step, (batch_x, batch_y) in enumerate(loader):          # for each training step
+        b_x = Variable(batch_x)
+        b_y = Variable(batch_y)
+
+        for net, opt, l_his in zip(nets, optimizers, losses_his):
+            output = net(b_x)              # get output for every net
+            loss = loss_func(output, b_y)  # compute loss for every net
+            opt.zero_grad()                # clear gradients for next train
+            loss.backward()                # backpropagation, compute gradients
+            opt.step()                     # apply gradients
+            l_his.append(loss.data[0])     # loss recoder
+
+labels = ['SGD', 'Momentum', 'RMSprop', 'Adam']
+for i, l_his in enumerate(losses_his):
+    plt.plot(l_his, label=labels[i])
+plt.legend(loc='best')
+plt.xlabel('Steps')
+plt.ylabel('Loss')
+plt.ylim((0, 0.2))
+plt.show()

pytorchTUT/README.md

@@ -26,6 +26,7 @@ If you speak Chinese, you can watch my [Youtube channel](https://www.youtube.com
   * [An easy way](https://github.com/MorvanZhou/tutorials/blob/master/pytorchTUT/303_build_nn_quickly.py)
   * [Save and reload](https://github.com/MorvanZhou/tutorials/blob/master/pytorchTUT/304_save_reload.py)
   * [Train on batch](https://github.com/MorvanZhou/tutorials/blob/master/pytorchTUT/305_batch_train.py)
+  * [Optimizers](https://github.com/MorvanZhou/tutorials/blob/master/pytorchTUT/306_optimizer.py)
 * Advanced neural network
   * [CNN](https://github.com/MorvanZhou/tutorials/blob/master/pytorchTUT/401_CNN.py)
   * [RNN-Classification](https://github.com/MorvanZhou/tutorials/blob/master/pytorchTUT/402_RNN_classifier.py)