remove gradient clipping i dont think its needed at this small scale

uccmen · Aug 20, 2022 · 4e0137d · 4e0137d
1 parent 35435ec
commit 4e0137d
Showing 1 changed file with 0 additions and 1 deletion.
diff --git a/makemore.py b/makemore.py
@@ -382,7 +382,6 @@ def create_datasets(input_file):
         # calculate the gradient, clip it, update the weights
         model.zero_grad(set_to_none=True)
         loss.backward()
-        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
         optimizer.step()
         if args.device.startswith('cuda'):
             torch.cuda.synchronize()