Skip to content

Commit

Permalink
tut
Browse files Browse the repository at this point in the history
  • Loading branch information
@k8lion
k8lion committed Jan 2, 2023
1 parent b0a58e0 commit 1af86de
Showing 1 changed file with 51 additions and 9 deletions.
60 changes: 51 additions & 9 deletions tutorial.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,46 @@
"Validation: Average loss: 0.0026, Accuracy: 5231/6000 (87.18%)\n",
"Train Epoch: 1 [0/54000 (0%)]\tLoss: 0.442268\n",
"Train Epoch: 1 [12800/54000 (24%)]\tLoss: 0.237986\n",
"Train Epoch: 1 [25600/54000 (47%)]\tLoss: 0.385638\n"
"Train Epoch: 1 [25600/54000 (47%)]\tLoss: 0.385638\n",
"Train Epoch: 1 [38400/54000 (71%)]\tLoss: 0.377946\n",
"Train Epoch: 1 [51200/54000 (95%)]\tLoss: 0.251457\n",
"Validation: Average loss: 0.0025, Accuracy: 5236/6000 (87.27%)\n",
"Layer 0 scores: mean 4.36, std 0.212, min 4.18, smallest 15%: []\n",
"Layer 1 scores: mean 9.09, std 0.583, min 8.61, smallest 15%: [6]\n",
"Layer 2 scores: mean 10, std 0.869, min 7.94, smallest 15%: [6]\n",
"Layer 3 scores: mean 3.54, std 0.284, min 3.11, smallest 15%: [10 11 13]\n",
"The pruned model now has 6720 effective parameters.\n",
"Validation after pruning: Average loss: 0.0038, Accuracy: 4851/6000 (80.85%)\n",
"Train Epoch: 0 [0/54000 (0%)]\tLoss: 0.372106\n",
"Train Epoch: 0 [12800/54000 (24%)]\tLoss: 0.313874\n",
"Train Epoch: 0 [25600/54000 (47%)]\tLoss: 0.311888\n",
"Train Epoch: 0 [38400/54000 (71%)]\tLoss: 0.319379\n",
"Train Epoch: 0 [51200/54000 (95%)]\tLoss: 0.299428\n",
"Validation: Average loss: 0.0026, Accuracy: 5237/6000 (87.28%)\n",
"Train Epoch: 1 [0/54000 (0%)]\tLoss: 0.306606\n",
"Train Epoch: 1 [12800/54000 (24%)]\tLoss: 0.405268\n",
"Train Epoch: 1 [25600/54000 (47%)]\tLoss: 0.341106\n",
"Train Epoch: 1 [38400/54000 (71%)]\tLoss: 0.237019\n",
"Train Epoch: 1 [51200/54000 (95%)]\tLoss: 0.266737\n",
"Validation: Average loss: 0.0025, Accuracy: 5235/6000 (87.25%)\n",
"Layer 0 scores: mean 4.42, std 0.193, min 4.24, smallest 15%: []\n",
"Layer 1 scores: mean 9.38, std 0.637, min 8.86, smallest 15%: [9]\n",
"Layer 2 scores: mean 9.6, std 0.649, min 8.4, smallest 15%: [7]\n",
"Layer 3 scores: mean 3.43, std 0.263, min 2.96, smallest 15%: [ 7 16]\n",
"The pruned model now has 5781 effective parameters.\n",
"Validation after pruning: Average loss: 0.0039, Accuracy: 4932/6000 (82.20%)\n",
"Train Epoch: 0 [0/54000 (0%)]\tLoss: 0.417025\n",
"Train Epoch: 0 [12800/54000 (24%)]\tLoss: 0.345414\n",
"Train Epoch: 0 [25600/54000 (47%)]\tLoss: 0.311447\n",
"Train Epoch: 0 [38400/54000 (71%)]\tLoss: 0.281655\n",
"Train Epoch: 0 [51200/54000 (95%)]\tLoss: 0.422159\n",
"Validation: Average loss: 0.0026, Accuracy: 5222/6000 (87.03%)\n",
"Train Epoch: 1 [0/54000 (0%)]\tLoss: 0.229719\n",
"Train Epoch: 1 [12800/54000 (24%)]\tLoss: 0.241673\n",
"Train Epoch: 1 [25600/54000 (47%)]\tLoss: 0.395002\n",
"Train Epoch: 1 [38400/54000 (71%)]\tLoss: 0.307319\n",
"Train Epoch: 1 [51200/54000 (95%)]\tLoss: 0.219365\n",
"Validation: Average loss: 0.0025, Accuracy: 5230/6000 (87.17%)\n"
]
}
],
Expand Down Expand Up @@ -382,28 +421,31 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Layer 0 score: 8/8, neurons to add: 1\n"
"Layer 0 score: 8/8, neurons to add: 1\n",
"Layer 1 score: 16/16, neurons to add: 1\n",
"Layer 2 score: 16/16, neurons to add: 1\n",
"Layer 3 score: 32/32, neurons to add: 2\n"
]
},
{
"ename": "TypeError",
"evalue": "cannot assign 'torch.FloatTensor' as parameter 'mask_vector' (torch.nn.Parameter or None expected)",
"ename": "RuntimeError",
"evalue": "shape '[34, 68]' is invalid for input of size 2176",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[8], line 12\u001b[0m\n\u001b[1;32m 10\u001b[0m to_add \u001b[39m=\u001b[39m \u001b[39mmax\u001b[39m(score\u001b[39m-\u001b[39m\u001b[39mint\u001b[39m(\u001b[39m0.95\u001b[39m\u001b[39m*\u001b[39mmax_rank), \u001b[39m0\u001b[39m)\n\u001b[1;32m 11\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mLayer \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m score: \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m/\u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m, neurons to add: \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(i, score, max_rank, to_add))\n\u001b[0;32m---> 12\u001b[0m modded_model_grow\u001b[39m.\u001b[39;49mgrow(i, to_add, fanin_weights\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39miterative_orthogonalization\u001b[39;49m\u001b[39m\"\u001b[39;49m, \n\u001b[1;32m 13\u001b[0m optimizer\u001b[39m=\u001b[39;49mmodded_optimizer_grow)\n\u001b[1;32m 14\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mThe grown model now has \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m effective parameters.\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(modded_model_grow\u001b[39m.\u001b[39mparameter_count(masked \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m)))\n\u001b[1;32m 15\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mValidation after growing: \u001b[39m\u001b[39m\"\u001b[39m, end \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m\"\u001b[39m)\n",
"File \u001b[0;32m~/repos/NeurOps/pytorch/src/models.py:146\u001b[0m, in \u001b[0;36mModSequential.grow\u001b[0;34m(self, layer_index, newneurons, fanin_weights, fanout_weights, optimizer, clear_activations, send_activations)\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[39mfor\u001b[39;00m i, module \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[1;32m 145\u001b[0m \u001b[39mif\u001b[39;00m i \u001b[39m==\u001b[39m layer_index \u001b[39mand\u001b[39;00m (\u001b[39misinstance\u001b[39m(module, ModLinear) \u001b[39mor\u001b[39;00m \u001b[39misinstance\u001b[39m(module, ModConv2d)):\n\u001b[0;32m--> 146\u001b[0m module\u001b[39m.\u001b[39;49mgrow(newneurons, \u001b[39m0\u001b[39;49m, fanin_weights \u001b[39m=\u001b[39;49m fanin_weights, optimizer\u001b[39m=\u001b[39;49moptimizer, \n\u001b[1;32m 147\u001b[0m activations\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mactivations[\u001b[39mstr\u001b[39;49m(layer_index\u001b[39m-\u001b[39;49m\u001b[39m1\u001b[39;49m)] \u001b[39mif\u001b[39;49;00m send_activations \u001b[39mor\u001b[39;49;00m fanin_weights \u001b[39m==\u001b[39;49m \u001b[39m\"\u001b[39;49m\u001b[39miterative_orthogonalization\u001b[39;49m\u001b[39m\"\u001b[39;49m \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m)\n\u001b[1;32m 148\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrack_activations:\n\u001b[1;32m 149\u001b[0m \u001b[39mif\u001b[39;00m clear_activations:\n",
"File \u001b[0;32m~/repos/NeurOps/pytorch/src/layers.py:699\u001b[0m, in \u001b[0;36mModConv2d.grow\u001b[0;34m(self, new_out_channels, new_in_channels, fanin_weights, fanout_weights, optimizer, activations)\u001b[0m\n\u001b[1;32m 697\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbias \u001b[39m=\u001b[39m new_bias\n\u001b[1;32m 698\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmasked:\n\u001b[0;32m--> 699\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmask_vector \u001b[39m=\u001b[39m new_mask_vector\n\u001b[1;32m 701\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mout_channels \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mout_channels \u001b[39m+\u001b[39m new_out_channels\n",
"File \u001b[0;32m/usr/local/lib/python3.10/site-packages/torch/nn/modules/module.py:1206\u001b[0m, in \u001b[0;36mModule.__setattr__\u001b[0;34m(self, name, value)\u001b[0m\n\u001b[1;32m 1204\u001b[0m \u001b[39melif\u001b[39;00m params \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m name \u001b[39min\u001b[39;00m params:\n\u001b[1;32m 1205\u001b[0m \u001b[39mif\u001b[39;00m value \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m-> 1206\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mTypeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mcannot assign \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m as parameter \u001b[39m\u001b[39m'\u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m \u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1207\u001b[0m \u001b[39m\"\u001b[39m\u001b[39m(torch.nn.Parameter or None expected)\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 1208\u001b[0m \u001b[39m.\u001b[39mformat(torch\u001b[39m.\u001b[39mtypename(value), name))\n\u001b[1;32m 1209\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mregister_parameter(name, value)\n\u001b[1;32m 1210\u001b[0m \u001b[39melse\u001b[39;00m:\n",
"\u001b[0;31mTypeError\u001b[0m: cannot assign 'torch.FloatTensor' as parameter 'mask_vector' (torch.nn.Parameter or None expected)"
"File \u001b[0;32m~/repos/NeurOps/pytorch/src/layers.py:313\u001b[0m, in \u001b[0;36mModLinear.grow\u001b[0;34m(self, new_out_features, new_in_features, fanin_weights, fanout_weights, optimizer, activations)\u001b[0m\n\u001b[1;32m 311\u001b[0m fanin_weights \u001b[39m=\u001b[39m kaiming_uniform(torch\u001b[39m.\u001b[39mzeros(new_out_features\u001b[39m+\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mout_features, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39min_features))[:new_out_features, :]\n\u001b[1;32m 312\u001b[0m \u001b[39melif\u001b[39;00m fanin_weights \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39miterative_orthogonalization\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m--> 313\u001b[0m fanin_weights \u001b[39m=\u001b[39m iterative_orthogonalization(torch\u001b[39m.\u001b[39;49mzeros(new_out_features\u001b[39m+\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mout_features, \n\u001b[1;32m 314\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49min_features), \n\u001b[1;32m 315\u001b[0m \u001b[39minput\u001b[39;49m\u001b[39m=\u001b[39;49mactivations)[:new_out_features, :]\n\u001b[1;32m 316\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39misinstance\u001b[39m(fanin_weights, torch\u001b[39m.\u001b[39mTensor) \u001b[39mand\u001b[39;00m \u001b[39mlen\u001b[39m(fanin_weights\u001b[39m.\u001b[39mshape) \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[1;32m 317\u001b[0m fanin_weights \u001b[39m=\u001b[39m fanin_weights\u001b[39m.\u001b[39munsqueeze(\u001b[39m1\u001b[39m)\n",
"File \u001b[0;32m~/repos/NeurOps/pytorch/src/initializations.py:21\u001b[0m, in \u001b[0;36miterative_orthogonalization\u001b[0;34m(weights, input, stride, output_normalize)\u001b[0m\n\u001b[1;32m 19\u001b[0m numneurons \u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39msize(\u001b[39m0\u001b[39m)\n\u001b[1;32m 20\u001b[0m u, s, v \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39msvd(\u001b[39minput\u001b[39m)\n\u001b[0;32m---> 21\u001b[0m weights \u001b[39m=\u001b[39m (u[:numneurons,:numneurons]\u001b[39m.\u001b[39;49mmm(torch\u001b[39m.\u001b[39;49mdiag(\u001b[39m1\u001b[39;49m\u001b[39m/\u001b[39;49mtorch\u001b[39m.\u001b[39;49msqrt(s[:numneurons])))\u001b[39m.\u001b[39;49mmm(v[:,:numneurons]\u001b[39m.\u001b[39;49mt()))\u001b[39m.\u001b[39;49mreshape(numneurons, \u001b[39m*\u001b[39;49mweights\u001b[39m.\u001b[39;49mshape[\u001b[39m1\u001b[39;49m:])\n\u001b[1;32m 22\u001b[0m \u001b[39mif\u001b[39;00m output_normalize:\n\u001b[1;32m 23\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(weights\u001b[39m.\u001b[39mshape) \u001b[39m==\u001b[39m \u001b[39m4\u001b[39m:\n",
"\u001b[0;31mRuntimeError\u001b[0m: shape '[34, 68]' is invalid for input of size 2176"
]
}
],
Expand Down

0 comments on commit 1af86de

Please sign in to comment.