Skip to content

Commit

Permalink
can merge?
Browse files Browse the repository at this point in the history
  • Loading branch information
@hwnam831
hwnam831 committed Dec 8, 2021
1 parent 02a7907 commit 316daf9
Show file tree
Hide file tree
Showing 7 changed files with 1,356 additions and 188 deletions.
227 changes: 227 additions & 0 deletions PCIE.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "b52655c1",
"metadata": {},
"outputs": [],
"source": [
"import PCIEDataset\n",
"import numpy as np\n",
"import pickle\n",
"import torch\n",
"from torch.utils.data import Dataset, DataLoader\n",
"import os\n",
"import torch.nn as nn\n",
"import torch.nn.functional as F\n",
"import time\n",
"\n",
"class GaussianGenerator(nn.Module):\n",
" def __init__(self, scale):\n",
" super().__init__()\n",
" self.bias = nn.Parameter(torch.zeros(1,dtype=torch.float))\n",
" self.scale = nn.Parameter(torch.ones(1, dtype=torch.float)*scale)\n",
" def forward(self,raw_x):\n",
" noise = torch.rand_like(raw_x)*self.scale - self.scale/2\n",
" perturb = noise - raw_x + self.bias\n",
" return torch.relu(perturb)\n",
"\n",
"class RandomSpike(nn.Module):\n",
" def __init__(self, scale, freq):\n",
" super().__init__()\n",
" self.threshold = 1/freq\n",
" #self.scale = nn.Parameter(torch.ones(1, dtype=torch.float)*scale)\n",
" self.scale = scale\n",
" def forward(self,raw_x):\n",
" nums = torch.rand_like(raw_x)\n",
" flags = (nums<self.threshold)\n",
" noise = (nums * flags)*self.scale/self.threshold\n",
" perturb = noise - raw_x\n",
" return torch.relu(perturb)\n",
"\n",
"class SpikeRemover(nn.Module):\n",
" def __init__(self, threshold, spikegen):\n",
" super().__init__()\n",
" self.spikegen = spikegen\n",
" self.threshold = threshold\n",
" def forward(self,raw_x):\n",
" perturbed = raw_x + self.spikegen(raw_x)\n",
" flags = raw_x > self.threshold\n",
" meanval = raw_x.mean()\n",
" perturb = meanval * flags - raw_x * flags\n",
" return perturb\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7e283aa1",
"metadata": {},
"outputs": [],
"source": [
"test_dataset = PCIEDataset.PCIEDataset('./nvmessd')\n",
"trainset = []\n",
"testset = []\n",
"for i in range(len(test_dataset)):\n",
" if i%7 == 0:\n",
" testset.append(i)\n",
" else:\n",
" trainset.append(i)\n",
"trainloader = DataLoader(test_dataset, batch_size=8, num_workers=4, sampler=\n",
" torch.utils.data.SubsetRandomSampler(trainset))\n",
"valloader = DataLoader(test_dataset, batch_size=8, num_workers=4, sampler=\n",
" torch.utils.data.SubsetRandomSampler(testset))\n",
"clf_test = PCIEDataset.RawClassifier(512,128,4).cuda()\n",
"#clf_test = PCIEDataset.AvgClassifier(512,128,4).cuda()\n",
"#gen = GaussianGenerator(7.0).cuda()\n",
"gen = RandomSpike(scale=250.0, freq=50).cuda()\n",
"#gen = SpikeRemover(threshold=50.0)\n",
"c_mat = PCIEDataset.Cooldown(clf_test, gen, trainloader, valloader, epochs=50)\n",
"print(capacity(c_mat))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6ecf92c1",
"metadata": {},
"outputs": [],
"source": [
"def capacity(pd_mat):\n",
" cap = 0.0\n",
" m_j = pd_mat.sum(dim=0)\n",
" base = np.log(len(pd_mat))\n",
" for row in pd_mat:\n",
" log_p_i = np.log(row.sum())\n",
" for j,q_j_i in enumerate(row):\n",
" if q_j_i == 0.0:\n",
" continue\n",
" cap += (q_j_i/base)*(np.log(q_j_i) - np.log(m_j[j]) - log_p_i)\n",
" return cap\n",
"def bin_cap(p):\n",
" pd_mat = torch.tensor([[p/2,0.5-p/2],[0.5-p/2,p/2]])\n",
" return capacity(pd_mat)\n",
"\n",
"bin_cap(0.508)/210"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4ef83c51",
"metadata": {},
"outputs": [],
"source": [
"import lz4.frame\n",
"preprocessed_x = []\n",
"labels = []\n",
"window=1024\n",
"rootdir = 'train'\n",
"filelist = os.listdir(rootdir)\n",
"threshold = 3.0\n",
"for fname in filelist:\n",
" with open(rootdir + '/'+ fname, 'rb') as f:\n",
" xarr, label = pickle.load(f)\n",
" labels.append(label)\n",
" #max, avg, freq\n",
" parr = np.zeros([(len(xarr)+1)//window, 3], dtype=np.float32)\n",
" for i in range(0, len(xarr), window):\n",
" subseq = xarr[i:i+window]\n",
" parr[i//window][0] = np.max(subseq)\n",
" parr[i//window][1] = np.average(subseq)\n",
" parr[i//window][2] = np.sum(subseq > threshold)/len(subseq)\n",
" preprocessed_x.append(parr)\n",
"with lz4.frame.open(rootdir + '.lz4', 'wb') as f:\n",
" pickle.dump((preprocessed_x,labels), f)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b0858dcc",
"metadata": {},
"outputs": [],
"source": [
"raw_dataset = PCIEDataset.PCIEDataset('train', mode='preprocess')\n",
"print(\"Dataset loading done\")\n",
"classifier = PCIEDataset.PreprocessClassifier(32, 128, 3).cuda()\n",
"gen = PCIEDataset.PreprocessCNN(32, 128, 4).cuda()\n",
"\n",
"trainset = []\n",
"testset = []\n",
"for i in range(len(raw_dataset)):\n",
" if i%7 == 0:\n",
" testset.append(i)\n",
" else:\n",
" trainset.append(i)\n",
"trainloader = DataLoader(raw_dataset, batch_size=8, num_workers=4, sampler=\n",
" torch.utils.data.SubsetRandomSampler(trainset))\n",
"valloader = DataLoader(raw_dataset, batch_size=8, num_workers=4, sampler=\n",
" torch.utils.data.SubsetRandomSampler(testset))\n",
"\n",
"criterion = nn.CrossEntropyLoss()\n",
"PCIEDataset.Warmup(classifier, gen, trainloader, valloader, 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "916365f9",
"metadata": {},
"outputs": [],
"source": [
"len(os.listdir('train'))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d1ec629c",
"metadata": {},
"outputs": [],
"source": [
"import lz4.frame"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5d6597e6",
"metadata": {},
"outputs": [],
"source": [
"!ls -lh"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0139ff40",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.3"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading

0 comments on commit 316daf9

Please sign in to comment.