memory-probe-examples.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2\"\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download and load custom op extension"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "response = urllib.request.urlopen(\"https://raw.githubusercontent.com/yaroslavvb/memory_probe_ops/master/linux.memory_probe_ops.so\")\n",
    "open(\"memory_probe_ops.so\", \"wb\").write(response.read())\n",
    "memory_probe_ops = tf.load_op_library(\"./memory_probe_ops.so\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check memory before and after initializing variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before init       1280 out of 12051264308 bytes\n",
      "After  init   67110656 out of 12051264308 bytes\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    probe_op = memory_probe_ops.bytes_in_use()\n",
    "    max_op = memory_probe_ops.bytes_limit()\n",
    "    mbs = 42\n",
    "    n = mbs*250000\n",
    "    inputs = tf.random_uniform((n,))\n",
    "\n",
    "    var = tf.Variable(inputs)\n",
    "    probe_op = memory_probe_ops.bytes_in_use()\n",
    "    max_op = memory_probe_ops.bytes_limit()\n",
    "    print(\"Before init %10d out of %10d bytes\" % tuple(sess.run([probe_op, max_op])))\n",
    "    sess.run(var.initializer)\n",
    "    print(\"After  init %10d out of %10d bytes\" % tuple(sess.run([probe_op, max_op])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Probe memory on multiple GPUs "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check that gpu:1 and gpu:2 memory are unaffected by variable initialized gpu:0 in previous cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gpu:1 bytes 1280, gpu:2 bytes 1280\n"
     ]
    }
   ],
   "source": [
    "with tf.device(\"/gpu:1\"):\n",
    "    probe_op_gpu1 = memory_probe_ops.bytes_in_use()\n",
    "with tf.device(\"/gpu:2\"):\n",
    "    probe_op_gpu2 = memory_probe_ops.bytes_in_use()\n",
    "print(\"gpu:1 bytes %d, gpu:2 bytes %d\" % tuple(sess.run([probe_op_gpu1, probe_op_gpu2])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Probe memory within single step"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can probe memory during step execution by using control dependencies to control when probe gets executed. Here's an example of tracing a computation that needs up to 100MB for the first half of computation, and then 50MB for the rest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fb2386f0160>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEDCAYAAADOc0QpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl4VPW9x/H3lyyEBAhkYRHIALKJKFsuilq01lq0rdTW\nWlBsa72yWKre6rW23nqt93av7bWtIuq1tqCCVqvYYt1q1WpRSdgEBAJCSNgCYQmErPO7f2TijSEh\nQzIzZ+bk83qePMyc+U3O9zmZfDj5nXO+x5xziIiIv3TxugAREYk8hbuIiA8p3EVEfEjhLiLiQwp3\nEREfUriLiPiQp+FuZo+Y2V4zez+MsXlm9pqZrTSzNWZ2aSxqFBFJRF7vuT8KTA1z7H8ATzrnxgPT\ngfujVZSISKLzNNydc28A5U2XmdmpZvZXMyswszfNbFTjcKBn6HEmsDOGpYqIJJRkrwtowYPAHOfc\nZjM7i4Y99AuBu4CXzOxbQAZwkXcliojEt7gKdzPrDpwDPGVmjYu7hv6dATzqnLvHzCYDC81sjHMu\n6EGpIiJxLa7CnYZpooPOuXEtvHYdofl559w/zSwNyAH2xrA+EZGE4PUB1Y9xzh0GPjSzLwNYg7Gh\nl4uBT4WWnwakAWWeFCoiEufMy66QZvYEcAENe+B7gP8E/gbMB/oDKcBi59zdZjYaeAjoTsPB1duc\ncy95UbeISLzzNNxFRCQ64mpaRkREIsOzA6o5OTlu8ODBXq1eRCQhFRQU7HPO5bY1zrNwHzx4MCtW\nrPBq9SIiCcnMtoczTtMyIiI+pHAXEfEhhbuIiA8p3EVEfEjhLiLiQwp3EREfUriLiPiQwl06lYqq\nWhYt305dvTpFi7/FW8tfkagJBh3/tmQ1r2zYw9DcDM45NcfrkkSiRnvu0mnc//ciXtmwB4Dt+ys9\nrkYkuhTu0in8feNe7nl5E58fewqpSV0U7uJ7CnfxvR3lldy0eBUj+/bgp186g4FZ3SguP+p1WSJR\npXAXXztWU8/shQU451hwzUTSU5MJZKWzbZ/23MXfFO7iW8457nh2LRt2H+be6eMJZGcAEMjOoLi8\nEt2oRvxM4S6+tWj5dp4pLOWmTw3nk6P6fLQ8kJ3Okeo6yo/WeFidSHQp3MWXCraX84Pn1/OpUX24\n8cLhH3stkJ0OwPZyTc2IfyncxXf2VlQxd1EhA3p345dfGUeXLvax1/OyGqZntu/XQVXxL4W7+Ept\nfZB5j62koqqOB2ZOJLNbynFjBmV1w0znuou/6QpV8ZUfLdvAu9vKuXf6OE7r37PFMV2TkzglsxvF\nCnfxMe25i288t6qU3721jW+cO4Rp4waccGxeVjrbNC0jPtZmuJvZI2a218zeb+V1M7Nfm1mRma0x\nswmRL1PkxDbsOsx3nl7DpCFZfPfSUW2OD2SnU6wDquJj4ey5PwpMPcHrlwDDQ1+zgPkdL0skfIcq\na5m9sIDMbin89qrxpCS1/bHOy05n35EajlTXxaBCkdhr87fAOfcGUH6CIdOAP7gGy4FeZtY/UgWK\nnEgw6Lh5yUp2HTrG/VdPpE+PtLDeNzh0QZPm3cWvIjHnPgDY0eR5SWjZccxslpmtMLMVZWVlEVi1\ndHb3vrqZ1zaWcefnRjMx0Dvs9+Vlhc5117y7+FRMD6g65x50zuU75/Jzc3NjuWrxoVc37OHeVzfz\npQkDmXl24KTeqwuZxO8iEe6lwKAmzweGlolEzbZ9R7l5ySpOP6UnP7x8DGbW9pua6JGWQnZGqs51\nF9+KRLgvBb4aOmvmbOCQc25XBL6vSIsqa+qYs6iApC7GAzMnkpaS1K7vk5edrmkZ8a02L2IysyeA\nC4AcMysB/hNIAXDOPQAsAy4FioBK4NpoFSvinOO7z6xl454Kfn/tJAaF5s7bI5CVznvbDkSwOpH4\n0Wa4O+dmtPG6A74ZsYpETuB3b23juVU7+ffPjGTKiI4dt8nLzmDp6p3U1AVJTdb1fOIv+kRLwnhn\n635+tGwDnx7dl7nnn9rh7zc4O52gg5IDmncX/1G4S0LYc7iKbz6+krysdO65cuxxnR7b46MzZnRQ\nVXxIjcMk7tXUBZm7qIDKmjqeuP4seqYd3+mxPdT6V/xM4S5x77/+vJ7C4oPcd9UEhvftEbHvm9M9\nlYzUJJ3rLr6kaRmJa38sKGHh8u3MmjKUz54Z2a4WZkZedoZaEIgvKdwlbr1feog7/rSWyUOzue0z\nI6OyjoBa/4pPKdwlLh04WsOcRQVkZaTym6vGkxxGp8f2CGSns+PAMYJBF5XvL+IVhbvEnfqg48bF\nK9l7uJr5MyeS071r1NYVyM6gpi7I7sNVUVuHiBcU7hJ3fvXyJt7cvI8fTDudcYN6RXVdjadDampG\n/EbhLnHlpXW7+e1rRUz/l0HMmJQX9fU1tv7VQVXxG4W7xI0tZUf49pOrOXNgJndddnpM1nlKr26k\nJJlOhxTfUbhLXDhaXcechQWkJndhfgc6PZ6spC7GoN7p2nMX31G4i+ecc9z2xzVsKTvCb2eMZ0Cv\nbjFdf162TocU/1G4i+ceenMrf1m7i9umjuKcYTkxX38gq2HPvaHBqYg/KNzFU29v2cdPXviAS8b0\nY/aUoZ7UEMjOoKK6jgOVtZ6sXyQaFO7imZ0Hj/Gtx1cyNLc7P//y2JO+VV6k/H93SE3NiH8o3MUT\n1XX1zH2skOq6IA/MnEj3rt71sFPrX/EjdYUUT9y1dD2rdxzkgZkTGdanu6e1DOydjpnCXfxFe+4S\nc0veK+aJd4u54YJTmTqmn9flkJaSRP+eaWwv17SM+IfCXWJqTclBvv/cOj4xPIdbLo5Op8f2yMtO\n1567+IrCXWJm/5Fq5iwsILd7V+6dPp6kCNwqL1ICWRkKd/EVhbvERF19kBsXr2Tf0RoWXDORrIxU\nr0v6mLzsdPYdqeZodZ3XpYhEhMJdYuLnL23kraL9/PALYxgzINPrco4zOLvhfqrF6jEjPqFwl6h7\nYe0uFry+lavPyuPL+YO8LqdFOtdd/EbhLlFVtLeCW59azfi8Xtz5+dFel9OqPJ3rLj6jcJeoqaiq\nZdbCArqlJjH/6ol0TY5Np8f26JmWQlZGqlr/im/oIiaJCucctz61mu37K3nsX8+iX2aa1yW1KS8r\nXdMy4hvac5eomP/6Fl5ct4fvXjKKs4dme11OWAI61118ROEuEffm5jJ+8eJGPj/2FK47b4jX5YQt\nkJXOzoPHqKkLel2KSIcp3CWiSg5UcuMTKxnepwc//dIZnnV6bI9AdgZBB6UHj3ldikiHKdwlYqpq\n65m7qJC6escD10wkPTWxDuk0ng6puzKJH4QV7mY21cw2mlmRmd3ewut5Zvaama00szVmdmnkS5V4\n5pzj+8++z9rSQ/zqK+MYkpPhdUknrfF0SN1PVfygzXA3syTgPuASYDQww8yan7D8H8CTzrnxwHTg\n/kgXKvHt8XeLeaqghBsvHMZFo/t6XU675HbvSnpqkg6qii+Es+c+CShyzm11ztUAi4FpzcY4oGfo\ncSawM3IlSrwrLD7AXUvXccHIXG66aITX5bSbmZGXlU6xWv+KD4QT7gOAHU2el4SWNXUXMNPMSoBl\nwLda+kZmNsvMVpjZirKysnaUK/GmrKKaGxYV0i8zjf/5yri46vTYHoHsdLZpz118IFIHVGcAjzrn\nBgKXAgvN7Ljv7Zx70DmX75zLz83NjdCqxSt19UHmPV7IwWM1LJiZT6/0+Or02B6B7AyKyysJBp3X\npYh0SDjhXgo07fY0MLSsqeuAJwGcc/8E0oCcSBQo8esnL3zAOx+W8+MvnsHoU3q2/YYEEMhOp6Yu\nyJ6KKq9LEemQcML9PWC4mQ0xs1QaDpgubTamGPgUgJmdRkO4a97Fx55fvZOH//EhXz9nMJePH+h1\nORETyGo4y2fbPk3NSGJrM9ydc3XAPOBFYAMNZ8WsM7O7zeyy0LBbgOvNbDXwBPB155z+rvWpjbsr\n+M7Ta8gP9OZ7l57mdTkR1Xiuuw6qSqIL6yoT59wyGg6UNl12Z5PH64FzI1uaxKPDVbXMWVRARtdk\n7r96AqnJ/roOrn9mGsldTKdDSsLz12+mRFUw6Pj2ktXsKK9k/tUT6NMz/js9nqzkpC4MykpX619J\neAp3Cdt9rxXxyoY9fP9zo8kfnOV1OVGj1r/iBwp3CcvfN+7ll69s4vLxA/jq5IDX5URVY+tfHTaS\nRKZwlzYV76/kpsWrGNWvJz+6PLE6PbZHIDuDiqo6DlbWel2KSLsp3OWEjtXUM2dRAc45FsycSLfU\n+L1VXqQEstQdUhKfwl1a5Zzjjj+tZcPuw9w7Y/xHXRP97v9Ph9RBVUlcCndp1cLl23lmZSn/dtEI\nPjmyj9flxMyg0J67ToeURKZwlxYVbC/n7ufXc9FpfZj3yWFelxNTaSlJ9M9MU7hLQlO4y3H2VlQx\nd1EhA3t3454rx9ElwTs9todOh5REp3CXj6mtD/LNxwqpqKpjwTX5ZHZL8bokTwSydSGTJDaFu3zM\nD/+ygfe2HeCnV5zJyH49vC7HM4HsDMoqqqmsqfO6FJF2UbjLR55dWcqjb2/juvOGcNnYU7wux1M6\nY0YSncJdAFi/8zC3P7OGSUOyuP2SUV6X4zm1/pVEp3AXDlU2dHrM7JbCfVdNICVJH4s8tf6VBBdW\ny1/xr2DQcfOSlew6dIwlsyeT26Or1yXFhcxuKfROT9HpkJKwtIvWyd376mZe21jGf37+dCbk9fa6\nnLiSl52hcJeEpXDvxF7dsId7X93MFRMHcvVZeV6XE3cCWels17SMJCiFeye1bd9Rbl6yijEDevLf\nXxjj+06P7RHITmfnwSpq64NelyJy0hTunVBlTR2zFxaQ1MWYf/VE0lL83+mxPQLZGdQHHaUHjnld\nishJU7h3Ms45bn96LZv2VvDr6eM/apIlx2s8112tfyURKdw7mUfe2sbS1Tu59eKRTBmR63U5ca2x\nr7suZJJEpHDvRJZv3c+Plm3g4tF9mXv+qV6XE/dye3SlW0qSzpiRhKRw7yR2H6pi3uOFBLLSuefK\nsZ2y0+PJMrOP7qcqkmgU7p1ATV2QuY8VUFlTz4JrJtIjrXN2emwPtf6VRKVw7wT+68/rWVl8kJ9f\nMZbhfTtvp8f2CGSnU1xeSTDovC5F5KQo3H3ujwUlLFy+ndlThvLZM/t7XU7CCWRnUF0XZG9Ftdel\niJwUhbuPvV96iDv+tJbJQ7P598+M9LqchKTTISVRKdx96sDRGmYvLCA7I5XfXjWeZHV6bJfG1r/F\nOqgqCUZdIX2oPui4cfFKyiqqeWrOZLK7q9Nje53SK43kLqYeM5JwFO4+9MuXN/Lm5n385ItnMHZQ\nL6/LSWjJSV0Y2LubToeUhKO/1X3mxXW7ue+1LUz/l0FMn6ROj5Gg1r+SiBTuPrKl7Ai3PLmaMwdm\nctdlp3tdjm8EdK67JKCwwt3MpprZRjMrMrPbWxlzpZmtN7N1ZvZ4ZMuUthyprmPOwgJSk7swf6Y6\nPUZSIDudw1V1HKys8boUkbC1OeduZknAfcCngRLgPTNb6pxb32TMcOC7wLnOuQNm1idaBcvxnHPc\n9sfVbCk7wqLrzmJAr25el+QrgezQzbL3VzIuPdXjakTCE86e+ySgyDm31TlXAywGpjUbcz1wn3Pu\nAIBzbm9ky5QTeejNrSxbu5vvTB3FOcNyvC7HdxrPddfUjCSScMJ9ALCjyfOS0LKmRgAjzOwtM1tu\nZlNb+kZmNsvMVpjZirKysvZVLB/zdtE+fvLCB1x6Rj9mTRnqdTm+lNfY+lcHVSWBROqAajIwHLgA\nmAE8ZGbHnYPnnHvQOZfvnMvPzVUv8Y7aefAY855YydDc7vzsirG6VV6UpKUk0a9nGtvV110SSDjh\nXgoMavJ8YGhZUyXAUudcrXPuQ2ATDWEvUVJVW8/cRQXU1AVZcM1EunfVJQvRlJetM2YksYQT7u8B\nw81siJmlAtOBpc3GPEvDXjtmlkPDNM3WCNYpzfzg+XWsLjnEL748llNzu3tdju81nA6pPXdJHG2G\nu3OuDpgHvAhsAJ50zq0zs7vN7LLQsBeB/Wa2HngN+Hfn3P5oFd3ZLX63mCfe3cENF5zK1DH9vC6n\nUwhkp7O3oppjNfVelyISlrD+lnfOLQOWNVt2Z5PHDvh26EuiaP3Ow9z53Do+MTyHWy5Wp8dYaTwd\nsri8kpH91BNf4p+uUE0w9766ibSULtw7fTxJulVezKj1ryQahXsC2Vp2hJfW7+GrkweTlaGLaWJJ\nrX8l0SjcE8hDb35ISlIXvnbOYK9L6XQy01PolZ6i1r+SMBTuCWJvRRVPF5bwpQkDye2h/uxe0Bkz\nkkgU7gni929vo7Y+yPWfGOJ1KZ2WWv9KIlG4J4Cj1XUs/Od2PjO6H0N1TrtnAlnplB48Rm190OtS\nRNqkcE8Ai9/bweGqOmafr94xXgpkp1MfdOw8eMzrUkTapHCPc7X1Qf73za1MGpLF+LzeXpfTqTVt\n/SsS7xTuce7Pa3ay81AVs9Xx0XON57oX61x3SQAK9zjmnGPB61sZ3qc7nxyp+594rU+PrqSldNFB\nVUkICvc49sbmfXywu4JZU4bSRVejes7MCGRlqPWvJASFexxb8PoW+vbsyrRxze+NIl5R619JFAr3\nOLW25BBvb9nPN84dQmqyfkzxIpCVTnF5JQ298kTil+7wEKcWvLGFHl2TmXFWntelSBOBnAyqaoO8\nuG43PdNSvC5HEtTQ3O70y0yL6joU7nGoeH8ly9bu4vopQxUgcWZEn4aLyOYsKvS4Eklk//2FMcw8\nOxDVdSjc49DD/9hKUhfjG+eq1UC8mTQki+fnnUdlTZ3XpUgCG5yTEfV1KNzjTPnRGp5csYPLxw+g\nb8/o/tkmJ8/MOGNgptdliLRJR+rizB/+uY2q2iCzdNGSiHSAwj2OHKup5/dvb+Oi0/owrI9u5SYi\n7adwjyNPFezgQGUts88/1etSRCTBKdzjRF19kIfe3MqEvF7kB9QgTEQ6RuEeJ/66bjc7yo8xa8qp\nmKnVgIh0jMI9DjQ2CBuak8GnR/f1uhwR8QGFexz455b9rC09xPVThpKkBmEiEgEK9zjwwBtbyene\nlcvHq0GYiESGwt1jG3Yd5o1NZVx77mDSUpK8LkdEfELh7rEH39hKemoSM8+Kbp8JEelcFO4eKjlQ\nydLVO5kxKY/MdDUIE5HIUbh76JF/bMOAb5ynBmEiElkKd48cqqxl8XvFXDb2FAb06uZ1OSLiMwp3\njyx6ZzuVNfXMOl8NwkQk8hTuHqiqred3b33I+SNyGdWvp9fliIgPhRXuZjbVzDaaWZGZ3X6CcV8y\nM2dm+ZEr0X+eKSxl35EaZmuvXUSipM1wN7Mk4D7gEmA0MMPMRrcwrgdwE/BOpIv0k/qg4+E3t3Lm\nwEwmD832uhwR8alw9twnAUXOua3OuRpgMTCthXH/BfwUqIpgfb7z7MpStu47ymw1CBORKAon3AcA\nO5o8Lwkt+4iZTQAGOef+cqJvZGazzGyFma0oKys76WIT3aY9FXz/ufeZkNeLqWP6eV2OiPhYhw+o\nmlkX4JfALW2Ndc496JzLd87l5+bmdnTVCeVwVS2zFxaQnprM/JkT1SBMRKIqnHAvBQY1eT4wtKxR\nD2AM8Hcz2wacDSzVQdX/Fww6vr1kNTvKK7n/6gm68bWIRF044f4eMNzMhphZKjAdWNr4onPukHMu\nxzk32Dk3GFgOXOacWxGVihPQ/X8v4pUNe/jepacxaUiW1+WISCfQZrg75+qAecCLwAbgSefcOjO7\n28wui3aBie71TWXc8/Impo07hWvPHex1OSLSSSSHM8g5twxY1mzZna2MvaDjZfnDjvJKbnxiJSP7\n9uDHXzxDZ8eISMzoCtUoOVZTz+yFBQ230LtmIumpYf0/KiISEUqcKHDOcceza1m/6zCPfD2fQHaG\n1yWJSCejPfcoWLR8O88UlnLzRcO5cJRueC0isadwj7CC7eX84Pn1XDiqDzdeONzrckSkk1K4R9De\niirmLipkQO9u/OrKcXTRhUoi4hGFe4TU1geZ99hKDlfV8sDMibptnoh4SgdUI+RHyzbw7rZy7p0+\njtP6q0e7iHhLe+4R8NyqUn731jauPXcw08YNaPsNIiJRpnDvoA27DvOdp9cwaXAW37v0NK/LEREB\nFO4dcqiyljmLCsjslsJvrx5PSpI2p4jEB825t1Mw6Lh5yUp2HjzG4lmT6dNDnR5FJH5oV7Odfv23\nzby2sYw7PzeaiYHeXpcjIvIxCvd2+NsHe/ifVzbzxQkDmHl2wOtyRESOo3A/Sdv3H+XmxasY3b8n\nP7pcnR5FJD4p3E9CZU0dsxcWYGYsuGYiaSlJXpckItIiHVANk3OO7z6zlo17Knj02kkMykr3uiQR\nkVZpzz1Mj769jedW7eSWT4/g/BGd6+beIpJ4FO5hePfDcn74lw18enRfbrhgmNfliIi0SeHehj2H\nq7jhsULystK558qx6vQoIglBc+4nUFMXZO6iAipr6nj8+rPomaZOjyKSGBTuJ/Dff1lPYfFB7rtq\nAiP69vC6HBGRsGlaphVPF5Twh39uZ9aUoXz2zP5elyMiclIU7i14v/QQ3/vTWiYPzea2z4z0uhwR\nkZOmcG/mwNEa5iwqICsjld9cNZ5kdXoUkQSkOfcm6oOOm5asYu/hap6cM5mc7l29LklEpF0U7k38\n6uVNvLGpjB9/8QzGDerldTkiIu2mOYeQl9bt5revFfGV/EHMmJTndTkiIh2icAe2lh3hlidXc+bA\nTH4w7XSvyxER6bBOH+5Hqxs6PaYkd2H+THV6FBF/6NTh7pzjtqfXsKXsCL+ZMZ4Bvbp5XZKISER0\n6nB/+M0P+cuaXdw2dRTnDsvxuhwRkYgJK9zNbKqZbTSzIjO7vYXXv21m681sjZm9amZxf++5t7fs\n4yd//YBLxvRj9pShXpcjIhJRbYa7mSUB9wGXAKOBGWY2utmwlUC+c+5M4I/AzyJdaCTtPHiMbz2+\nksHZ6fz8y2N1qzwR8Z1w9twnAUXOua3OuRpgMTCt6QDn3GvOucrQ0+XAwMiWGTnVdfXMfayQ6rog\nC67Jp3tXneovIv4TTrgPAHY0eV4SWtaa64AXOlJUNN21dD2rdxzkF18ey7A+3b0uR0QkKiK622pm\nM4F84PxWXp8FzALIy4v9hUJL3ivmiXeLmXvBqUwd0y/m6xcRiZVw9txLgUFNng8MLfsYM7sIuAO4\nzDlX3dI3cs496JzLd87l5+bG9j6ka0oO8v3n1nHesBxuvVidHkXE38IJ9/eA4WY2xMxSgenA0qYD\nzGw8sICGYN8b+TI7Zv+RauYsLCC3e1d+PWM8SbpVnoj4XJvh7pyrA+YBLwIbgCedc+vM7G4zuyw0\n7OdAd+ApM1tlZktb+XYxV1cf5MbFK9l3tIYHZk4kKyPV65JERKIurDl359wyYFmzZXc2eXxRhOuK\nmF+8tIm3ivbzsyvO5IyBmV6XIyISE76+QvWFtbt44PUtXH1WHlfmD2r7DSIiPuHbcC/aW8GtT61m\n3KBe3Pn55tdciYj4my/DvaKqllkLC+iWmsT8mRPomqxOjyLSufju8kznHLc+tZrt+yt57F/Pon+m\nOj2KSOfjuz33+a9v4cV1e/juJaM4e2i21+WIiHjCV+H+5uYyfvHiRj53Zn+uO2+I1+WIiHjGN+Fe\ncqCSG59YyfA+PfjZFWeq06OIdGq+CPeq2nrmLCqgrt7xwDUTSU/13aEEEZGTkvAp6Jzj+8++z/ul\nh3n4q/kMycnwuiQREc8l/J774+8W81RBCTdeOIyLRvf1uhwRkbiQ0OFeWHyAu5au4/wRudx00Qiv\nyxERiRsJG+5lFdXcsKiQfplp3Dt9nDo9iog0kZBz7nX1QeY9XsiByhqeueEceqWr06OISFMJGe4/\neeED3vmwnF99ZSynn6JOjyIizSXctMzzq3fy8D8+5GuTA1w+Pm7vwy0i4qmEC/esjFQ+Pbovd3xW\nnR5FRFqTcNMy5w7L4dxhOV6XISIS1xJuz11ERNqmcBcR8SGFu4iIDyncRUR8SOEuIuJDCncRER9S\nuIuI+JDCXUTEh8w5582KzcqA7e18ew6wL4LlRJrq6xjV13HxXqPqa7+Acy63rUGehXtHmNkK51y+\n13W0RvV1jOrruHivUfVFn6ZlRER8SOEuIuJDiRruD3pdQBtUX8eovo6L9xpVX5Ql5Jy7iIicWKLu\nuYuIyAko3EVEfCiuw93MpprZRjMrMrPbW3i9q5ktCb3+jpkNjmFtg8zsNTNbb2brzOymFsZcYGaH\nzGxV6OvOWNUXWv82M1sbWveKFl43M/t1aPutMbMJMaxtZJPtssrMDpvZzc3GxHz7mdkjZrbXzN5v\nsizLzF42s82hf3u38t6vhcZsNrOvxai2n5vZB6Gf35/MrFcr7z3hZyHKNd5lZqVNfo6XtvLeE/6+\nR7G+JU1q22Zmq1p5b0y2YcQ45+LyC0gCtgBDgVRgNTC62ZgbgAdCj6cDS2JYX39gQuhxD2BTC/Vd\nAPzZw224Dcg5weuXAi8ABpwNvOPhz3o3DRdneLr9gCnABOD9Jst+Btweenw78NMW3pcFbA392zv0\nuHcMarsYSA49/mlLtYXzWYhyjXcBt4bxGTjh73u06mv2+j3AnV5uw0h9xfOe+ySgyDm31TlXAywG\npjUbMw34fejxH4FPmZnFojjn3C7nXGHocQWwARgQi3VH0DTgD67BcqCXmfX3oI5PAVucc+29Yjli\nnHNvAOXNFjf9nP0e+EILb/0M8LJzrtw5dwB4GZga7dqccy855+pCT5cDnt41vpXtF45wft877ET1\nhbLjSuCJSK/XC/Ec7gOAHU2el3B8eH40JvQBPwRkx6S6JkLTQeOBd1p4ebKZrTazF8zs9JgWBg54\nycwKzGxWC6+Hs41jYTqt/0J5uf0a9XXO7Qo93g30bWFMPGzLb9Dwl1hL2vosRNu80NTRI61Ma8XD\n9vsEsMc5t7mV173ehiclnsM9IZhZd+Bp4Gbn3OFmLxfSMNUwFvgN8GyMyzvPOTcBuAT4pplNifH6\n22RmqcCixO9VAAACR0lEQVRlwFMtvOz19juOa/j7PO7OHzazO4A64LFWhnj5WZgPnAqMA3bRMPUR\nj2Zw4r32uP99aiqew70UGNTk+cDQshbHmFkykAnsj0l1DetMoSHYH3POPdP8defcYefckdDjZUCK\nmeXEqj7nXGno373An2j407epcLZxtF0CFDrn9jR/wevt18Sexumq0L97Wxjj2bY0s68DnwOuDv3n\nc5wwPgtR45zb45yrd84FgYdaWbenn8VQfnwRWNLaGC+3YXvEc7i/Bww3syGhvbvpwNJmY5YCjWcl\nXAH8rbUPd6SF5uf+F9jgnPtlK2P6NR4DMLNJNGzvmPznY2YZZtaj8TENB97ebzZsKfDV0FkzZwOH\nmkw/xEqre0tebr9mmn7OvgY818KYF4GLzax3aNrh4tCyqDKzqcBtwGXOucpWxoTzWYhmjU2P41ze\nyrrD+X2PpouAD5xzJS296PU2bBevj+ie6IuGszk20XAU/Y7Qsrtp+CADpNHw53wR8C4wNIa1nUfD\nn+drgFWhr0uBOcCc0Jh5wDoajvwvB86JYX1DQ+tdHaqhcfs1rc+A+0Lbdy2QH+OfbwYNYZ3ZZJmn\n24+G/2h2AbU0zPteR8NxnFeBzcArQFZobD7wcJP3fiP0WSwCro1RbUU0zFU3fgYbzx47BVh2os9C\nDLffwtDnaw0Ngd2/eY2h58f9vseivtDyRxs/d03GerINI/Wl9gMiIj4Uz9MyIiLSTgp3EREfUriL\niPiQwl1ExIcU7iIiPqRwFxHxIYW7iIgP/R/BlIDc7ZsGogAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fb26808b550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# makes chain of ops\n",
    "def make_chain(length, mbs_per_node):\n",
    "    n = mbs_per_node*250000\n",
    "    node0 = tf.random_uniform((n,))\n",
    "    node = node0\n",
    "    leaf = node0\n",
    "    probes = []\n",
    "    nodes = [node]\n",
    "    for i in range(length):\n",
    "        with tf.control_dependencies([node, leaf]):\n",
    "            probe = memory_probe_ops.bytes_in_use()\n",
    "        probes.append(probe)\n",
    "        with tf.control_dependencies([probe]):\n",
    "            leaf = tf.random_uniform((n,))\n",
    "            if i < length//2:\n",
    "                node = tf.add_n(nodes+[leaf])\n",
    "            else:\n",
    "                node = tf.add_n(nodes[-length//4:]+[leaf])\n",
    "        nodes.append(node)\n",
    "    return nodes, probes\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "with tf.device(\"/gpu:0\"):\n",
    "    nodes, probes = make_chain(20, 10)\n",
    "initial_memory = sess.run(memory_probe_ops.bytes_in_use())\n",
    "_, probes0 = sess.run([nodes[-1], probes])\n",
    "plt.plot(np.array(probes0)-initial_memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:tim-jan17]",
   "language": "python",
   "name": "conda-env-tim-jan17-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "84px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": false,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}