tensorflow/tf-scope.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. v=tf.get_variable(name,shape,dtype,initizlizer) #给定值求\n",
    "\n",
    "2. tf.variable_scope('scope_name') #\n",
    "\n",
    "3. tf.get_variable_scope().reuse==False #会使那些范围内的变量不共享？\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. variable_scope and get_variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"foo\") as foo_scope: \n",
    "    v = tf.get_variable(\"v\", [1]) \n",
    "with tf.variable_scope(foo_scope): \n",
    "    w = tf.get_variable(\"w\", [1]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "whos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  1.1 variable_scope 会直接给get_variable加上名字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"foo\") as foo_scope:\n",
    "    assert foo_scope.name == \"foo\"\n",
    "with tf.variable_scope(\"bar\"):\n",
    "    with tf.variable_scope(\"baz\") as other_scope:\n",
    "        assert other_scope.name == \"bar/baz\"\n",
    "        with tf.variable_scope(foo_scope) as foo_scope2:\n",
    "            assert foo_scope2.name == \"foo\" # 保持不变"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2变量作用域的初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo/v:0\n",
      "foo/w:0\n",
      "foo/bar/v:0\n",
      "foo/baz/v:0\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope(\"foo\", initializer=tf.constant_initializer(0.4)):\n",
    "    v = tf.get_variable(\"v\", [1])\n",
    "    print(v.name)\n",
    "    #assert v.eval() == 0.4 # 被作用域初始化\n",
    "    w = tf.get_variable(\"w\", [1], initializer=tf.constant_initializer(0.3))\n",
    "    print(w.name)\n",
    "    #assert w.eval() == 0.3 # 重写初始化器的值\n",
    "    with tf.variable_scope(\"bar\"):\n",
    "        v = tf.get_variable(\"v\", [1])\n",
    "        print(v.name)\n",
    "        #assert v.eval() == 0.4 # 继承默认的初始化器\n",
    "    with tf.variable_scope(\"baz\", initializer=tf.constant_initializer(0.2)):\n",
    "        v = tf.get_variable(\"v\", [1])\n",
    "        print(v.name)\n",
    "        #assert v.eval() == 0.2 # 重写父作用域的初始化器的值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "variable_scope 对op_name有怎么样的影响？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xsj_6/add\n"
     ]
    }
   ],
   "source": [
    "with tf.variable_scope('xsj',):\n",
    "    tf.get_variable_scope().reuse_variables()\n",
    "    x=1.0+tf.get_variable('v',[1])\n",
    "    print(x.op.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "为什么设置可以重复使用后，在空间后边加上了编号？但是没事，还是只有一个显示出来"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "variable_scope 主要用在循环神经网络（RNN）的操作中，其中需要大量的共享变量。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable   Type        Data/Info\n",
      "--------------------------------\n",
      "tf         module      <module 'tensorflow' from<...>tensorflow\\\\__init__.py'>\n",
      "v          Variable    <tf.Variable 'foo/baz/v:0<...>e=(1,) dtype=float32_ref>\n",
      "w          Variable    <tf.Variable 'foo/w:0' sh<...>e=(1,) dtype=float32_ref>\n",
      "x          Tensor      Tensor(\"xsj_6/add:0\", shape=(1,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "whos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. name_scope\n",
    "    name_scope会影响 op_name，不会影响用 get_variable()创建的变量，而会影响通过 Variable()创建的变量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('foo4/v:0', 'foo4/bar/b:0', 'foo4/bar/add', 'foo4/bar/b')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with tf.variable_scope(\"foo4\"):\n",
    "     with tf.name_scope(\"bar\"):\n",
    "        v = tf.get_variable(\"v\", [1])\n",
    "        b = tf.Variable(tf.zeros([1]), name='b')\n",
    "        x = 1.0 + v\n",
    "        \n",
    "v.name,b.name,x.op.name,b.op.name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 批标准化（batch normalization，BN）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "批标准化一般用在非线性映射（激活函数）之前，对 x=Wu+b 做规范化，使结果（输出\n",
    "\n",
    "信号各个维度）的均值为 0，方差为 1。让每一层的输入有一个稳定的分布会有利于网络的\n",
    "\n",
    "训练。\n"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 计算 Wx_plus_b 的均值和方差，其中 axes=[0]表示想要标准化的维度\n",
    "fc_mean, fc_var = tf.nn.moments(Wx_plus_b, axes=[0], )\n",
    "scale = tf.Variable(tf.ones([out_size]))\n",
    "shift = tf.Variable(tf.zeros([out_size]))\n",
    "epsilon = 0.001\n",
    "Wx_plus_b = tf.nn.batch_normalization(Wx_plus_b, fc_mean, fc_var, shift,\n",
    "scale, epsilon)\n",
    "# 也就是在做：\n",
    "# Wx_plus_b = (Wx_plus_b - fc_mean) / tf.sqrt(fc_var + 0.001)\n",
    "# Wx_plus_b = Wx_plus_b * scale + shift"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf.nn.relu()\n",
    "\n",
    "tf.nn.sigmoid()\n",
    "\n",
    "tf.nn.tanh()\n",
    "\n",
    "tf.nn.elu()\n",
    "\n",
    "tf.nn.bias_add()\n",
    "\n",
    "tf.nn.crelu()\n",
    "\n",
    "tf.nn.relu6()\n",
    "\n",
    "tf.nn.softplus()\n",
    "\n",
    "tf.nn.softsign()\n",
    "\n",
    "tf.nn.dropout() # 防止过拟合，用来舍弃某些神经元"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. 卷积"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf.nn.convolution(input, filter, padding, strides=None,dilation_rate=None, name=None, data_format=None)\n",
    "\n",
    "tf.nn.conv2d(input, filter, strides, padding, use_cudnn_on_gpu=None,data_format= None, name=None)\n",
    "tf.nn.depthwise_conv2d (input, filter, strides, padding, rate=None, name=None,data_format=None)\n",
    "\n",
    "tf.nn.separable_conv2d (input, depthwise_filter, pointwise_filter, strides, padding,rate=None, name=None, data_format=None)\n",
    "\n",
    "tf.nn.atrous_conv2d(value, filters, rate, padding, name=None)\n",
    "\n",
    "tf.nn.conv2d_transpose(value, filter, output_shape, strides, padding='SAME',data_format='NHWC', name=None)\n",
    "\n",
    "tf.nn.conv1d(value, filters, stride, padding, use_cudnn_on_gpu=None,data_format= None, name=None)\n",
    "\n",
    "tf.nn.conv3d(input, filter, strides, padding, name=None)\n",
    "\n",
    "tf.nn.conv3d_transpose(value, filter, output_shape, strides, padding='SAME', name=None) "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "def conv2d(input, filter, strides, padding, use_cudnn_on_gpu=None,\n",
    "data_format=None, name=None)\n",
    "# 输入：\n",
    "# input：一个 Tensor。数据类型必须是 float32 或者 float64 \n",
    "# filter：一个 Tensor。数据类型必须是 input 相同\n",
    "# strides：一个长度是 4 的一维整数类型数组，每一维度对应的是 input 中每一维的对应移动步数，\n",
    "# 比如，strides[1]对应 input[1]的移动步数\n",
    "# padding：一个字符串，取值为 SAME 或者 VALID\n",
    "# padding='SAME'：仅适用于全尺寸操作，即输入数据维度和输出数据维度相同\n",
    "# padding='VALID：适用于部分窗口，即输入数据维度和输出数据维度不同\n",
    "# use_cudnn_on_gpu：一个可选布尔值，默认情况下是 True\n",
    "# name：（可选）为这个操作取一个名字"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf.nn.conv1d(value, filters, stride, padding, use_cudnn_on_gpu=None, data_format=None,name=None)\n",
    "    \n",
    "    这个函数是用来计算给定三维的输入和过滤器的情况下的一维卷积。\n",
    "    不同的是，它的输入是三维，如[batch, in_width, in_channels]。\n",
    "    卷积核的维度也是三维，少了一维 filter_height，如 [filter_width, in_channels, out_channels]。stride 是一个正整数，代表卷积核向右移动每一步的长度。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. 池化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf.nn.avg_pool(value, ksize, strides, padding, data_format='NHWC', name=None)\n",
    "\n",
    "tf.nn.max_pool(value, ksize, strides, padding, data_format='NHWC', name=None)\n",
    "\n",
    "tf.nn.max_pool_with_argmax(input, ksize, strides, padding, Targmax=None, name=None)\n",
    "\n",
    "tf.nn.avg_pool3d(input, ksize, strides, padding, name=None)\n",
    "\n",
    "tf.nn.max_pool3d(input, ksize, strides, padding, name=None)\n",
    "\n",
    "tf.nn.fractional_avg_pool(value, pooling_ratio, pseudo_random=None, overlapping=None,\n",
    "deterministic=None, seed=None, seed2=None, name=None)\n",
    "\n",
    "tf.nn.fractional_max_pool(value, pooling_ratio, pseudo_random=None, overlapping=None,\n",
    "deterministic=None, seed=None, seed2=None, name=None)\n",
    "\n",
    "tf.nn.pool(input, window_shape, pooling_type, padding, dilation_rate=None, strides=None,\n",
    "name=None, data_format=None) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. 分类函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " sigmoid_cross_entropy_with_logits、softmax、log_softmax、\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. 优化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "class tf.train.GradientDescentOptimizer\n",
    "\n",
    "class tf.train.AdadeltaOptimizer\n",
    "\n",
    "class tf.train.AdagradOptimizer\n",
    "\n",
    "class tf.train.AdagradDAOptimizer\n",
    "\n",
    "class tf.train.MomentumOptimizer\n",
    "\n",
    "class tf.train.AdamOptimizer\n",
    "\n",
    "class tf.train.FtrlOptimizer\n",
    "\n",
    "class tf.train.RMSPropOptimizer "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这 8 个优化器对应 8 种优化方法，分别是梯度下降法（BGD 和 SGD）、Adadelta 法、Adagrad\n",
    "法（Adagrad 和 AdagradDAO）、Momentum 法（Momentum 和 Nesterov Momentum）、Adam、\n",
    "Ftrl 法和 RMSProp 法，其中 BGD、SGD、Momentum 和 Nesterov Momentum 是手动指定学习\n",
    "率的，其余算法能够自动调节学习率。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3．Momentum 法\n",
    "Momentum 是模拟物理学中动量的概念，更新时在一定程度上保留之前的更新方向，利用\n",
    "当前的批次再微调本次的更新参数，因此引入了一个新的变量 v（速度），作为前几次梯度的累\n",
    "加。因此，Momentum 能够更新学习率，在下降初期，前后梯度方向一致时，能够加速学习；\n",
    "在下降的中后期，在局部最小值的附近来回震荡时，能够抑制震荡，加很收敛。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9.模型的存储与加载 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TensorFlow 的 API 提供了以下两种方式来存储和加载模型。\n",
    "\n",
    "1. 生成检查点文件（checkpoint file），扩展名一般为.ckpt，通过在 tf.train.Saver 对象上调\n",
    "用 Saver.save()生成。它包含权重和其他在程序中定义的变量，不包含图结构。如果需要在另一\n",
    "个程序中使用，需要重新创建图形结构，并告诉 TensorFlow 如何处理这些权重。\n",
    "2. 生成图协议文件（graph proto file），这是一个二进制文件，扩展名一般为.pb，用\n",
    "tf.train.write_graph()保存，只包含图形结构，不包含权重，然后使用 tf.import_graph_def()来加载\n",
    "图形"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.1 变量参数"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 在声明完所有变量后，调用 tf.train.Saver\n",
    "saver = tf.train.Saver() \n",
    "\n",
    "with tf.Session() as sess:\n",
    "    tf.initialize_all_variables().run() \n",
    "    saver.save(sess,  \"/model.ckpt\", global_step=global_step) # 存储模型\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path) # 加载所有的参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.2 图的存储于加载"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 写入"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "sess = tf.Session()\n",
    "tf.train.write_graph(sess.graph_def, '/tmp/tfmodel', 'train.pbtxt')  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 读取"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "with tf.Session() as _sess:\n",
    "    with gfile.FastGFile(\"/tmp/tfmodel/train.pbtxt\",'rb') as f:\n",
    "    graph_def = tf.GraphDef()\n",
    "    graph_def.ParseFromString(f.read())\n",
    "    _sess.graph.as_default()\n",
    "    tf.import_graph_def(graph_def, name='tfgraph') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10 队列"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. FIFOQueue 创建一个先入先出队列。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "# 创建一个先入先出队列,初始化队列插入 0.1、0.2、0.3 三个数字\n",
    "q = tf.FIFOQueue(3, \"float\")\n",
    "init = q.enqueue_many(([0.1, 0.2, 0.3],))\n",
    "# 定义出队、+1、入队操作\n",
    "x = q.dequeue()#x=0.1\n",
    "y = x + 1#y=1.1\n",
    "q_inc = q.enqueue([y]) #"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29 3\n",
      "10.1\n",
      "10.2\n",
      "10.3\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    quelen = sess.run(q.size())\n",
    "    \n",
    "    for i in range(30):\n",
    "        sess.run(q_inc) # 执行 2 次操作，队列中的值变为 0.3,1.1,1.2\n",
    "    quelen = sess.run(q.size())\n",
    "    print(i,quelen)\n",
    "    for i in range(quelen):\n",
    "        print (sess.run(q.dequeue())) # 输出队列的值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2. RandomShuffleQueue 创建一个随机队列，在出队列时，是以随机的顺序产生元素的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "q = tf.RandomShuffleQueue(capacity=10, min_after_dequeue=2, dtypes=\"float\") \n",
    "sess = tf.Session()\n",
    "for i in range(0, 10): #10 次入队\n",
    "    sess.run(q.enqueue(i))\n",
    "#for i in range(0, 8): # 8 次出队\n",
    "#    print(sess.run(q.dequeue())) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11. 线程和协调器"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# 主线程\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())\n",
    "# Coordinator：协调器，协调线程间的关系可以视为一种信号量，用来做同步\n",
    "coord = tf.train.Coordinator()\n",
    "# 启动入队线程，协调器是线程的参数\n",
    "enqueue_threads =qr.create_threads(sess, coord = coord,start=True)\n",
    "# 主线程\n",
    "for i in range(0, 10):\n",
    "    print(sess.run(q.dequeue()))\n",
    "coord.request_stop()# 通知其他线程关闭\n",
    "coord.join(enqueue_threads) # join 操作等待其他线程结束，其他所有线程关闭之后，这一函数才能返回"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6 3 5]\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "# 设计图\n",
    "x1 = tf.constant([2, 3, 4])\n",
    "x2 = tf.constant([4, 0, 1])\n",
    "y = tf.add(x1, x2) \n",
    "a1 = tf.placeholder(tf.int16)\n",
    "a2 = tf.placeholder(tf.int16)\n",
    "b = tf.add(x1, x2)\n",
    "# 用 Python 产生数据\n",
    "li1 = [2, 3, 4]\n",
    "li2 = [4, 0, 1]\n",
    "# 打开一个会话，将数据填充给后端\n",
    "with tf.Session() as sess:\n",
    "    print(sess.run(b, feed_dict={a1: li1, a2: li2}) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}