Tutorial_7 - Full understanding of name_scope and variable_scope.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "# 充分理解 name / variable_scope \n",
    "@author: huangyongye<br/>\n",
    "@creat_date: 2017-03-08<br/><br/>\n",
    "refer to: https://www.tensorflow.org/programmers_guide/variable_scope <br/>\n",
    "** 起因：在运行 RNN LSTM 实例代码的时候出现 ValueError。 ** <br/>\n",
    "在 TensorFlow 中，经常会看到这两个东东出现，这到底是什么鬼，是用来干嘛的。在做 LSTM 的时候遇到了下面的错误：<br/>\n",
    " <font color='red'>ValueError: Variable rnn/basic_lstm_cell/weights already exists, disallowed.</font><br/>\n",
    "然后谷歌百度都查了一遍，结果也不知是咋回事。我是在 jupyter notebook 运行的示例程序，第一次运行的时候没错，然后就总是出现上面的错误。后来才知道是 get_variable() 和 variable_scope() 搞的鬼。 <br/>\n",
    "下面就来分析一下 TensorFlow 中到底用这来干啥。<br/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 实验1.三种方式创建变量： tf.placeholder, tf.Variable, tf.get_variable\n",
    "实验目的：探索三种方式定义的变量之间的区别。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "# 设置GPU按需增长\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Placeholder:0\n",
      "ph:0\n",
      "ph_1:0\n",
      "<class 'tensorflow.python.framework.ops.Tensor'>\n",
      "Tensor(\"ph_1:0\", shape=(2, 3, 4), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 1.placeholder \n",
    "v1 = tf.placeholder(tf.float32, shape=[2,3,4])\n",
    "print v1.name\n",
    "v1 = tf.placeholder(tf.float32, shape=[2,3,4], name='ph')\n",
    "print v1.name\n",
    "v1 = tf.placeholder(tf.float32, shape=[2,3,4], name='ph')\n",
    "print v1.name\n",
    "print type(v1)\n",
    "print v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable:0\n",
      "V:0\n",
      "V_1:0\n",
      "<class 'tensorflow.python.ops.variables.Variable'>\n",
      "Tensor(\"V_1/read:0\", shape=(2,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 2. tf.Variable()\n",
    "v2 = tf.Variable([1,2], dtype=tf.float32)\n",
    "print v2.name\n",
    "v2 = tf.Variable([1,2], dtype=tf.float32, name='V')\n",
    "print v2.name\n",
    "v2 = tf.Variable([1,2], dtype=tf.float32, name='V')\n",
    "print v2.name\n",
    "print type(v2)\n",
    "print v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gv:0\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Variable gv already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n  File \"<ipython-input-7-29efaac2d76c>\", line 2, in <module>\n    v3 = tf.get_variable(name='gv', shape=[])\n  File \"/home/common/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/home/common/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-29efaac2d76c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mv3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'gv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mv3\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mv4\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_variable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'gv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0mv4\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m    986\u001b[0m       \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    987\u001b[0m       \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 988\u001b[0;31m       custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m    989\u001b[0m get_variable_or_local_docstring = (\n\u001b[1;32m    990\u001b[0m     \"\"\"%s\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, var_store, name, shape, dtype, initializer, regularizer, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m    888\u001b[0m           \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    889\u001b[0m           \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidate_shape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate_shape\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 890\u001b[0;31m           custom_getter=custom_getter)\n\u001b[0m\u001b[1;32m    891\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    892\u001b[0m   def _get_partitioned_variable(self,\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36mget_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape, custom_getter)\u001b[0m\n\u001b[1;32m    346\u001b[0m           \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    347\u001b[0m           \u001b[0mcaching_device\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaching_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpartitioner\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpartitioner\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 348\u001b[0;31m           validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m    349\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    350\u001b[0m   def _get_partitioned_variable(\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_true_getter\u001b[0;34m(name, shape, dtype, initializer, regularizer, reuse, trainable, collections, caching_device, partitioner, validate_shape)\u001b[0m\n\u001b[1;32m    331\u001b[0m           \u001b[0minitializer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitializer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mregularizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreuse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreuse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    332\u001b[0m           \u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcollections\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcollections\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m           caching_device=caching_device, validate_shape=validate_shape)\n\u001b[0m\u001b[1;32m    334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    335\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcustom_getter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/common/anaconda2/lib/python2.7/site-packages/tensorflow/python/ops/variable_scope.pyc\u001b[0m in \u001b[0;36m_get_single_variable\u001b[0;34m(self, name, shape, dtype, initializer, regularizer, partition_info, reuse, trainable, collections, caching_device, validate_shape)\u001b[0m\n\u001b[1;32m    637\u001b[0m                          \u001b[0;34m\" Did you mean to set reuse=True in VarScope? \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m                          \"Originally defined at:\\n\\n%s\" % (\n\u001b[0;32m--> 639\u001b[0;31m                              name, \"\".join(traceback.format_list(tb))))\n\u001b[0m\u001b[1;32m    640\u001b[0m       \u001b[0mfound_var\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_vars\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_compatible_with\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfound_var\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Variable gv already exists, disallowed. Did you mean to set reuse=True in VarScope? Originally defined at:\n\n  File \"<ipython-input-7-29efaac2d76c>\", line 2, in <module>\n    v3 = tf.get_variable(name='gv', shape=[])\n  File \"/home/common/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"/home/common/anaconda2/lib/python2.7/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n"
     ]
    }
   ],
   "source": [
    "# 3.tf.get_variable() 创建变量的时候必须要提供 name\n",
    "v3 = tf.get_variable(name='gv', shape=[])  \n",
    "print v3.name\n",
    "v4 = tf.get_variable(name='gv', shape=[2])\n",
    "print v4.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tensorflow.python.ops.variables.Variable'>\n",
      "Tensor(\"gv/read:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print type(v3)\n",
    "print v3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "还记得有这么个函数吗？ tf.trainable_variables(), 它能够将我们定义的所有的 trainable=True 的所有变量以一个list的形式返回。 very good, 现在要派上用场了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "Tensor(\"Variable/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"V/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"V_1/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"gv/read:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "vs = tf.trainable_variables()\n",
    "print len(vs)\n",
    "for v in vs:\n",
    "    print v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "var1.name:  var1:0\n",
      "var2.name:  var1_1:0\n",
      "Tensor(\"var1/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"var1_1/read:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# 补充： tf.Variable 和 tf.get_variable 不会发生命名冲突。\n",
    "var1 = tf.Variable([1,2], dtype=tf.float32, name='var1')\n",
    "print 'var1.name: ', var1.name\n",
    "var2 = tf.get_variable(name='var1', shape=[])\n",
    "print 'var2.name: ', var2.name\n",
    "print var1\n",
    "print var2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "Tensor(\"Variable/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"V/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"V_1/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"gv/read:0\", shape=(), dtype=float32)\n",
      "Tensor(\"var1/read:0\", shape=(2,), dtype=float32)\n",
      "Tensor(\"var1_1/read:0\", shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "vs = tf.trainable_variables()\n",
    "print len(vs)\n",
    "for v in vs:\n",
    "    print v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实验1结论：从上面的实验结果来看，这三种方式所定义的变量具有相同的类型。而且只有 tf.get_variable() 创建的变量之间会发生命名冲突。其中\n",
    "- tf.placeholder() 定义的变量 trainable==False\n",
    "- tf.Variable() 可以选择 trainable 类型\n",
    "- tf.get_variable() 一定 trainable == True. \n",
    "\n",
    "在实际使用中，三种创建变量方式的用途也是分工非常明确的，主要说说 tf.get_variable()。<br/>\n",
    "tf.get_variable() 一般都是和 tf.variable_scope() 配合使用，从而实现变量共享的功能。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 实验2. 探索 name_scope 和 variable_scope"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "# 设置GPU按需增长\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v1.name:  nsc1/v1:0\n",
      "v2.name:  nsc1/vsc1/v2:0\n",
      "v3.name:  vsc1/v3:0\n"
     ]
    }
   ],
   "source": [
    "with tf.name_scope('nsc1'):\n",
    "    v1 = tf.Variable([1], name='v1')\n",
    "    with tf.variable_scope('vsc1'):\n",
    "        v2 = tf.Variable([1], name='v2')\n",
    "        v3 = tf.get_variable(name='v3', shape=[])\n",
    "print 'v1.name: ', v1.name\n",
    "print 'v2.name: ', v2.name\n",
    "print 'v3.name: ', v3.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "v4.name:  nsc1_1/v4:0\n"
     ]
    }
   ],
   "source": [
    "with tf.name_scope('nsc1'):\n",
    "    v4 = tf.Variable([1], name='v4')\n",
    "print 'v4.name: ', v4.name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "tf.name_scope() 并不会对 tf.get_variable() 创建的变量有任何影响。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "tf.name_scope() 主要是用来管理命名空间的，这样子让我们的整个模型更加有条理。而 tf.variable_scope() 的作用是为了实现变量共享，它和 tf.get_variable() 来完成变量共享的功能。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1.第一组，用 tf.Variable() 的方式来定义。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 8 train_able_variables in the Graph: \n",
      "Tensor(\"conv1_weights/read:0\", shape=(5, 5, 32, 32), dtype=float32)\n",
      "Tensor(\"conv1_biases/read:0\", shape=(32,), dtype=float32)\n",
      "Tensor(\"conv2_weights/read:0\", shape=(5, 5, 32, 32), dtype=float32)\n",
      "Tensor(\"conv2_biases/read:0\", shape=(32,), dtype=float32)\n",
      "Tensor(\"conv1_weights_1/read:0\", shape=(5, 5, 32, 32), dtype=float32)\n",
      "Tensor(\"conv1_biases_1/read:0\", shape=(32,), dtype=float32)\n",
      "Tensor(\"conv2_weights_1/read:0\", shape=(5, 5, 32, 32), dtype=float32)\n",
      "Tensor(\"conv2_biases_1/read:0\", shape=(32,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "# 设置GPU按需增长\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "# 拿官方的例子改动一下\n",
    "def my_image_filter():\n",
    "    conv1_weights = tf.Variable(tf.random_normal([5, 5, 32, 32]),\n",
    "        name=\"conv1_weights\")\n",
    "    conv1_biases = tf.Variable(tf.zeros([32]), name=\"conv1_biases\")\n",
    "    conv2_weights = tf.Variable(tf.random_normal([5, 5, 32, 32]),\n",
    "        name=\"conv2_weights\")\n",
    "    conv2_biases = tf.Variable(tf.zeros([32]), name=\"conv2_biases\")\n",
    "    return None\n",
    "\n",
    "# First call creates one set of 4 variables.\n",
    "result1 = my_image_filter()\n",
    "# Another set of 4 variables is created in the second call.\n",
    "result2 = my_image_filter()\n",
    "# 获取所有的可训练变量\n",
    "vs = tf.trainable_variables()\n",
    "print 'There are %d train_able_variables in the Graph: ' % len(vs)\n",
    "for v in vs:\n",
    "    print v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "2.第二种方式，用 tf.get_variable() 的方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 4 train_able_variables in the Graph: \n",
      "Tensor(\"image_filters/conv1/weights/read:0\", shape=(5, 5, 32, 32), dtype=float32)\n",
      "Tensor(\"image_filters/conv1/biases/read:0\", shape=(32,), dtype=float32)\n",
      "Tensor(\"image_filters/conv2/weights/read:0\", shape=(5, 5, 32, 32), dtype=float32)\n",
      "Tensor(\"image_filters/conv2/biases/read:0\", shape=(32,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "# 设置GPU按需增长\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config=config)\n",
    "\n",
    "# 下面是定义一个卷积层的通用方式\n",
    "def conv_relu(kernel_shape, bias_shape):\n",
    "    # Create variable named \"weights\".\n",
    "    weights = tf.get_variable(\"weights\", kernel_shape, initializer=tf.random_normal_initializer())\n",
    "    # Create variable named \"biases\".\n",
    "    biases = tf.get_variable(\"biases\", bias_shape, initializer=tf.constant_initializer(0.0))\n",
    "    return None\n",
    "\n",
    "\n",
    "def my_image_filter():\n",
    "    # 按照下面的方式定义卷积层，非常直观，而且富有层次感\n",
    "    with tf.variable_scope(\"conv1\"):\n",
    "        # Variables created here will be named \"conv1/weights\", \"conv1/biases\".\n",
    "        relu1 = conv_relu([5, 5, 32, 32], [32])\n",
    "    with tf.variable_scope(\"conv2\"):\n",
    "        # Variables created here will be named \"conv2/weights\", \"conv2/biases\".\n",
    "        return conv_relu( [5, 5, 32, 32], [32])\n",
    "\n",
    "    \n",
    "with tf.variable_scope(\"image_filters\") as scope:\n",
    "    # 下面我们两次调用 my_image_filter 函数，但是由于引入了 变量共享机制\n",
    "    # 可以看到我们只是创建了一遍网络结构。\n",
    "    result1 = my_image_filter()\n",
    "    scope.reuse_variables()\n",
    "    result2 = my_image_filter()\n",
    "\n",
    "\n",
    "# 看看下面，完美地实现了变量共享！！！\n",
    "vs = tf.trainable_variables()\n",
    "print 'There are %d train_able_variables in the Graph: ' % len(vs)\n",
    "for v in vs:\n",
    "    print v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "实验 2 结论：首先我们要确立一种 Graph 的思想。在 TensorFlow 中，我们定义一个变量，相当于往 Graph 中添加了一个节点。和普通的 python 函数不一样，在一般的函数中，我们对输入进行处理，然后返回一个结果，而函数里边定义的一些局部变量我们就不管了。但是在 TensorFlow 中，我们在函数里边创建了一个变量，就是往 Graph 中添加了一个节点。出了这个函数后，这个节点还是存在于 Graph 中的。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "在深度学习中，通常说到 变量共享 我们都会想到 RNN 。下面我找了两个源码中非常漂亮的例子，可以参考学习学习。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 例1：MultiRNNCell(RNNCell) 中这样来创建 各层 Cell\n",
    "# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\n",
    "for i, cell in enumerate(self._cells):\n",
    "    with vs.variable_scope(\"cell_%d\" % i):\n",
    "        if self._state_is_tuple:\n",
    "            ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 例2：tf.contrib.rnn.static_bidirectional_rnn 双端 LSTM \n",
    "with vs.variable_scope(scope or \"bidirectional_rnn\"):\n",
    "    # Forward direction\n",
    "    with vs.variable_scope(\"fw\") as fw_scope:\n",
    "        output_fw, output_state_fw = static_rnn(\n",
    "        cell_fw, inputs, initial_state_fw, dtype,\n",
    "        sequence_length, scope=fw_scope)\n",
    "\n",
    "    # Backward direction\n",
    "    with vs.variable_scope(\"bw\") as bw_scope:\n",
    "        reversed_inputs = _reverse_seq(inputs, sequence_length)\n",
    "        tmp, output_state_bw = static_rnn(\n",
    "              cell_bw, reversed_inputs, initial_state_bw,\n",
    "              dtype, sequence_length, scope=bw_scope)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}