{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "name": "IST718_WK9_TensorFlow.ipynb", "provenance": [], "collapsed_sections": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" } }, "cells": [ { "cell_type": "code", "metadata": { "id": "h0ntV7eLSQmr", "colab_type": "code", "outputId": "2cfab896-2308-4ebd-ced5-8ac19d9e3e7b", "colab": { "base_uri": "https://localhost:8080/", "height": 62 } }, "source": [ "# SINGLE LAYER AND MULTI LAYER NETWORKS FOR MNIST\n", "# BASED ON CODE FROM TENSORFLOW TUTORIAL\n", "\n", "import tensorflow as tf" ], "execution_count": 0, "outputs": [ { "output_type": "display_data", "data": { "text/html": [ "

\n", "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.
\n", "We recommend you upgrade now \n", "or ensure your notebook will continue to use TensorFlow 1.x via the %tensorflow_version 1.x magic:\n", "more info.

\n" ], "text/plain": [ "" ] }, "metadata": { "tags": [] } } ] }, { "cell_type": "markdown", "metadata": { "id": "OUIxbSv9Sas_", "colab_type": "text" }, "source": [ "# OBTAIN \n", "## (& SCRUB -- this data comes scrubbed)" ] }, { "cell_type": "code", "metadata": { "id": "B8JapYFVSaF7", "colab_type": "code", "outputId": "c532d84f-4e77-443d-d72a-93920ab50b9b", "colab": { "base_uri": "https://localhost:8080/", "height": 545 } }, "source": [ "from tensorflow.examples.tutorials.mnist import input_data\n", "mnist = input_data.read_data_sets(\"MNIST_data/\", one_hot=True)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "WARNING:tensorflow:From :2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please write your own downloading logic.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/base.py:252: _internal_retry..wrap..wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use urllib or similar directly.\n", "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use tf.data to implement this functionality.\n", "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use tf.data to implement this functionality.\n", "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use tf.one_hot on tensors.\n", "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n", "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n", "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n", "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "meJJIMKeSkjl", "colab_type": "text" }, "source": [ "# MODEL" ] }, { "cell_type": "code", "metadata": { "id": "MojnOBo2SdeB", "colab_type": "code", "colab": {} }, "source": [ "# MODEL\n", "# CREATE PLACEHOLDER VARIABLES FOR OPERATION MANIPULATION\n", "# THE 784 MATCHES THE VECTOR SIZE OF THE MNIST IMAGES - 28*28\n", "\n", "x = tf.placeholder(tf.float32, [None, 784])\n", "\n", "# MODEL\n", "# CREATE WEIGHTS & BIASES VARIABLES\n", "# IN TF, OUR MODEL PARAMETERS ARE OFTEN MANAGED AS VARIABLES\n", "\n", "W = tf.Variable(tf.zeros([784, 10]))\n", "b = tf.Variable(tf.zeros([10]))\n", "\n", "# MODEL\n", "# CREATE MODEL - USES SOFTMAX AS THE ACTIVATION FUNCTION\n", "# REMEMBER GOAL FOR ACTIVATION FUNCTION IS TO \"SHAPE\" THE \n", "# OUTPUT INTO A PROBABILITY DISTRO OVER THE 10 CLASSES\n", "\n", "y = tf.nn.softmax(tf.matmul(x, W) + b)\n", "\n", "# MODEL\n", "# CREATE PREDICTED VARIABLE Y-HAT\n", "# AND USE CROSS ENTROPY TO DETERMINE LOSS\n", "# CROSS ENTROPY - HOW INEFFICIENT ARE OUR PREDICTIONS?\n", "\n", "y_ = tf.placeholder(tf.float32, [None, 10])\n", "cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))\n", "\n", "# MODEL\n", "# TRAIN USING GRADIENT DESCENT\n", "# LEARNING RATE AT MIDPOINT - .5 - MAKE SMALL ADJUSTMENTS TO MINIMIZE COST\n", "\n", "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "McuTduNnTF_G", "colab_type": "text" }, "source": [ "## MODEL -- RUN MODEL" ] }, { "cell_type": "code", "metadata": { "id": "vwczv4wMSl8W", "colab_type": "code", "colab": {} }, "source": [ "# MODEL - RUN\n", "\n", "sess = tf.InteractiveSession()\n", "tf.global_variables_initializer().run()\n", "for _ in range(10000):\n", " batch_xs, batch_ys = mnist.train.next_batch(100)\n", " sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})" ], "execution_count": 0, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "UKFcM_2_TJsF", "colab_type": "text" }, "source": [ "# EVALUATE" ] }, { "cell_type": "code", "metadata": { "id": "giuRB68TTIKf", "colab_type": "code", "outputId": "cb226fac-6419-4fdf-b17a-abf138fde952", "colab": { "base_uri": "https://localhost:8080/", "height": 35 } }, "source": [ "# EVALUATE MODEL\n", "\n", "correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", "print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))\n" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "0.9255\n" ], "name": "stdout" } ] }, { "cell_type": "markdown", "metadata": { "id": "jWBKIc28TSBl", "colab_type": "text" }, "source": [ "# BLOCK TWO" ] }, { "cell_type": "markdown", "metadata": { "id": "5u8g1XfpTRrd", "colab_type": "text" }, "source": [ "Alternative Approach" ] }, { "cell_type": "code", "metadata": { "id": "vfs4fpggTLHJ", "colab_type": "code", "colab": {} }, "source": [ "# WEIGHT INITIALIZATION\n", "\n", "def weight_variable(shape):\n", " initial = tf.truncated_normal(shape, stddev=0.1)\n", " return tf.Variable(initial)\n", "\n", "def bias_variable(shape):\n", " initial = tf.constant(0.1, shape=shape)\n", " return tf.Variable(initial)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "MgsoyfksTaLW", "colab_type": "code", "colab": {} }, "source": [ "# CREATE CONVOLUTION AND POOLING LAYERS\n", "\n", "def conv2d(x, W):\n", " return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')\n", "\n", "def max_pool_2x2(x):\n", " return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],\n", " strides=[1, 2, 2, 1], padding='SAME')" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "0qFHN1jFTarC", "colab_type": "code", "colab": {} }, "source": [ "# FIRST CONVOLUTION LAYER\n", "\n", "W_conv1 = weight_variable([5, 5, 1, 32])\n", "b_conv1 = bias_variable([32])\n", "\n", "x_image = tf.reshape(x, [-1, 28, 28, 1]) # BASE IMAGE SIZE OF 28 * 28\n", "\n", "h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)\n", "h_pool1 = max_pool_2x2(h_conv1) # RESULTING IMAGE SIZE IS 14 * 14\n" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "jM80nUZ8TcTO", "colab_type": "code", "colab": {} }, "source": [ "# SECOND CONOLUTION LAYER \n", "# MORE THAN ONE LAYER? DEEP LEARNING\n", "\n", "W_conv2 = weight_variable([5, 5, 32, 64])\n", "b_conv2 = bias_variable([64])\n", "\n", "h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)\n", "h_pool2 = max_pool_2x2(h_conv2)" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "iw_9anXSTeSU", "colab_type": "code", "colab": {} }, "source": [ "# FULLY CONNECTED LAYER - BEFORE OUTPUT\n", "\n", "W_fc1 = weight_variable([7 * 7 * 64, 1024])\n", "b_fc1 = bias_variable([1024])\n", "\n", "h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])\n", "h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) # ADD THE RECTIFIED LINEAR UNIT\n" ], "execution_count": 0, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "DuIUN-apTfil", "colab_type": "code", "outputId": "e71a06ef-87f0-4f39-98a4-3f34077888bb", "colab": { "base_uri": "https://localhost:8080/", "height": 90 } }, "source": [ "# DROP LAYER - REDUCE OVERFITTING\n", "# USE OF rate IS AN UPDATE BASED ON TF\n", "\n", "keep_prob = tf.placeholder(tf.float32)\n", "rate = 1 - keep_prob\n", "h_fc1_drop = tf.nn.dropout(h_fc1, rate)" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "WARNING:tensorflow:From :4: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "stYCQSeLTg7o", "colab_type": "code", "colab": {} }, "source": [ "# LAST LAYER - OUTPUT\n", "\n", "W_fc2 = weight_variable([1024, 10])\n", "b_fc2 = bias_variable([10])\n", "\n", "y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2" ], "execution_count": 0, "outputs": [] }, { "cell_type": "markdown", "metadata": { "id": "SE8Yd91WTlIi", "colab_type": "text" }, "source": [ "## MODEL -- RUN MODEL" ] }, { "cell_type": "code", "metadata": { "id": "yvFyzfJnTijj", "colab_type": "code", "outputId": "d28de041-3ba3-4894-df6f-3ed214b7bee3", "colab": { "base_uri": "https://localhost:8080/", "height": 1000 } }, "source": [ "# RUN THE MODEL\n", "\n", "cross_entropy = tf.reduce_mean(\n", " tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))\n", "train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)\n", "correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))\n", "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n", "\n", "with tf.Session() as sess:\n", " sess.run(tf.global_variables_initializer())\n", " for i in range(10000):\n", " batch = mnist.train.next_batch(50)\n", " if i % 100 == 0:\n", " train_accuracy = accuracy.eval(feed_dict={\n", " x: batch[0], y_: batch[1], rate: 1.0})\n", " print('step %d, training accuracy %g' % (i, train_accuracy))\n", " train_step.run(feed_dict={x: batch[0], y_: batch[1], rate: 0.5})\n", "\n", " print('test accuracy %g' % accuracy.eval(feed_dict={\n", " x: mnist.test.images, y_: mnist.test.labels, rate: 1.0}))" ], "execution_count": 0, "outputs": [ { "output_type": "stream", "text": [ "WARNING:tensorflow:From :3: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "\n", "Future major versions of TensorFlow will allow gradients to flow\n", "into the labels input on backprop by default.\n", "\n", "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n", "\n", "step 0, training accuracy 0.08\n", "step 100, training accuracy 0.84\n", "step 200, training accuracy 0.92\n", "step 300, training accuracy 0.9\n", "step 400, training accuracy 0.94\n", "step 500, training accuracy 0.98\n", "step 600, training accuracy 0.92\n", "step 700, training accuracy 0.96\n", "step 800, training accuracy 0.94\n", "step 900, training accuracy 0.98\n", "step 1000, training accuracy 0.94\n", "step 1100, training accuracy 0.96\n", "step 1200, training accuracy 1\n", "step 1300, training accuracy 0.98\n", "step 1400, training accuracy 1\n", "step 1500, training accuracy 0.96\n", "step 1600, training accuracy 0.98\n", "step 1700, training accuracy 0.94\n", "step 1800, training accuracy 0.96\n", "step 1900, training accuracy 1\n", "step 2000, training accuracy 1\n", "step 2100, training accuracy 0.98\n", "step 2200, training accuracy 1\n", "step 2300, training accuracy 1\n", "step 2400, training accuracy 0.96\n", "step 2500, training accuracy 1\n", "step 2600, training accuracy 0.92\n", "step 2700, training accuracy 1\n", "step 2800, training accuracy 0.98\n", "step 2900, training accuracy 1\n", "step 3000, training accuracy 0.94\n", "step 3100, training accuracy 1\n", "step 3200, training accuracy 0.94\n", "step 3300, training accuracy 0.96\n", "step 3400, training accuracy 1\n", "step 3500, training accuracy 1\n", "step 3600, training accuracy 0.98\n", "step 3700, training accuracy 0.98\n", "step 3800, training accuracy 1\n", "step 3900, training accuracy 0.98\n", "step 4000, training accuracy 1\n", "step 4100, training accuracy 0.98\n", "step 4200, training accuracy 1\n", "step 4300, training accuracy 1\n", "step 4400, training accuracy 1\n", "step 4500, training accuracy 1\n", "step 4600, training accuracy 1\n", "step 4700, training accuracy 1\n", "step 4800, training accuracy 1\n", "step 4900, training accuracy 1\n", "step 5000, training accuracy 0.98\n", "step 5100, training accuracy 1\n", "step 5200, training accuracy 1\n", "step 5300, training accuracy 0.98\n", "step 5400, training accuracy 0.98\n", "step 5500, training accuracy 1\n", "step 5600, training accuracy 1\n", "step 5700, training accuracy 1\n", "step 5800, training accuracy 1\n", "step 5900, training accuracy 1\n", "step 6000, training accuracy 0.96\n", "step 6100, training accuracy 1\n", "step 6200, training accuracy 1\n", "step 6300, training accuracy 1\n", "step 6400, training accuracy 1\n", "step 6500, training accuracy 1\n", "step 6600, training accuracy 1\n", "step 6700, training accuracy 0.98\n", "step 6800, training accuracy 0.98\n", "step 6900, training accuracy 1\n", "step 7000, training accuracy 1\n", "step 7100, training accuracy 0.96\n", "step 7200, training accuracy 1\n", "step 7300, training accuracy 1\n", "step 7400, training accuracy 1\n", "step 7500, training accuracy 1\n", "step 7600, training accuracy 1\n", "step 7700, training accuracy 1\n", "step 7800, training accuracy 1\n", "step 7900, training accuracy 0.98\n", "step 8000, training accuracy 1\n", "step 8100, training accuracy 1\n", "step 8200, training accuracy 0.98\n", "step 8300, training accuracy 1\n", "step 8400, training accuracy 0.98\n", "step 8500, training accuracy 1\n", "step 8600, training accuracy 1\n", "step 8700, training accuracy 1\n", "step 8800, training accuracy 1\n", "step 8900, training accuracy 1\n", "step 9000, training accuracy 1\n", "step 9100, training accuracy 0.98\n", "step 9200, training accuracy 1\n", "step 9300, training accuracy 1\n", "step 9400, training accuracy 0.98\n", "step 9500, training accuracy 1\n", "step 9600, training accuracy 1\n", "step 9700, training accuracy 1\n", "step 9800, training accuracy 1\n", "step 9900, training accuracy 1\n", "test accuracy 0.9915\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "xpzUkEh8TpLz", "colab_type": "code", "colab": {} }, "source": [ "" ], "execution_count": 0, "outputs": [] } ] }