import tensorflow as tf

from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

# X is our placeholder value for our input
x = tf.placeholder(tf.float32, [None, 784])

W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))

y = tf.nn.softmax(tf.matmul(x, W) + b)

y_ = tf.placeholder(tf.float32, [None, 10])

cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))

train_step = tf.train.GradientDescentOptimizer(0.05).minimize(cross_entropy)

# Create an InteractiveSession 
sess = tf.InteractiveSession()

# Create an operation to initialize the variables we created
tf.global_variables_initializer().run()

for _ in range(1000):
  batch_xs, batch_ys = mnist.train.next_batch(100)
  sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})

correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))

accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print(sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels}))

import pandas
from IPython.display import display, HTML

data = [["500",0.8908],
         ["1,000",0.901],
         ["2,000",0.9143],
         ["5,000",0.9181],
         ["10,000",0.9213]]
df = pandas.DataFrame(data, ['500', '1,000', '2,000', '5,000', '10,000'], ['Epochs', 'Percentage'])
display(HTML(df.to_html(index=False)))

import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data

#
# Data processing
#
# Here mnist is a lightweight class which stores the training, validation,
# and testing sets as NumPy arrays. It also provides a function for iterating
# through data minibatches, which we will use below.
#
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

#
# TF Session
#
# InteractiveSession allows you to interleave operations which build a computation graph 
# with ones that run the graph. This is particularly convenient when working in 
# interactive contexts like IPython.
#
sess = tf.InteractiveSession()

#
# TF Placeholders
#
# Placeholders are values we input when asking TF to run a computation
#
# The input images `x` will consist of a 2d tensor of floating point numbers.
# Here we assign it a shape of [None, 784], where 784 is the dimensionality
# of a single flattened 28 by 28 pixel MNIST image, and None indicates that
# the first dimension, corresponding to the batch size, can be of any size.
# The target output classes y_ will also consist of a 2d tensor, where each
# row is a one-hot 10-dimensional vector indicating which digit class
# (zero through nine) the corresponding MNIST image belongs to.
#
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])

#
# TF Variables
# 
# A Variable is a value that lives in TensorFlow's computation graph.
# It can be used and even modified by the computation.
#
# The model paramaters are generally variables also.
#
# We initialize both as tensors full of zeros.
#
# `W` is a 784x10 matrix (because we have 784 input features and 10 outputs)
# `b` is a 10-dimensional vector (because we have 10 classes)
#
W = tf.Variable(tf.zeros([784,10]))
b = tf.Variable(tf.zeros([10]))

#
# Initialize variables
#
# Before Variables can be used within a session, they must be initialized using that session.
# This step takes the initial values (in this case tensors full of zeros) that have already
# been specified, and assigns them to each Variable.
#
# This can be done for all Variables at once:
#
sess.run(tf.global_variables_initializer())

#
# Regression Model
#
# "input times weight, add a bias, activate" - @Siraj Raval
#
# We multiply the vectorized input images `x` by the weight matrix `W`, add the bias `b`.
y = tf.matmul(x,W) + b

#
# Loss function
#
# Loss indicates how bad the model's prediction was on a single example;
# we try to minimize that while training across all the examples. Here,
# our loss function is the cross-entropy between the target and the softmax
# activation function applied to the model's prediction.
#
# Note that tf.nn.softmax_cross_entropy_with_logits internally applies the
# softmax on the model's unnormalized model prediction and sums across all
# classes, and tf.reduce_mean takes the average over these sums.
#
# As in the beginners tutorial, we use the stable formulation:
#
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y))

#
# Train the model
#
# Because TensorFlow knows the entire computation graph, it can use automatic
# differentiation to find the gradients of the loss with respect to each of the
# variables.
#
# TensorFlow has a variety of built-in optimization algorithms.
#
# For this example, we will use steepest gradient descent, with a step length of 0.5,
# to descend the cross entropy.
#
# What TensorFlow actually does in this single line is add new operations to the
# computation graph. These operations include ones to compute gradients, compute
# parameter update steps, and apply update steps to the parameters.
#
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)

#
# Run the training steps
#
# The returned operation train_step, when run, will apply the gradient descent updates
# to the parameters. Training the model can therefore be accomplished by repeatedly
# running train_step.
#
for _ in range(1000):
  # We load 100 training examples in each training iteration  
  batch = mnist.train.next_batch(100)
  # We then run the train_step operation, using feed_dict to replace the placeholder
  # tensors x and y_ with the training examples.
  train_step.run(feed_dict={x: batch[0], y_: batch[1]})

#
# Evaluate the model
#

# tf.argmax gives you the index of the highest entry in a tensor along some axis.
#
# For example, tf.argmax(y,1) is the label our model thinks is most likely for each input,
# while tf.argmax(y_,1) is the true label.
#
# We can use tf.equal to check if our prediction matches the truth.
#
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))

#
# 
#
# That gives us a list of booleans. To determine what fraction are correct, we cast to
# floating point numbers and then take the mean.
#
# For example, [True, False, True, True] would become [1,0,1,1] which would become 0.75.
#
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

#
# Finally, we can evaluate our accuracy on the test data. This should be about 92% correct.
#
print(accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels}))

#
# Weight Initialization
#
# To create this model, we're going to need to create a lot of weights and biases.
# One should generally initialize weights with a small amount of noise for symmetry
# breaking, and to prevent 0 gradients. Since we're using ReLU neurons, it is also
# good practice to initialize them with a slightly positive initial bias to avoid
# "dead neurons". Instead of doing this repeatedly while we build the model,
# let's create two handy functions to do it for us.
#
def weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

#
# Convolution and Pooling
# 
# TensorFlow also gives us a lot of flexibility in convolution and pooling operations.
# How do we handle the boundaries? What is our stride size?
# In this example, we're always going to choose the vanilla version.
#
# Our convolutions use a stride of one and are zero padded so that the output is the 
# same size as the input.
#
# Our pooling is plain old max pooling over 2x2 blocks.
#
# We'll also abstract those operations into functions.
#
def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

#
# First Convolutional Layer
#
# Our first layer will consist of a convolution, followed by max pooling.
#
# The convolution will compute 32 features for each 5x5 patch.
# Its weight tensor will have a shape of [5, 5, 1, 32].
#
# The first two dimensions are the patch size (5)
# The next is the number of input channels (1)
# The last is the number of output channels (32)
#
# We will also have a bias vector with a component for each output channel.
#
W_conv1 = weight_variable([5, 5, 1, 32])
b_conv1 = bias_variable([32])

# To apply the layer, we first reshape x to a 4d tensor, with the second and third dimensions
# corresponding to image width and height, and the final dimension corresponding to the
# number of color channels.
x_image = tf.reshape(x, [-1, 28, 28, 1])

# We then convolve x_image with the weight tensor, add the bias, apply the ReLU function,
# and finally max pool.
# The max_pool_2x2 method will reduce the image size to 14x14.
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)
h_pool1 = max_pool_2x2(h_conv1)

#
# Second Convolutional Layer
#
# In order to build a deep network, we stack several layers of this type.
#
# The second layer will have 64 features for each 5x5 patch.
# The max_pool_2x2 method will reduce the image size to 7x7.
#
W_conv2 = weight_variable([5, 5, 32, 64])
b_conv2 = bias_variable([64])

h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
h_pool2 = max_pool_2x2(h_conv2)

#
# Densely Connected Layer
#
# Now that the image size has been reduced to 7x7, we add a fully-connected layer
# with 1024 neurons to allow processing on the entire image.
#
# We reshape the tensor from the pooling layer into a batch of vectors,
# multiply by a weight matrix, add a bias, and apply a ReLU.
#
W_fc1 = weight_variable([7 * 7 * 64, 1024])
b_fc1 = bias_variable([1024])

h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64])
h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)

#
# Dropout
#
# To reduce overfitting, we will apply dropout before the readout layer.
# We create a placeholder for the probability that a neuron's output is kept during dropout.
# This allows us to turn dropout on during training, and turn it off during testing.
# TensorFlow's tf.nn.dropout op automatically handles scaling neuron outputs in addition
# to masking them, so dropout just works without any additional scaling.
#
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

#
# Readout Layer
#
# Finally, we add a layer, just like for the one layer softmax regression above.
#
W_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])

y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2

#
# Train and evaluate
#
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_, logits=y_conv))

# We will replace the steepest gradient descent optimizer with the more sophisticated ADAM optimizer.
# We will add logging to every 100th iteration in the training process.
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

if False: # Don't execute for compiling blog    
   
    # We will also use tf.Session rather than tf.InteractiveSession.
    # This better separates the process of creating the graph (model sepecification)
    # and the process of evaluating the graph (model fitting).
    with tf.Session() as sess:
      sess.run(tf.global_variables_initializer())
      for i in range(2000):
        batch = mnist.train.next_batch(50)
        if i % 100 == 0:
          train_accuracy = accuracy.eval(feed_dict={x: batch[0], y_: batch[1], keep_prob: 1.0})
          print('step %d, training accuracy %g' % (i, train_accuracy))
        train_step.run(feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})

      # Save the model --------------------------------------------
        
      # Build the signature_def_map.
      classification_inputs = tf.saved_model.utils.build_tensor_info(x)
      classification_outputs_classes = tf.saved_model.utils.build_tensor_info(y)
      classification_outputs_scores = tf.saved_model.utils.build_tensor_info(y_conv)

      classification_signature = (
        tf.saved_model.signature_def_utils.build_signature_def(
            inputs={tf.saved_model.signature_constants.CLASSIFY_INPUTS: classification_inputs},
            outputs={tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES: classification_outputs_classes,
                tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES: classification_outputs_scores},
            method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME))

      tensor_info_x = tf.saved_model.utils.build_tensor_info(x)
      tensor_info_y = tf.saved_model.utils.build_tensor_info(y)

      prediction_signature = (
          tf.saved_model.signature_def_utils.build_signature_def(
              inputs={'images': tensor_info_x},
              outputs={'scores': tensor_info_y},
              method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))

      export_path = './export_model'
      print('Exporting trained model to: %r' % export_path)
      builder = tf.saved_model.builder.SavedModelBuilder(export_path)
      builder.add_meta_graph_and_variables(
              sess, [tf.saved_model.tag_constants.SERVING],
              signature_def_map={
                   'predict_images':
                       prediction_signature,
                   tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
                       classification_signature,
              })
      builder.save()
   
      # Complete --------------------------------------------

      print('test accuracy %g' % accuracy.eval(feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))