import warnings
warnings.filterwarnings(action="ignore", module="scipy", message="^internal gelsd")

import pandas as pd
from sklearn import linear_model
import matplotlib.pyplot as plt

%matplotlib inline

# Read our data into a pandas data-frame object
# which is a 2d data structure of rows and columns
dataframe = pd.read_fwf('brain_body.txt')
x_values = dataframe[['Brain']]
y_values = dataframe[['Body']]

# linear regression helps find the relationship between our 2 vars to find the only line of best fit
# Use scikits linear_model object to init our linear regression model
body_reg = linear_model.LinearRegression()

# Fit our model on our x,y value pairs
body_reg.fit(x_values, y_values)

# Now we have the line of best fit, we can plot it on a graph

# Scatter plot
plt.scatter(x_values, y_values)

# Plot our regression line
# "for ever x value we have, predict the associated y value
#  and draw a line that intersects all those points"
plt.plot(x_values, body_reg.predict(x_values))

# Save it to a file
# plt.savefig('best-fit.png', bbox_inches='tight')

# Show it
plt.show()

import pandas as pd
from sklearn import linear_model
import matplotlib.pyplot as plt
import numpy as np
import warnings
warnings.filterwarnings(action="ignore", module="scipy", message="^internal gelsd")
from IPython.display import display

global df

# Read our data into a pandas data-frame object
df = pd.read_csv('challenge.csv', names=['X', 'Y'])

# Split our data into a test and training set (so we can calculate the error)
# http://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = np.asarray(train_test_split(df['X'], df['Y'], test_size=0.1))

"Dataframe length: %s, X_train: %s, X_test: %s, y_train: %s, y_test: %s" % (len(df), len(X_train), len(X_test), len(y_train), len(y_test))

# linear regression helps find the relationship between our 2 vars to find the only line of best fit
# Use scikits linear_model object to init our linear regression model
reg = linear_model.LinearRegression()

# Fit our model on our x,y value pairs
# https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.reshape.html
x = reg.fit(X_train.values.reshape(-1, 1), y_train.values.reshape(-1, 1))

# Returns the mean accuracy on the given test data and labels.
score = reg.score(X_test.values.reshape(-1, 1), y_test.values.reshape(-1, 1))

score

# Now we have the line of best fit, we can plot it on a graph

# Create our horizontal axis ranging from 5-25
x_line = np.arange(5, 25).reshape(-1,1)

# Scatter plot
plt.scatter(df['X'], df['Y'])

# Plot our regression line
# "for ever x value we have, predict the associated y value
#  and draw a line that intersects all those points"
plt.plot(x_line, reg.predict(x_line))

# Show it
plt.show()

salary_data = {
    'year': [1,2,3,4,5,6,7,8,9,10],
    'salary': [40000,42000,43000,46000,48000,52000,58000,62000,65500,68000]
}
df = pd.DataFrame(data=salary_data)
df

# No train/test splitting as we only have 10 values
X_train = df['year'].values.reshape(-1, 1)
y_train = df['salary'].values.reshape(-1, 1)

# Create our linear model
reg = linear_model.LinearRegression()

# Fit our model on our x,y value pairs
x = reg.fit(X_train, y_train)

b = int(round(reg.intercept_[0]))
m = int(round(reg.coef_[0][0]))

"y = {}x + {}".format(m, b)

import numpy as np
from numpy import exp, array, random, dot
import matplotlib.pyplot as plt
from past.builtins import xrange
from IPython.display import display, Markdown

import numpy as np

# Custom logger to log one-line arrays
def logger(msg, log_array, nl=False):
    if not isinstance(log_array, np.ndarray):
        print('{message}'.format(message=msg))
    else:
        string_array = ', '.join(map(str, log_array))
        print('{message: <{fill}}: {array}'.format(message=msg, array=string_array, fill='35'))

    if nl is True:
        print('\n')

class NeuralNetwork():
    def __init__(self):
        # Seed the random generator so we get the same number each time
        np.random.seed(1)
        
        # We model a single neuron with 3 input connections 1 output connection
        # we assign random weights to a 3 x 1 matrix, with values in the range -1 to 1
        self.synaptic_weights = 2 * np.random.random((3,1)) -1
        display(Markdown('## Initializing Single Layer Feed Forward Network'))
        display(Markdown('Starting our network with the following random synaptic weights:'))
        display(Markdown('> ``%s``' % self.synaptic_weights))

    # The Sigmoid function, which describes an S shaped curve.
    # We pass the weighted sum of the inputs through this function to
    # normalise them between 0 and 1.
    # We can pass a np.array here to apply it to all values
    def __sigmoid(self, x):
        return 1 / (1 + np.exp(-x))
    
    # The derivative of the Sigmoid function.
    # This is the gradient of the Sigmoid curve. The slope.
    # It indicates how confident we are about the existing weight.
    def __sigmoid_derivative(self, x):
        return x * (1-x)
    
    # We train the neural network through a process of trial and error.
    # Adjusting the synaptic weights each time.
    def train(self, training_set_inputs, training_set_outputs, number_of_training_iterations):
        for iteration in xrange(number_of_training_iterations):
            if (iteration % 2000 == 0):
                logger('Iteration: %s' % iteration, False)
                logger('Current Synaptic Weights:', self.synaptic_weights)
          
            # Pass the training set through our neural network (a single neuron).
            output = self.predict(training_set_inputs)
            
            # Calculate the error (The difference between the desired output
            # and the predicted output).
            error = training_set_outputs - output

            # Calculate the adjustment by multiplying the error by the input and again by the gradient of the Sigmoid curve.
            # This means less confident weights are adjusted more.
            # This means inputs, which are zero, do not cause changes to the weights.
            adjustment = np.dot(training_set_inputs.T, error * self.__sigmoid_derivative(output))

            # Adjust the weights.
            self.synaptic_weights += adjustment

            if (iteration % 2000 == 0):
                logger('Sigmoid of dot inputs and weights:', output)
                logger('Error of our training to outputs', error, False)
                logger('adjustment for the weights', adjustment, False)
                logger('The new synaptic weights are:', self.synaptic_weights, True)
    
    def predict(self, inputs):
        # Pass inputs through our neural network (our single layer).
        return self.__sigmoid(np.dot(inputs, self.synaptic_weights))

    
if __name__ == '__main__':
    
    # Initialise our single neuron neural network
    neural_network = NeuralNetwork()
    
    # The training set
    # We have 4 examples, each consisting of 3 input values and 1 output value
    training_set_inputs = np.array([[0,0,1], [1,1,1], [1,0,1], [0,1,1]])
    training_set_outputs = np.array([[0,1,1,0]]).T

    # This is running in a ipython notebook, so we can use markdown
    display(Markdown('We have the following training set inputs:'))
    display(Markdown('> ``%s``' % training_set_inputs))
    display(Markdown('and the expected outputs:'))
    display(Markdown('> ``%s``' % training_set_outputs))
    display(Markdown("Let's step through our network to better understand what's going on."))
    
    display(Markdown('-------------'))
    
    # Train the network using a training set
    # Do it 10,000 times and make small adjustments each time
    neural_network.train(training_set_inputs, training_set_outputs, 10000)
    
    display(Markdown('The final Synaptic weights after training for 10,000 epochs:'))
    display(Markdown('> ``%s``' % neural_network.synaptic_weights))

    # Test the network with a new situation
    display(Markdown("Let's now make a prediction with [1, 0, 0]."))
    display(Markdown('> ``%s``' % neural_network.predict(np.array([1, 0, 0]))))

    display(Markdown("The script is complete"))
    display(Markdown('-------------'))

import numpy as np
np.seterr(over='ignore')

class NeuralNetwork():
    def __init__(self):
        np.random.seed(1)  # Seed the random number generator
        self.weights = {}  # Create dict to hold weights (renamed)
        self.num_layers = 1  # Set initial number of layer to one (input layer)
        self.adjustments = {}  # Create dict to hold adjustments
        self.log = False;

    def add_layer(self, shape):
        # Create weights with shape specified + biases
        self.weights[self.num_layers] = np.vstack((2 * np.random.random(shape) - 1, 2 * np.random.random((1, shape[1])) - 1))
        # Initialize the adjustments for these weights to zero
        self.adjustments[self.num_layers] = np.zeros(shape)
        self.num_layers += 1

    def __sigmoid(self, x):
        return 1 / (1 + np.exp(-x))

    def __sigmoid_derivative(self, x):
        return x * (1 - x)

    def predict(self, data):
        # Pass data through pretrained network
        for layer in range(1, self.num_layers+1):
            data = np.dot(data, self.weights[layer-1][:, :-1]) + self.weights[layer-1][:, -1] # + self.biases[layer]
            data = self.__sigmoid(data)
        return data

    def __forward_propagate(self, data):
        # Progapagate through network and hold values for use in back-propagation
        activation_values = {}
        activation_values[1] = data
        for layer in range(2, self.num_layers+1):
            data = np.dot(data.T, self.weights[layer-1][:-1, :]) + self.weights[layer-1][-1, :].T # + self.biases[layer]
            data = self.__sigmoid(data).T
            activation_values[layer] = data
        return activation_values

    def simple_error(self, outputs, targets):
        return targets - outputs

    def sum_squared_error(self, outputs, targets):
        return 0.5 * np.mean(np.sum(np.power(outputs - targets, 2), axis=1))

    def __back_propagate(self, output, target):
        deltas = {}
        # Delta of output Layer
        deltas[self.num_layers] = output[self.num_layers] - target

        # Delta of hidden Layers
        for layer in reversed(range(2, self.num_layers)):  # All layers except input/output
            a_val = output[layer]
            weights = self.weights[layer][:-1, :]
            prev_deltas = deltas[layer+1]
            deltas[layer] = np.multiply(np.dot(weights, prev_deltas), self.__sigmoid_derivative(a_val))

        # Caclculate total adjustments based on deltas
        for layer in range(1, self.num_layers):
            self.adjustments[layer] += np.dot(deltas[layer+1], output[layer].T).T

    def __gradient_descent(self, batch_size, learning_rate):
        # Calculate partial derivative and take a step in that direction
        for layer in range(1, self.num_layers):
            partial_d = (1/batch_size) * self.adjustments[layer]

            # Update the first 2 weight dimensions
            self.weights[layer][:-1, :] += learning_rate * -partial_d

            # Update the last weight dimension with a 0.001 difference
            self.weights[layer][-1, :] += learning_rate*1e-3 * -partial_d[-1, :]

    def train(self, inputs, targets, num_epochs, learning_rate=1, stop_accuracy=1e-5):
        error = []
        for iteration in range(num_epochs):
            for i in range(len(inputs)):
                x = inputs[i]
                y = targets[i]

                # Pass the training set through our neural network
                output = self.__forward_propagate(x)

                # Calculate the error
                loss = self.sum_squared_error(output[self.num_layers], y)
                error.append(loss)

                # Calculate Adjustments
                self.__back_propagate(output, y)

            self.__gradient_descent(i, learning_rate)
            
            # Check if accuracy criterion is satisfied
            if np.mean(error[-(i+1):]) < stop_accuracy and iteration > 0:
                break

        return(np.asarray(error), iteration+1)

if __name__ == "__main__":
    # ----------- XOR Function -----------------

    # Create instance of a neural network
    nn = NeuralNetwork()

    # Add Layers (Input layer is created by default)
    nn.add_layer((2, 9))
    nn.add_layer((9, 1))

    # XOR function
    training_data = np.asarray([[0, 0], [0, 1], [1, 0], [1, 1]]).reshape(4, 2, 1)
    training_labels = np.asarray([[0], [1], [1], [0]])

    error, iteration = nn.train(training_data, training_labels, 5000)
    # nn.predict(testing_data)
    print('Error: %s' % np.mean(error[-4:]))
    print('Epoches needed to train: %s' % iteration)
    print('Predicting 1,0,0: %s' % neural_network.predict(np.array([1, 0, 0])))
    print('Predicting 0,0,0: %s' % neural_network.predict(np.array([0, 0, 0])))
    print('Predicting 0,0,1: %s' % neural_network.predict(np.array([0, 0, 1])))