📑 Digit Recognition Models
Libraries¶
In [1]:
%%html
<style>
@import url('https://fonts.googleapis.com/css?family=Orbitron|Roboto');
body {background-color: gainsboro;}
a {color: #37c9e1; font-family: 'Roboto';}
h1 {color: #37c9e1; font-family: 'Orbitron'; text-shadow: 4px 4px 4px #ccc;}
h2, h3 {color: slategray; font-family: 'Orbitron'; text-shadow: 4px 4px 4px #ccc;}
h4 {color: #818286; font-family: 'Roboto';}
span {text-shadow: 4px 4px 4px #ccc;}
div.output_prompt, div.output_area pre {color: slategray;}
div.input_prompt, div.output_subarea {color: #37c9e1;}
div.output_stderr pre {background-color: gainsboro;}
div.output_stderr {background-color: slategrey;}
</style>
In [2]:
import numpy as np
import scipy as sp
import pandas as pd
from time import time
import warnings
warnings.filterwarnings('ignore')
from IPython.core.display import HTML, display
import matplotlib.pylab as plt
import matplotlib.cm as cm
import cv2
%matplotlib inline
In [3]:
from sklearn.neural_network import MLPClassifier
from sklearn import linear_model, neighbors, svm, ensemble
from sklearn import datasets, metrics
from sklearn.model_selection import train_test_split
In [4]:
import keras as ks
import tensorflow as tf
from keras.models import Sequential, load_model, Model
from keras.preprocessing.image import ImageDataGenerator
from keras.optimizers import SGD, RMSprop
from keras.layers import Dense, Dropout, LSTM, ConvLSTM2D, Reshape
from keras.layers import Activation, Flatten, Input, BatchNormalization
from keras.layers import Conv1D, MaxPooling1D, Conv2D, MaxPooling2D
from keras.layers import AveragePooling2D, GlobalMaxPooling2D, GlobalAveragePooling2D
from keras.layers.embeddings import Embedding
from keras.layers.advanced_activations import PReLU, LeakyReLU
from keras.callbacks import ModelCheckpoint
from keras.callbacks import EarlyStopping
from keras.applications.inception_v3 import InceptionV3
from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19
In [5]:
from keras import __version__
print('keras version:', __version__)
print('tensorflow version:', tf.__version__)
Dataset¶
In [6]:
df_train = pd.read_csv("kaggle_digits_train.csv")
df_test = pd.read_csv("kaggle_digits_test.csv")
df_train.shape
Out[6]:
In [7]:
df_train.ix[15:20,15:20]
Out[7]:
In [8]:
images = ["%s%s" %("pixel",pixel_no) for pixel_no in range(0,784)]
train_images = np.array(df_train[images])
train_images = train_images.astype('float32')/255
train_images.shape
Out[8]:
In [9]:
train_labels = df_train['label']
train_labels_cat = ks.utils.to_categorical(train_labels, num_classes=10)
train_labels_cat.shape
Out[9]:
In [10]:
test_images = np.array(df_test[images])
test_images = test_images.astype('float32')/255
test_images.shape
Out[10]:
In [11]:
X_train, X_test, y_train, y_test = train_test_split(train_images, train_labels_cat,
test_size=0.2, random_state=32)
In [12]:
n = int(len(X_test)/2)
X_valid, y_valid = X_test[:n], y_test[:n]
X_test, y_test = X_test[n:], y_test[n:]
X_train.shape, X_test.shape, X_valid.shape, y_train.shape, y_test.shape, y_valid.shape
Out[12]:
In [13]:
y_train_num = np.array([np.argmax(x) for x in y_train])
y_test_num = np.array([np.argmax(x) for x in y_test])
y_valid_num = np.array([np.argmax(x) for x in y_valid])
In [14]:
X_train_app = [cv2.cvtColor(cv2.resize(i, (32,32)), cv2.COLOR_GRAY2BGR) for i in X_train]
X_train_app = np.concatenate([arr[np.newaxis] for arr in X_train_app]).astype('float32')
X_test_app = [cv2.cvtColor(cv2.resize(i, (32,32)), cv2.COLOR_GRAY2BGR) for i in X_test]
X_test_app = np.concatenate([arr[np.newaxis] for arr in X_test_app]).astype('float32')
X_valid_app = [cv2.cvtColor(cv2.resize(i, (32,32)), cv2.COLOR_GRAY2BGR) for i in X_valid]
X_valid_app = np.concatenate([arr[np.newaxis] for arr in X_valid_app]).astype('float32')
In [15]:
X_train_app.shape, X_test_app.shape, X_valid_app.shape, y_train.shape, y_test.shape, y_valid.shape
Out[15]:
Examples¶
In [16]:
fig, ax = plt.subplots(figsize=(18, 2), nrows=1, ncols=15, sharex=True, sharey=True,)
ax = ax.flatten()
for i in range(15):
image = train_images[i].reshape(28,28)
ax[i].imshow(image, cmap=cm.bone)
ax[0].set_xticks([])
ax[0].set_yticks([])
plt.tight_layout()
plt.gcf()
ax[7].set_title('Examples of the 784-dimensional digits', fontsize=25);
In [16]:
X_train_tensor = tf.image.grayscale_to_rgb(X_train.reshape(-1, 28, 28, 1), name=None)
X_test_tensor = tf.image.grayscale_to_rgb(X_test.reshape(-1, 28, 28, 1), name=None)
X_valid_tensor = tf.image.grayscale_to_rgb(X_valid.reshape(-1, 28, 28, 1), name=None)
In [17]:
sess = tf.Session()
with sess.as_default():
X_train_color = X_train_tensor.eval()
X_test_color = X_test_tensor.eval()
X_valid_color = X_valid_tensor.eval()
In [18]:
X_train_color.shape, X_test_color.shape, X_valid_color.shape
Out[18]:
In [19]:
plt.imshow(X_test_color[0]);
Neural Network Models¶
Model #1. Convolutional Neural Network. Keras¶
In [17]:
def cnn_mc_model():
model_input = Input(shape=(28, 28, 1))
x = BatchNormalization()(model_input)
x = Conv2D(28, (5, 5), padding='same')(x)
x = LeakyReLU(alpha=0.02)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Dropout(0.25)(x)
x = Conv2D(128, (5, 5))(x)
x = LeakyReLU(alpha=0.02)(x)
x = MaxPooling2D(strides=(2, 2))(x)
x = Dropout(0.25)(x)
x = GlobalMaxPooling2D()(x)
x = Dense(1024)(x)
x = LeakyReLU(alpha=0.02)(x)
x = Dropout(0.5)(x)
# x = Reshape((1, 128), input_shape=(128,))(x)
# x = LSTM(128, activation='relu')(x)
y = Dense(10, activation='softmax')(x)
model = Model(input=model_input, output=y)
model.compile(loss='categorical_crossentropy', optimizer='nadam', metrics=['accuracy'])
return model
cnn_mc_model = cnn_mc_model()
In [29]:
cnn_checkpointer = ModelCheckpoint(filepath='weights.best3.cnn_mc.digits.hdf5',
verbose=2, save_best_only=True)
fit_cnn = cnn_mc_model.fit(X_train.reshape(-1, 28, 28, 1), y_train,
validation_data=(X_valid.reshape(-1, 28, 28, 1), y_valid),
epochs=50, batch_size=128, verbose=2, callbacks=[cnn_checkpointer])
In [32]:
plt.figure(figsize=(18, 6))
plt.plot(fit_cnn.history['loss'][2:], '-o', c='#37c9e1', lw=1, label = 'train')
plt.plot(fit_cnn.history['val_loss'][2:], '-o', c='#39d4be', lw=1, label = 'test')
plt.legend()
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.title('CNN Loss Function');
In [18]:
cnn_mc_model.load_weights('weights.best3.cnn_mc.digits.hdf5')
cnn_scores = cnn_mc_model.evaluate(X_test.reshape(-1, 28, 28, 1), y_test, verbose=0)
print("CNN Scores: " , (cnn_scores))
print("CNN Error: %.2f%%" % (100 - cnn_scores[1]*100))
print(cnn_mc_model.summary())
In [ ]:
steps, epochs = 1000, 50
data_generator = ImageDataGenerator(zoom_range=0.2, shear_range=0.2, rotation_range=20)
fit_dg_cnn = cnn_mc_model.fit_generator(data_generator.flow(X_train.reshape(-1, 28, 28, 1),
y_train,
batch_size=128),
steps_per_epoch = steps, epochs = epochs,
validation_data = (X_valid.reshape(-1, 28, 28, 1), y_valid),
callbacks=[cnn_checkpointer], verbose=2)
In [21]:
plt.figure(figsize=(18, 6))
plt.plot(fit_dg_cnn.history['loss'][2:], '-o', c='#37c9e1', lw=1, label = 'train')
plt.plot(fit_dg_cnn.history['val_loss'][2:], '-o', c='#39d4be', lw=1, label = 'test')
plt.legend()
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.title('CNN with Data Generator; Loss Function');
In [22]:
cnn_mc_model.load_weights('weights.best3.cnn_mc.digits.hdf5')
cnn_scores = cnn_mc_model.evaluate(X_test.reshape(-1, 28, 28, 1), y_test, verbose=0)
print("CNN with Data Generator; Scores: " , (cnn_scores))
print("CNN with Data Generator; Error: %.2f%%" % (100 - cnn_scores[1]*100))
In [79]:
cnn_mc_model.save('kaggle_digits_cnn_mc_model3.h5')
Model #2. Multi-layer Perceptron. Keras¶
In [44]:
def mlp_mc_model():
model = Sequential()
model.add(Dense(1568, activation='relu', input_shape = X_train.shape[1:]))
model.add(Dropout(0.2))
model.add(Dense(392, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='nadam', loss='categorical_crossentropy', metrics=['accuracy'])
return model
mlp_mc_model = mlp_mc_model()
In [45]:
mlp_checkpointer = ModelCheckpoint(filepath='weights.best.mlp_mc.digits.hdf5',
verbose=2, save_best_only=True)
fit_mlp = mlp_mc_model.fit(X_train, y_train, validation_data=(X_valid, y_valid),
epochs=15, batch_size=128, verbose=2, callbacks=[mlp_checkpointer]);
In [47]:
plt.figure(figsize=(18,5))
plt.plot(fit_mlp.history['acc'][2:], '-o', c='#37c9e1', lw=1, label = 'train')
plt.plot(fit_mlp.history['val_acc'][2:], '-o', c='#39d4be', lw=1, label = 'test')
plt.legend()
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.title('MLP Accuracy');
In [48]:
mlp_mc_model.load_weights('weights.best.mlp_mc.digits.hdf5')
mlp_scores = mlp_mc_model.evaluate(X_test, y_test)
print("\nMLP Scores: ", (mlp_scores))
print("MLP Error: %.2f%%" % (100 - mlp_scores[1] * 100))
print(mlp_mc_model.summary())
In [49]:
mlp_mc_model.save('kaggle_digits_mlp_mc_model.h5')
Model #3. Recurrent Neural Network. Keras¶
In [50]:
def rnn_mc_model():
model = Sequential()
model.add(LSTM(196, return_sequences=True, input_shape=(1, 784)))
model.add(LSTM(196, return_sequences=True))
model.add(LSTM(784))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='nadam', metrics=['accuracy'])
return model
In [51]:
rnn_mc_model = rnn_mc_model()
rnn_checkpointer = ModelCheckpoint(filepath='weights.best.rnn_mc.digits.hdf5',
verbose=2, save_best_only=True)
fit_rnn = rnn_mc_model.fit(X_train.reshape(X_train.shape[0], 1, X_train.shape[1]), y_train,
epochs=10, batch_size=128, verbose=2, callbacks=[rnn_checkpointer],
validation_data=(X_valid.reshape(X_valid.shape[0], 1, X_valid.shape[1]), y_valid))
In [52]:
plt.figure(figsize=(18,5))
plt.plot(fit_rnn.history['acc'][2:], '-o', c='#37c9e1', lw=1, label = 'train')
plt.plot(fit_rnn.history['val_acc'][2:], '-o', c='#39d4be', lw=1, label = 'test')
plt.legend()
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.title('RNN Accuracy');
In [53]:
rnn_mc_model.load_weights('weights.best.rnn_mc.digits.hdf5')
rnn_scores = rnn_mc_model.evaluate(X_test.reshape(X_test.shape[0], 1, X_test.shape[1]), y_test)
print("\nRNN Scores: ", (rnn_scores))
print("RNN Error: %.2f%%" % (100 - rnn_scores[1] * 100))
print(rnn_mc_model.summary())
In [54]:
rnn_mc_model.save('kaggle_digits_rnn_mc_model.h5')
Model #4. MLPClassifier. Scikit-Learn¶
In [91]:
clf = MLPClassifier(hidden_layer_sizes=(784,), max_iter=50, alpha=1e-4,
solver='adam', verbose=1, tol=1e-6, random_state=1,
learning_rate_init=7e-4, batch_size=128)
clf.fit(X_train, y_train);
In [92]:
print("MNIST. MLPClassifier. Train score: %f" % (clf.score(X_train, y_train)*100),'%')
print("MNIST. MLPClassifier. Test score: %f" % (clf.score(X_test, y_test)*100),'%')
In [93]:
clf2 = MLPClassifier(hidden_layer_sizes=(784,), max_iter=50, alpha=1e-4,
solver='lbfgs', verbose=1, tol=1e-6, random_state=1,
learning_rate_init=7e-4, batch_size=128)
clf2.fit(X_train, y_train_num);
In [94]:
print("MNIST. MLPClassifier. Train score: %f" % (clf2.score(X_train, y_train_num)*100),'%')
print("MNIST. MLPClassifier. Test score: %f" % (clf2.score(X_test, y_test_num)*100),'%')
Classifiers¶
In [175]:
classifier_list = []
classifier_names = []
clf_datasets = []
acc_train, acc_test = [], []
loss_train, loss_test = [], []
In [176]:
def classifier_fit_score(classifier, classifier_name, clf_dataset, x_train, x_test, y_train, y_test):
classifier_list.append(str(classifier))
classifier_names.append(str(classifier_name))
clf_datasets.append(str(clf_dataset))
clf = classifier
clf.fit(x_train, y_train)
y_clf_train = clf.predict(x_train)
y_clf_test = clf.predict(x_test)
acc_clf_train = round(metrics.accuracy_score(y_train, y_clf_train), 4)
acc_train.append(acc_clf_train)
acc_clf_test = round(metrics.accuracy_score(y_test, y_clf_test), 4)
acc_test.append(acc_clf_test)
loss_clf_train = round(metrics.hamming_loss(y_train, y_clf_train), 4)
loss_train.append(loss_clf_train)
loss_clf_test = round(metrics.hamming_loss(y_test, y_clf_test), 4)
loss_test.append(loss_clf_test)
return [y_clf_train, y_clf_test, acc_clf_train, acc_clf_test, loss_clf_train, loss_clf_test]
In [177]:
def get_classifier_results():
df_results = pd.DataFrame({'classifier' : classifier_list,
'classifier_name': classifier_names,
'clf_dataset': clf_datasets,
'acc_train': acc_train,
'acc_test': acc_test,
'loss_train': loss_train,
'loss_test': loss_test})
df_list = ['classifier_name', 'acc_train', 'acc_test', 'loss_train', 'loss_test']
return df_results, df_results[df_list]
In [178]:
clf1 = neighbors.KNeighborsClassifier()
clf2 = linear_model.LogisticRegressionCV()
clf3 = ensemble.GradientBoostingClassifier()
In [179]:
y_knc_train, y_knc_test = classifier_fit_score(clf1, 'KNeighborsClassifier', 'digits',
X_train, X_test, y_train_num, y_test_num)[:2]
In [180]:
y_lrcvc_train, y_lrcvc_test = classifier_fit_score(clf2, 'LogisticRegressionCV', 'digits',
X_train, X_test, y_train_num, y_test_num)[:2]
In [181]:
y_gbc_train, y_gbc_test = classifier_fit_score(clf3, 'GradientBoostingClassifier', 'digits',
X_train, X_test, y_train_num, y_test_num)[:2]
In [182]:
df_results, df_results2 = get_classifier_results()
df_results2.sort_values('acc_test', ascending=False)
Out[182]:
In [208]:
plt.figure(figsize=(18, 10))
n=100
x=range(n)
plt.scatter(x, y_test_num[:n], marker='*', s=150, color='#37c9e1', alpha=0.7, label='Real data')
plt.scatter(x, y_knc_test[:n], marker='v', s=30, color='steelblue', label='K Neighbors Classifier')
# plt.scatter(x, y_lrcvc_test[:n], marker='s', s=30, color='darkblue', label='Logistic RegressionCV')
# plt.scatter(x, y_gbc_test[:n], marker='o', s=30, color='darkgreen', label='GradientBoostingClassifier')
plt.xlabel('Observations', fontsize=12)
plt.ylabel('Target', fontsize=12)
plt.title('Classifiers, Test Data', fontsize=15)
plt.legend(loc=5, fontsize=12);
Applications¶
In [52]:
vgg16_base_model = VGG16(weights='imagenet', include_top=False)
X_train_bn = vgg16_base_model.predict(X_train_app)
X_test_bn = vgg16_base_model.predict(X_test_app)
In [57]:
X_train_bn.shape
Out[57]:
In [54]:
np.save('X_train_bn.npy', X_train_bn.reshape(X_train_bn.shape[0], X_train_bn.shape[3]))
np.save('X_test_bn.npy', X_test_bn.reshape(X_test_bn.shape[0], X_test_bn.shape[3]))
In [56]:
X_train_bn = np.load('X_train_bn.npy')
X_test_bn = np.load('X_test_bn.npy')
X_train_bn = X_train_bn.reshape(X_train_bn.shape[0], 1, 1, X_train_bn.shape[1])
X_test_bn = X_test_bn.reshape(X_test_bn.shape[0], 1, 1, X_test_bn.shape[1])
In [79]:
def vgg16_add_model():
model = Sequential()
model.add(GlobalMaxPooling2D(input_shape=X_train_bn.shape[1:]))
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.25))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='nadam', metrics=['accuracy'])
return model
In [80]:
vgg16_add_model = vgg16_add_model()
vgg16_checkpointer = ModelCheckpoint(filepath='weights.best.vgg16.digits.hdf5',
verbose=2, save_best_only=True)
In [81]:
vgg16_add_history = vgg16_add_model.fit(X_train_bn, y_train,
validation_data=(X_test_bn, y_test),
epochs=50, batch_size=128,
callbacks=[vgg16_checkpointer], verbose=0);
In [83]:
vgg16_add_model.load_weights('weights.best.vgg16.digits.hdf5')
vgg16_test_scores = vgg16_add_model.evaluate(X_test_bn, y_test)
print("Accuracy: %.2f%%" % (vgg16_test_scores[1]*100))
Predictions¶
In [84]:
predict_labels_cnn = cnn_mc_model.predict(test_images.reshape(28000,28,28,1))
predict_labels_cnn = predict_labels_cnn.argmax(axis=-1)
predict_labels_cnn[:100]
Out[84]:
In [132]:
predict_labels_mlp = mlp_mc_model.predict_classes(test_images)
predict_labels_mlp[:100]
Out[132]:
In [114]:
predict_labels_rnn = rnn_mc_model.predict_classes(test_images.reshape(28000,1,784))
predict_labels_rnn[:100]
Out[114]:
In [120]:
predict_labels_mlpc = clf.predict(test_images)
predict_labels_mlpc_num = np.array([np.argmax(x) for x in predict_labels_mlpc])
predict_labels_mlpc_num[:100]
Out[120]:
In [85]:
submission_cnn = pd.DataFrame({"ImageId": range(1, len(predict_labels_cnn)+1),
"Label": predict_labels_cnn})
print(submission_cnn[0:20])
submission_cnn.to_csv('kaggle_digits_cnn_mc_model.csv', index=False)
In [44]:
submission_mlp = pd.DataFrame({"ImageId": range(1, len(predict_labels_mlp)+1),
"Label": predict_labels_mlp})
print(submission_mlp[:20])
submission_mlp.to_csv('kaggle_digits_mlp_mc_model.csv', index=False)
In [118]:
submission_rnn = pd.DataFrame({"ImageId": range(1, len(predict_labels_rnn)+1),
"Label": predict_labels_rnn})
print(submission_rnn[0:20])
submission_rnn.to_csv('kaggle_digits_rnn_mc_model.csv', index=False)
In [122]:
submission_mlpc = pd.DataFrame({"ImageId": range(1, len(predict_labels_mlpc_num)+1),
"Label": predict_labels_mlpc_num})
print(submission_mlpc[0:20])
submission_mlpc.to_csv('kaggle_digits_mlpc_model.csv', index=False)
In [86]:
real_data = np.array([2, 0, 9, 0, 3, 7, 0, 3, 0, 3,
5, 7, 4, 0, 4, 3, 3, 1, 9, 0,
9, 1, 1, 5, 7, 4, 2, 7, 4, 7,
7, 5, 4, 2, 6, 2, 5, 5, 1, 6,
7, 7, 4, 9, 8, 7, 8, 2, 6, 7,
6, 8, 8, 3, 8, 2, 1, 2, 2, 0,
4, 1, 7, 0, 0, 0, 1, 9, 0, 1,
6, 5, 8, 8, 2, 8, 9, 9, 2, 3,
5, 4, 1, 0, 9, 2, 4, 3, 6, 7,
2, 0, 6, 6, 1, 4, 3, 9, 7, 4,
0, 9, 2, 0, 7, 3, 0, 5, 0, 8,
0, 0, 4, 7, 1, 7, 1, 1, 3, 3,
3, 7, 2, 8, 6, 3, 8, 7, 7, 4,
3, 5, 4, 0, 0, 0, 3, 1, 3, 6,
4, 3, 4, 5, 5, 8, 7, 4, 2, 8,
4, 3, 5, 6, 5, 3, 7, 5, 7, 8,
3, 0, 4, 5, 1, 3, 7, 6, 3, 0,
2, 7, 8, 6, 1, 3, 7, 4, 1, 2,
4, 8, 5, 2, 4, 9, 2, 1, 6, 0,
6, 1, 4, 9, 6, 0, 9, 7, 6, 9,
1, 9, 0, 9, 9, 0, 8, 4, 6, 2,
0, 9, 3, 6, 3, 2, 1, 6, 3, 4,
2, 3, 1, 2, 2, 0, 4, 6, 1, 0,
0, 4, 9, 1, 7, 3, 2, 3, 8, 6,
8, 6, 2, 8, 5, 5, 4, 8, 3, 5,
9, 7, 1, 3, 8, 4, 5, 1, 4, 5,
6, 3, 3, 5, 7, 0, 6, 8, 3, 1,
6, 0, 6, 3, 9, 5, 1, 5, 8, 4,
0, 9, 2, 0, 5, 3, 7, 1, 9, 9,
5, 7, 7, 9, 9, 6, 3, 0, 3, 3])
In [87]:
plt.figure(figsize = (18, 10))
plt.plot(real_data[0:300], '*', ms=12, c='#37c9e1', label='Real Data')
plt.plot(submission_cnn['Label'][0:300], 'bo', ms=5, label='CNN')
# plt.plot(submission_mlp['Label'][0:200], 'v', ms=5, label='MLP')
# plt.plot(submission_rnn['Label'][0:200], 's', ms=5, label='RNN')
# plt.plot(submission_mlpc['Label'][0:200], 'p', ms=5, label='MLPC')
plt.legend(loc=6)
plt.title("Predictions");
In [74]:
fig, ax = plt.subplots(figsize=(18, 4), nrows=2, ncols=10, sharex=True, sharey=True,)
ax = ax.flatten()
for i in range(20):
image = test_images[100:120][i].reshape(28,28)
ax[i].imshow(image, cmap=cm.bone)
ax[0].set_xticks([])
ax[0].set_yticks([])
plt.tight_layout()
plt.gcf()
ax[4].set_title('Examples of the 784-dimensional digits. Test datapoints', fontsize=25);
