📑 Keras Cookbook
0. Code Library and Links¶
In [13]:
%%html
<style>
@import url('https://fonts.googleapis.com/css?family=Orbitron|Roboto');
body {background-color: gainsboro;}
a {color: darkgreen; font-family: 'Roboto';}
h1 {color: slategray; font-family: 'Orbitron'; text-shadow: 4px 4px 4px #aaa;}
h2, h3 {color: darkslategray; font-family: 'Orbitron'; text-shadow: 4px 4px 4px #aaa;}
h4 {color: slategray; font-family: 'Roboto';}
span {text-shadow: 4px 4px 4px #ddd;}
div.input_prompt, div.output_area pre {color: slategray;}
div.output_prompt {color: green;}
</style>
In [14]:
import numpy as np
import pandas as pd
import os
import json
import scipy
import cv2
import seaborn
import time
import keras
from PIL import ImageFile
from IPython.display import SVG
from tqdm import tqdm
from skimage import data
import matplotlib.pylab as plt
from matplotlib import cm
plt.style.use('seaborn-whitegrid')
%matplotlib inline
In [15]:
from keras.datasets import mnist, cifar10, imdb, boston_housing
In [16]:
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_files
In [17]:
from keras.preprocessing import sequence as keras_sequence
from keras.preprocessing import image as keras_image
from keras.utils import to_categorical, np_utils
from keras.utils.vis_utils import model_to_dot
In [18]:
from keras.models import Sequential, Model
from keras.models import load_model, model_from_json
In [19]:
from keras.layers import Conv1D, MaxPooling1D, GlobalAveragePooling1D
from keras.layers import Conv2D, MaxPooling2D, GlobalAveragePooling2D
from keras.layers.embeddings import Embedding
from keras.layers.advanced_activations import PReLU
from keras.layers import Dense, LSTM, GlobalMaxPooling2D
from keras.layers import Activation, Flatten, Dropout
from keras.layers import Input, BatchNormalization
In [20]:
from keras.applications.resnet50 import ResNet50
from keras.applications.resnet50 import preprocess_input as resnet50_preprocess_input
from keras.applications.vgg16 import VGG16
from keras.applications.vgg16 import preprocess_input as vgg16_preprocess_input
from keras.applications.vgg19 import VGG19
from keras.applications.vgg19 import preprocess_input as vgg19_preprocess_input
In [21]:
from keras.applications.inception_v3 import InceptionV3
from keras.applications.inception_v3 import preprocess_input as inception_v3_preprocess_input
from keras.applications.xception import Xception
from keras.applications.xception import preprocess_input as xception_preprocess_input
In [23]:
def loss_plot(fit_history):
plt.figure(figsize=(18, 4))
plt.plot(fit_history.history['loss'], label = 'train')
plt.plot(fit_history.history['val_loss'], label = 'test')
plt.legend()
plt.title('Loss Function');
def mae_plot(fit_history):
plt.figure(figsize=(18, 4))
plt.plot(fit_history.history['mean_absolute_error'], label = 'train')
plt.plot(fit_history.history['val_mean_absolute_error'], label = 'test')
plt.legend()
plt.title('Mean Absolute Error');
def acc_plot(fit_history):
plt.figure(figsize=(18, 4))
plt.plot(fit_history.history['acc'], label = 'train')
plt.plot(fit_history.history['val_acc'], label = 'test')
plt.legend()
plt.title('Accuracy');
In [24]:
def decode_predictions2(preds, fpath, top=5):
global CLASS_INDEX
if len(preds.shape) != 2 or preds.shape[1] != 1000:
raise ValueError('`decode_predictions` expects '
'a batch of predictions '
'(i.e. a 2D array of shape (samples, 1000)). '
'Found array with shape: ' + str(preds.shape))
if CLASS_INDEX is None:
CLASS_INDEX = json.load(open(fpath))
results = []
for pred in preds:
top_indices = pred.argsort()[-top:][::-1]
result = [tuple(CLASS_INDEX[str(i)]) + (pred[i],) for i in top_indices]
results.append(result)
return results
1. Backend¶
In [25]:
'''
/.keras/keras.json
{"image_data_format": "channels_last",
"epsilon": 1e-07,
"floatx": "float32",
"backend": "tensorflow"}
'''
''''''
Out[25]:
In [26]:
# variants: "theano", "tensorflow", "cntk"
keras.backend.backend()
Out[26]:
In [27]:
keras.backend.image_dim_ordering()
Out[27]:
In [28]:
# 32x32 color images; 50,000 - the train set; 10,000 - the test set; labeled over 10 categories
(x_train1, y_train1), (x_test1, y_test1) = cifar10.load_data()
In [29]:
n = int(len(x_test1)/2)
x_valid1, y_valid1 = x_test1[:n], y_test1[:n]
x_test1, y_test1 = x_test1[n:], y_test1[n:]
print(x_train1.shape, x_valid1.shape, x_test1.shape)
print(y_train1.shape, y_valid1.shape, y_test1.shape)
In [30]:
print('Label: ', y_train1[1][0])
plt.figure(figsize=(3,3)); plt.xticks([]); plt.yticks([])
plt.imshow(x_train1[1]);
In [31]:
# 28x28 grayscale images; 55,000 - the train set; 10,000 - the test set; labeled over 10 categories
(x_train2, y_train2), (x_test2, y_test2) = mnist.load_data()
In [32]:
n = int(len(x_test2)/2)
x_valid2, y_valid2 = x_test2[:n], y_test2[:n]
x_test2, y_test2 = x_test2[n:], y_test2[n:]
print(x_train2.shape, x_valid2.shape, x_test2.shape)
print(y_train2.shape, y_valid2.shape, y_test2.shape)
In [33]:
print('Label: ', y_train2[1])
plt.figure(figsize=(3,3)); plt.xticks([]); plt.yticks([])
plt.imshow(x_train2[1], cmap='tab10');
In [34]:
# 25,000 movies reviews from IMDB, labeled by sentiment (positive/negative)
num_words, max_length, embedding_vector_length = 10000, 1000, 32
(x_train3, y_train3), (x_test3, y_test3) = imdb.load_data(path="imdb_full.pkl", num_words=num_words,
skip_top=0, maxlen=max_length, seed=113,
start_char=1, oov_char=2, index_from=3)
In [35]:
n = int(len(x_test3)/2)
x_valid3, y_valid3 = x_test3[:n], y_test3[:n]
x_test3, y_test3 = x_test3[n:], y_test3[n:]
print(x_train3.shape, x_valid3.shape, x_test3.shape)
print(y_train3.shape, y_valid3.shape, y_test3.shape)
In [36]:
print('Label: ', y_train3[1])
print('Sequence of word indexes: \n', x_train3[1])
In [37]:
# 13 attributes of houses at different locations, targets are the median values of the houses at a location (in k$)
(x_train4, y_train4), (x_test4, y_test4) = boston_housing.load_data()
In [38]:
n = int(len(x_test4)/2)
x_valid4, y_valid4 = x_test4[:n], y_test4[:n]
x_test4, y_test4 = x_test4[n:], y_test4[n:]
print(x_train4.shape, x_valid4.shape, x_test4.shape)
print(y_train4.shape, y_valid4.shape, y_test4.shape)
In [39]:
print('Target value: ', y_train4[1])
print("Features' values: \n", x_train4[1])
In [40]:
plt.figure(figsize=(14, 3))
plt.hist(y_train4, bins=50, alpha=0.7);
artificial datasets¶
In [41]:
# Classification; the artificial set, labeled over 2 categories
X5, Y5 = make_classification(n_samples=5000, n_features=2, n_redundant=0, n_informative=2)
x_train5, x_test5, y_train5, y_test5 = train_test_split(X5, Y5, test_size = 0.2, random_state = 1)
In [42]:
n = int(len(x_test5)/2)
x_valid5, y_valid5 = x_test5[:n], y_test5[:n]
x_test5, y_test5 = x_test5[n:], y_test5[n:]
print(x_train5.shape, x_valid5.shape, x_test5.shape)
print(y_train5.shape, y_valid5.shape, y_test5.shape)
In [43]:
print('Label: ', y_train5[1])
print('Features: \n', x_train5[1])
In [44]:
plt.figure(figsize=(14,3))
plt.scatter(X5[:, 0], X5[:, 1], marker='o', s=5, c=Y5, cmap='tab10');
external datasets¶
In [45]:
# 150x150 grayscale face images; labeled over 15 categories(persons)
yalefaces_paths = []
yalefaces_images = []
yalefaces_labels = []
for element in os.listdir("yale_faces"):
if element != 'Readme.txt':
yalefaces_paths.append(os.path.join("yalefaces", element))
for path in yalefaces_paths:
image = data.imread(path, as_gray=True)
yalefaces_images.append(image)
label = int(os.path.split(path)[1].split(".")[0].replace("subject", "")) - 1
yalefaces_labels.append(label)
yalefaces_cut_images = []
face_detector = cv2.CascadeClassifier("haarcascade_frontalface_default.xml")
for i in range(len(yalefaces_images)):
image = yalefaces_images[i]
face = face_detector.detectMultiScale(image)
x, y = face[0][:2]
cut_image = image[y: y+150, x: x+150]
yalefaces_cut_images.append(cut_image)
In [46]:
yalefaces_labels = np.array(yalefaces_labels)
yalefaces_cut_images = np.array(yalefaces_cut_images) / 255
print('Label: ', yalefaces_labels[11])
plt.imshow(yalefaces_cut_images[11], cmap='bone');
In [47]:
x_train6, x_test6, y_train6, y_test6 = \
train_test_split(yalefaces_cut_images, yalefaces_labels, test_size = 0.2, random_state = 1)
n = int(len(x_test6)/2)
x_valid6, y_valid6 = x_test6[:n], y_test6[:n]
x_test6, y_test6 = x_test6[n:], y_test6[n:]
print(x_train6.shape, x_valid6.shape, x_test6.shape)
print(y_train6.shape, y_valid6.shape, y_test6.shape)
In [48]:
# 210 128x128 flower color images; labeled over 10 categories
flowers = pd.read_csv("flower_images/flower_labels.csv")
flower_files = flowers['file']
flower_targets = flowers['label'].values
In [49]:
def path_to_tensor(img_path):
img = keras_image.load_img("flower_images/"+img_path, target_size=(128, 128))
x = keras_image.img_to_array(img)
return np.expand_dims(x, axis=0)
def paths_to_tensor(img_paths):
list_of_tensors = [path_to_tensor(img_path) for img_path in tqdm(img_paths)]
return np.vstack(list_of_tensors)
ImageFile.LOAD_TRUNCATED_IMAGES = True
flower_tensors = paths_to_tensor(flower_files) / 255;
In [50]:
x_train7, x_test7, y_train7, y_test7 = \
train_test_split(flower_tensors, flower_targets, test_size = 0.1, random_state = 1)
n = int(len(x_test7)/2)
x_valid7, y_valid7 = x_test7[:n], y_test7[:n]
x_test7, y_test7 = x_test7[n:], y_test7[n:]
print(x_train7.shape, x_valid7.shape, x_test7.shape)
print(y_train7.shape, y_valid7.shape, y_test7.shape)
In [51]:
print('Label: ', flower_targets[1])
flower_image = cv2.imread('flower_images/'+flower_files[1])
rgb_flower_image = cv2.cvtColor(flower_image, cv2.COLOR_BGR2RGB)
plt.xticks([]); plt.yticks([])
plt.imshow(rgb_flower_image);
3. Preprocessing¶
In [52]:
# Sequence Padding
p_x_train3 = keras_sequence.pad_sequences(x_train3, maxlen=max_length)
p_x_valid3 = keras_sequence.pad_sequences(x_valid3, maxlen=max_length)
p_x_test3 = keras_sequence.pad_sequences(x_test3, maxlen=max_length)
In [53]:
# One-Hot Encoding
c_y_train1 = to_categorical(y_train1, 10)
c_y_valid1 = to_categorical(y_valid1, 10)
c_y_test1 = to_categorical(y_test1, 10)
c_y_train2 = to_categorical(y_train2, 10)
c_y_valid2 = to_categorical(y_valid2, 10)
c_y_test2 = to_categorical(y_test2, 10)
c_y_train6 = to_categorical(y_train6, 15)
c_y_valid6 = to_categorical(y_valid6, 15)
c_y_test6 = to_categorical(y_test6, 15)
c_y_train7 = to_categorical(y_train7, 10)
c_y_valid7 = to_categorical(y_valid7, 10)
c_y_test7 = to_categorical(y_test7, 10)
4. Basic Information¶
Optimizers:
- 'adam', 'nadam',
- 'adagrad', 'adadelta', 'adamax',
- 'sgd', 'rmsprop',
- TFOptimizer.
Loss functions:
- 'mean_squared_error'('mse'), 'mean_absolute_error'('mae'),
- 'mean_absolute_percentage_error', 'mean_squared_logarithmic_error',
- 'squared_hinge', 'hinge', 'categorical_hinge',
- 'categorical_crossentropy', 'sparse_categorical_crossentropy',
- 'binary_crossentropy',
- 'kullback_leibler_divergence',
- 'poisson',
- 'cosine_proximity',
- 'logcosh'.
Metrics:
- mae, mse, acc,
- binary_accuracy,
- categorical_accuracy, sparse_categorical_accuracy,
- top_k_categorical_accuracy, sparse_top_k_categorical_accuracy.
Compilation¶
For a multi-class classification problem:
- model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
- model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
For a binary classification problem:
- model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
- model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
For a mean squared error regression problem:
- model.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
- model.compile(optimizer='sgd', loss='mse', metrics=['mae'])
Activation¶
Types:
- 'elu', 'selu', 'relu', 'tanh', 'linear', 'softmax', 'softplus', 'softsign', 'sigmoid', 'hard_sigmoid'
- LeakyReLU, PReLU, ELU, ThresholdedReLU
For a multi-class classification problem:
- 'softmax'
For a binary classification problem:
- 'sigmoid'
For a mean squared error regression problem:
- 'linear'
Layers¶
Core Layers
- Keras Documentation. Core Layers
Inputlayers hold an input tensor (for example, the pixel values of the image with width 32, height 32, and 3 color channels).Dense(fully connected) layers compute the class scores, resulting in volume of size. For example, the size [11] corresponds to class scores, such as 10 digits and 1 empty place. Each neuron in these layers are connected to all the numbers in the previous volume.Activationapplies an activation function to an output. This leaves the size of the volume unchanged.Dropoutlayers consist in randomly setting a fraction rate of input units to 0 at each update during training time, which helps prevent overfitting. This leaves the size of the volume unchanged.Reshapelayers convert an output to a certain shape.Flattenlayers flatten the input and collapses it into the one-dimensional feature vector. This results in the volume such as [2, 30, 64] -> [3840].- etc.
Convolutional Layers
- Keras Documentation. Convolutional Layers
- Convolutional Neural Networks. Fei-Fei Li & Andrej Karpathy & Justin Johnson
- Convolutional Neural Networks. Mirella Lapata
Conv1Dlayers (temporal convolution) convolve the filter with the signal, i.e. “is sliding over the signal vector, computing dot products”. Here the filter is an integer, the dimensionality of the output space (i.e. the number output of filters in the convolution) and the kernel size is an integer, specifying the length of the 1D convolution window.Conv2Dlayers (spatial convolution over images) compute the output of neurons that are connected to local regions in the input each computing a dot product between their weights and a small region they are connected to in the input volume. This results in the volume such as [64, 64, 1] -> [64, 64, 32] when it was used 32 filters.Conv3Dlayers (spatial convolution over volumes).SeparableConv2D,Conv2DTranspose.- etc.
Recurrent Layers
SimpleRNNlayers are fully-connected RNN. This leaves the size of the volume unchanged.LSTMlayers.GRUlayers.
Pooling Layers
- Convolutional Neural Networks. CS231
MaxPooling2Dlayers perform a downsampling operation along the spatial dimensions (width, height). Max-pooling partitions the input image into a set of non-overlapping rectangles and, for each such subregion, outputs the maximum value. This results in the volume such as [28, 28, 32] -> [14, 14, 32].MaxPooling1D&MaxPooling3Dis a max pooling operation for temporal (1D) data and spatial or spatio-temporal (3D) data respectively.
Advanced Activations Layers
LeakyReLU.- $\mathbf {f(x) = alpha * x for x < 0, f(x) = x for x >= 0}$, alpha is a small gradient.
PReLU.- $\mathbf {f(x) = alpha * x for x < 0, f(x) = x for x >= 0}$, alpha is a learned array.
ELU.- $\mathbf {f(x) = alpha * (exp(x) - 1.) for x < 0, f(x) = x for x >= 0}$, alpha is a small gradient.
ThresholdedReLU.- $\mathbf {f(x) = x for x > theta, f(x) = 0}$ otherwise.
etc.
5. Basic Examples¶
In [57]:
# The basic model for binary classification
basic_model = Sequential([Dense(16, input_dim=2), Activation('relu'), Dense(1), Activation('sigmoid')])
basic_model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
In [58]:
# Train
basic_model.fit(x_train5, y_train5,
validation_data=(x_valid5, y_valid5),
epochs=100, batch_size=128, verbose=0)
# Predict classes
y_test5_predictions = basic_model.predict_classes(x_test5)
In [59]:
plt.figure(figsize=(14,2))
plt.scatter(range(100), y_test5[:100], s=100)
plt.scatter(range(100), y_test5_predictions[:100], s=25);
In [60]:
# Evaluate
basic_score = basic_model.evaluate(x_test5, y_test5)
basic_score
Out[60]:
In [61]:
# Model parameters
basic_model.input, basic_model.outputs
Out[61]:
In [62]:
basic_model.input_shape, basic_model.output_shape
Out[62]:
In [63]:
basic_model.summary()
In [64]:
basic_model.get_config()
Out[64]:
In [65]:
basic_model.get_weights()
Out[65]:
In [66]:
# Save/reload models
# basic_model.save('basic_model.h5')
# basic_model = load_model('basic_model.h5')
In [67]:
# Save/reload weights
# basic_model.save_weights('basic_model_weights.h5')
# basic_model.load_weights('basic_model_weights.h5', by_name=False)
In [69]:
# Choose optimization
optimizer = keras.optimizers.rmsprop(lr=0.001, decay=1e-5)
optimizer = keras.optimizers.Nadam(lr=0.005, beta_1=0.99, beta_2=0.9999,
epsilon=None, schedule_decay=0.005)
basic_model = Sequential([Dense(16, input_dim=2), Activation('relu'), Dense(1), Activation('sigmoid')])
basic_model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
basic_model.fit(x_train5, y_train5,
validation_data=(x_valid5, y_valid5),
epochs=100, batch_size=128, verbose=0)
basic_score = basic_model.evaluate(x_test5, y_test5)
basic_score
Out[69]:
In [71]:
# Improve activation
# encoder
inp = Input(shape=(2,))
act = keras.layers.LeakyReLU(alpha=0.4)
lay = Dense(16, name='encoder')(inp)
lay = act(lay)
# decoder
out = Dense(1, activation='sigmoid', name='decoder')(lay)
# model
basic_model = Model(inputs=inp, outputs=out, name='cae')
basic_model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
basic_model.fit(x_train5, y_train5,
validation_data=(x_valid5, y_valid5),
epochs=100, batch_size=128, verbose=0)
basic_score = basic_model.evaluate(x_test5, y_test5)
basic_score
Out[71]:
In [72]:
# Use callbacks
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)
checkpointer = keras.callbacks.ModelCheckpoint(filepath='weights.best.basic_model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=5, factor=0.5)
basic_model = Model(inputs=inp, outputs=out, name='cae')
basic_model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])
basic_model.fit(x_train5, y_train5, validation_data=(x_valid5, y_valid5),
epochs=200, batch_size=128, verbose=0,
callbacks=[early_stopping, checkpointer, lr_reduction])
basic_model.load_weights('weights.best.basic_model.hdf5')
basic_score = basic_model.evaluate(x_test5, y_test5)
basic_score
Out[72]:
In [54]:
# Reshape image arrays
x_train6 = (x_train6).reshape(-1, 150*150)
x_valid6 = (x_valid6).reshape(-1, 150*150)
x_test6 = (x_test6).reshape(-1, 150*150)
In [56]:
# Multi-Class Classification
def model():
model = Sequential()
model.add(Dense(128, activation='relu', input_shape=(22500,)))
model.add(Dropout(0.1))
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(15, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
return model
model = model()
checkpointer = keras.callbacks.ModelCheckpoint(filepath='weights.best.model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = model.fit(x_train6, c_y_train6, validation_data=(x_valid6, c_y_valid6),
epochs=70, batch_size=64, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test6, c_y_test6)
test_score
Out[56]:
In [57]:
loss_plot(history)
acc_plot(history)
In [58]:
y_test6_predictions = model.predict_classes(x_test6)
plt.figure(figsize=(14,5))
plt.scatter(range(17), y_test6, s=100)
plt.scatter(range(17), y_test6_predictions, s=25);
In [62]:
# MLP: Multi-Class Classification
def model():
model = Sequential()
model.add(Dense(49, activation='relu', input_shape=(784,)))
model.add(Dropout(0.1))
model.add(Dense(8*784, activation='relu'))
model.add(Dropout(0.1))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
return model
model = model()
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=20)
checkpointer = keras.callbacks.ModelCheckpoint(filepath='weights.best.model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', verbose=2, patience=5, factor=0.5)
history = model.fit(x_train2.reshape(-1,784), c_y_train2,
validation_data=(x_valid2.reshape(-1,784), c_y_valid2),
epochs=70, batch_size=256, verbose=0,
callbacks=[early_stopping, checkpointer, lr_reduction]);
In [63]:
loss_plot(history)
acc_plot(history)
In [64]:
model.load_weights('weights.best.model.hdf5')
y_test2_predictions = model.predict_classes(x_test2.reshape(-1,784))
plt.figure(figsize=(14,5))
plt.scatter(range(100), y_test2[:100], s=100)
plt.scatter(range(100), y_test2_predictions[:100], s=25)
test_score = model.evaluate(x_test2.reshape(-1,784), c_y_test2)
test_score
Out[64]:
In [161]:
# Regression
def model():
model = Sequential()
model.add(Dense(52, activation='relu', input_shape=(13,)))
model.add(Dense(52, activation='relu'))
model.add(Dense(208, activation='relu'))
model.add(Dense(208, activation='relu'))
model.add(Dense(832, activation='relu'))
model.add(Dense(1))
model.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
return model
model = model()
checkpointer = keras.callbacks.ModelCheckpoint(filepath='weights.best.model.hdf5', verbose=2, save_best_only=True)
lr_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = model.fit(x_train4, y_train4, validation_data=(x_valid4, y_valid4),
epochs=100, batch_size=16, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test4, y_test4)
test_score
Out[161]:
In [162]:
loss_plot(history)
mae_plot(history)
In [163]:
y_test4_predictions = model.predict(x_test4)
plt.figure(figsize=(14,3))
plt.plot(range(len(y_test4)), y_test4, '-o', label = 'real data')
plt.plot(range(len(y_test4)), y_test4_predictions, '-o', label = 'predictions');
Convolutional Neural Networks (CNN)¶
In [65]:
# CNN: Binary Classification
def model():
model = Sequential()
model.add(Conv1D(filters=8, kernel_size=5, padding='same',
activation='relu', input_shape=(2,1)))
model.add(Conv1D(filters=8, kernel_size=5,
padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.1))
model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
return model
model = model()
checkpointer = keras.callbacks.ModelCheckpoint(filepath='weights.best.model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = model.fit(x_train5.reshape(-1, 2, 1), y_train5.reshape(-1, 1, 1),
validation_data=(x_valid5.reshape(-1, 2, 1), y_valid5.reshape(-1, 1, 1)),
epochs=20, batch_size=64, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test5.reshape(-1, 2, 1), y_test5.reshape(-1, 1, 1))
test_score
Out[65]:
In [66]:
y_test5_predictions = model.predict_classes(x_test5.reshape(-1, 2, 1))
plt.figure(figsize=(14,2))
plt.scatter(range(100), y_test5[:100], s=100)
plt.scatter(range(100), y_test5_predictions[:100], s=25);
In [67]:
# VGG-like CNN: Multi-Class Classification
def model():
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', input_shape=x_train1.shape[1:]))
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(96, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Conv2D(96, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(GlobalMaxPooling2D())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
In [68]:
model = model()
checkpointer = keras.callbacks\
.ModelCheckpoint(filepath='weights.best.model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = model.fit(x_train1, c_y_train1, validation_data=(x_valid1, c_y_valid1),
epochs=30, batch_size=128, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test1, c_y_test1)
test_score
Out[68]:
In [69]:
loss_plot(history)
acc_plot(history)
In [70]:
steps, epochs = 1000, 5
data_generator = keras_image.ImageDataGenerator(zoom_range=0.2, horizontal_flip=True)
dg_history = \
model.fit_generator(data_generator.flow(x_train1, c_y_train1, batch_size=128),
steps_per_epoch = steps, epochs = epochs, verbose=2,
validation_data = (x_valid1, c_y_valid1),
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test1, c_y_test1)
test_score
Out[70]:
In [71]:
loss_plot(dg_history)
acc_plot(dg_history)
In [72]:
y_valid1_predictions = model.predict_classes(x_valid1)
y_test1_predictions = model.predict_classes(x_test1)
In [73]:
plt.figure(1, figsize=(18, 5))
plt.subplot(121)
plt.scatter(range(100), y_valid1[:100], s=100)
plt.scatter(range(100), y_valid1_predictions[:100], s=25)
plt.title("Validation Predictions")
plt.subplot(122)
plt.scatter(range(100), y_test1[:100], s=100)
plt.scatter(range(100), y_test1_predictions[:100], s=25)
plt.title("Test Predictions");
In [74]:
# CNN: Regression
def model():
model = Sequential()
model.add(Conv1D(52, 5, padding='valid', activation='relu', input_shape=(13,1)))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.2))
model.add(Conv1D(208, 3, padding='valid', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(1024, kernel_initializer='normal', activation='relu'))
model.add(Dropout(0.4))
model.add(Dense(1, kernel_initializer='normal'))
model.compile(loss='mse', optimizer='rmsprop', metrics=['mae'])
return model
In [75]:
model = model()
checkpointer = keras.callbacks\
.ModelCheckpoint(filepath='weights.best.model.hdf5', verbose=0, save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=10, verbose=2, factor=0.95)
history = model.fit(x_train4.reshape(-1, 13, 1), y_train4,
validation_data=(x_valid4.reshape(-1, 13, 1), y_valid4),
epochs=300, batch_size=16, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test4.reshape(-1, 13, 1), y_test4)
test_score
Out[75]:
In [76]:
loss_plot(history)
mae_plot(history)
In [78]:
y_test4_predictions = model.predict(x_test4.reshape(-1, 13, 1))
plt.figure(figsize=(14,3))
plt.plot(range(len(y_test4)), y_test4, '-o', label = 'real data')
plt.plot(range(len(y_test4)), y_test4_predictions, '-o', label = 'predictions')
plt.legend();
Recurrent Neural Networks (RNN)¶
In [84]:
# RNN: Binary Classification
def model():
model = Sequential()
model.add(LSTM(32, dropout=0.2, recurrent_dropout=0.2, input_shape=(1, 2)))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
In [85]:
model = model()
checkpointer = keras.callbacks\
.ModelCheckpoint(filepath='weights.best.model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = model.fit(x_train5.reshape(-1, 1, 2), y_train5,
validation_data=(x_valid5.reshape(-1, 1, 2), y_valid5),
epochs=20, batch_size=128, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test5.reshape(-1, 1, 2), y_test5)
test_score
Out[85]:
In [86]:
loss_plot(history)
acc_plot(history)
In [87]:
y_test5_predictions = model.predict_classes(x_test5.reshape(-1, 1, 2))
plt.figure(figsize=(14,2))
plt.scatter(range(100), y_test5[:100], s=100)
plt.scatter(range(100), y_test5_predictions[:100], s=25);
In [88]:
# RNN: Multi-Class Classification
def model():
model = Sequential()
model.add(LSTM(112, return_sequences=True, input_shape=(1,784)))
model.add(LSTM(112, return_sequences=True))
model.add(LSTM(112))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='nadam', metrics=['accuracy'])
return model
In [89]:
model = model()
checkpointer = keras.callbacks\
.ModelCheckpoint(filepath='weights.best.model.hdf5', save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = model.fit(x_train2.reshape(-1, 1, 784), c_y_train2,
validation_data=(x_valid2.reshape(-1, 1, 784), c_y_valid2),
epochs=10, batch_size=128, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test2.reshape(-1, 1, 784), c_y_test2)
test_score
Out[89]:
In [90]:
loss_plot(history)
acc_plot(history)
In [92]:
y_test2_predictions = model.predict_classes(x_test2.reshape(-1, 1, 784))
plt.figure(figsize=(14, 5))
plt.scatter(range(100), y_test2[:100], s=100)
plt.scatter(range(100), y_test2_predictions[:100], s=25);
In [94]:
# RNN: Regression
def model():
model = Sequential()
model.add(LSTM(52, return_sequences=True, input_shape=(1, 13)))
model.add(LSTM(208, return_sequences=False))
model.add(Dense(1))
model.compile(optimizer='rmsprop', loss='mse', metrics=['mae'])
return model
In [95]:
model = model()
checkpointer = keras.callbacks\
.ModelCheckpoint(filepath='weights.best.model.hdf5', verbose=0, save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=10, verbose=2, factor=0.95)
history = model.fit(x_train4.reshape(-1, 1, 13), y_train4,
validation_data=(x_valid4.reshape(-1, 1, 13), y_valid4),
epochs=400, batch_size=16, verbose=0,
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(x_test4.reshape(-1, 1, 13), y_test4)
test_score
Out[95]:
In [96]:
loss_plot(history)
mae_plot(history)
In [97]:
y_test4_predictions = model.predict(x_test4.reshape(-1, 1, 13))
plt.figure(figsize=(14,3))
plt.plot(range(len(y_test4)), y_test4, '-o', label = 'real data')
plt.plot(range(len(y_test4)), y_test4_predictions, '-o', label = 'predictions')
plt.legend();
Combined Neural Networks¶
In [98]:
# CNN & LSTM For Sequence Classification
def model():
model = Sequential()
model.add(Embedding(num_words, embedding_vector_length, input_length=max_length))
model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
model.add(MaxPooling1D(pool_size=2))
model.add(LSTM(32))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy'])
return model
In [99]:
model = model()
checkpointer = keras.callbacks\
.ModelCheckpoint(filepath='weights.best.model.hdf5', verbose=0, save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=10, verbose=2, factor=0.5)
history = model.fit(p_x_train3, y_train3, epochs=3, batch_size=128,
validation_data=(p_x_valid3, y_valid3),
callbacks=[checkpointer, lr_reduction])
model.load_weights('weights.best.model.hdf5')
test_score = model.evaluate(p_x_test3, y_test3)
test_score
Out[99]:
In [100]:
loss_plot(history)
acc_plot(history)
7. Applications¶
In [102]:
resnet50_model = ResNet50(weights='imagenet')
# resnet50_model_json = resnet50_model.to_json()
# with open("resnet50.json", "w") as json_file:
# json_file.write(resnet50_model_json)
# json_file = open('resnet50.json', 'r')
# resnet50_model_json = json_file.read()
# json_file.close()
# resnet50_model = model_from_json(resnet50_model_json)
In [103]:
# Classify imagenet_class_index.json classes with ResNet50
cat_image = keras_image.load_img('cat.png', target_size=(224, 224))
CLASS_INDEX = None
x = keras_image.img_to_array(cat_image)
x = np.expand_dims(x, axis=0)
x = resnet50_preprocess_input(x)
cat_predictions = resnet50_model.predict(x)
print('Predictions: \n', decode_predictions2(cat_predictions, 'imagenet_class_index.json')[0])
cv_cat_image = cv2.imread('cat.png')
rgb_cat_image = cv2.cvtColor(cv_cat_image, cv2.COLOR_BGR2RGB)
plt.imshow(rgb_cat_image);
In [104]:
# Extract image features with VGG16
# https://github.com/fchollet/deep-learning-models/releases/download/v0.1/
# vgg16_weights_tf_dim_ordering_tf_kernels.h5
# vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5
vgg16_model = VGG16(weights='imagenet', include_top=False)
cat_image = keras_image.load_img('cat.png', target_size=(224, 224))
y = keras_image.img_to_array(cat_image)
y = np.expand_dims(y, axis=0)
y = vgg16_preprocess_input(y)
cat_features16 = vgg16_model.predict(y)
np.argmax(cat_features16)
Out[104]:
In [106]:
# Create bottleneck features with VGG16
VGG16_model = VGG16(weights='imagenet', include_top=False)
bn_x_train1 = VGG16_model.predict(x_train1)
bn_x_valid1 = VGG16_model.predict(x_valid1)
bn_x_test1 = VGG16_model.predict(x_test1)
In [107]:
#np.save('bn_x_train1.npy', bn_x_train1.reshape(bn_x_train1.shape[0], bn_x_train1.shape[3]))
#np.save('bn_x_valid1.npy', bn_x_valid1.reshape(bn_x_valid1.shape[0], bn_x_valid1.shape[3]))
#np.save('bn_x_test1.npy', bn_x_test1.reshape(bn_x_test1.shape[0], bn_x_test1.shape[3]))
In [108]:
#bn_x_trainl = np.load('bn_x_train1.npy')
#bn_x_validl = np.load('bn_x_valid1.npy')
#bn_x_test1 = np.load('bn_x_test1.npy')
bn_x_train1 = bn_x_train1.reshape(bn_x_train1.shape[0], 1, 1, bn_x_train1.shape[3])
bn_x_valid1 = bn_x_valid1.reshape(bn_x_valid1.shape[0], 1, 1, bn_x_valid1.shape[3])
bn_x_testl = bn_x_test1.reshape(bn_x_test1.shape[0], 1, 1, bn_x_test1.shape[3])
In [113]:
def vgg16_add_model():
model = Sequential()
model.add(GlobalAveragePooling2D(input_shape=bn_x_train1.shape[1:]))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(384, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
return model
In [114]:
vgg16_add_model = vgg16_add_model()
checkpointer = keras.callbacks.\
ModelCheckpoint(filepath='weights.best.model.hdf5', verbose=2, save_best_only=True)
lr_reduction = keras.callbacks\
.ReduceLROnPlateau(monitor='val_loss', patience=10, verbose=2, factor=0.5)
In [115]:
vgg16_add_history = vgg16_add_model.fit(bn_x_train1, c_y_train1,
validation_data=(bn_x_valid1, c_y_valid1),
epochs=50, batch_size=128, verbose=0,
callbacks=[checkpointer, lr_reduction]);
In [116]:
loss_plot(vgg16_add_history)
acc_plot(vgg16_add_history)
In [117]:
vgg16_add_model.load_weights('weights.best.model.hdf5')
vgg16_add_predictions = \
[np.argmax(vgg16_add_model.predict(np.expand_dims(x, axis=0))) for x in bn_x_test1]
vgg16_add_test_accuracy = \
100*np.sum(np.array(vgg16_add_predictions)==np.argmax(c_y_test1, axis=1))/len(vgg16_add_predictions)
print('Test accuracy: %.4f%%' % vgg16_add_test_accuracy)
In [118]:
# Extract image features with VGG19
# https://github.com/fchollet/deep-learning-models/releases/download/v0.1/
# vgg19_weights_tf_dim_ordering_tf_kernels.h5
# vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5
vgg19_model = VGG19(weights='imagenet')
cat_image = keras_image.load_img('cat.png', target_size=(224, 224))
z = keras_image.img_to_array(cat_image)
z = np.expand_dims(z, axis=0)
z = vgg19_preprocess_input(z)
cat_features19 = vgg19_model.predict(z)
np.argmax(cat_features19)
Out[118]:
In [122]:
# InceptionV3 for a new dataset
# https://github.com/fchollet/deep-learning-models/releases/download/v0.1/
# inception_v3_weights_tf_dim_ordering_tf_kernels.h5
# inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5
iv3_base_model = InceptionV3(weights='imagenet', include_top=False)
x = iv3_base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(512, activation='relu')(x)
y = Dense(10, activation='softmax')(x)
iv3_model = Model(inputs=iv3_base_model.input, outputs=y)
In [123]:
# Freeze InceptionV3 convolutional layers
for layer in iv3_base_model.layers:
layer.trainable = False
iv3_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
In [124]:
# Train
steps, epochs = 189, 10
data_generator = \
keras_image.ImageDataGenerator(shear_range=0.2, zoom_range=0.2, horizontal_flip=True)
checkpointer = \
keras.callbacks.ModelCheckpoint(filepath='weights.best.iv3_model.hdf5', verbose=2, save_best_only=True)
lr_reduction = \
keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=5, verbose=2, factor=0.5)
history = iv3_model.fit_generator(data_generator.flow(x_train7, c_y_train7, batch_size=64),
steps_per_epoch = steps, epochs = epochs,
callbacks=[checkpointer, lr_reduction],
validation_data = (x_valid7, c_y_valid7))
In [126]:
loss_plot(history)
acc_plot(history)
In [127]:
for i, layer in enumerate(iv3_base_model.layers[173:]):
print(i, layer.name)
In [128]:
# Unfreeze InceptionV3 convolutional layers
for layer in iv3_model.layers[:173]:
layer.trainable = False
for layer in iv3_model.layers[173:]:
layer.trainable = True
iv3_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
In [129]:
# Train
history = iv3_model.fit_generator(data_generator.flow(x_train7, c_y_train7, batch_size=64),
steps_per_epoch = steps, epochs = epochs,
callbacks=[checkpointer, lr_reduction],
validation_data = (x_valid7, c_y_valid7))
In [130]:
# Evaluate
iv3_model.load_weights('weights.best.iv3_model.hdf5')
iv3_test_scores = iv3_model.evaluate(x_test7, c_y_test7)
print("Accuracy: %.2f%%" % (iv3_test_scores[1]*100))
In [40]:
# Xception
# https://github.com/fchollet/deep-learning-models/releases/download/v0.1/
# xception_weights_tf_dim_ordering_tf_kernels.h5
# xception_weights_tf_dim_ordering_tf_kernels_notop.h5
xce_base_model = Xception(include_top=True, weights='imagenet')
xce_base_model.summary()