%%html
<style>
@import url('https://fonts.googleapis.com/css?family=Orbitron|Roboto');
body {background-color: aliceblue;}
a {color: #4876ff; font-family: 'Roboto';}
h1 {color: #348ABD; font-family: 'Orbitron'; text-shadow: 4px 4px 4px #ccc;}
h2, h3 {color: slategray; font-family: 'Roboto'; text-shadow: 4px 4px 4px #ccc;}
h4 {color: #348ABD; font-family: 'Orbitron';}
span {text-shadow: 4px 4px 4px #ccc;}
div.output_prompt, div.output_area pre {color: slategray;}
div.input_prompt, div.output_subarea {color: #4876ff;}
div.output_stderr pre {background-color: aliceblue;}
div.output_stderr {background-color: slategrey;}
</style>
<script>
code_show = true;
function code_display() {
if (code_show) {
$('div.input').each(function(id) {
if (id == 0 || $(this).html().indexOf('hide_code') > -1) {$(this).hide();}
});
$('div.output_prompt').css('opacity', 0);
} else {
$('div.input').each(function(id) {$(this).show();});
$('div.output_prompt').css('opacity', 1);
};
code_show = !code_show;
}
$(document).ready(code_display);
</script>
<form action="javascript: code_display()">
<input style="color: #348ABD; background: aliceblue; opacity: 0.8;" \
type="submit" value="Click to display or hide code cells">
</form>
hide_code = ''
import numpy as np
import pandas as pd
import tensorflow as tf
from IPython.core.magic import (register_line_magic, register_cell_magic)
from PIL import ImageFile
from tqdm import tqdm
import h5py
import cv2
import matplotlib.pylab as plt
from matplotlib import cm
%matplotlib inline
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier
from keras.utils import to_categorical
from keras.preprocessing import image as keras_image
from keras.callbacks import ModelCheckpoint, TensorBoard
from keras.preprocessing.image import ImageDataGenerator
from keras.preprocessing.image import array_to_img, img_to_array, load_img
from keras import backend
from keras import losses
from keras.engine.topology import Layer
from keras.optimizers import Adam, Nadam
from keras.engine import InputLayer
from keras.models import Sequential, load_model, Model
from keras.layers import Input, BatchNormalization, Flatten, Dropout
from keras.layers import Dense, LSTM, Activation, LeakyReLU
from keras.layers import Conv2D, MaxPool2D, MaxPooling2D, GlobalMaxPooling2D
from keras.layers import UpSampling2D, Conv2DTranspose
from keras.layers.core import RepeatVector, Permute
from keras.layers import Reshape, concatenate, merge
# from keras.applications.inception_v3 import InceptionV3, preprocess_input
from keras.applications.inception_resnet_v2 import InceptionResNetV2, preprocess_input
from skimage.color import rgb2lab, lab2rgb, rgb2gray, gray2rgb
from skimage import color, measure
from skimage.transform import resize as skimage_resize
from skimage.io import imsave
hide_code
from keras import __version__
print('keras version:', __version__)
print('tensorflow version:', tf.__version__)
hide_code
# Plot the neural network fitting history
def history_plot(fit_history):
plt.figure(figsize=(18, 12))
plt.subplot(211)
plt.plot(fit_history.history['loss'], color='slategray', label = 'train')
plt.plot(fit_history.history['val_loss'], color='#4876ff', label = 'valid')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.legend()
plt.title('Loss Function');
plt.subplot(212)
plt.plot(fit_history.history['acc'], color='slategray', label = 'train')
plt.plot(fit_history.history['val_acc'], color='#4876ff', label = 'valid')
plt.xlabel("Epochs")
plt.ylabel("Accuracy")
plt.legend()
plt.title('Accuracy');
For this project, I have created the dataset of color images (150x150x3) with traditional patterns. Run the following cells to download the data.
hide_code
# Function for processing an image
def image_to_tensor(img_path, folder_path):
img = keras_image.load_img(folder_path + img_path, target_size=(150, 150))
x = keras_image.img_to_array(img)
return np.expand_dims(x, axis=0)
# Function for creating the data tensor
def data_to_tensor(img_paths, folder_path):
list_of_tensors = [image_to_tensor(img_path, folder_path) for img_path in tqdm(img_paths)]
return np.vstack(list_of_tensors)
ImageFile.LOAD_TRUNCATED_IMAGES = True
hide_code
# Load the dataset
data = pd.read_csv("decor.txt")
files = data['file']
countries = data['country_label'].as_matrix()
decors = data['decor_label'].as_matrix()
types = data['type_label'].as_matrix()
images = data_to_tensor(files, "data/");
hide_code
# Print the shape
print ('Image shape:', images.shape)
print ('Country shape:', countries.shape)
print ('Decor shape:', decors.shape)
print ('Type shape:', types.shape)
hide_code
# Read from files and display images using OpenCV
def display_images(img_path, ax):
img = cv2.imread("data/" + img_path)
ax.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
fig = plt.figure(figsize=(18, 6))
for i in range(10):
ax = fig.add_subplot(2, 5, i + 1, xticks=[], yticks=[],
title=data['country'][i*48]+'; '+data['decor'][i*48]+'; '+data['type'][i*48])
display_images(files[i*48], ax)
hide_code
# Normalize the tensors
images = images.astype('float32')/255
hide_code
# Read and display a tensor using Matplotlib
pattern_number = 106
print('Country: ', countries[pattern_number], '-', data['country'][pattern_number])
print('Decor: ', decors[pattern_number], '-', data['decor'][pattern_number])
print('Type: ', types[pattern_number], '-', data['type'][pattern_number])
plt.figure(figsize=(5,5))
plt.imshow(images[pattern_number]);
hide_code
# Grayscaled tensors
gray_images = np.dot(images[...,:3], [0.299, 0.587, 0.114])
print ("Shape of grayscaled images:", gray_images.shape)
hide_code
# Read and display a grayscaled tensor using Matplotlib
print('Country: ', countries[pattern_number], '-', data['country'][pattern_number])
print('Decor: ', decors[pattern_number], '-', data['decor'][pattern_number])
print('Type: ', types[pattern_number], '-', data['type'][pattern_number])
plt.figure(figsize=(5,5))
plt.imshow(gray_images[pattern_number], cmap=cm.bone);
hide_code
# Vectorize an image example / Just for fun
@register_line_magic
def vector(number):
example = images[int(number)]
gray_example = color.colorconv.rgb2grey(example)
contours = measure.find_contours(gray_example, 0.85)
plt.figure(figsize=(8,8))
plt.gca().invert_yaxis()
for n, contour in enumerate(contours):
plt.plot(contour[:, 1], contour[:, 0], lw=1)
hide_code
# Display a vector image using the magic command
%vector 106
hide_code
# Split the data / Color images
x_train, x_test = train_test_split(images,
test_size = 0.4,
random_state = 1)
n = int(len(x_test)/2)
x_valid = x_test[:n]
x_test = x_test[n:]
hide_code
# Split the data / Grayscaled images
x_train2, x_test2 = train_test_split(gray_images,
test_size = 0.4,
random_state = 1)
n = int(len(x_test2)/2)
x_valid2 = x_test2[:n]
x_test2 = x_test2[n:]
hide_code
# Resize the train and valid set
resized_x_train = []
for i in x_train:
i = skimage_resize(i, (256, 256, 3), mode='constant')
resized_x_train.append(i)
resized_x_train = np.array(resized_x_train)
resized_x_valid = []
for i in x_valid:
i = skimage_resize(i, (256, 256, 3), mode='constant')
resized_x_valid.append(i)
resized_x_valid = np.array(resized_x_valid)
resized_x_test = []
for i in x_test:
i = skimage_resize(i, (256, 256, 3), mode='constant')
resized_x_test.append(i)
resized_x_test = np.array(resized_x_test)
resized_x_train.shape, resized_x_valid.shape, resized_x_test.shape
hide_code
# Grayscale the train set
grayscaled_x_train = gray2rgb(rgb2gray(resized_x_train))
grayscaled_x_valid = gray2rgb(rgb2gray(resized_x_valid))
grayscaled_x_test = gray2rgb(rgb2gray(resized_x_test))
hide_code
# Save images
train_example = resized_x_train[2]
test_example = resized_x_test[2]
grayscaled_train_example = grayscaled_x_train[2]
grayscaled_test_example = grayscaled_x_test[2]
hide_code
# Display the grayscaled example
plt.figure(figsize=(12, 4))
plt.subplot(121)
plt.title('Color Image. Shape:'+str(train_example.shape))
plt.imshow(train_example)
plt.subplot(122)
plt.title('Gray Image. Shape:'+str(grayscaled_train_example[:, :, 0].shape))
plt.imshow(grayscaled_train_example[:, :, 0]*100, cmap=cm.bone);
hide_code
# Convert rgb2lab
lab_x_train = rgb2lab(resized_x_train)
hide_code
# Convert rgb2lab
lab_x_valid = rgb2lab(resized_x_valid)
hide_code
# Convert rgb2lab
lab_x_test = rgb2lab(resized_x_test)
hide_code
# Save images
lab_test_example = lab_x_test[2]
lab_train_example = lab_x_train[2]
hide_code
# Display the lab preprocessing example
plt.figure(figsize=(18, 4))
plt.subplot(131)
plt.title('Lab Image. Channel #1. Shape:'+str(lab_train_example[:,:,0].shape))
plt.imshow(lab_train_example[:,:,0], cmap=cm.bone)
plt.subplot(132)
plt.title('Lab Image. Channel #2. Shape:'+str(lab_train_example[:,:,1].shape))
plt.imshow(lab_train_example[:,:,1], cmap=cm.RdYlGn)
plt.subplot(133)
plt.title('Lab Image. Channel #3. Shape:'+str(lab_train_example[:,:,2].shape))
plt.imshow(lab_train_example[:,:,2], cmap=cm.YlGnBu);
hide_code
# Delete variables
del x_train, x_valid, x_test
del resized_x_train, resized_x_valid, resized_x_test
hide_code
# Create inputs and outputs / Model #2
input_x_train = lab_x_train[:,:,:,0]/100
input_x_train = input_x_train.reshape(input_x_train.shape+(1,))
output_y_train = lab_x_train[:,:,:,1:]/127.5
hide_code
# Create inputs and outputs / Model #2
input_x_valid = lab_x_valid[:,:,:,0]/100
input_x_valid = input_x_valid.reshape(input_x_valid.shape+(1,))
output_y_valid = lab_x_valid[:,:,:,1:]/127.5
hide_code
# Create inputs and outputs / Model #2
input_x_test = lab_x_test[:,:,:,0]/100
input_x_test = input_x_test.reshape(input_x_test.shape+(1,))
output_y_test = lab_x_test[:,:,:,1:]/127.5
hide_code
# Display difference distribution
plt.figure(figsize=(18, 4))
plt.hist(grayscaled_train_example[:,:,0] - np.squeeze(input_x_train[2]));
hide_code
# Display a combined image
rgb_train_example = np.zeros((256, 256, 3))
rgb_train_example[:,:,0] = grayscaled_train_example[:, :, 0]*100
rgb_train_example[:,:,1:] = output_y_train[2]*127.5
rgb_train_example = lab2rgb(rgb_train_example)
plt.title('Combined RGB Image. Shape:'+str(rgb_train_example.shape))
plt.imshow(rgb_train_example);
hide_code
# Delete variables
del lab_x_train, lab_x_valid, lab_x_test
hide_code
def eacc(y_true, y_pred):
return backend.mean(backend.equal(backend.round(y_true), backend.round(y_pred)))
def l1(y_true, y_pred):
return backend.mean(backend.abs(y_pred - y_true))
def create_conv(filters, kernel_size, inputs, name=None, bn=True,
dropout=0., padding='same', activation='relu'):
conv = Conv2D(filters, kernel_size, padding=padding,
kernel_initializer='he_normal', name=name)(inputs)
if bn:
conv = BatchNormalization()(conv)
if activation == 'relu':
conv = Activation(activation)(conv)
elif activation == 'leakyrelu':
conv = LeakyReLU()(conv)
if dropout != 0:
conv = Dropout(dropout)(conv)
return conv
hide_code
def dcgan_generator(input_shape, output_channels):
inputs = Input(input_shape)
conv1 = create_conv(64, (3, 3), inputs, 'conv1_1', activation='leakyrelu')
conv1 = create_conv(64, (3, 3), conv1, 'conv1_2', activation='leakyrelu')
pool1 = MaxPool2D((2, 2))(conv1)
conv2 = create_conv(128, (3, 3), pool1, 'conv2_1', activation='leakyrelu')
conv2 = create_conv(128, (3, 3), conv2, 'conv2_2', activation='leakyrelu')
pool2 = MaxPool2D((2, 2))(conv2)
conv3 = create_conv(256, (3, 3), pool2, 'conv3_1', activation='leakyrelu')
conv3 = create_conv(256, (3, 3), conv3, 'conv3_2', activation='leakyrelu')
pool3 = MaxPool2D((2, 2))(conv3)
conv4 = create_conv(512, (3, 3), pool3, 'conv4_1', activation='leakyrelu')
conv4 = create_conv(512, (3, 3), conv4, 'conv4_2', activation='leakyrelu')
pool4 = MaxPool2D((2, 2))(conv4)
conv5 = create_conv(1024, (3, 3), pool4, 'conv5_1', activation='leakyrelu')
conv5 = create_conv(1024, (3, 3), conv5, 'conv5_2', activation='leakyrelu')
up6 = create_conv(512, (2, 2), UpSampling2D((2, 2))(conv5), 'up6')
merge6 = concatenate([conv4, up6], axis=3)
conv6 = create_conv(512, (3, 3), merge6, 'conv6_1', activation='relu')
conv6 = create_conv(512, (3, 3), conv6, 'conv6_2', activation='relu')
up7 = create_conv(256, (2, 2), UpSampling2D((2, 2))(conv6), 'up7')
merge7 = concatenate([conv3, up7], axis=3)
conv7 = create_conv(256, (3, 3), merge7, 'conv7_1', activation='relu')
conv7 = create_conv(256, (3, 3), conv7, 'conv7_2', activation='relu')
up8 = create_conv(128, (2, 2), UpSampling2D((2, 2))(conv7), 'up8')
merge8 = concatenate([conv2, up8], axis=3)
conv8 = create_conv(128, (3, 3), merge8, 'conv8_1', activation='relu')
conv8 = create_conv(128, (3, 3), conv8, 'conv8_2', activation='relu')
up9 = create_conv(64, (2, 2), UpSampling2D((2, 2))(conv8))
merge9 = concatenate([conv1, up9], axis=3)
conv9 = create_conv(64, (3, 3), merge9, 'conv9_1', activation='relu')
conv9 = create_conv(64, (3, 3), conv9, 'conv9_2', activation='relu')
conv9 = Conv2D(output_channels, (1, 1), padding='same', name='conv9_3')(conv9)
model = Model(inputs=inputs, outputs=conv9, name='generator')
return model
hide_code
dcgan_generator = dcgan_generator((256, 256, 1), 3)
dcgan_generator.summary()
hide_code
def dcgan_discriminator(input_shape):
inputs = Input(input_shape)
conv1 = create_conv(64, (3, 3), inputs, 'conv1', activation='leakyrelu', dropout=.8)
pool1 = MaxPool2D((2, 2))(conv1)
conv2 = create_conv(128, (3, 3), pool1, 'conv2', activation='leakyrelu', dropout=.8)
pool2 = MaxPool2D((2, 2))(conv2)
conv3 = create_conv(256, (3, 3), pool2, 'conv3', activation='leakyrelu', dropout=.8)
pool3 = MaxPool2D((2, 2))(conv3)
conv4 = create_conv(512, (3, 3), pool3, 'conv4', activation='leakyrelu', dropout=.8)
pool4 = MaxPool2D((2, 2))(conv4)
conv5 = create_conv(512, (3, 3), pool4, 'conv5', activation='leakyrelu', dropout=.8)
flat = Flatten()(conv5)
dense6 = Dense(1, activation='sigmoid')(flat)
model = Model(inputs=inputs, outputs=dense6, name='discriminator')
return model
hide_code
dcgan_discriminator = dcgan_discriminator((256, 256, 3))
dcgan_discriminator.summary()
hide_code
def dcgan(input_shape, generator, discriminator):
dcgan_inputs = Input(input_shape)
generator_outputs = generator(inputs)
discriminator_outputs = discriminator(concatenate([generator_outputs, inputs], axis=3))
model = Model(inputs=[dcgan_inputs],
outputs=[discriminator_outputs, generator_outputs],
name='dcgan')
return model
hide_code
def gan_model(input_shape_generator, input_shape_discriminator, output_channels,
lr, momentum, loss_weights):
optimizer = Adam(lr=lr, beta_1=momentum)
model_generator = dcgan_generator(input_shape=input_shape_generator, output_channels=output_channels)
model_generator.compile(loss=losses.mean_absolute_error, optimizer=optimizer)
model_discriminator = dcgan_discriminator(input_shape=input_shape_discriminator)
model_discriminator.trainable = False
model_gan = dcgan(input_shape=input_shape_generator,
generator=model_generator, discriminator=model_discriminator)
model_gan.compile(loss=[losses.binary_crossentropy, l1], metrics=[eacc, 'accuracy'],
loss_weights=loss_weights, optimizer=optimizer)
model_discriminator.trainable = True
model_discriminator.compile(loss=losses.binary_crossentropy, optimizer=optimizer)
return model_generator, model_discriminator, model_gan
hide_code
# Application Model
INCRNV2_model = InceptionResNetV2(weights=None, include_top=True)
INCRNV2_model.load_weights('inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5')
# INCRNV2_model.graph = tf.get_default_graph()
hide_code
# The Model 'Encoder-Fusion-Decoder'
model_input = Input(shape=(1000,))
# Encoding
encoder_input = Input(shape=(256, 256, 1,))
x = Conv2D(64, (3, 3), strides=2,
activation='relu',
padding='same')(encoder_input)
x = Conv2D(128, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(128, (3, 3), strides=2, activation='relu', padding='same')(x)
x = Conv2D(256, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(256, (3, 3), strides=2, activation='relu', padding='same')(x)
x = Conv2D(512, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(512, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(256, (3, 3), strides=1, activation='relu', padding='same')(x)
# Fusion
y = RepeatVector(32 * 32)(model_input)
y = Reshape(([32, 32, 1000]))(y)
y = concatenate([y, x], axis=3)
y = Conv2D(256, (1, 1), activation='relu')(y)
# Decoding
z = Conv2D(128, (3, 3), strides=1, activation='relu', padding='same')(y)
z = UpSampling2D((2, 2))(z)
z = Conv2D(64, (3, 3), strides=1, activation='relu', padding='same')(z)
z = UpSampling2D((2, 2))(z)
z = Conv2D(32, (3, 3), strides=1, activation='relu', padding='same')(z)
z = Conv2D(16, (3, 3), strides=1, activation='relu', padding='same')(z)
z = Conv2D(2, (2, 2), activation='tanh', padding='same')(z)
decoder_output = UpSampling2D((2, 2))(z)
model = Model(inputs=[encoder_input, model_input], outputs=decoder_output)
hide_code
# Display the
model.summary()
hide_code
# Preprocess for InceptionResNetV2
preprocessed_x_train = preprocess_input(grayscaled_x_train)
hide_code
# Preprocess for InceptionResNetV2
preprocessed_x_valid = preprocess_input(grayscaled_x_valid)
hide_code
# Preprocess for InceptionResNetV2
preprocessed_x_test = preprocess_input(grayscaled_x_test)
hide_code
# Predict with InceptionResNetV2
embedded_x_train = INCRNV2_model.predict(preprocessed_x_train)*255
hide_code
# Predict with InceptionResNetV2
embedded_x_valid = INCRNV2_model.predict(preprocessed_x_valid)*255
hide_code
# Predict with InceptionResNetV2
embedded_x_test = INCRNV2_model.predict(preprocessed_x_test)*255
hide_code
# Delete variables
del preprocessed_x_train, preprocessed_x_valid, preprocessed_x_test
hide_code
# Train the model
model.compile(optimizer='nadam', loss='mse')
checkpointer = ModelCheckpoint(filepath='weights.best.decor.model.hdf5',
verbose=2, save_best_only=True)
history = model.fit([input_x_train, embedded_x_train], output_y_train,
epochs=10, batch_size=32, verbose=2,
validation_data=([input_x_valid, embedded_x_valid], output_y_valid),
callbacks=[checkpointer])
hide_code
# Predict
predict_y_test = model.predict([input_x_test, embedded_x_test])
predict_y_test[0][0][0]
output_y_test[0][0][0]
hide_code
# Embedding function
def INCRNV2_embedding(batch):
grayscaled_batch = gray2rgb(rgb2gray(batch))
preprocessed = preprocess_input(grayscaled_batch)
with INCRNV2_model.graph.as_default():
embedded = INCRNV2_model.predict(preprocessed)
return embedded
# Image generator
image_generator = ImageDataGenerator(shear_range=0.3, zoom_range=0.3, rotation_range=30, horizontal_flip=True)
hide_code
# Generate training data
batch_size = 4
def train_generator(batch_size):
for batch in image_generator.flow(resized_x_train, batch_size=batch_size):
x_batch_embedded = INCRNV2_embedding(batch)
lab_batch = rgb2lab(batch)
x_batch = lab_batch[:,:,:,0]
x_batch = x_batch.reshape(x_batch.shape+(1,))
y_batch = lab_batch[:,:,:,1:]/128
yield ([x_batch, x_batch_embedded], y_batch)
hide_code
# Train the model
filepath="weights-improvement-{epoch:02d}-{loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1,
save_weights_only=True, period=20)
callbacks_list = [checkpoint]
model.compile(optimizer='adam', loss='mse')
model.fit_generator(train_generator(batch_size),
epochs=10, steps_per_epoch=1,
callbacks=callbacks_list, verbose=1);
hide_code
# Save the model
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
model.save_weights("weights.model.h5")
hide_code
# Display the grayscaled example
plt.figure(figsize=(12, 4))
plt.subplot(121)
plt.title('Color Image. Shape:'+str(test_example.shape))
plt.imshow(test_example)
plt.subplot(122)
plt.title('Gray Image. Shape:'+str(grayscaled_test_example.shape))
plt.imshow(grayscaled_test_example, cmap=cm.bone);
hide_code
# Display the lab preprocessing example
plt.figure(figsize=(18, 4))
plt.subplot(131)
plt.title('Lab Image. Channel #1. Shape:'+str(lab_test_example[:,:,0].shape))
plt.imshow(lab_test_example[:,:,0], cmap=cm.bone)
plt.subplot(132)
plt.title('Lab Image. Channel #2. Shape:'+str(lab_test_example[:,:,1].shape))
plt.imshow(lab_test_example[:,:,1], cmap=cm.RdYlGn)
plt.subplot(133)
plt.title('Lab Image. Channel #3. Shape:'+str(lab_test_example[:,:,2].shape))
plt.imshow(lab_test_example[:,:,2], cmap=cm.YlGnBu);
hide_code
# Display a combined image
rgb_test_example = np.zeros((256, 256, 3))
rgb_test_example[:,:,0] = grayscaled_test_example[:, :, 0]*100
rgb_test_example[:,:,1:] = output_y_test[2]
rgb_test_example = lab2rgb(rgb_test_example)
plt.title('Combined RGB Image. Shape:'+str(rgb_test_example.shape))
plt.imshow(rgb_test_example);