Deep Learning

Practice Projects

P5: Decor Colorization. Part 2

Step 0. Style and Libraries

Let's choose a style of the Jupyter notebook and import the software libraries. The command hide_code will hide the code cells.

In [1]:
%%html
<style>
@import url('https://fonts.googleapis.com/css?family=Orbitron|Roboto');
body {background-color: aliceblue;} 
a {color: #4876ff; font-family: 'Roboto';} 
h1 {color: #348ABD; font-family: 'Orbitron'; text-shadow: 4px 4px 4px #ccc;} 
h2, h3 {color: slategray; font-family: 'Roboto'; text-shadow: 4px 4px 4px #ccc;}
h4 {color: #348ABD; font-family: 'Orbitron';}
span {text-shadow: 4px 4px 4px #ccc;}
div.output_prompt, div.output_area pre {color: slategray;}
div.input_prompt, div.output_subarea {color: #4876ff;}      
div.output_stderr pre {background-color: aliceblue;}  
div.output_stderr {background-color: slategrey;}                        
</style>
<script>
code_show = true; 
function code_display() {
    if (code_show) {
        $('div.input').each(function(id) {
            if (id == 0 || $(this).html().indexOf('hide_code') > -1) {$(this).hide();}
        });
        $('div.output_prompt').css('opacity', 0);
    } else {
        $('div.input').each(function(id) {$(this).show();});
        $('div.output_prompt').css('opacity', 1);
    };
    code_show = !code_show;
} 
$(document).ready(code_display);
</script>
<form action="javascript: code_display()">
<input style="color: #348ABD; background: aliceblue; opacity: 0.8;" \ 
type="submit" value="Click to display or hide code cells">
</form>                  
In [2]:
hide_code = ''
import numpy as np 
import pandas as pd
import tensorflow as tf

from IPython.core.magic import (register_line_magic, register_cell_magic)
from PIL import ImageFile
from tqdm import tqdm
import h5py
import cv2

import matplotlib.pylab as plt
from matplotlib import cm
%matplotlib inline

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler
from sklearn.ensemble import GradientBoostingClassifier, RandomForestClassifier

from keras.utils import to_categorical
from keras.preprocessing import image as keras_image
from keras.callbacks import ModelCheckpoint, TensorBoard 
from keras.preprocessing.image import ImageDataGenerator
from keras.preprocessing.image import array_to_img, img_to_array, load_img

from keras import backend
from keras import losses
from keras.engine.topology import Layer
from keras.optimizers import Adam, Nadam
from keras.engine import InputLayer
from keras.models import Sequential, load_model, Model

from keras.layers import Input, BatchNormalization, Flatten, Dropout
from keras.layers import Dense, LSTM, Activation, LeakyReLU
from keras.layers import Conv2D, MaxPool2D, MaxPooling2D, GlobalMaxPooling2D
from keras.layers import UpSampling2D, Conv2DTranspose
from keras.layers.core import RepeatVector, Permute
from keras.layers import Reshape, concatenate, merge

# from keras.applications.inception_v3 import InceptionV3, preprocess_input
from keras.applications.inception_resnet_v2 import InceptionResNetV2, preprocess_input

from skimage.color import rgb2lab, lab2rgb, rgb2gray, gray2rgb
from skimage import color, measure
from skimage.transform import resize as skimage_resize
from skimage.io import imsave
Using TensorFlow backend.
In [3]:
hide_code
from keras import __version__
print('keras version:', __version__)
print('tensorflow version:', tf.__version__)
keras version: 2.1.3
tensorflow version: 1.0.0
In [4]:
hide_code
# Plot the neural network fitting history
def history_plot(fit_history):
    plt.figure(figsize=(18, 12))
    
    plt.subplot(211)
    plt.plot(fit_history.history['loss'], color='slategray', label = 'train')
    plt.plot(fit_history.history['val_loss'], color='#4876ff', label = 'valid')
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.legend()
    plt.title('Loss Function');  
    
    plt.subplot(212)
    plt.plot(fit_history.history['acc'], color='slategray', label = 'train')
    plt.plot(fit_history.history['val_acc'], color='#4876ff', label = 'valid')
    plt.xlabel("Epochs")
    plt.ylabel("Accuracy")    
    plt.legend()
    plt.title('Accuracy');

Step 1. Load and Explore the Data

For this project, I have created the dataset of color images (150x150x3) with traditional patterns. Run the following cells to download the data.

In [5]:
hide_code
# Function for processing an image
def image_to_tensor(img_path, folder_path):
    img = keras_image.load_img(folder_path + img_path, target_size=(150, 150))
    x = keras_image.img_to_array(img)
    return np.expand_dims(x, axis=0)
# Function for creating the data tensor
def data_to_tensor(img_paths, folder_path):
    list_of_tensors = [image_to_tensor(img_path, folder_path) for img_path in tqdm(img_paths)]
    return np.vstack(list_of_tensors)
ImageFile.LOAD_TRUNCATED_IMAGES = True 
In [6]:
hide_code
# Load the dataset 
data = pd.read_csv("decor.txt")
files = data['file']

countries = data['country_label'].as_matrix()
decors = data['decor_label'].as_matrix()
types = data['type_label'].as_matrix()

images = data_to_tensor(files, "data/");
100%|██████████| 485/485 [00:11<00:00, 39.37it/s]
In [7]:
hide_code
# Print the shape 
print ('Image shape:', images.shape)
print ('Country shape:', countries.shape)
print ('Decor shape:', decors.shape)
print ('Type shape:', types.shape)
Image shape: (485, 150, 150, 3)
Country shape: (485,)
Decor shape: (485,)
Type shape: (485,)
In [8]:
hide_code
# Read from files and display images using OpenCV
def display_images(img_path, ax):
    img = cv2.imread("data/" + img_path)
    ax.imshow(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
    
fig = plt.figure(figsize=(18, 6))
for i in range(10):
    ax = fig.add_subplot(2, 5, i + 1, xticks=[], yticks=[], 
                         title=data['country'][i*48]+'; '+data['decor'][i*48]+'; '+data['type'][i*48])
    display_images(files[i*48], ax)

Step 2. Implement Preprocess Functions

In [9]:
hide_code
# Normalize the tensors
images = images.astype('float32')/255
In [10]:
hide_code
# Read and display a tensor using Matplotlib
pattern_number = 106
print('Country: ', countries[pattern_number], '-', data['country'][pattern_number])
print('Decor: ', decors[pattern_number], '-', data['decor'][pattern_number])
print('Type: ', types[pattern_number], '-', data['type'][pattern_number])
plt.figure(figsize=(5,5))
plt.imshow(images[pattern_number]);
Country:  1 - Russia
Decor:  2 - Khokhloma
Type:  1 - pattern
In [11]:
hide_code
# Grayscaled tensors
gray_images = np.dot(images[...,:3], [0.299, 0.587, 0.114])
print ("Shape of grayscaled images:", gray_images.shape)
Shape of grayscaled images: (485, 150, 150)
In [12]:
hide_code
# Read and display a grayscaled tensor using Matplotlib
print('Country: ', countries[pattern_number], '-', data['country'][pattern_number])
print('Decor: ', decors[pattern_number], '-', data['decor'][pattern_number])
print('Type: ', types[pattern_number], '-', data['type'][pattern_number])
plt.figure(figsize=(5,5))
plt.imshow(gray_images[pattern_number], cmap=cm.bone);
Country:  1 - Russia
Decor:  2 - Khokhloma
Type:  1 - pattern
In [13]:
hide_code
# Vectorize an image example / Just for fun
@register_line_magic
def vector(number):
    example = images[int(number)]
    gray_example = color.colorconv.rgb2grey(example)
    contours = measure.find_contours(gray_example, 0.85)
    plt.figure(figsize=(8,8))
    plt.gca().invert_yaxis()
    for n, contour in enumerate(contours):
        plt.plot(contour[:, 1], contour[:, 0], lw=1)
In [14]:
hide_code
# Display a vector image using the magic command
%vector 106
In [15]:
hide_code
# Split the data / Color images
x_train, x_test = train_test_split(images, 
                                   test_size = 0.4, 
                                   random_state = 1)
n = int(len(x_test)/2)
x_valid = x_test[:n]
x_test = x_test[n:]
In [16]:
hide_code
# Split the data / Grayscaled images 
x_train2, x_test2 = train_test_split(gray_images, 
                                     test_size = 0.4, 
                                     random_state = 1)

n = int(len(x_test2)/2)
x_valid2 = x_test2[:n]
x_test2 = x_test2[n:]
In [17]:
hide_code
# Resize the train and valid set
resized_x_train = []
for i in x_train:
    i = skimage_resize(i, (256, 256, 3), mode='constant')
    resized_x_train.append(i)
resized_x_train = np.array(resized_x_train)

resized_x_valid = []
for i in x_valid:
    i = skimage_resize(i, (256, 256, 3), mode='constant')
    resized_x_valid.append(i)
resized_x_valid = np.array(resized_x_valid)

resized_x_test = []
for i in x_test:
    i = skimage_resize(i, (256, 256, 3), mode='constant')
    resized_x_test.append(i)
resized_x_test = np.array(resized_x_test)

resized_x_train.shape, resized_x_valid.shape, resized_x_test.shape
Out[17]:
((291, 256, 256, 3), (97, 256, 256, 3), (97, 256, 256, 3))
In [18]:
hide_code
# Grayscale the train set
grayscaled_x_train = gray2rgb(rgb2gray(resized_x_train))
grayscaled_x_valid = gray2rgb(rgb2gray(resized_x_valid))
grayscaled_x_test = gray2rgb(rgb2gray(resized_x_test))
In [19]:
hide_code
# Save images
train_example = resized_x_train[2]
test_example = resized_x_test[2]
grayscaled_train_example = grayscaled_x_train[2]
grayscaled_test_example = grayscaled_x_test[2]
In [20]:
hide_code
# Display the grayscaled example
plt.figure(figsize=(12, 4))    
plt.subplot(121)
plt.title('Color Image. Shape:'+str(train_example.shape))
plt.imshow(train_example)
plt.subplot(122)
plt.title('Gray Image. Shape:'+str(grayscaled_train_example[:, :, 0].shape))
plt.imshow(grayscaled_train_example[:, :, 0]*100, cmap=cm.bone);
In [21]:
hide_code
# Convert rgb2lab
lab_x_train = rgb2lab(resized_x_train)
In [22]:
hide_code
# Convert rgb2lab
lab_x_valid = rgb2lab(resized_x_valid)
In [23]:
hide_code
# Convert rgb2lab
lab_x_test = rgb2lab(resized_x_test)
In [24]:
hide_code
# Save images
lab_test_example = lab_x_test[2]
lab_train_example = lab_x_train[2]
In [25]:
hide_code
# Display the lab preprocessing example

plt.figure(figsize=(18, 4))    
plt.subplot(131)
plt.title('Lab Image. Channel #1. Shape:'+str(lab_train_example[:,:,0].shape))
plt.imshow(lab_train_example[:,:,0], cmap=cm.bone)
plt.subplot(132)
plt.title('Lab Image. Channel #2. Shape:'+str(lab_train_example[:,:,1].shape))
plt.imshow(lab_train_example[:,:,1], cmap=cm.RdYlGn)
plt.subplot(133)
plt.title('Lab Image. Channel #3. Shape:'+str(lab_train_example[:,:,2].shape))
plt.imshow(lab_train_example[:,:,2], cmap=cm.YlGnBu);
In [25]:
hide_code
# Delete variables
del x_train, x_valid, x_test
del resized_x_train, resized_x_valid, resized_x_test 
In [26]:
hide_code
# Create inputs and outputs / Model #2
input_x_train = lab_x_train[:,:,:,0]/100
input_x_train = input_x_train.reshape(input_x_train.shape+(1,))
output_y_train = lab_x_train[:,:,:,1:]/127.5
In [27]:
hide_code
# Create inputs and outputs / Model #2
input_x_valid = lab_x_valid[:,:,:,0]/100
input_x_valid = input_x_valid.reshape(input_x_valid.shape+(1,))
output_y_valid = lab_x_valid[:,:,:,1:]/127.5
In [28]:
hide_code
# Create inputs and outputs / Model #2
input_x_test = lab_x_test[:,:,:,0]/100
input_x_test = input_x_test.reshape(input_x_test.shape+(1,))
output_y_test = lab_x_test[:,:,:,1:]/127.5
In [30]:
hide_code
# Display difference distribution
plt.figure(figsize=(18, 4)) 
plt.hist(grayscaled_train_example[:,:,0] - np.squeeze(input_x_train[2]));
In [31]:
hide_code
# Display a combined image
rgb_train_example = np.zeros((256, 256, 3))
rgb_train_example[:,:,0] = grayscaled_train_example[:, :, 0]*100
rgb_train_example[:,:,1:] = output_y_train[2]*127.5
rgb_train_example = lab2rgb(rgb_train_example)
plt.title('Combined RGB Image. Shape:'+str(rgb_train_example.shape))
plt.imshow(rgb_train_example);
In [32]:
hide_code
# Delete variables
del lab_x_train, lab_x_valid, lab_x_test 

Step 3. DCGAN Colorization

Model #1

Based on the model

!!! In progress !!!

In [86]:
hide_code
def eacc(y_true, y_pred):
    return backend.mean(backend.equal(backend.round(y_true), backend.round(y_pred)))

def l1(y_true, y_pred):
    return backend.mean(backend.abs(y_pred - y_true))

def create_conv(filters, kernel_size, inputs, name=None, bn=True, 
                dropout=0., padding='same', activation='relu'):
    conv = Conv2D(filters, kernel_size, padding=padding,
                  kernel_initializer='he_normal', name=name)(inputs)

    if bn:
        conv = BatchNormalization()(conv)

    if activation == 'relu':
        conv = Activation(activation)(conv)
    elif activation == 'leakyrelu':
        conv = LeakyReLU()(conv)

    if dropout != 0:
        conv = Dropout(dropout)(conv)

    return conv
In [87]:
hide_code
def dcgan_generator(input_shape, output_channels):
    inputs = Input(input_shape)
    conv1 = create_conv(64, (3, 3), inputs, 'conv1_1', activation='leakyrelu')
    conv1 = create_conv(64, (3, 3), conv1, 'conv1_2', activation='leakyrelu')
    pool1 = MaxPool2D((2, 2))(conv1)

    conv2 = create_conv(128, (3, 3), pool1, 'conv2_1', activation='leakyrelu')
    conv2 = create_conv(128, (3, 3), conv2, 'conv2_2', activation='leakyrelu')
    pool2 = MaxPool2D((2, 2))(conv2)

    conv3 = create_conv(256, (3, 3), pool2, 'conv3_1', activation='leakyrelu')
    conv3 = create_conv(256, (3, 3), conv3, 'conv3_2', activation='leakyrelu')
    pool3 = MaxPool2D((2, 2))(conv3)

    conv4 = create_conv(512, (3, 3), pool3, 'conv4_1', activation='leakyrelu')
    conv4 = create_conv(512, (3, 3), conv4, 'conv4_2', activation='leakyrelu')
    pool4 = MaxPool2D((2, 2))(conv4)

    conv5 = create_conv(1024, (3, 3), pool4, 'conv5_1', activation='leakyrelu')
    conv5 = create_conv(1024, (3, 3), conv5, 'conv5_2', activation='leakyrelu')

    up6 = create_conv(512, (2, 2), UpSampling2D((2, 2))(conv5), 'up6')
    merge6 = concatenate([conv4, up6], axis=3)
    conv6 = create_conv(512, (3, 3), merge6, 'conv6_1', activation='relu')
    conv6 = create_conv(512, (3, 3), conv6, 'conv6_2', activation='relu')

    up7 = create_conv(256, (2, 2), UpSampling2D((2, 2))(conv6), 'up7')
    merge7 = concatenate([conv3, up7], axis=3)
    conv7 = create_conv(256, (3, 3), merge7, 'conv7_1', activation='relu')
    conv7 = create_conv(256, (3, 3), conv7, 'conv7_2', activation='relu')

    up8 = create_conv(128, (2, 2), UpSampling2D((2, 2))(conv7), 'up8')
    merge8 = concatenate([conv2, up8], axis=3)
    conv8 = create_conv(128, (3, 3), merge8, 'conv8_1', activation='relu')
    conv8 = create_conv(128, (3, 3), conv8, 'conv8_2', activation='relu')

    up9 = create_conv(64, (2, 2), UpSampling2D((2, 2))(conv8))
    merge9 = concatenate([conv1, up9], axis=3)
    conv9 = create_conv(64, (3, 3), merge9, 'conv9_1', activation='relu')
    conv9 = create_conv(64, (3, 3), conv9, 'conv9_2', activation='relu')
    conv9 = Conv2D(output_channels, (1, 1), padding='same', name='conv9_3')(conv9)

    model = Model(inputs=inputs, outputs=conv9, name='generator')

    return model
In [88]:
hide_code
dcgan_generator = dcgan_generator((256, 256, 1), 3)
dcgan_generator.summary()
____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_1 (InputLayer)             (None, 256, 256, 1)   0                                            
____________________________________________________________________________________________________
conv1_1 (Conv2D)                 (None, 256, 256, 64)  640         input_1[0][0]                    
____________________________________________________________________________________________________
batch_normalization_1 (BatchNorm (None, 256, 256, 64)  256         conv1_1[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_13 (LeakyReLU)       (None, 256, 256, 64)  0           batch_normalization_1[0][0]      
____________________________________________________________________________________________________
conv1_2 (Conv2D)                 (None, 256, 256, 64)  36928       leaky_re_lu_13[0][0]             
____________________________________________________________________________________________________
batch_normalization_2 (BatchNorm (None, 256, 256, 64)  256         conv1_2[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_14 (LeakyReLU)       (None, 256, 256, 64)  0           batch_normalization_2[0][0]      
____________________________________________________________________________________________________
max_pooling2d_17 (MaxPooling2D)  (None, 128, 128, 64)  0           leaky_re_lu_14[0][0]             
____________________________________________________________________________________________________
conv2_1 (Conv2D)                 (None, 128, 128, 128) 73856       max_pooling2d_17[0][0]           
____________________________________________________________________________________________________
batch_normalization_3 (BatchNorm (None, 128, 128, 128) 512         conv2_1[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_15 (LeakyReLU)       (None, 128, 128, 128) 0           batch_normalization_3[0][0]      
____________________________________________________________________________________________________
conv2_2 (Conv2D)                 (None, 128, 128, 128) 147584      leaky_re_lu_15[0][0]             
____________________________________________________________________________________________________
batch_normalization_4 (BatchNorm (None, 128, 128, 128) 512         conv2_2[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_16 (LeakyReLU)       (None, 128, 128, 128) 0           batch_normalization_4[0][0]      
____________________________________________________________________________________________________
max_pooling2d_18 (MaxPooling2D)  (None, 64, 64, 128)   0           leaky_re_lu_16[0][0]             
____________________________________________________________________________________________________
conv3_1 (Conv2D)                 (None, 64, 64, 256)   295168      max_pooling2d_18[0][0]           
____________________________________________________________________________________________________
batch_normalization_5 (BatchNorm (None, 64, 64, 256)   1024        conv3_1[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_17 (LeakyReLU)       (None, 64, 64, 256)   0           batch_normalization_5[0][0]      
____________________________________________________________________________________________________
conv3_2 (Conv2D)                 (None, 64, 64, 256)   590080      leaky_re_lu_17[0][0]             
____________________________________________________________________________________________________
batch_normalization_6 (BatchNorm (None, 64, 64, 256)   1024        conv3_2[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_18 (LeakyReLU)       (None, 64, 64, 256)   0           batch_normalization_6[0][0]      
____________________________________________________________________________________________________
max_pooling2d_19 (MaxPooling2D)  (None, 32, 32, 256)   0           leaky_re_lu_18[0][0]             
____________________________________________________________________________________________________
conv4_1 (Conv2D)                 (None, 32, 32, 512)   1180160     max_pooling2d_19[0][0]           
____________________________________________________________________________________________________
batch_normalization_7 (BatchNorm (None, 32, 32, 512)   2048        conv4_1[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_19 (LeakyReLU)       (None, 32, 32, 512)   0           batch_normalization_7[0][0]      
____________________________________________________________________________________________________
conv4_2 (Conv2D)                 (None, 32, 32, 512)   2359808     leaky_re_lu_19[0][0]             
____________________________________________________________________________________________________
batch_normalization_8 (BatchNorm (None, 32, 32, 512)   2048        conv4_2[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_20 (LeakyReLU)       (None, 32, 32, 512)   0           batch_normalization_8[0][0]      
____________________________________________________________________________________________________
max_pooling2d_20 (MaxPooling2D)  (None, 16, 16, 512)   0           leaky_re_lu_20[0][0]             
____________________________________________________________________________________________________
conv5_1 (Conv2D)                 (None, 16, 16, 1024)  4719616     max_pooling2d_20[0][0]           
____________________________________________________________________________________________________
batch_normalization_9 (BatchNorm (None, 16, 16, 1024)  4096        conv5_1[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_21 (LeakyReLU)       (None, 16, 16, 1024)  0           batch_normalization_9[0][0]      
____________________________________________________________________________________________________
conv5_2 (Conv2D)                 (None, 16, 16, 1024)  9438208     leaky_re_lu_21[0][0]             
____________________________________________________________________________________________________
batch_normalization_10 (BatchNor (None, 16, 16, 1024)  4096        conv5_2[0][0]                    
____________________________________________________________________________________________________
leaky_re_lu_22 (LeakyReLU)       (None, 16, 16, 1024)  0           batch_normalization_10[0][0]     
____________________________________________________________________________________________________
up_sampling2d_1 (UpSampling2D)   (None, 32, 32, 1024)  0           leaky_re_lu_22[0][0]             
____________________________________________________________________________________________________
up6 (Conv2D)                     (None, 32, 32, 512)   2097664     up_sampling2d_1[0][0]            
____________________________________________________________________________________________________
batch_normalization_11 (BatchNor (None, 32, 32, 512)   2048        up6[0][0]                        
____________________________________________________________________________________________________
activation_13 (Activation)       (None, 32, 32, 512)   0           batch_normalization_11[0][0]     
____________________________________________________________________________________________________
concatenate_1 (Concatenate)      (None, 32, 32, 1024)  0           leaky_re_lu_20[0][0]             
                                                                   activation_13[0][0]              
____________________________________________________________________________________________________
conv6_1 (Conv2D)                 (None, 32, 32, 512)   4719104     concatenate_1[0][0]              
____________________________________________________________________________________________________
batch_normalization_12 (BatchNor (None, 32, 32, 512)   2048        conv6_1[0][0]                    
____________________________________________________________________________________________________
activation_14 (Activation)       (None, 32, 32, 512)   0           batch_normalization_12[0][0]     
____________________________________________________________________________________________________
conv6_2 (Conv2D)                 (None, 32, 32, 512)   2359808     activation_14[0][0]              
____________________________________________________________________________________________________
batch_normalization_13 (BatchNor (None, 32, 32, 512)   2048        conv6_2[0][0]                    
____________________________________________________________________________________________________
activation_15 (Activation)       (None, 32, 32, 512)   0           batch_normalization_13[0][0]     
____________________________________________________________________________________________________
up_sampling2d_2 (UpSampling2D)   (None, 64, 64, 512)   0           activation_15[0][0]              
____________________________________________________________________________________________________
up7 (Conv2D)                     (None, 64, 64, 256)   524544      up_sampling2d_2[0][0]            
____________________________________________________________________________________________________
batch_normalization_14 (BatchNor (None, 64, 64, 256)   1024        up7[0][0]                        
____________________________________________________________________________________________________
activation_16 (Activation)       (None, 64, 64, 256)   0           batch_normalization_14[0][0]     
____________________________________________________________________________________________________
concatenate_2 (Concatenate)      (None, 64, 64, 512)   0           leaky_re_lu_18[0][0]             
                                                                   activation_16[0][0]              
____________________________________________________________________________________________________
conv7_1 (Conv2D)                 (None, 64, 64, 256)   1179904     concatenate_2[0][0]              
____________________________________________________________________________________________________
batch_normalization_15 (BatchNor (None, 64, 64, 256)   1024        conv7_1[0][0]                    
____________________________________________________________________________________________________
activation_17 (Activation)       (None, 64, 64, 256)   0           batch_normalization_15[0][0]     
____________________________________________________________________________________________________
conv7_2 (Conv2D)                 (None, 64, 64, 256)   590080      activation_17[0][0]              
____________________________________________________________________________________________________
batch_normalization_16 (BatchNor (None, 64, 64, 256)   1024        conv7_2[0][0]                    
____________________________________________________________________________________________________
activation_18 (Activation)       (None, 64, 64, 256)   0           batch_normalization_16[0][0]     
____________________________________________________________________________________________________
up_sampling2d_3 (UpSampling2D)   (None, 128, 128, 256) 0           activation_18[0][0]              
____________________________________________________________________________________________________
up8 (Conv2D)                     (None, 128, 128, 128) 131200      up_sampling2d_3[0][0]            
____________________________________________________________________________________________________
batch_normalization_17 (BatchNor (None, 128, 128, 128) 512         up8[0][0]                        
____________________________________________________________________________________________________
activation_19 (Activation)       (None, 128, 128, 128) 0           batch_normalization_17[0][0]     
____________________________________________________________________________________________________
concatenate_3 (Concatenate)      (None, 128, 128, 256) 0           leaky_re_lu_16[0][0]             
                                                                   activation_19[0][0]              
____________________________________________________________________________________________________
conv8_1 (Conv2D)                 (None, 128, 128, 128) 295040      concatenate_3[0][0]              
____________________________________________________________________________________________________
batch_normalization_18 (BatchNor (None, 128, 128, 128) 512         conv8_1[0][0]                    
____________________________________________________________________________________________________
activation_20 (Activation)       (None, 128, 128, 128) 0           batch_normalization_18[0][0]     
____________________________________________________________________________________________________
conv8_2 (Conv2D)                 (None, 128, 128, 128) 147584      activation_20[0][0]              
____________________________________________________________________________________________________
batch_normalization_19 (BatchNor (None, 128, 128, 128) 512         conv8_2[0][0]                    
____________________________________________________________________________________________________
activation_21 (Activation)       (None, 128, 128, 128) 0           batch_normalization_19[0][0]     
____________________________________________________________________________________________________
up_sampling2d_4 (UpSampling2D)   (None, 256, 256, 128) 0           activation_21[0][0]              
____________________________________________________________________________________________________
conv2d_17 (Conv2D)               (None, 256, 256, 64)  32832       up_sampling2d_4[0][0]            
____________________________________________________________________________________________________
batch_normalization_20 (BatchNor (None, 256, 256, 64)  256         conv2d_17[0][0]                  
____________________________________________________________________________________________________
activation_22 (Activation)       (None, 256, 256, 64)  0           batch_normalization_20[0][0]     
____________________________________________________________________________________________________
concatenate_4 (Concatenate)      (None, 256, 256, 128) 0           leaky_re_lu_14[0][0]             
                                                                   activation_22[0][0]              
____________________________________________________________________________________________________
conv9_1 (Conv2D)                 (None, 256, 256, 64)  73792       concatenate_4[0][0]              
____________________________________________________________________________________________________
batch_normalization_21 (BatchNor (None, 256, 256, 64)  256         conv9_1[0][0]                    
____________________________________________________________________________________________________
activation_23 (Activation)       (None, 256, 256, 64)  0           batch_normalization_21[0][0]     
____________________________________________________________________________________________________
conv9_2 (Conv2D)                 (None, 256, 256, 64)  36928       activation_23[0][0]              
____________________________________________________________________________________________________
batch_normalization_22 (BatchNor (None, 256, 256, 64)  256         conv9_2[0][0]                    
____________________________________________________________________________________________________
activation_24 (Activation)       (None, 256, 256, 64)  0           batch_normalization_22[0][0]     
____________________________________________________________________________________________________
conv9_3 (Conv2D)                 (None, 256, 256, 3)   195         activation_24[0][0]              
====================================================================================================
Total params: 31,058,115
Trainable params: 31,044,419
Non-trainable params: 13,696
____________________________________________________________________________________________________
In [89]:
hide_code
def dcgan_discriminator(input_shape):
    inputs = Input(input_shape)
    conv1 = create_conv(64, (3, 3), inputs, 'conv1', activation='leakyrelu', dropout=.8)
    pool1 = MaxPool2D((2, 2))(conv1)

    conv2 = create_conv(128, (3, 3), pool1, 'conv2', activation='leakyrelu', dropout=.8)
    pool2 = MaxPool2D((2, 2))(conv2)

    conv3 = create_conv(256, (3, 3), pool2, 'conv3', activation='leakyrelu', dropout=.8)
    pool3 = MaxPool2D((2, 2))(conv3)

    conv4 = create_conv(512, (3, 3), pool3, 'conv4', activation='leakyrelu', dropout=.8)
    pool4 = MaxPool2D((2, 2))(conv4)

    conv5 = create_conv(512, (3, 3), pool4, 'conv5', activation='leakyrelu', dropout=.8)

    flat = Flatten()(conv5)
    dense6 = Dense(1, activation='sigmoid')(flat)

    model = Model(inputs=inputs, outputs=dense6, name='discriminator')

    return model
In [90]:
hide_code
dcgan_discriminator = dcgan_discriminator((256, 256, 3))
dcgan_discriminator.summary()
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_2 (InputLayer)         (None, 256, 256, 3)       0         
_________________________________________________________________
conv1 (Conv2D)               (None, 256, 256, 64)      1792      
_________________________________________________________________
batch_normalization_23 (Batc (None, 256, 256, 64)      256       
_________________________________________________________________
leaky_re_lu_23 (LeakyReLU)   (None, 256, 256, 64)      0         
_________________________________________________________________
dropout_25 (Dropout)         (None, 256, 256, 64)      0         
_________________________________________________________________
max_pooling2d_21 (MaxPooling (None, 128, 128, 64)      0         
_________________________________________________________________
conv2 (Conv2D)               (None, 128, 128, 128)     73856     
_________________________________________________________________
batch_normalization_24 (Batc (None, 128, 128, 128)     512       
_________________________________________________________________
leaky_re_lu_24 (LeakyReLU)   (None, 128, 128, 128)     0         
_________________________________________________________________
dropout_26 (Dropout)         (None, 128, 128, 128)     0         
_________________________________________________________________
max_pooling2d_22 (MaxPooling (None, 64, 64, 128)       0         
_________________________________________________________________
conv3 (Conv2D)               (None, 64, 64, 256)       295168    
_________________________________________________________________
batch_normalization_25 (Batc (None, 64, 64, 256)       1024      
_________________________________________________________________
leaky_re_lu_25 (LeakyReLU)   (None, 64, 64, 256)       0         
_________________________________________________________________
dropout_27 (Dropout)         (None, 64, 64, 256)       0         
_________________________________________________________________
max_pooling2d_23 (MaxPooling (None, 32, 32, 256)       0         
_________________________________________________________________
conv4 (Conv2D)               (None, 32, 32, 512)       1180160   
_________________________________________________________________
batch_normalization_26 (Batc (None, 32, 32, 512)       2048      
_________________________________________________________________
leaky_re_lu_26 (LeakyReLU)   (None, 32, 32, 512)       0         
_________________________________________________________________
dropout_28 (Dropout)         (None, 32, 32, 512)       0         
_________________________________________________________________
max_pooling2d_24 (MaxPooling (None, 16, 16, 512)       0         
_________________________________________________________________
conv5 (Conv2D)               (None, 16, 16, 512)       2359808   
_________________________________________________________________
batch_normalization_27 (Batc (None, 16, 16, 512)       2048      
_________________________________________________________________
leaky_re_lu_27 (LeakyReLU)   (None, 16, 16, 512)       0         
_________________________________________________________________
dropout_29 (Dropout)         (None, 16, 16, 512)       0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 131072)            0         
_________________________________________________________________
dense_17 (Dense)             (None, 1)                 131073    
=================================================================
Total params: 4,047,745
Trainable params: 4,044,801
Non-trainable params: 2,944
_________________________________________________________________
In [91]:
hide_code
def dcgan(input_shape, generator, discriminator):
    dcgan_inputs = Input(input_shape)

    generator_outputs = generator(inputs)
    discriminator_outputs = discriminator(concatenate([generator_outputs, inputs], axis=3))

    model = Model(inputs=[dcgan_inputs], 
                  outputs=[discriminator_outputs, generator_outputs], 
                  name='dcgan')

    return model
In [92]:
hide_code
def gan_model(input_shape_generator, input_shape_discriminator, output_channels, 
              lr, momentum, loss_weights):
    optimizer = Adam(lr=lr, beta_1=momentum)

    model_generator = dcgan_generator(input_shape=input_shape_generator, output_channels=output_channels)
    model_generator.compile(loss=losses.mean_absolute_error, optimizer=optimizer)

    model_discriminator = dcgan_discriminator(input_shape=input_shape_discriminator)
    model_discriminator.trainable = False

    model_gan = dcgan(input_shape=input_shape_generator, 
                      generator=model_generator, discriminator=model_discriminator)
    model_gan.compile(loss=[losses.binary_crossentropy, l1], metrics=[eacc, 'accuracy'],
                      loss_weights=loss_weights, optimizer=optimizer)

    model_discriminator.trainable = True
    model_discriminator.compile(loss=losses.binary_crossentropy, optimizer=optimizer)

    return model_generator, model_discriminator, model_gan

Model #2

Based on the model

!!! In progress !!!

In [36]:
hide_code
# Application Model
INCRNV2_model = InceptionResNetV2(weights=None, include_top=True)
INCRNV2_model.load_weights('inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5')
# INCRNV2_model.graph = tf.get_default_graph()
In [37]:
hide_code
# The Model 'Encoder-Fusion-Decoder'
model_input = Input(shape=(1000,))

# Encoding
encoder_input = Input(shape=(256, 256, 1,))
x = Conv2D(64, (3, 3), strides=2, 
           activation='relu', 
           padding='same')(encoder_input)
x = Conv2D(128, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(128, (3, 3), strides=2, activation='relu', padding='same')(x)
x = Conv2D(256, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(256, (3, 3), strides=2, activation='relu', padding='same')(x)
x = Conv2D(512, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(512, (3, 3), strides=1, activation='relu', padding='same')(x)
x = Conv2D(256, (3, 3), strides=1, activation='relu', padding='same')(x)

# Fusion
y = RepeatVector(32 * 32)(model_input) 
y = Reshape(([32, 32, 1000]))(y)
y = concatenate([y, x], axis=3) 
y = Conv2D(256, (1, 1), activation='relu')(y) 

# Decoding
z = Conv2D(128, (3, 3), strides=1, activation='relu', padding='same')(y)
z = UpSampling2D((2, 2))(z)
z = Conv2D(64, (3, 3), strides=1, activation='relu', padding='same')(z)
z = UpSampling2D((2, 2))(z)
z = Conv2D(32, (3, 3), strides=1, activation='relu', padding='same')(z)
z = Conv2D(16, (3, 3), strides=1, activation='relu', padding='same')(z)
z = Conv2D(2, (2, 2), activation='tanh', padding='same')(z)
decoder_output = UpSampling2D((2, 2))(z)

model = Model(inputs=[encoder_input, model_input], outputs=decoder_output)
In [38]:
hide_code
# Display the 
model.summary()
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_3 (InputLayer)            (None, 256, 256, 1)  0                                            
__________________________________________________________________________________________________
conv2d_204 (Conv2D)             (None, 128, 128, 64) 640         input_3[0][0]                    
__________________________________________________________________________________________________
conv2d_205 (Conv2D)             (None, 128, 128, 128 73856       conv2d_204[0][0]                 
__________________________________________________________________________________________________
conv2d_206 (Conv2D)             (None, 64, 64, 128)  147584      conv2d_205[0][0]                 
__________________________________________________________________________________________________
conv2d_207 (Conv2D)             (None, 64, 64, 256)  295168      conv2d_206[0][0]                 
__________________________________________________________________________________________________
conv2d_208 (Conv2D)             (None, 32, 32, 256)  590080      conv2d_207[0][0]                 
__________________________________________________________________________________________________
input_2 (InputLayer)            (None, 1000)         0                                            
__________________________________________________________________________________________________
conv2d_209 (Conv2D)             (None, 32, 32, 512)  1180160     conv2d_208[0][0]                 
__________________________________________________________________________________________________
repeat_vector_1 (RepeatVector)  (None, 1024, 1000)   0           input_2[0][0]                    
__________________________________________________________________________________________________
conv2d_210 (Conv2D)             (None, 32, 32, 512)  2359808     conv2d_209[0][0]                 
__________________________________________________________________________________________________
reshape_1 (Reshape)             (None, 32, 32, 1000) 0           repeat_vector_1[0][0]            
__________________________________________________________________________________________________
conv2d_211 (Conv2D)             (None, 32, 32, 256)  1179904     conv2d_210[0][0]                 
__________________________________________________________________________________________________
concatenate_1 (Concatenate)     (None, 32, 32, 1256) 0           reshape_1[0][0]                  
                                                                 conv2d_211[0][0]                 
__________________________________________________________________________________________________
conv2d_212 (Conv2D)             (None, 32, 32, 256)  321792      concatenate_1[0][0]              
__________________________________________________________________________________________________
conv2d_213 (Conv2D)             (None, 32, 32, 128)  295040      conv2d_212[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_1 (UpSampling2D)  (None, 64, 64, 128)  0           conv2d_213[0][0]                 
__________________________________________________________________________________________________
conv2d_214 (Conv2D)             (None, 64, 64, 64)   73792       up_sampling2d_1[0][0]            
__________________________________________________________________________________________________
up_sampling2d_2 (UpSampling2D)  (None, 128, 128, 64) 0           conv2d_214[0][0]                 
__________________________________________________________________________________________________
conv2d_215 (Conv2D)             (None, 128, 128, 32) 18464       up_sampling2d_2[0][0]            
__________________________________________________________________________________________________
conv2d_216 (Conv2D)             (None, 128, 128, 16) 4624        conv2d_215[0][0]                 
__________________________________________________________________________________________________
conv2d_217 (Conv2D)             (None, 128, 128, 2)  130         conv2d_216[0][0]                 
__________________________________________________________________________________________________
up_sampling2d_3 (UpSampling2D)  (None, 256, 256, 2)  0           conv2d_217[0][0]                 
==================================================================================================
Total params: 6,541,042
Trainable params: 6,541,042
Non-trainable params: 0
__________________________________________________________________________________________________
In [39]:
hide_code
# Preprocess for InceptionResNetV2
preprocessed_x_train = preprocess_input(grayscaled_x_train)
In [40]:
hide_code
# Preprocess for InceptionResNetV2
preprocessed_x_valid = preprocess_input(grayscaled_x_valid)
In [41]:
hide_code
# Preprocess for InceptionResNetV2
preprocessed_x_test = preprocess_input(grayscaled_x_test)
In [42]:
hide_code
# Predict with InceptionResNetV2
embedded_x_train = INCRNV2_model.predict(preprocessed_x_train)*255
In [43]:
hide_code
# Predict with InceptionResNetV2
embedded_x_valid = INCRNV2_model.predict(preprocessed_x_valid)*255
In [44]:
hide_code
# Predict with InceptionResNetV2
embedded_x_test = INCRNV2_model.predict(preprocessed_x_test)*255
In [45]:
hide_code
# Delete variables
del preprocessed_x_train, preprocessed_x_valid, preprocessed_x_test
In [46]:
hide_code
# Train the model
model.compile(optimizer='nadam', loss='mse')
checkpointer = ModelCheckpoint(filepath='weights.best.decor.model.hdf5', 
                               verbose=2, save_best_only=True)
history = model.fit([input_x_train, embedded_x_train], output_y_train, 
                    epochs=10, batch_size=32, verbose=2,
                    validation_data=([input_x_valid, embedded_x_valid], output_y_valid),
                    callbacks=[checkpointer])
Train on 291 samples, validate on 97 samples
Epoch 1/10

Epoch 00001: val_loss improved from inf to 0.02113, saving model to weights.best.decor.model.hdf5
 - 4545s - loss: 0.3046 - val_loss: 0.0211
Epoch 2/10

Epoch 00002: val_loss did not improve
 - 1042s - loss: 0.0211 - val_loss: 0.0231
Epoch 3/10

Epoch 00003: val_loss improved from 0.02113 to 0.02103, saving model to weights.best.decor.model.hdf5
 - 845s - loss: 0.0218 - val_loss: 0.0210
Epoch 4/10

Epoch 00004: val_loss did not improve
 - 670s - loss: 0.0209 - val_loss: 0.0217
Epoch 5/10

Epoch 00005: val_loss did not improve
 - 666s - loss: 0.0210 - val_loss: 0.0214
Epoch 6/10

Epoch 00006: val_loss did not improve
 - 645s - loss: 0.0209 - val_loss: 0.0260
Epoch 7/10

Epoch 00007: val_loss improved from 0.02103 to 0.02100, saving model to weights.best.decor.model.hdf5
 - 722s - loss: 0.0215 - val_loss: 0.0210
Epoch 8/10

Epoch 00008: val_loss did not improve
 - 630s - loss: 0.0209 - val_loss: 0.0210
Epoch 9/10

Epoch 00009: val_loss did not improve
 - 622s - loss: 0.0887 - val_loss: 0.0226
Epoch 10/10

Epoch 00010: val_loss did not improve
 - 628s - loss: 0.0247 - val_loss: 0.0214
In [ ]:
hide_code
# Predict
predict_y_test = model.predict([input_x_test, embedded_x_test])
In [ ]:
predict_y_test[0][0][0]
In [ ]:
output_y_test[0][0][0]

With an Image Geberator

In [ ]:
hide_code
# Embedding function
def INCRNV2_embedding(batch):
    grayscaled_batch = gray2rgb(rgb2gray(batch))
    preprocessed = preprocess_input(grayscaled_batch)
    with INCRNV2_model.graph.as_default():
        embedded = INCRNV2_model.predict(preprocessed)
    return embedded

# Image generator
image_generator = ImageDataGenerator(shear_range=0.3, zoom_range=0.3, rotation_range=30, horizontal_flip=True)
In [ ]:
hide_code
# Generate training data
batch_size = 4

def train_generator(batch_size):
    for batch in image_generator.flow(resized_x_train, batch_size=batch_size):
        x_batch_embedded = INCRNV2_embedding(batch)
        lab_batch = rgb2lab(batch)
        x_batch = lab_batch[:,:,:,0]
        x_batch = x_batch.reshape(x_batch.shape+(1,))
        y_batch = lab_batch[:,:,:,1:]/128
        yield ([x_batch, x_batch_embedded], y_batch)
In [ ]:
hide_code
# Train the model
filepath="weights-improvement-{epoch:02d}-{loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, 
                             save_weights_only=True, period=20)
callbacks_list = [checkpoint]

model.compile(optimizer='adam', loss='mse')
model.fit_generator(train_generator(batch_size), 
                    epochs=10, steps_per_epoch=1, 
                    callbacks=callbacks_list, verbose=1);
In [ ]:
hide_code
# Save the model
model_json = model.to_json()
with open("model.json", "w") as json_file:
    json_file.write(model_json)
model.save_weights("weights.model.h5")

Step 4. Compare Results

In [33]:
hide_code
# Display the grayscaled example
plt.figure(figsize=(12, 4))    
plt.subplot(121)
plt.title('Color Image. Shape:'+str(test_example.shape))
plt.imshow(test_example)
plt.subplot(122)
plt.title('Gray Image. Shape:'+str(grayscaled_test_example.shape))
plt.imshow(grayscaled_test_example, cmap=cm.bone);
In [34]:
hide_code
# Display the lab preprocessing example
plt.figure(figsize=(18, 4))    
plt.subplot(131)
plt.title('Lab Image. Channel #1. Shape:'+str(lab_test_example[:,:,0].shape))
plt.imshow(lab_test_example[:,:,0], cmap=cm.bone)
plt.subplot(132)
plt.title('Lab Image. Channel #2. Shape:'+str(lab_test_example[:,:,1].shape))
plt.imshow(lab_test_example[:,:,1], cmap=cm.RdYlGn)
plt.subplot(133)
plt.title('Lab Image. Channel #3. Shape:'+str(lab_test_example[:,:,2].shape))
plt.imshow(lab_test_example[:,:,2], cmap=cm.YlGnBu);
In [35]:
hide_code
# Display a combined image
rgb_test_example = np.zeros((256, 256, 3))
rgb_test_example[:,:,0] = grayscaled_test_example[:, :, 0]*100
rgb_test_example[:,:,1:] = output_y_test[2]
rgb_test_example = lab2rgb(rgb_test_example)
plt.title('Combined RGB Image. Shape:'+str(rgb_test_example.shape))
plt.imshow(rgb_test_example);