In this project, you’re going to take a peek into the realm of neural network machine translation. You’ll be training a sequence to sequence model on a dataset of English and French sentences that can translate new sentences from English to French.
%%html
<style>
@import url('https://fonts.googleapis.com/css?family=Orbitron|Roboto');
body {background-color: azure;}
a {color: cadetblue; font-family: Roboto;}
h1, h2 {color: #48D1CC; font-family: Orbitron; text-shadow: 4px 4px 4px #ccc;}
h3, h4 {color: cadetblue; font-family: Roboto; text-shadow: 4px 4px 4px #ccc;}
span {text-shadow: 4px 4px 4px #ccc;}
div.output_prompt, div.output_area pre {color: slategray;}
div.input_prompt, div.output_subarea {color: cadetblue;}
div.output_stderr pre {background-color: azure;}
div.output_stderr {background-color: slategrey;}
</style>
<script>
code_show = true;
function code_display() {
if (code_show) {
$('div.input').each(function(id) {
if (id == 0 || $(this).html().indexOf('hide_code') > -1) {$(this).hide();}
});
$('div.output_prompt').css('opacity', 0);
} else {
$('div.input').each(function(id) {$(this).show();});
$('div.output_prompt').css('opacity', 1);
};
code_show = !code_show;
}
$(document).ready(code_display);
</script>
<form action="javascript: code_display()">
<input style="color: cadetblue; background: azure; text-shadow: 4px 4px 4px #ccc;" \
type="submit" value="Click to display or hide code cells">
</form>
hide_code = ''
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import os
import pickle
import copy
import numpy as np
import tensorflow as tf
from tensorflow.python.layers.core import Dense
from distutils.version import LooseVersion
import warnings
import itertools
import collections
hide_code = ''
# https://github.com/udacity/deep-learning/blob/master/language-translation/helper.py
CODES = {'<PAD>': 0, '<EOS>': 1, '<UNK>': 2, '<GO>': 3 }
def load_data(path):
""" Load Dataset from File"""
with open(os.path.join(path), 'r', encoding='utf-8') as f:
return f.read()
def preprocess_and_save_data(source_path, target_path, text_to_ids):
"""Preprocess Text Data. Save to to file."""
# Preprocess
source_text = load_data(source_path)
target_text = load_data(target_path)
source_text = source_text.lower()
target_text = target_text.lower()
source_vocab_to_int, source_int_to_vocab = create_lookup_tables(source_text)
target_vocab_to_int, target_int_to_vocab = create_lookup_tables(target_text)
source_text, target_text = text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int)
# Save Data
with open('preprocess.p', 'wb') as out_file:
pickle.dump((
(source_text, target_text),
(source_vocab_to_int, target_vocab_to_int),
(source_int_to_vocab, target_int_to_vocab)), out_file)
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/helper.py
def load_preprocess():
"""Load the Preprocessed Training data and return them in batches of <batch_size> or less"""
with open('preprocess.p', mode='rb') as in_file:
return pickle.load(in_file)
def create_lookup_tables(text):
"""Create lookup tables for vocabulary"""
vocab = set(text.split())
vocab_to_int = copy.copy(CODES)
for v_i, v in enumerate(vocab, len(CODES)):
vocab_to_int[v] = v_i
int_to_vocab = {v_i: v for v, v_i in vocab_to_int.items()}
return vocab_to_int, int_to_vocab
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/helper.py
def save_params(params):
"""Save parameters to file"""
with open('params.p', 'wb') as out_file:
pickle.dump(params, out_file)
def load_params():
"""Load parameters from file"""
with open('params.p', mode='rb') as in_file:
return pickle.load(in_file)
def batch_data(source, target, batch_size):
"""Batch source and target together"""
for batch_i in range(0, len(source)//batch_size):
start_i = batch_i * batch_size
source_batch = source[start_i:start_i + batch_size]
target_batch = target[start_i:start_i + batch_size]
yield np.array(pad_sentence_batch(source_batch)), np.array(pad_sentence_batch(target_batch))
def pad_sentence_batch(sentence_batch):
"""Pad sentence with <PAD> id"""
max_sentence = max([len(sentence) for sentence in sentence_batch])
return [sentence + [CODES['<PAD>']] * (max_sentence - len(sentence))
for sentence in sentence_batch]
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def _print_success_message():
print('Tests Passed')
def test_text_to_ids(text_to_ids):
test_source_text = 'new jersey is sometimes quiet during autumn , \
and it is snowy in april .\nthe united states is usually chilly during july , \
and it is usually freezing in november .\ncalifornia is usually quiet during march , \
and it is usually hot in june .\nthe united states is sometimes mild during june , \
and it is cold in september .'
test_target_text = 'new jersey est parfois calme pendant l\' automne , \
et il est neigeux en avril .\nles états-unis est généralement froid en juillet , \
et il gèle habituellement en novembre .\ncalifornia est généralement calme en mars , \
et il est généralement chaud en juin .\nles états-unis est parfois légère en juin , \
et il fait froid en septembre .'
test_source_text = test_source_text.lower()
test_target_text = test_target_text.lower()
source_vocab_to_int, source_int_to_vocab = create_lookup_tables(test_source_text)
target_vocab_to_int, target_int_to_vocab = create_lookup_tables(test_target_text)
test_source_id_seq, test_target_id_seq = text_to_ids(test_source_text, test_target_text,
source_vocab_to_int, target_vocab_to_int)
assert len(test_source_id_seq) == len(test_source_text.split('\n')),\
'source_id_text has wrong length, it should be {}.'.format(len(test_source_text.split('\n')))
assert len(test_target_id_seq) == len(test_target_text.split('\n')), \
'target_id_text has wrong length, it should be {}.'.format(len(test_target_text.split('\n')))
target_not_iter = [type(x) for x in test_source_id_seq if not isinstance(x, collections.Iterable)]
assert not target_not_iter,\
'Element in source_id_text is not iteratable. Found type {}'.format(target_not_iter[0])
target_not_iter = [type(x) for x in test_target_id_seq if not isinstance(x, collections.Iterable)]
assert not target_not_iter, \
'Element in target_id_text is not iteratable. Found type {}'.format(target_not_iter[0])
source_changed_length = [(words, word_ids)
for words, word_ids in zip(test_source_text.split('\n'), test_source_id_seq)
if len(words.split()) != len(word_ids)]
assert not source_changed_length,\
'Source text changed in size from {} word(s) to {} id(s): {}'\
.format(len(source_changed_length[0][0].split()),
len(source_changed_length[0][1]), source_changed_length[0][1])
target_missing_end = \
[word_ids for word_ids in test_target_id_seq if word_ids[-1] != target_vocab_to_int['<EOS>']]
assert not target_missing_end,\
'Missing <EOS> id at the end of {}'.format(target_missing_end[0])
target_bad_size = [(words.split(), word_ids)
for words, word_ids in zip(test_target_text.split('\n'), test_target_id_seq)
if len(word_ids) != len(words.split()) + 1]
assert not target_bad_size,\
'Target text incorrect size. {} should be length {}'.format(
target_bad_size[0][1], len(target_bad_size[0][0]) + 1)
source_bad_id = [(word, word_id)
for word, word_id in zip(
[word for sentence in test_source_text.split('\n') for word in sentence.split()],
itertools.chain.from_iterable(test_source_id_seq))
if source_vocab_to_int[word] != word_id]
assert not source_bad_id,\
'Source word incorrectly converted from {} to id {}.'\
.format(source_bad_id[0][0], source_bad_id[0][1])
target_bad_id = [(word, word_id)
for word, word_id in zip(
[word for sentence in test_target_text.split('\n') for word in sentence.split()],
[word_id for word_ids in test_target_id_seq for word_id in word_ids[:-1]])
if target_vocab_to_int[word] != word_id]
assert not target_bad_id,\
'Target word incorrectly converted from {} to id {}.'\
.format(target_bad_id[0][0], target_bad_id[0][1])
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_model_inputs(model_inputs):
with tf.Graph().as_default():
input_data, targets, lr, keep_prob, target_sequence_length, max_target_sequence_length, \
source_sequence_length = model_inputs()
# Check type
assert input_data.op.type == 'Placeholder',\
'Input is not a Placeholder.'
assert targets.op.type == 'Placeholder',\
'Targets is not a Placeholder.'
assert lr.op.type == 'Placeholder',\
'Learning Rate is not a Placeholder.'
assert keep_prob.op.type == 'Placeholder', \
'Keep Probability is not a Placeholder.'
assert target_sequence_length.op.type == 'Placeholder', \
'Target Sequence Length is not a Placeholder.'
assert max_target_sequence_length.op.type == 'Max', \
'Max Target Sequence Length is not a Placeholder.'
assert source_sequence_length.op.type == 'Placeholder', \
'Source Sequence Length is not a Placeholder.'
# Check name
assert input_data.name == 'input:0',\
'Input has bad name. Found name {}'.format(input_data.name)
assert target_sequence_length.name == 'target_sequence_length:0',\
'Target Sequence Length has bad name. Found name {}'.format(target_sequence_length.name)
assert source_sequence_length.name == 'source_sequence_length:0',\
'Source Sequence Length has bad name. Found name {}'.format(source_sequence_length.name)
assert keep_prob.name == 'keep_prob:0', \
'Keep Probability has bad name. Found name {}'.format(keep_prob.name)
assert tf.assert_rank(input_data, 2, message='Input data has wrong rank')
assert tf.assert_rank(targets, 2, message='Targets has wrong rank')
assert tf.assert_rank(lr, 0, message='Learning Rate has wrong rank')
assert tf.assert_rank(keep_prob, 0, message='Keep Probability has wrong rank')
assert tf.assert_rank(target_sequence_length, 1,
message='Target Sequence Length has wrong rank')
assert tf.assert_rank(max_target_sequence_length, 0,
message='Max Target Sequence Length has wrong rank')
assert tf.assert_rank(source_sequence_length, 1,
message='Source Sequence Lengthhas wrong rank')
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_encoding_layer(encoding_layer):
rnn_size = 512
batch_size = 64
num_layers = 3
source_sequence_len = 22
source_vocab_size = 20
encoding_embedding_size = 30
with tf.Graph().as_default():
rnn_inputs = tf.placeholder(tf.int32, [batch_size,
source_sequence_len])
source_sequence_length = tf.placeholder(tf.int32,
(None,),
name='source_sequence_length')
keep_prob = tf.placeholder(tf.float32)
enc_output, states = encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob,
source_sequence_length, source_vocab_size,
encoding_embedding_size)
assert len(states) == num_layers,\
'Found {} state(s). It should be {} states.'.format(len(states), num_layers)
bad_types = [type(state) for state in states if not isinstance(state, tf.contrib.rnn.LSTMStateTuple)]
assert not bad_types,\
'Found wrong type: {}'.format(bad_types[0])
bad_shapes = [state_tensor.get_shape()
for state in states
for state_tensor in state
if state_tensor.get_shape().as_list() not in [[None, rnn_size], [batch_size, rnn_size]]]
assert not bad_shapes,\
'Found wrong shape: {}'.format(bad_shapes[0])
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_decoding_layer(decoding_layer):
batch_size = 64
vocab_size = 1000
embedding_size = 200
sequence_length = 22
rnn_size = 512
num_layers = 3
target_vocab_to_int = {'<EOS>': 1, '<GO>': 3}
with tf.Graph().as_default():
target_sequence_length_p = tf.placeholder(tf.int32, (None,), name='target_sequence_length')
max_target_sequence_length = tf.reduce_max(target_sequence_length_p, name='max_target_len')
dec_input = tf.placeholder(tf.int32, [batch_size, sequence_length])
dec_embed_input = tf.placeholder(tf.float32, [batch_size, sequence_length, embedding_size])
dec_embeddings = tf.placeholder(tf.float32, [vocab_size, embedding_size])
keep_prob = tf.placeholder(tf.float32)
state = tf.contrib.rnn.LSTMStateTuple(
tf.placeholder(tf.float32, [None, rnn_size]),
tf.placeholder(tf.float32, [None, rnn_size]))
encoder_state = (state, state, state)
train_decoder_output, infer_logits_output = decoding_layer( dec_input,
encoder_state,
target_sequence_length_p,
max_target_sequence_length,
rnn_size,
num_layers,
target_vocab_to_int,
vocab_size,
batch_size,
keep_prob,
embedding_size)
assert isinstance(train_decoder_output, tf.contrib.seq2seq.BasicDecoderOutput),\
'Found wrong type: {}'.format(type(train_decoder_output))
assert isinstance(infer_logits_output, tf.contrib.seq2seq.BasicDecoderOutput),\
'Found wrong type: {}'.format(type(infer_logits_output))
assert train_decoder_output.rnn_output.get_shape().as_list() == [batch_size, None, vocab_size], \
'Wrong shape returned. Found {}'.format(train_decoder_output.rnn_output.get_shape())
assert infer_logits_output.sample_id.get_shape().as_list() == [batch_size, None], \
'Wrong shape returned. Found {}'.format(infer_logits_output.sample_id.get_shape())
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_seq2seq_model(seq2seq_model):
batch_size = 64
vocab_size = 300
embedding_size = 100
sequence_length = 22
rnn_size = 512
num_layers = 3
target_vocab_to_int = {'<EOS>': 1, '<GO>': 3}
with tf.Graph().as_default():
dec_input = tf.placeholder(tf.int32, [batch_size, sequence_length])
dec_embed_input = tf.placeholder(tf.float32, [batch_size, sequence_length, embedding_size])
dec_embeddings = tf.placeholder(tf.float32, [vocab_size, embedding_size])
keep_prob = tf.placeholder(tf.float32)
enc_state = tf.contrib.rnn.LSTMStateTuple(
tf.placeholder(tf.float32, [None, rnn_size]),
tf.placeholder(tf.float32, [None, rnn_size]))
input_data = tf.placeholder(tf.int32, [batch_size, sequence_length])
target_data = tf.placeholder(tf.int32, [batch_size, sequence_length])
keep_prob = tf.placeholder(tf.float32)
source_sequence_length = tf.placeholder(tf.int32, (None,), name='source_sequence_length')
target_sequence_length_p = tf.placeholder(tf.int32, (None,), name='target_sequence_length')
max_target_sequence_length = tf.reduce_max(target_sequence_length_p, name='max_target_len')
train_decoder_output, infer_logits_output = seq2seq_model( input_data,
target_data,
keep_prob,
batch_size,
source_sequence_length,
target_sequence_length_p,
max_target_sequence_length,
vocab_size,
vocab_size,
embedding_size,
embedding_size,
rnn_size,
num_layers,
target_vocab_to_int)
# input_data, target_data, keep_prob, batch_size, sequence_length,
# 200, target_vocab_size, 64, 80, rnn_size, num_layers, target_vocab_to_int)
assert isinstance(train_decoder_output, tf.contrib.seq2seq.BasicDecoderOutput),\
'Found wrong type: {}'.format(type(train_decoder_output))
assert isinstance(infer_logits_output, tf.contrib.seq2seq.BasicDecoderOutput),\
'Found wrong type: {}'.format(type(infer_logits_output))
assert train_decoder_output.rnn_output.get_shape().as_list() == [batch_size, None, vocab_size], \
'Wrong shape returned. Found {}'.format(train_decoder_output.rnn_output.get_shape())
assert infer_logits_output.sample_id.get_shape().as_list() == [batch_size, None], \
'Wrong shape returned. Found {}'.format(infer_logits_output.sample_id.get_shape())
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_sentence_to_seq(sentence_to_seq):
sentence = 'this is a test sentence'
vocab_to_int = {'<PAD>': 0, '<EOS>': 1, '<UNK>': 2, 'this': 3, 'is': 6, 'a': 5, 'sentence': 4}
output = sentence_to_seq(sentence, vocab_to_int)
assert len(output) == 5,\
'Wrong length. Found a length of {}'.format(len(output))
assert output[3] == 2,\
'Missing <UNK> id.'
assert np.array_equal(output, [3, 6, 5, 2, 4]),\
'Incorrect ouput. Found {}'.format(output)
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_process_encoding_input(process_encoding_input):
batch_size = 2
seq_length = 3
target_vocab_to_int = {'<GO>': 3}
with tf.Graph().as_default():
target_data = tf.placeholder(tf.int32, [batch_size, seq_length])
dec_input = process_encoding_input(target_data, target_vocab_to_int, batch_size)
assert dec_input.get_shape() == (batch_size, seq_length),\
'Wrong shape returned. Found {}'.format(dec_input.get_shape())
test_target_data = [[10, 20, 30], [40, 18, 23]]
with tf.Session() as sess:
test_dec_input = sess.run(dec_input, {target_data: test_target_data})
assert test_dec_input[0][0] == target_vocab_to_int['<GO>'] and\
test_dec_input[1][0] == target_vocab_to_int['<GO>'],\
'Missing GO Id.'
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_decoding_layer_train(decoding_layer_train):
batch_size = 64
vocab_size = 1000
embedding_size = 200
sequence_length = 22
rnn_size = 512
num_layers = 3
with tf.Graph().as_default():
with tf.variable_scope("decoding") as decoding_scope:
# dec_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.BasicLSTMCell(rnn_size)] * num_layers)
dec_embed_input = tf.placeholder(tf.float32, [batch_size, sequence_length, embedding_size])
keep_prob = tf.placeholder(tf.float32)
target_sequence_length_p = tf.placeholder(tf.int32, (None,), name='target_sequence_length')
max_target_sequence_length = tf.reduce_max(target_sequence_length_p, name='max_target_len')
for layer in range(num_layers):
with tf.variable_scope('decoder_{}'.format(layer)):
lstm = tf.contrib.rnn.LSTMCell(rnn_size,
initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))
dec_cell = tf.contrib.rnn.DropoutWrapper(lstm,
input_keep_prob=keep_prob)
output_layer = Dense(vocab_size,
kernel_initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
name='output_layer')
# output_fn = lambda x: tf.contrib.layers.fully_connected(x, vocab_size, None, scope=decoding_scope)
encoder_state = tf.contrib.rnn.LSTMStateTuple(
tf.placeholder(tf.float32, [None, rnn_size]),
tf.placeholder(tf.float32, [None, rnn_size]))
train_decoder_output = decoding_layer_train(encoder_state, dec_cell,
dec_embed_input,
target_sequence_length_p,
max_target_sequence_length,
output_layer,
keep_prob)
# encoder_state, dec_cell, dec_embed_input, sequence_length,
# decoding_scope, output_fn, keep_prob)
assert isinstance(train_decoder_output, tf.contrib.seq2seq.BasicDecoderOutput),\
'Found wrong type: {}'.format(type(train_decoder_output))
assert train_decoder_output.rnn_output.get_shape().as_list() == [batch_size, None, vocab_size], \
'Wrong shape returned. Found {}'.format(train_decoder_output.rnn_output.get_shape())
_print_success_message()
hide_code
# https://github.com/udacity/deep-learning/blob/master/language-translation/problem_unittests.py
def test_decoding_layer_infer(decoding_layer_infer):
batch_size = 64
vocab_size = 1000
sequence_length = 22
embedding_size = 200
rnn_size = 512
num_layers = 3
with tf.Graph().as_default():
with tf.variable_scope("decoding") as decoding_scope:
dec_embeddings = tf.Variable(tf.random_uniform([vocab_size, embedding_size]))
dec_embed_input = tf.placeholder(tf.float32, [batch_size, sequence_length, embedding_size])
keep_prob = tf.placeholder(tf.float32)
target_sequence_length_p = tf.placeholder(tf.int32, (None,), name='target_sequence_length')
max_target_sequence_length = tf.reduce_max(target_sequence_length_p, name='max_target_len')
for layer in range(num_layers):
with tf.variable_scope('decoder_{}'.format(layer)):
lstm = tf.contrib.rnn.LSTMCell(rnn_size,
initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2))
dec_cell = tf.contrib.rnn.DropoutWrapper(lstm,
input_keep_prob=keep_prob)
output_layer = Dense(vocab_size,
kernel_initializer=tf.truncated_normal_initializer(mean=0.0, stddev=0.1),
name='output_layer')
# output_fn = lambda x: tf.contrib.layers.fully_connected(x, vocab_size, None, scope=decoding_scope)
encoder_state = tf.contrib.rnn.LSTMStateTuple(
tf.placeholder(tf.float32, [None, rnn_size]),
tf.placeholder(tf.float32, [None, rnn_size]))
infer_logits_output = decoding_layer_infer( encoder_state,
dec_cell,
dec_embeddings,
1,
2,
max_target_sequence_length,
vocab_size,
output_layer,
batch_size,
keep_prob)
# encoder_state, dec_cell, dec_embeddings, 10, 20,
# sequence_length, vocab_size, decoding_scope, output_fn, keep_prob)
assert isinstance(infer_logits_output, tf.contrib.seq2seq.BasicDecoderOutput),\
'Found wrong type: {}'.format(type(infer_logits_output))
assert infer_logits_output.sample_id.get_shape().as_list() == [batch_size, None], \
'Wrong shape returned. Found {}'.format(infer_logits_output.sample_id.get_shape())
_print_success_message()
Since translating the whole language of English to French will take lots of time to train, we have provided you with a small portion of the English corpus.
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
source_path = 'data/small_vocab_en'
target_path = 'data/small_vocab_fr'
source_text = load_data(source_path)
target_text = load_data(target_path)
Play around with view_sentence_range to view different parts of the data.
hide_code
view_sentence_range = (0, 10)
"""DON'T MODIFY ANYTHING IN THIS CELL"""
print('Dataset Stats')
print('Roughly the number of unique words: {}'.format(len({word: None for word in source_text.split()})))
sentences = source_text.split('\n')
word_counts = [len(sentence.split()) for sentence in sentences]
print('Number of sentences: {}'.format(len(sentences)))
print('Average number of words in a sentence: {}'.format(np.average(word_counts)))
print()
print('English sentences {} to {}:'.format(*view_sentence_range))
print('\n'.join(source_text.split('\n')[view_sentence_range[0]:view_sentence_range[1]]))
print()
print('French sentences {} to {}:'.format(*view_sentence_range))
print('\n'.join(target_text.split('\n')[view_sentence_range[0]:view_sentence_range[1]]))
As you did with other RNNs, you must turn the text into a number so the computer can understand it. In the function text_to_ids(), you'll turn source_text and target_text from words to ids. However, you need to add the <EOS> word id at the end of target_text. This will help the neural network predict when the sentence should end.
You can get the <EOS> word id by doing:
target_vocab_to_int['<EOS>']
You can get other word ids using source_vocab_to_int and target_vocab_to_int.
hide_code
def text_to_ids(source_text, target_text, source_vocab_to_int, target_vocab_to_int):
"""
Convert source and target text to proper word ids
:param source_text: String that contains all the source text.
:param target_text: String that contains all the target text.
:param source_vocab_to_int: Dictionary to go from the source words to an id
:param target_vocab_to_int: Dictionary to go from the target words to an id
:return: A tuple of lists (source_id_text, target_id_text)
"""
# TODO: Implement Function
source_id_text = [[source_vocab_to_int[word] for word in sentence.split()] \
for sentence in source_text.split('\n')]
target_id_text = [[target_vocab_to_int[word] for word in (sentence + ' <EOS>').split()]\
for sentence in target_text.split('\n')]
return source_id_text, target_id_text
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_text_to_ids(text_to_ids)
Running the code cell below will preprocess all the data and save it to file.
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
preprocess_and_save_data(source_path, target_path, text_to_ids)
This is your first checkpoint. If you ever decide to come back to this notebook or have to restart the notebook, you can start from here. The preprocessed data has been saved to disk.
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
(source_int_text, target_int_text), (source_vocab_to_int, target_vocab_to_int), _ = load_preprocess()
hide_code
print('Example of source_int_text: \n', source_int_text[0], '\n')
print('Example of target_int_text: \n', target_int_text[0], '\n')
print('Example of source_vocab_to_int: \n', \
{k:v for k,v in source_vocab_to_int.items() \
if v in [98, 188, 182, 78, 199, 101, 126, 115, 102, 5, 182, 44, 100, 193, 213]}, '\n')
print('Example of target_vocab_to_int \n', \
{k:v for k,v in target_vocab_to_int.items() \
if v in [285, 69, 52, 100, 160, 311, 236, 89, 125, 221, 344, 52, 39, 220, 120, 162, 1]})
This will check to make sure you have the correct version of TensorFlow and access to a GPU
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
# Check TensorFlow Version
assert LooseVersion(tf.__version__) >= LooseVersion('1.1'), 'Please use TensorFlow version 1.1 or newer'
print('TensorFlow Version: {}'.format(tf.__version__))
# Check for a GPU
if not tf.test.gpu_device_name():
warnings.warn('No GPU found. Please use a GPU to train your neural network.')
else:
print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
You'll build the components necessary to build a Sequence-to-Sequence model by implementing the following functions below:
model_inputsprocess_decoder_inputencoding_layerdecoding_layer_traindecoding_layer_inferdecoding_layerseq2seq_modelImplement the model_inputs() function to create TF Placeholders for the Neural Network. It should create the following placeholders:
tf.reduce_max on the target_sequence_length placeholder. Rank 0.Return the placeholders in the following the tuple (input, targets, learning rate, keep probability, target sequence length, max target sequence length, source sequence length)
hide_code
def model_inputs():
"""
Create TF Placeholders for input, targets, learning rate, and lengths of source and target sequences.
:return: Tuple (input, targets, learning rate, keep probability, target sequence length,
max target sequence length, source sequence length)
"""
# TODO: Implement Function
inputs = tf.placeholder(tf.int32, shape=[None,None], name="input") # rank 2
targets = tf.placeholder(tf.int32, shape=[None, None], name="target") # rank 2
learning_rate = tf.placeholder(tf.float32, shape=[], name="learning_rate") # rank 0
keep_probability = tf.placeholder(tf.float32, shape=[], name="keep_prob") # rank 0
target_sequence_length = tf.placeholder(tf.int32, shape=[None], name="target_sequence_length") # rank 1
max_target_sequence_length = tf.reduce_max(target_sequence_length, name='max_target_len') # rank 0
source_sequence_length = tf.placeholder(tf.int32, shape=[None], name="source_sequence_length") # rank 1
return (inputs, targets, learning_rate, keep_probability,
target_sequence_length, max_target_sequence_length, source_sequence_length)
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_model_inputs(model_inputs)
Implement process_decoder_input by removing the last word id from each batch in target_data and concat the GO ID to the begining of each batch.
tf.strided_slice; tf.concat; tf.fill.hide_code
def process_decoder_input(target_data, target_vocab_to_int, batch_size):
"""
Preprocess target data for encoding
:param target_data: Target Placehoder
:param target_vocab_to_int: Dictionary to go from the target words to an id
:param batch_size: Batch Size
:return: Preprocessed target data
"""
# TODO: Implement Function
target_endings = tf.strided_slice(target_data, [0, 0], [batch_size, -1], [1, 1])
decoded_target_data = tf.concat([tf.fill([batch_size, 1], target_vocab_to_int['<GO>']),
target_endings], 1)
return decoded_target_data
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_process_encoding_input(process_decoder_input)
Implement encoding_layer() to create a Encoder RNN layer:
tf.contrib.layers.embed_sequencetf.contrib.rnn.LSTMCell wrapped in a tf.contrib.rnn.DropoutWrappertf.nn.dynamic_rnn()hide_code
def encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob,
source_sequence_length, source_vocab_size,
encoding_embedding_size):
"""
Create encoding layer
:param rnn_inputs: Inputs for the RNN
:param rnn_size: RNN Size
:param num_layers: Number of layers
:param keep_prob: Dropout keep probability
:param source_sequence_length: a list of the lengths of each sequence in the batch
:param source_vocab_size: vocabulary size of source data
:param encoding_embedding_size: embedding size of source data
:return: tuple (RNN output, RNN state)
"""
# TODO: Implement Function
emb_outputs = tf.contrib.layers.embed_sequence(rnn_inputs,
vocab_size=source_vocab_size,
embed_dim=encoding_embedding_size)
stacked_cell = tf.contrib.rnn.MultiRNNCell([tf.contrib.rnn.LSTMCell(rnn_size) \
for _ in range(num_layers)])
stacked_cell = tf.contrib.rnn.DropoutWrapper(stacked_cell,
input_keep_prob=1.0,
output_keep_prob=keep_prob)
enc_outputs, enc_state = tf.nn.dynamic_rnn(stacked_cell, emb_outputs, dtype=tf.float32)
return enc_outputs, enc_state
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_encoding_layer(encoding_layer)
Create a training decoding layer:
tf.contrib.seq2seq.TrainingHelper tf.contrib.seq2seq.BasicDecodertf.contrib.seq2seq.dynamic_decodehide_code
def decoding_layer_train(encoder_state, dec_cell, dec_embed_input,
target_sequence_length, max_summary_length,
output_layer, keep_prob):
"""
Create a decoding layer for training
:param encoder_state: Encoder State
:param dec_cell: Decoder RNN Cell
:param dec_embed_input: Decoder embedded input
:param target_sequence_length: The lengths of each sequence in the target batch
:param max_summary_length: The length of the longest sequence in the batch
:param output_layer: Function to apply the output layer
:param keep_prob: Dropout keep probability
:return: BasicDecoderOutput containing training logits and sample_id
"""
# TODO: Implement Function
dec_trainhelper = tf.contrib.seq2seq.TrainingHelper(inputs=dec_embed_input,
sequence_length=target_sequence_length)
dec_basic = tf.contrib.seq2seq.BasicDecoder(cell=dec_cell,
helper=dec_trainhelper,
initial_state=encoder_state,
output_layer=output_layer)
outputs, _ = tf.contrib.seq2seq.dynamic_decode(decoder=dec_basic,
output_time_major=False,
impute_finished=True,
maximum_iterations=20)
return outputs
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_decoding_layer_train(decoding_layer_train)
Create inference decoder:
tf.contrib.seq2seq.GreedyEmbeddingHelpertf.contrib.seq2seq.BasicDecodertf.contrib.seq2seq.dynamic_decodehide_code
def decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, start_of_sequence_id,
end_of_sequence_id, max_target_sequence_length,
vocab_size, output_layer, batch_size, keep_prob):
"""
Create a decoding layer for inference
:param encoder_state: Encoder state
:param dec_cell: Decoder RNN Cell
:param dec_embeddings: Decoder embeddings
:param start_of_sequence_id: GO ID
:param end_of_sequence_id: EOS Id
:param max_target_sequence_length: Maximum length of target sequences
:param vocab_size: Size of decoder/target vocabulary
:param decoding_scope: TenorFlow Variable Scope for decoding
:param output_layer: Function to apply the output layer
:param batch_size: Batch size
:param keep_prob: Dropout keep probability
:return: BasicDecoderOutput containing inference logits and sample_id
"""
# TODO: Implement Function
tf.contrib.seq2seq.GreedyEmbeddingHelper
tf.contrib.seq2seq.BasicDecoder
tf.contrib.seq2seq.dynamic_decode
return None
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_decoding_layer_infer(decoding_layer_infer)
Implement decoding_layer() to create a Decoder RNN layer.
decoding_layer_train(encoder_state, dec_cell, dec_embed_input, target_sequence_length, max_target_sequence_length, output_layer, keep_prob) function to get the training logits.decoding_layer_infer(encoder_state, dec_cell, dec_embeddings, start_of_sequence_id, end_of_sequence_id, max_target_sequence_length, vocab_size, output_layer, batch_size, keep_prob) function to get the inference logits.Note: You'll need to use tf.variable_scope to share variables between training and inference.
hide_code
def decoding_layer(dec_input, encoder_state,
target_sequence_length, max_target_sequence_length,
rnn_size,
num_layers, target_vocab_to_int, target_vocab_size,
batch_size, keep_prob, decoding_embedding_size):
"""
Create decoding layer
:param dec_input: Decoder input
:param encoder_state: Encoder state
:param target_sequence_length: The lengths of each sequence in the target batch
:param max_target_sequence_length: Maximum length of target sequences
:param rnn_size: RNN Size
:param num_layers: Number of layers
:param target_vocab_to_int: Dictionary to go from the target words to an id
:param target_vocab_size: Size of target vocabulary
:param batch_size: The size of the batch
:param keep_prob: Dropout keep probability
:param decoding_embedding_size: Decoding embedding size
:return: Tuple of (Training BasicDecoderOutput, Inference BasicDecoderOutput)
"""
# TODO: Implement Function
return None, None
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_decoding_layer(decoding_layer)
Apply the functions you implemented above to:
encoding_layer(rnn_inputs, rnn_size, num_layers, keep_prob, source_sequence_length, source_vocab_size, encoding_embedding_size).process_decoder_input(target_data, target_vocab_to_int, batch_size) function.decoding_layer(dec_input, enc_state, target_sequence_length, max_target_sentence_length, rnn_size, num_layers, target_vocab_to_int, target_vocab_size, batch_size, keep_prob, dec_embedding_size) function.hide_code
def seq2seq_model(input_data, target_data, keep_prob, batch_size,
source_sequence_length, target_sequence_length,
max_target_sentence_length,
source_vocab_size, target_vocab_size,
enc_embedding_size, dec_embedding_size,
rnn_size, num_layers, target_vocab_to_int):
"""
Build the Sequence-to-Sequence part of the neural network
:param input_data: Input placeholder
:param target_data: Target placeholder
:param keep_prob: Dropout keep probability placeholder
:param batch_size: Batch Size
:param source_sequence_length: Sequence Lengths of source sequences in the batch
:param target_sequence_length: Sequence Lengths of target sequences in the batch
:param source_vocab_size: Source vocabulary size
:param target_vocab_size: Target vocabulary size
:param enc_embedding_size: Decoder embedding size
:param dec_embedding_size: Encoder embedding size
:param rnn_size: RNN Size
:param num_layers: Number of layers
:param target_vocab_to_int: Dictionary to go from the target words to an id
:return: Tuple of (Training BasicDecoderOutput, Inference BasicDecoderOutput)
"""
# TODO: Implement Function
return None, None
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_seq2seq_model(seq2seq_model)
Tune the following parameters:
epochs to the number of epochs.batch_size to the batch size.rnn_size to the size of the RNNs.num_layers to the number of layers.encoding_embedding_size to the size of the embedding for the encoder.decoding_embedding_size to the size of the embedding for the decoder.learning_rate to the learning rate.keep_probability to the Dropout keep probabilitydisplay_step to state how many steps between each debug output statementhide_code
epochs = None
batch_size = None
rnn_size = None
num_layers = None
encoding_embedding_size = None
decoding_embedding_size = None
learning_rate = None
keep_probability = None
display_step = None
Build the graph using the neural network you implemented.
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
save_path = 'checkpoints/dev'
(source_int_text, target_int_text), (source_vocab_to_int, target_vocab_to_int), _ = load_preprocess()
max_target_sentence_length = max([len(sentence) for sentence in source_int_text])
train_graph = tf.Graph()
with train_graph.as_default():
input_data, targets, lr, keep_prob,
target_sequence_length, max_target_sequence_length, source_sequence_length = model_inputs()
#sequence_length = tf.placeholder_with_default(max_target_sentence_length, None, name='sequence_length')
input_shape = tf.shape(input_data)
train_logits, inference_logits = seq2seq_model(tf.reverse(input_data, [-1]),
targets,
keep_prob,
batch_size,
source_sequence_length,
target_sequence_length,
max_target_sequence_length,
len(source_vocab_to_int),
len(target_vocab_to_int),
encoding_embedding_size,
decoding_embedding_size,
rnn_size,
num_layers,
target_vocab_to_int)
training_logits = tf.identity(train_logits.rnn_output, name='logits')
inference_logits = tf.identity(inference_logits.sample_id, name='predictions')
masks = tf.sequence_mask(target_sequence_length, max_target_sequence_length, dtype=tf.float32, name='masks')
with tf.name_scope("optimization"):
# Loss function
cost = tf.contrib.seq2seq.sequence_loss(
training_logits,
targets,
masks)
# Optimizer
optimizer = tf.train.AdamOptimizer(lr)
# Gradient Clipping
gradients = optimizer.compute_gradients(cost)
capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]
train_op = optimizer.apply_gradients(capped_gradients)
Batch and pad the source and target sequences
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
def pad_sentence_batch(sentence_batch, pad_int):
"""Pad sentences with <PAD> so that each sentence of a batch has the same length"""
max_sentence = max([len(sentence) for sentence in sentence_batch])
return [sentence + [pad_int] * (max_sentence - len(sentence)) for sentence in sentence_batch]
def get_batches(sources, targets, batch_size, source_pad_int, target_pad_int):
"""Batch targets, sources, and the lengths of their sentences together"""
for batch_i in range(0, len(sources)//batch_size):
start_i = batch_i * batch_size
# Slice the right amount for the batch
sources_batch = sources[start_i:start_i + batch_size]
targets_batch = targets[start_i:start_i + batch_size]
# Pad
pad_sources_batch = np.array(pad_sentence_batch(sources_batch, source_pad_int))
pad_targets_batch = np.array(pad_sentence_batch(targets_batch, target_pad_int))
# Need the lengths for the _lengths parameters
pad_targets_lengths = []
for target in pad_targets_batch:
pad_targets_lengths.append(len(target))
pad_source_lengths = []
for source in pad_sources_batch:
pad_source_lengths.append(len(source))
yield pad_sources_batch, pad_targets_batch, pad_source_lengths, pad_targets_lengths
Train the neural network on the preprocessed data. If you have a hard time getting a good loss, check the forms to see if anyone is having the same problem.
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
def get_accuracy(target, logits):
"""
Calculate accuracy
"""
max_seq = max(target.shape[1], logits.shape[1])
if max_seq - target.shape[1]:
target = np.pad(
target,
[(0,0),(0,max_seq - target.shape[1])],
'constant')
if max_seq - logits.shape[1]:
logits = np.pad(
logits,
[(0,0),(0,max_seq - logits.shape[1])],
'constant')
return np.mean(np.equal(target, logits))
# Split data to training and validation sets
train_source = source_int_text[batch_size:]
train_target = target_int_text[batch_size:]
valid_source = source_int_text[:batch_size]
valid_target = target_int_text[:batch_size]
(valid_sources_batch, valid_targets_batch, valid_sources_lengths, valid_targets_lengths ) = \
next(get_batches(valid_source, valid_target, batch_size,
source_vocab_to_int['<PAD>'], target_vocab_to_int['<PAD>']))
with tf.Session(graph=train_graph) as sess:
sess.run(tf.global_variables_initializer())
for epoch_i in range(epochs):
for batch_i, (source_batch, target_batch, sources_lengths, targets_lengths) in enumerate(
get_batches(train_source, train_target, batch_size,
source_vocab_to_int['<PAD>'],
target_vocab_to_int['<PAD>'])):
_, loss = sess.run(
[train_op, cost],
{input_data: source_batch,
targets: target_batch,
lr: learning_rate,
target_sequence_length: targets_lengths,
source_sequence_length: sources_lengths,
keep_prob: keep_probability})
if batch_i % display_step == 0 and batch_i > 0:
batch_train_logits = sess.run(
inference_logits,
{input_data: source_batch,
source_sequence_length: sources_lengths,
target_sequence_length: targets_lengths,
keep_prob: 1.0})
batch_valid_logits = sess.run(
inference_logits,
{input_data: valid_sources_batch,
source_sequence_length: valid_sources_lengths,
target_sequence_length: valid_targets_lengths,
keep_prob: 1.0})
train_acc = get_accuracy(target_batch, batch_train_logits)
valid_acc = get_accuracy(valid_targets_batch, batch_valid_logits)
print('Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.4f}, \
Validation Accuracy: {:>6.4f}, Loss: {:>6.4f}'\
.format(epoch_i, batch_i, len(source_int_text) // batch_size, train_acc, valid_acc, loss))
# Save Model
saver = tf.train.Saver()
saver.save(sess, save_path)
print('Model Trained and Saved')
Save the batch_size and save_path parameters for inference.
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
# Save parameters for checkpoint
save_params(save_path)
hide_code
"""DON'T MODIFY ANYTHING IN THIS CELL"""
_, (source_vocab_to_int, target_vocab_to_int), (source_int_to_vocab, target_int_to_vocab) = load_preprocess()
load_path = load_params()
To feed a sentence into the model for translation, you first need to preprocess it. Implement the function sentence_to_seq() to preprocess new sentences.
vocab_to_int<UNK> word id.hide_code
def sentence_to_seq(sentence, vocab_to_int):
"""
Convert a sentence to a sequence of ids
:param sentence: String
:param vocab_to_int: Dictionary to go from the words to an id
:return: List of word ids
"""
# TODO: Implement Function
return None
"""DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE"""
test_sentence_to_seq(sentence_to_seq)
This will translate translate_sentence from English to French.
hide_code
translate_sentence = 'he saw a old yellow truck .'
"""DON'T MODIFY ANYTHING IN THIS CELL"""
translate_sentence = sentence_to_seq(translate_sentence, source_vocab_to_int)
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
# Load saved model
loader = tf.train.import_meta_graph(load_path + '.meta')
loader.restore(sess, load_path)
input_data = loaded_graph.get_tensor_by_name('input:0')
logits = loaded_graph.get_tensor_by_name('predictions:0')
target_sequence_length = loaded_graph.get_tensor_by_name('target_sequence_length:0')
source_sequence_length = loaded_graph.get_tensor_by_name('source_sequence_length:0')
keep_prob = loaded_graph.get_tensor_by_name('keep_prob:0')
translate_logits = sess.run(logits, {input_data: [translate_sentence]*batch_size,
target_sequence_length: [len(translate_sentence)*2]*batch_size,
source_sequence_length: [len(translate_sentence)]*batch_size,
keep_prob: 1.0})[0]
print('Input')
print(' Word Ids: {}'.format([i for i in translate_sentence]))
print(' English Words: {}'.format([source_int_to_vocab[i] for i in translate_sentence]))
print('\nPrediction')
print(' Word Ids: {}'.format([i for i in translate_logits]))
print(' French Words: {}'.format(" ".join([target_int_to_vocab[i] for i in translate_logits])))
You might notice that some sentences translate better than others. Since the dataset you're using only has a vocabulary of 227 English words of the thousands that you use, you're only going to see good results using these words. For this project, you don't need a perfect translation. However, if you want to create a better translation model, you'll need better data.
You can train on the WMT10 French-English corpus. This dataset has more vocabulary and richer in topics discussed. However, this will take you days to train, so make sure you've a GPU and the neural network is performing well on dataset we provided. Just make sure you play with the WMT10 corpus after you've submitted this project.
When submitting this project, make sure to run all the cells before saving the notebook. Save the notebook file as "dlnd_language_translation.ipynb" and save it as a HTML file under "File" -> "Download as". Include the "helper.py" and "problem_unittests.py" files in your submission.