DAY 29
0
AI & Data

## Day 29 利用transformer自己實作一個翻譯程式(十一) Decoder layer

3. Point wise feed forward networks

Transformer中共有N個Encoder layer

``````class DecoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(DecoderLayer, self).__init__()

self.ffn = point_wise_feed_forward_network(d_model, dff)

self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)

self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
self.dropout3 = tf.keras.layers.Dropout(rate)

def call(self, x, enc_output, training,
# enc_output.shape == (batch_size, input_seq_len, d_model)

attn1 = self.dropout1(attn1, training=training)
out1 = self.layernorm1(attn1 + x)

attn2, attn_weights_block2 = self.mha2(
attn2 = self.dropout2(attn2, training=training)
out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)

ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)
ffn_output = self.dropout3(ffn_output, training=training)
out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)

return out3, attn_weights_block1, attn_weights_block2
``````
``````sample_decoder_layer = DecoderLayer(512, 8, 2048)

sample_decoder_layer_output, _, _ = sample_decoder_layer(
tf.random.uniform((64, 50, 512)), sample_encoder_layer_output,
False, None, None)

sample_decoder_layer_output.shape  # (batch_size, target_seq_len, d_model)
``````
``````TensorShape([64, 50, 512])
``````