DAY 22
0
AI & Data

【22】正規化方法 L1 與 L2 Regularizers 的比較實驗

Colab連結

``````def get_mobilenetV2(shape, regularizer):
input_node = tf.keras.layers.Input(shape=shape)

net = tf.keras.layers.Conv2D(32, 3, (2, 2), use_bias=False, padding='same', kernel_regularizer=regularizer, bias_regularizer=regularizer)(input_node)
net = tf.keras.layers.BatchNormalization()(net)
net = tf.keras.layers.ReLU(max_value=6)(net)

net = tf.keras.layers.DepthwiseConv2D(3, use_bias=False, padding='same', depthwise_regularizer=regularizer, bias_regularizer=regularizer)(net)
net = tf.keras.layers.BatchNormalization()(net)
net = tf.keras.layers.ReLU(max_value=6)(net)
net = tf.keras.layers.Conv2D(16, 1, use_bias=False, padding='same', kernel_regularizer=regularizer, bias_regularizer=regularizer)(net)
net = tf.keras.layers.BatchNormalization()(net)

net = bottleneck(net, 16, 24, (2, 2), regularizer, shortcut=False, zero_pad=True)  # block_1
net = bottleneck(net, 24, 24, (1, 1), regularizer, shortcut=True)  # block_2

net = bottleneck(net, 24, 32, (2, 2), regularizer, shortcut=False, zero_pad=True)  # block_3
net = bottleneck(net, 32, 32, (1, 1), regularizer, shortcut=True)  # block_4
net = bottleneck(net, 32, 32, (1, 1), regularizer, shortcut=True)  # block_5

net = bottleneck(net, 32, 64, (2, 2), regularizer, shortcut=False, zero_pad=True)  # block_6
net = bottleneck(net, 64, 64, (1, 1), regularizer, shortcut=True)  # block_7
net = bottleneck(net, 64, 64, (1, 1), regularizer, shortcut=True)  # block_8
net = bottleneck(net, 64, 64, (1, 1), regularizer, shortcut=True)  # block_9

net = bottleneck(net, 64, 96, (1, 1), regularizer, shortcut=False)  # block_10
net = bottleneck(net, 96, 96, (1, 1), regularizer, shortcut=True)  # block_11
net = bottleneck(net, 96, 96, (1, 1), regularizer, shortcut=True)  # block_12

net = bottleneck(net, 96, 160, (2, 2), regularizer, shortcut=False, zero_pad=True)  # block_13
net = bottleneck(net, 160, 160, (1, 1), regularizer, shortcut=True)  # block_14
net = bottleneck(net, 160, 160, (1, 1), regularizer, shortcut=True)  # block_15

net = bottleneck(net, 160, 320, (1, 1), regularizer, shortcut=False)  # block_16

net = tf.keras.layers.Conv2D(1280, 1, use_bias=False, padding='same', kernel_regularizer=regularizer, bias_regularizer=regularizer)(net)
net = tf.keras.layers.BatchNormalization()(net)
net = tf.keras.layers.ReLU(max_value=6)(net)

return input_node, net

def bottleneck(net, filters, out_ch, strides, regularizer, shortcut=True, zero_pad=False):

shortcut_net = net

net = tf.keras.layers.Conv2D(filters * 6, 1, use_bias=False, padding='same', kernel_regularizer=regularizer, bias_regularizer=regularizer)(net)
net = tf.keras.layers.BatchNormalization()(net)
net = tf.keras.layers.ReLU(max_value=6)(net)

net = tf.keras.layers.BatchNormalization()(net)
net = tf.keras.layers.ReLU(max_value=6)(net)

net = tf.keras.layers.Conv2D(out_ch, 1, use_bias=False, padding='same', kernel_regularizer=regularizer, bias_regularizer=regularizer)(net)
net = tf.keras.layers.BatchNormalization()(net)

if shortcut:

return net
``````

``````REGULARIZER=None

input_node, net = get_mobilenetV2((224,224,3), REGULARIZER)
net = tf.keras.layers.GlobalAveragePooling2D()(net)
net = tf.keras.layers.Dense(NUM_OF_CLASS)(net)

model = tf.keras.Model(inputs=[input_node], outputs=[net])

model.compile(
optimizer=tf.keras.optimizers.SGD(LR),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
)

history = model.fit(
ds_train,
epochs=EPOCHS,
validation_data=ds_test,
verbose=True)
``````

``````loss: 0.2075 - sparse_categorical_accuracy: 0.9257 - val_loss: 0.6781 - val_sparse_categorical_accuracy: 0.8164
``````

``````REGULARIZER=tf.keras.regularizers.l1(0.001)

input_node, net = get_mobilenetV2((224,224,3), REGULARIZER)
net = tf.keras.layers.GlobalAveragePooling2D()(net)
net = tf.keras.layers.Dense(NUM_OF_CLASS)(net)

model = tf.keras.Model(inputs=[input_node], outputs=[net])

reg_losses = []
class LogRegCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
reg_losses.append(reg_loss.numpy())

model.compile(
optimizer=tf.keras.optimizers.SGD(LR),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
)

history = model.fit(
ds_train,
epochs=EPOCHS,
validation_data=ds_test,
verbose=True,
callbacks=[LogRegCallback()])
``````

``````loss: 1.7935 - sparse_categorical_accuracy: 0.4739 - val_loss: 2.2774 - val_sparse_categorical_accuracy: 0.3328
``````

reg loss值在10個 epochs 中分別是：
[1.1237922, 0.58376014, 0.4727106, 0.3930303, 0.37723392, 0.37397617, 0.36347908, 0.3558346, 0.347651, 0.3464979]

``````REGULARIZER=tf.keras.regularizers.l2(0.001)

input_node, net = get_mobilenetV2((224,224,3), REGULARIZER)
net = tf.keras.layers.GlobalAveragePooling2D()(net)
net = tf.keras.layers.Dense(NUM_OF_CLASS, kernel_regularizer=REGULARIZER, bias_regularizer=REGULARIZER)(net)

model = tf.keras.Model(inputs=[input_node], outputs=[net])

reg_losses = []
class LogRegCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
reg_losses.append(reg_loss.numpy())

model.compile(
optimizer=tf.keras.optimizers.SGD(LR),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
)

history = model.fit(
ds_train,
epochs=EPOCHS,
validation_data=ds_test,
verbose=True,
callbacks=[LogRegCallback()])
``````

``````loss: 1.0696 - sparse_categorical_accuracy: 0.8059 - val_loss: 1.8243 - val_sparse_categorical_accuracy: 0.5945
``````

reg loss值在10個 epochs 中分別是：
[2.9584365, 1.793744, 1.1741993, 0.8488613, 0.67782825, 0.5916853, 0.54554325, 0.52234, 0.51154846, 0.50444514]