摘要

1. 交叉驗證不同方法組合的模型準確率

   1.1 參數說明

   1.2 程式碼

2. 選擇模型組合方法

   2.1 交叉比對結果

   2.2 結論

內容

1. 交叉驗證不同方法組合的模型準確率。

   1.1 參數說明

   • s1 → 加權依據: 1 每個字的準確度占比 ; 2 單獨計算800個中文字機率占比
   • s2 → isnull判斷依據： 1 模型投票(多數決) ; 2 每個字機率加權，取最大機率
   • s3 → isnull閾值: 1 機率最小值; 2 機率平均值

   1.2 程式碼

   # 產出各模型組合(交叉比對2*2*2共8種)
   k <- c(1,2,3,4,5,6,7)
   selectMatrix <- BitMatrix(length(k))
   model_count = apply(selectMatrix, 1, function(x){ k[which(x==1)] })
   model_count = model_count[-1]
   
   # 定義參數初始值
   mm_name = NULL
   
   in800 = NULL
   out800 = NULL
   test = NULL
   
   in800_acc = NULL
   out800_acc = NULL
   test_acc = NULL
   
   in800_confus = NULL
   test_confus = NULL
   
   test_o800_acc = NULL
   
   s1list = NULL
   s2list = NULL
   s3list = NULL
   
   count2 = 1
   
   # 交叉比較
   for(count in 1:length(model_count)){
     nmodel = length(model_count[[count]])
     model_name = c(1,2,3,'ex3',"ex4",'ex5','ex6')[model_count[[count]]]
     print(model_name)
     # 各字準確度 & 各字機率占比
     for(s1 in 1:2){
       # 各字準確度(產權重表+新機率表)
       if(s1 == 1){
         new_model_i800 = get_new_model(namesmodel = model_name,stat = 'acc',dataset = "offical_in800")
         new_model_o800 = get_new_model(namesmodel = model_name,stat = 'acc',dataset = "offical_noin800")
         new_model_test = get_new_model(namesmodel = model_name,stat = 'acc',dataset = "test_data")
         new_model_i800_new = new_model_i800[[2]]
         new_model_o800_new = new_model_o800[[2]]
         new_model_test_new = new_model_test[[2]]
         new_model_i800 = new_model_i800[[1]]
         new_model_o800 = new_model_o800[[1]]
         new_model_test = new_model_test[[1]]
         # 投票判斷 & 組合後判斷
         for(s2 in 1:2){
           # 投票判斷
           if(s2 == 1){
             # 閾值用最小值 & 用平均機率
             for(s3 in 1:2){
                 # 用最小值
               if(s3 == 1){
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_in800")
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_noin800")
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "test_data")
            }
            else{
                 # 用平均機率 
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_in800")
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_noin800")
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "test_data")
            }
   
               # 模型組合名稱
               mm_name[count2] = paste(model_name,collapse = ",")
   
               s1list[count2] = s1
               s2list[count2] = s2
               s3list[count2] = s3
   
               # 幾筆資料
               in800[count2] = nrow(new_model_i800)
               out800[count2] = nrow(new_model_o800)
               test[count2] = nrow(new_model_test)
   
               in800_acc[count2] = mean(new_model_i800$acc)
               out800_acc[count2] = mean(new_model_o800$acc_null)
               test_acc[count2] = mean(new_model_test$acc[new_model_test$origin_word != "isnull"])
   
               test_o800_acc[count2] = mean(new_model_test$acc_null[new_model_test$origin_word == 'isnull'])
   
               in800_confus[count2] = mean(new_model_i800$acc_null[new_model_i800$acc == 1])
               test_confus[count2] = mean(new_model_test$acc_null[new_model_test$acc == 1])
               count2 = count2 + 1
             }
   
           }
           else{
             # 組合後判斷
             # 閾值用最小值 & 用平均機率  
             for(s3 in 1:2){
                 # 用最小值
               if(s3 == 1){
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_in800",
                                                  new_data = new_model_i800,new_stat = new_model_i800_new)
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_noin800",
                                                  new_data = new_model_o800,new_stat = new_model_o800_new)
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "test_data",
                                                  new_data = new_model_test,new_stat = new_model_test_new)
               }
               else{
                 # 用平均機率 
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_in800",
                                                  new_data = new_model_i800,new_stat = new_model_i800_new)
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_noin800",
                                                  new_data = new_model_o800,new_stat = new_model_o800_new)
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "test_data",
                                                  new_data = new_model_test,new_stat = new_model_test_new)
               }
               mm_name[count2] = paste(model_name,collapse = ",")
               s1list[count2] = s1
               s2list[count2] = s2
               s3list[count2] = s3
   
               in800[count2] = nrow(new_model_i800)
               out800[count2] = nrow(new_model_o800)
               test[count2] = nrow(new_model_test)
   
               in800_acc[count2] = mean(new_model_i800$acc)
               out800_acc[count2] = mean(new_model_o800$acc_null)
               test_acc[count2] = mean(new_model_test$acc[new_model_test$origin_word != 'isnull'])
   
               test_o800_acc[count2] = mean(new_model_test$acc_null[new_model_test$origin_word == 'isnull'])
   
               in800_confus[count2] = mean(new_model_i800$acc_null[new_model_i800$acc == 1])
               test_confus[count2] = mean(new_model_test$acc_null[new_model_test$acc == 1])
               count2 = count2 + 1
             }
           }
         }
       }
       else{
   
         # 各字機率平均
         new_model_i800 = get_new_model(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_in800")
         new_model_o800 = get_new_model(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_noin800")
         new_model_test = get_new_model(namesmodel = model_name,stat = 'mean_prob',dataset = "test_data")
         new_model_i800_new = new_model_i800[[2]]
         new_model_o800_new = new_model_o800[[2]]
         new_model_test_new = new_model_test[[2]]
         new_model_i800 = new_model_i800[[1]]
         new_model_o800 = new_model_o800[[1]]
         new_model_test = new_model_test[[1]]
         # 投票判斷 & 組合後判斷
         for(s2 in 1:2){
           # 投票判斷
           if(s2 == 1){
             # 閾值用最小值 & 用平均機率
             for(s3 in 1:2){
               # 用最小值
               if(s3 == 1){
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_in800")
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_noin800")
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "test_data")
               }
               else{
                 # 用平均機率
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_in800")
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_noin800")
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "test_data")
            }
               mm_name[count2] = paste(model_name,collapse = ",")
               s1list[count2] = s1
               s2list[count2] = s2
               s3list[count2] = s3
   
               in800[count2] = nrow(new_model_i800)
               out800[count2] = nrow(new_model_o800)
               test[count2] = nrow(new_model_test)
   
               in800_acc[count2] = mean(new_model_i800$acc)
               out800_acc[count2] = mean(new_model_o800$acc_null)
               test_acc[count2] = mean(new_model_test$acc[new_model_test$origin_word != 'isnull'])
   
               test_o800_acc[count2] = mean(new_model_test$acc_null[new_model_test$origin_word == 'isnull'])
   
               in800_confus[count2] = mean(new_model_i800$acc_null[new_model_i800$acc == 1])
               test_confus[count2] = mean(new_model_test$acc_null[new_model_test$acc == 1])
               count2 = count2 + 1
             }        
           }
           else{
             # 整併後判斷
             # 閾值用最小值 & 用平均機率
             for(s3 in 1:2){
               # 用最小值
               if(s3 == 1){
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_in800",
                                                  new_data = new_model_i800,new_stat = new_model_i800_new)
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "offical_noin800",
                                                  new_data = new_model_o800,new_stat = new_model_o800_new)
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'min_prob',dataset = "test_data",
                                                  new_data = new_model_test,new_stat = new_model_test_new)
               }else{
                 # 用平均機率
                 new_model_i800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_in800",
                                                  new_data = new_model_i800,new_stat = new_model_i800_new)
                 new_model_o800$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "offical_noin800",
                                                  new_data = new_model_o800,new_stat = new_model_o800_new)
                 new_model_test$acc_null = get_min01(namesmodel = model_name,stat = 'mean_prob',dataset = "test_data",
                                                  new_data = new_model_test,new_stat = new_model_test_new)
               }
               mm_name[count2] = paste(model_name,collapse = ",")
               s1list[count2] = s1
               s2list[count2] = s2
               s3list[count2] = s3
   
               in800[count2] = nrow(new_model_i800)
               out800[count2] = nrow(new_model_o800)
               test[count2] = nrow(new_model_test)
   
               in800_acc[count2] = mean(new_model_i800$acc)
               out800_acc[count2] = mean(new_model_o800$acc_null)
               test_acc[count2] = mean(new_model_test$acc[new_model_test$origin_word != 'isnull'])
   
               test_o800_acc[count2] = mean(new_model_test$acc_null[new_model_test$origin_word == 'isnull'])
   
               in800_confus[count2] = mean(new_model_i800$acc_null[new_model_i800$acc == 1])
               test_confus[count2] = mean(new_model_test$acc_null[new_model_test$acc == 1])
               count2 = count2 + 1
             }
           }
         }
       }
     }
   }
   
   result = data.frame(
     mm_name = mm_name,
     s1list = s1list,
     s2list = s2list,
     s3list = s3list,
     in800 = in800,
     in800_acc = in800_acc,
     in800_confus = in800_confus,
     out800 = out800,
     out800_acc = out800_acc,
     test = test,
     test_acc = test_acc,
     test_confus = test_confus,
     test_null = test_o800_acc
   )
   write.csv(result,file = "C:/Users/wooden/Desktop/dl/model/model_statement.csv",row.names = F)
   

2. 選擇模型組合方法

   2.1 交叉比對結果

   

   2.2 考量

   • 辨識時間

     • 從下圖可得知，4個模型組合(A處)較3個模型組合(B處)辨識效果佳。
     • 但正式賽收到官方請求後，需在1秒內回傳辨識結果，4個模型組合可能面臨辨識時間過長，導致來不及回傳結果的困境，故優先選擇3個模型的組合。

     

   • 辨識準確度

     • 綜合考量官方800字內、官方800字外、測試賽準確度，我們挑選出3個模型組合。
     • 辨識表現：A > C > B，模型組合方法採用[s1list]=2、[s3list]=2、[s3list]=2 (如下圖)

     

   2.2 結論

   • 選定模型組合方法

     • s1_加權依據：單獨計算800個中文字機率占比(2)
     • s2_isnull判斷依據：每個字機率加權，取最大機率(2)
     • s3_isnull閾值：機率平均值(2)

   • 選定組合哪幾個模型

     • 模型3：InceptionResNetV2
     • 模型ex3：Xception
     • 模型ex4：Resnet152V2

   

小結

1. 交叉驗證後，我們選擇了模型組合方法與模型。雖然區分isnull的準確率是100%，考量到測試賽資料集樣本少，且出現isnull的頻率低，正式賽表現應該會打個折扣。
2. 接下來，我們需要將模型打包部署到GCP上，以API的形式提供手寫中文字辨識服務。故下一站，目標是：「分享如何啟用GCP服務，並以Computer Engine API架設VMware」。

讓我們繼續看下去...