做个记录：
1，架构是3个残差块：
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 5, 32, 32, 32, 3, 1, 1));
layers.emplace_back(std::make_shared<BN>(cudnn, batch, 32, 32, 32));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, 32, 32, 32));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 32, 64, 32, 32, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 64, 32, 32));
layers.emplace_back(std::make_shared<MaxPool2D>(cudnn, batch, 64, 32, 32, 2, 2, 0, 2));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 64, 128, 16, 16, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 128, 16, 16));
layers.emplace_back(std::make_shared<MaxPool2D>(cudnn, batch, 128, 16, 16, 2, 2, 0, 2));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 128, 256, 8, 8, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 256, 8, 8));
layers.emplace_back(std::make_shared<MaxPool2D>(cudnn, batch, 256, 8, 8, 2, 2, 0, 2));
// layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 256, 300, 4, 4, 4));
layers.emplace_back(std::make_shared<Linear>(cublas, batch, 256*4*4, 300));
layers.emplace_back(std::make_shared<BN>(cudnn, batch, 300, 1, 1));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, 300, 1, 1));
layers.emplace_back(std::make_shared<Linear>(cublas, batch, 300, 10));
————————————————
方差和均值爆料情况：
时间: 27535.343750 ms
train Classification result: 97.72% ok (used 49984 images)
时间: 2174.266113 ms
Test Classification result: 81.56% ok (used 9984 images)
learn rate：0.0001
轮次：24
rb均值: 1.6140453815,rb方差:5.120904445648
均值: -93.1453933716,方差:16.6951541901
均值: -89.5411300659,方差:13.9249677658
均值: -91.6143341064,方差:10.9254045486
均值: -88.4864883423,方差:17.3460960388
均值: -92.7311325073,方差:16.6858062744
均值: -90.4656372070,方差:6.9966602325
均值: -91.2220077515,方差:20.6955375671
均值: -92.4978637695,方差:28.9232978821
均值: -95.8759613037,方差:15.1841077805
均值: -88.2818069458,方差:13.0533695221
均值: -84.8050689697,方差:11.1592187881
均值: -90.9559707642,方差:26.2159500122
均值: -90.2095413208,方差:21.0307292938
均值: -94.9991760254,方差:18.5383968353
均值: -88.7679290771,方差:15.3791704178
均值: -94.6937179565,方差:24.2541046143
均值: -87.7234725952,方差:7.5897364616
均值: -89.1307373047,方差:18.1437034607
均值: -89.9150695801,方差:13.0529365540
均值: -92.5738677979,方差:17.3618869781
均值: -95.2071914673,方差:20.6090621948
均值: -92.9055862427,方差:11.8797712326
均值: -91.4030151367,方差:24.5985851288
均值: -90.6616897583,方差:18.5187110901
均值: -89.8819732666,方差:19.0706024170
均值: -86.7467422485,方差:11.9170856476
均值: -88.7020034790,方差:14.6680479050
均值: -93.5892105103,方差:21.0500984192
均值: -89.5009918213,方差:15.7073564529
均值: -88.4498672485,方差:9.8511447906
均值: -92.2534332275,方差:11.2799034119
均值: -89.2119445801,方差:14.4264163971
时间: 27513.873047 ms
train Classification result: 98.23% ok (used 49984 images)
时间: 2177.084961 ms
Test Classification result:84.59% ok (used 9984 images)
learn rate：1e-05
轮次：25
————————————————
2，5个 残差块架构：
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 5, 32, 32, 32, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 32, 32, 32));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 32, 64, 32, 32, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 64, 32, 32));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 64, 128, 32, 32, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22>(cudnn, batch, 128, 32, 32));
layers.emplace_back(std::make_shared<MaxPool2D>(cudnn, batch, 128, 32, 32, 2, 2, 0, 2));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 128, 256, 16, 16, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22SE>(cudnn, batch, 256, 16, 16));
layers.emplace_back(std::make_shared<MaxPool2D>(cudnn, batch, 256, 16, 16, 2, 2, 0, 2));
layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 256, 512, 8, 8, 3, 1, 1));
layers.emplace_back(std::make_shared<residualExt22SE>(cudnn, batch, 512, 8, 8));
layers.emplace_back(std::make_shared<averPool2D>(cudnn, batch, 512, 8, 8, 2, 2, 0, 2));
// layers.emplace_back(std::make_shared<Conv2D>(cudnn, batch, 256, 300, 4, 4, 4));
layers.emplace_back(std::make_shared<Linear>(cublas, batch, 512*16, 350));
layers.emplace_back(std::make_shared<BN>(cudnn, batch, 350, 1, 1));
layers.emplace_back(std::make_shared<LeakyRL>(cudnn, batch, 350, 1, 1));
layers.emplace_back(std::make_shared<Linear>(cublas, batch, 350, 10));
————————————————
方差和均值爆料情况：很明显，稳定很多！增加了两层senet也稳如泰山！
轮次：17
learn rate：1e-06
时间: 74121.484375 ms
train Classification result: 98.05% ok (used 49984 images)
时间: 5473.725098 ms
Test Classification result: 84.42% ok (used 9984 images)
轮次：18
learn rate：1e-06
时间: 77872.156250 ms
train Classification result: 98.12% ok (used 49984 images)
时间: 5715.910156 ms
Test Classification result:84.48% ok (used 9984 images)
轮次：19
learn rate：1e-07
时间: 77190.242188 ms
train Classification result: 97.97% ok (used 49984 images)
时间: 5709.939941 ms
Test Classification result: 84.47% ok (used 9984 images)
轮次：20
learn rate：1e-07
时间: 77821.828125 ms
train Classification result: 98.12% ok (used 49984 images)
时间: 5641.894043 ms
Test Classification result: 84.46% ok (used 9984 images)
轮次：21
learn rate：1e-07
时间: 75851.140625 ms
train Classification result: 98.09% ok (used 49984 images)
时间: 5531.998047 ms
Test Classification result: 84.40% ok (used 9984 images)
轮次：22
learn rate：1e-07
时间: 75607.789063 ms
train Classification result: 98.13% ok (used 49984 images)
时间: 5596.937988 ms
Test Classification result: 84.45% ok (used 9984 images)
轮次：23
learn rate：1e-07
时间: 76939.070313 ms
train Classification result: 97.98% ok (used 49984 images)
时间: 5557.229004 ms
Test Classification result: 84.46% ok (used 9984 images)
人生路上，我们有时求稳，有时激进，只有回过头看，才看明白！
这两天，又学会了标签平滑，的确有效！比上面更平稳，上3个senet块也能跑，只是3060显卡不够用了，开始变慢，一轮上80秒了！得分test=83.65！还行！
