Skip to content

Commit

Permalink
more precise about sigmoid / softmas
Browse files Browse the repository at this point in the history
  • Loading branch information
@cbernet
cbernet committed Feb 11, 2019
1 parent 7c0301d commit 1b4b81d
Show file tree
Hide file tree
Showing 5 changed files with 722 additions and 279 deletions.
153 changes: 81 additions & 72 deletions handwritten_digits_keras/handwritten_digits_keras.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -109,7 +109,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 2,
"metadata": {},
"outputs": [
{
Expand All @@ -118,7 +118,7 @@
"(1797, 64)"
]
},
"execution_count": 7,
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -137,7 +137,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 3,
"metadata": {},
"outputs": [
{
Expand All @@ -146,7 +146,7 @@
"array([0, 1, 2, ..., 8, 9, 8])"
]
},
"execution_count": 8,
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -175,9 +175,16 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Using TensorFlow backend.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
Expand Down Expand Up @@ -209,7 +216,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
Expand Down Expand Up @@ -243,7 +250,7 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -252,7 +259,7 @@
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 9,
"metadata": {},
"outputs": [
{
Expand Down Expand Up @@ -301,7 +308,7 @@
"# It must have 10 neurons, corresponding to \n",
"# the 10 digit categories \n",
"output = layers.Dense(10, \n",
" activation='sigmoid')(tmp)\n",
" activation='softmax')(tmp)\n",
"\n",
"# create the neural network from the layers\n",
"model = Model(img_input, output)\n",
Expand Down Expand Up @@ -346,7 +353,7 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 10,
"metadata": {},
"outputs": [
{
Expand All @@ -355,105 +362,105 @@
"text": [
"Train on 1000 samples, validate on 797 samples\n",
"Epoch 1/50\n",
"1000/1000 [==============================] - 1s 776us/step - loss: 2.2106 - acc: 0.1180 - val_loss: 2.1137 - val_acc: 0.3225\n",
"1000/1000 [==============================] - 0s 430us/step - loss: 2.2570 - acc: 0.2060 - val_loss: 1.9799 - val_acc: 0.4391\n",
"Epoch 2/50\n",
"1000/1000 [==============================] - 0s 26us/step - loss: 1.9848 - acc: 0.5350 - val_loss: 1.8760 - val_acc: 0.6524\n",
"1000/1000 [==============================] - 0s 27us/step - loss: 1.7615 - acc: 0.5700 - val_loss: 1.5124 - val_acc: 0.6173\n",
"Epoch 3/50\n",
"1000/1000 [==============================] - 0s 29us/step - loss: 1.7036 - acc: 0.7480 - val_loss: 1.5570 - val_acc: 0.7591\n",
"1000/1000 [==============================] - 0s 57us/step - loss: 1.2999 - acc: 0.7120 - val_loss: 1.2052 - val_acc: 0.7327\n",
"Epoch 4/50\n",
"1000/1000 [==============================] - 0s 32us/step - loss: 1.3446 - acc: 0.8330 - val_loss: 1.1898 - val_acc: 0.8294\n",
"1000/1000 [==============================] - 0s 53us/step - loss: 0.9755 - acc: 0.8360 - val_loss: 0.9633 - val_acc: 0.7917\n",
"Epoch 5/50\n",
"1000/1000 [==============================] - 0s 31us/step - loss: 0.9934 - acc: 0.8750 - val_loss: 0.9386 - val_acc: 0.8557\n",
"1000/1000 [==============================] - 0s 47us/step - loss: 0.7352 - acc: 0.8570 - val_loss: 0.8170 - val_acc: 0.8105\n",
"Epoch 6/50\n",
"1000/1000 [==============================] - 0s 42us/step - loss: 0.7446 - acc: 0.8970 - val_loss: 0.7563 - val_acc: 0.8632\n",
"1000/1000 [==============================] - 0s 48us/step - loss: 0.5977 - acc: 0.8900 - val_loss: 0.6957 - val_acc: 0.8306\n",
"Epoch 7/50\n",
"1000/1000 [==============================] - 0s 39us/step - loss: 0.5650 - acc: 0.9380 - val_loss: 0.6428 - val_acc: 0.8758\n",
"1000/1000 [==============================] - 0s 50us/step - loss: 0.4799 - acc: 0.9130 - val_loss: 0.5716 - val_acc: 0.8770\n",
"Epoch 8/50\n",
"1000/1000 [==============================] - 0s 38us/step - loss: 0.4594 - acc: 0.9330 - val_loss: 0.5415 - val_acc: 0.8946\n",
"1000/1000 [==============================] - 0s 33us/step - loss: 0.4047 - acc: 0.9270 - val_loss: 0.5378 - val_acc: 0.8720\n",
"Epoch 9/50\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.3943 - acc: 0.9360 - val_loss: 0.4755 - val_acc: 0.9134\n",
"1000/1000 [==============================] - 0s 55us/step - loss: 0.3405 - acc: 0.9460 - val_loss: 0.5185 - val_acc: 0.8808\n",
"Epoch 10/50\n",
"1000/1000 [==============================] - 0s 46us/step - loss: 0.3290 - acc: 0.9500 - val_loss: 0.4788 - val_acc: 0.8971\n",
"1000/1000 [==============================] - 0s 56us/step - loss: 0.2989 - acc: 0.9480 - val_loss: 0.5236 - val_acc: 0.8657\n",
"Epoch 11/50\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.2844 - acc: 0.9610 - val_loss: 0.4306 - val_acc: 0.8996\n",
"1000/1000 [==============================] - 0s 48us/step - loss: 0.2653 - acc: 0.9590 - val_loss: 0.4815 - val_acc: 0.8908\n",
"Epoch 12/50\n",
"1000/1000 [==============================] - 0s 45us/step - loss: 0.2489 - acc: 0.9650 - val_loss: 0.3858 - val_acc: 0.9122\n",
"1000/1000 [==============================] - 0s 30us/step - loss: 0.2489 - acc: 0.9490 - val_loss: 0.4888 - val_acc: 0.8795\n",
"Epoch 13/50\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.2215 - acc: 0.9700 - val_loss: 0.3930 - val_acc: 0.9021\n",
"1000/1000 [==============================] - 0s 58us/step - loss: 0.2339 - acc: 0.9600 - val_loss: 0.4344 - val_acc: 0.8971\n",
"Epoch 14/50\n",
"1000/1000 [==============================] - 0s 43us/step - loss: 0.2031 - acc: 0.9730 - val_loss: 0.3678 - val_acc: 0.9009\n",
"1000/1000 [==============================] - 0s 59us/step - loss: 0.2089 - acc: 0.9620 - val_loss: 0.4893 - val_acc: 0.8632\n",
"Epoch 15/50\n",
"1000/1000 [==============================] - 0s 35us/step - loss: 0.2024 - acc: 0.9650 - val_loss: 0.3528 - val_acc: 0.9134\n",
"1000/1000 [==============================] - 0s 48us/step - loss: 0.2012 - acc: 0.9580 - val_loss: 0.4107 - val_acc: 0.8934\n",
"Epoch 16/50\n",
"1000/1000 [==============================] - 0s 35us/step - loss: 0.1845 - acc: 0.9680 - val_loss: 0.3517 - val_acc: 0.9147\n",
"1000/1000 [==============================] - 0s 29us/step - loss: 0.1801 - acc: 0.9640 - val_loss: 0.4007 - val_acc: 0.8921\n",
"Epoch 17/50\n",
"1000/1000 [==============================] - 0s 40us/step - loss: 0.1589 - acc: 0.9740 - val_loss: 0.3789 - val_acc: 0.9059\n",
"1000/1000 [==============================] - 0s 60us/step - loss: 0.1633 - acc: 0.9720 - val_loss: 0.3830 - val_acc: 0.9021\n",
"Epoch 18/50\n",
"1000/1000 [==============================] - 0s 38us/step - loss: 0.1503 - acc: 0.9800 - val_loss: 0.3330 - val_acc: 0.9134\n",
"1000/1000 [==============================] - 0s 52us/step - loss: 0.1475 - acc: 0.9710 - val_loss: 0.4084 - val_acc: 0.8946\n",
"Epoch 19/50\n",
"1000/1000 [==============================] - 0s 33us/step - loss: 0.1313 - acc: 0.9800 - val_loss: 0.3099 - val_acc: 0.9172\n",
"1000/1000 [==============================] - 0s 50us/step - loss: 0.1449 - acc: 0.9720 - val_loss: 0.4290 - val_acc: 0.8770\n",
"Epoch 20/50\n",
"1000/1000 [==============================] - 0s 41us/step - loss: 0.1147 - acc: 0.9880 - val_loss: 0.3235 - val_acc: 0.9159\n",
"1000/1000 [==============================] - 0s 57us/step - loss: 0.1393 - acc: 0.9720 - val_loss: 0.4002 - val_acc: 0.8959\n",
"Epoch 21/50\n",
"1000/1000 [==============================] - 0s 65us/step - loss: 0.1031 - acc: 0.9900 - val_loss: 0.2926 - val_acc: 0.9210\n",
"1000/1000 [==============================] - 0s 45us/step - loss: 0.1190 - acc: 0.9790 - val_loss: 0.3784 - val_acc: 0.8934\n",
"Epoch 22/50\n",
"1000/1000 [==============================] - 0s 55us/step - loss: 0.0985 - acc: 0.9930 - val_loss: 0.2900 - val_acc: 0.9272\n",
"1000/1000 [==============================] - 0s 50us/step - loss: 0.1172 - acc: 0.9770 - val_loss: 0.3580 - val_acc: 0.8971\n",
"Epoch 23/50\n",
"1000/1000 [==============================] - 0s 45us/step - loss: 0.0867 - acc: 0.9950 - val_loss: 0.2773 - val_acc: 0.9247\n",
"1000/1000 [==============================] - 0s 50us/step - loss: 0.1304 - acc: 0.9740 - val_loss: 0.3794 - val_acc: 0.8971\n",
"Epoch 24/50\n",
"1000/1000 [==============================] - 0s 46us/step - loss: 0.0841 - acc: 0.9940 - val_loss: 0.2813 - val_acc: 0.9235\n",
"1000/1000 [==============================] - 0s 48us/step - loss: 0.1127 - acc: 0.9810 - val_loss: 0.3616 - val_acc: 0.8996\n",
"Epoch 25/50\n",
"1000/1000 [==============================] - 0s 37us/step - loss: 0.0777 - acc: 0.9950 - val_loss: 0.2863 - val_acc: 0.9197\n",
"1000/1000 [==============================] - 0s 54us/step - loss: 0.0884 - acc: 0.9890 - val_loss: 0.3453 - val_acc: 0.9072\n",
"Epoch 26/50\n",
"1000/1000 [==============================] - 0s 31us/step - loss: 0.0890 - acc: 0.9880 - val_loss: 0.3006 - val_acc: 0.9210\n",
"1000/1000 [==============================] - 0s 48us/step - loss: 0.0823 - acc: 0.9880 - val_loss: 0.3525 - val_acc: 0.9021\n",
"Epoch 27/50\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.0865 - acc: 0.9950 - val_loss: 0.2640 - val_acc: 0.9297\n",
"1000/1000 [==============================] - 0s 29us/step - loss: 0.0911 - acc: 0.9830 - val_loss: 0.3605 - val_acc: 0.9034\n",
"Epoch 28/50\n",
"1000/1000 [==============================] - 0s 33us/step - loss: 0.0751 - acc: 0.9920 - val_loss: 0.2812 - val_acc: 0.9260\n",
"1000/1000 [==============================] - 0s 58us/step - loss: 0.1099 - acc: 0.9790 - val_loss: 0.4343 - val_acc: 0.8770\n",
"Epoch 29/50\n",
"1000/1000 [==============================] - 0s 29us/step - loss: 0.0676 - acc: 0.9940 - val_loss: 0.2818 - val_acc: 0.9235\n",
"1000/1000 [==============================] - 0s 49us/step - loss: 0.1251 - acc: 0.9720 - val_loss: 0.3649 - val_acc: 0.9009\n",
"Epoch 30/50\n",
"1000/1000 [==============================] - 0s 34us/step - loss: 0.0701 - acc: 0.9920 - val_loss: 0.3042 - val_acc: 0.9134\n",
"1000/1000 [==============================] - 0s 47us/step - loss: 0.1068 - acc: 0.9740 - val_loss: 0.3727 - val_acc: 0.8946\n",
"Epoch 31/50\n",
"1000/1000 [==============================] - 0s 32us/step - loss: 0.0640 - acc: 0.9940 - val_loss: 0.3073 - val_acc: 0.9197\n",
"1000/1000 [==============================] - 0s 43us/step - loss: 0.0902 - acc: 0.9830 - val_loss: 0.3856 - val_acc: 0.8795\n",
"Epoch 32/50\n",
"1000/1000 [==============================] - 0s 28us/step - loss: 0.0592 - acc: 0.9970 - val_loss: 0.2980 - val_acc: 0.9260\n",
"1000/1000 [==============================] - 0s 47us/step - loss: 0.1281 - acc: 0.9630 - val_loss: 0.3476 - val_acc: 0.9072\n",
"Epoch 33/50\n",
"1000/1000 [==============================] - 0s 28us/step - loss: 0.0553 - acc: 0.9970 - val_loss: 0.2896 - val_acc: 0.9260\n",
"1000/1000 [==============================] - 0s 33us/step - loss: 0.0833 - acc: 0.9880 - val_loss: 0.3495 - val_acc: 0.9109\n",
"Epoch 34/50\n",
"1000/1000 [==============================] - 0s 32us/step - loss: 0.0514 - acc: 0.9970 - val_loss: 0.2756 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 48us/step - loss: 0.0858 - acc: 0.9880 - val_loss: 0.3507 - val_acc: 0.9084\n",
"Epoch 35/50\n",
"1000/1000 [==============================] - 0s 32us/step - loss: 0.0470 - acc: 0.9970 - val_loss: 0.2742 - val_acc: 0.9235\n",
"1000/1000 [==============================] - 0s 63us/step - loss: 0.0862 - acc: 0.9890 - val_loss: 0.3509 - val_acc: 0.9021\n",
"Epoch 36/50\n",
"1000/1000 [==============================] - 0s 34us/step - loss: 0.0451 - acc: 0.9960 - val_loss: 0.2678 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 87us/step - loss: 0.0698 - acc: 0.9890 - val_loss: 0.3407 - val_acc: 0.9059\n",
"Epoch 37/50\n",
"1000/1000 [==============================] - 0s 32us/step - loss: 0.0433 - acc: 0.9970 - val_loss: 0.2741 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 54us/step - loss: 0.0628 - acc: 0.9910 - val_loss: 0.3355 - val_acc: 0.9059\n",
"Epoch 38/50\n",
"1000/1000 [==============================] - 0s 30us/step - loss: 0.0403 - acc: 0.9970 - val_loss: 0.2619 - val_acc: 0.9297\n",
"1000/1000 [==============================] - 0s 51us/step - loss: 0.0602 - acc: 0.9890 - val_loss: 0.3490 - val_acc: 0.9046\n",
"Epoch 39/50\n",
"1000/1000 [==============================] - 0s 27us/step - loss: 0.0389 - acc: 0.9980 - val_loss: 0.2695 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 49us/step - loss: 0.0612 - acc: 0.9940 - val_loss: 0.3303 - val_acc: 0.9134\n",
"Epoch 40/50\n",
"1000/1000 [==============================] - 0s 28us/step - loss: 0.0372 - acc: 0.9980 - val_loss: 0.2786 - val_acc: 0.9247\n",
"1000/1000 [==============================] - 0s 39us/step - loss: 0.0664 - acc: 0.9880 - val_loss: 0.3398 - val_acc: 0.9084\n",
"Epoch 41/50\n",
"1000/1000 [==============================] - 0s 30us/step - loss: 0.0352 - acc: 0.9980 - val_loss: 0.2656 - val_acc: 0.9272\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.0663 - acc: 0.9890 - val_loss: 0.3624 - val_acc: 0.9021\n",
"Epoch 42/50\n",
"1000/1000 [==============================] - 0s 23us/step - loss: 0.0340 - acc: 0.9980 - val_loss: 0.2644 - val_acc: 0.9310\n",
"1000/1000 [==============================] - 0s 25us/step - loss: 0.0634 - acc: 0.9840 - val_loss: 0.3461 - val_acc: 0.9147\n",
"Epoch 43/50\n",
"1000/1000 [==============================] - 0s 26us/step - loss: 0.0338 - acc: 0.9990 - val_loss: 0.2694 - val_acc: 0.9235\n",
"1000/1000 [==============================] - 0s 35us/step - loss: 0.0532 - acc: 0.9920 - val_loss: 0.3476 - val_acc: 0.9134\n",
"Epoch 44/50\n",
"1000/1000 [==============================] - 0s 34us/step - loss: 0.0332 - acc: 0.9980 - val_loss: 0.2747 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 27us/step - loss: 0.0482 - acc: 0.9950 - val_loss: 0.3428 - val_acc: 0.9159\n",
"Epoch 45/50\n",
"1000/1000 [==============================] - 0s 28us/step - loss: 0.0328 - acc: 0.9990 - val_loss: 0.2643 - val_acc: 0.9235\n",
"1000/1000 [==============================] - 0s 31us/step - loss: 0.0445 - acc: 0.9950 - val_loss: 0.3268 - val_acc: 0.9147\n",
"Epoch 46/50\n",
"1000/1000 [==============================] - 0s 35us/step - loss: 0.0317 - acc: 0.9990 - val_loss: 0.2646 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.0435 - acc: 0.9970 - val_loss: 0.3308 - val_acc: 0.9059\n",
"Epoch 47/50\n",
"1000/1000 [==============================] - 0s 28us/step - loss: 0.0304 - acc: 0.9990 - val_loss: 0.2674 - val_acc: 0.9297\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.0405 - acc: 0.9970 - val_loss: 0.3252 - val_acc: 0.9172\n",
"Epoch 48/50\n",
"1000/1000 [==============================] - 0s 36us/step - loss: 0.0305 - acc: 0.9990 - val_loss: 0.2666 - val_acc: 0.9285\n",
"1000/1000 [==============================] - 0s 27us/step - loss: 0.0426 - acc: 0.9950 - val_loss: 0.3404 - val_acc: 0.9122\n",
"Epoch 49/50\n",
"1000/1000 [==============================] - 0s 31us/step - loss: 0.0292 - acc: 0.9990 - val_loss: 0.2636 - val_acc: 0.9310\n",
"1000/1000 [==============================] - 0s 32us/step - loss: 0.0430 - acc: 0.9940 - val_loss: 0.3306 - val_acc: 0.9122\n",
"Epoch 50/50\n",
"1000/1000 [==============================] - 0s 34us/step - loss: 0.0284 - acc: 0.9990 - val_loss: 0.2642 - val_acc: 0.9247\n"
"1000/1000 [==============================] - 0s 30us/step - loss: 0.0421 - acc: 0.9960 - val_loss: 0.3342 - val_acc: 0.9109\n"
]
}
],
Expand All @@ -478,15 +485,16 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[2.3307739e-04 2.4624971e-05 2.5646106e-04 1.7080925e-03 4.5834472e-06\n",
" 5.5023044e-01 9.7322202e-04 5.3183449e-04 2.3377737e-05 4.0023492e-04]\n"
"[7.22454715e-05 1.40154385e-04 1.14361326e-04 1.87753886e-03\n",
" 5.77343690e-06 9.97086465e-01 3.70808011e-05 4.92060144e-06\n",
" 3.18983046e-04 3.42447049e-04]\n"
]
}
],
Expand All @@ -508,7 +516,7 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -534,16 +542,17 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"predicted probabilities:\n",
"[2.3307739e-04 2.4624971e-05 2.5646106e-04 1.7080925e-03 4.5834472e-06\n",
" 5.5023044e-01 9.7322202e-04 5.3183449e-04 2.3377737e-05 4.0023492e-04]\n",
"[7.22454715e-05 1.40154385e-04 1.14361326e-04 1.87753886e-03\n",
" 5.77343690e-06 9.97086465e-01 3.70808011e-05 4.92060144e-06\n",
" 3.18983046e-04 3.42447049e-04]\n",
"predicted category 5\n",
"true probabilities:\n",
"[0. 0. 0. 0. 0. 1. 0. 0. 0. 0.]\n",
Expand Down Expand Up @@ -578,7 +587,7 @@
},
{
"cell_type": "code",
"execution_count": 17,
"execution_count": 14,
"metadata": {},
"outputs": [
{
Expand All @@ -591,10 +600,10 @@
{
"data": {
"text/plain": [
"0.9247176913425345"
"0.9109159347553325"
]
},
"execution_count": 17,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -619,7 +628,7 @@
"source": [
"You should obtain an accuracy around 91%, similar to the one we had obtained in the same conditions with scikit-learn. \n",
"\n",
"Please note that the result is not deterministic, so the accuracy will vary every time you train the network. I usually get an accuracy between 90 and 93%. \n",
"Please note that the result is not deterministic, so the accuracy will vary every time you train the network. I usually get an accuracy between 90 and 93%, but I sometimes get a value as low as 87%. \n",
"\n",
"Please repeat the exercise starting from the creation of the neural network to see what happens. \n",
"\n",
Expand Down
Loading

0 comments on commit 1b4b81d

Please sign in to comment.