neural networks part ii feed-forward neural networks

41
Neural Networks Part II Feed-Forward Neural Networks

Upload: others

Post on 04-May-2022

43 views

Category:

Documents


0 download

TRANSCRIPT

Page 1: Neural Networks Part II Feed-Forward Neural Networks

Neural Networks Part II

Feed-Forward Neural Networks

Page 2: Neural Networks Part II Feed-Forward Neural Networks

Previously on CSCI 4622

2

The Perceptron as a NN: Can learn linearly separable things like OR, AND, NAND, etc

The Multilevel Perceptron: Can learn non-linearly separable things like XOR

Layer 1 to 2:

Layer 2 to 3:

y =

⇢0 if wTx+ b 01 if wTx+ b > 0

<latexit sha1_base64="d3Pw5q+8jIlGM4xpzrnffbc+yco=">AAACgXicjVFNb9QwEHVCgRK+FjhyGbEtQiAFZy8FIVAlLhyL1KWV1svK8U52rTpOsCfQKMoP4W9x49eAd5sDtBwYydLTe/PlN3lttCfOf0bxtZ3rN27u3kpu37l77/7owcNPvmqcwqmqTOVOc+nRaItT0mTwtHYoy9zgSX72fqOffEXndWWPqa1xXsqV1YVWkgK1GH3fS8RaUtf28BaEwYJEl4gcV9p20jnZ9p1zfcLhKQjCc3JlB7qAHjqRF/Ct/3y8Bec9vIAcNh2+AAchIMn+r+Qd8ESgXQ7TEuH0ak1psrcYjXnKtwFXQTaAMRviaDH6IZaVakq0pIz0fpbxmuahL2llMHRuPNZSnckVzgK0skQ/77YW9rAfmCUUlQvPEmzZPys6WXrflnnILCWt/WVtQ/5LmzVUvJp32tYNoVUXg4rGAFWwuQcstUNFpg1AKqfDrqDW0klF4WpJMCG7/OWrYDpJX6f842R8eDC4scsesyfsGcvYATtkH9gRmzLFfkX7URq9jHfi5zGPJxepcTTUPGJ/RfzmN2SzvSc=</latexit><latexit sha1_base64="d3Pw5q+8jIlGM4xpzrnffbc+yco=">AAACgXicjVFNb9QwEHVCgRK+FjhyGbEtQiAFZy8FIVAlLhyL1KWV1svK8U52rTpOsCfQKMoP4W9x49eAd5sDtBwYydLTe/PlN3lttCfOf0bxtZ3rN27u3kpu37l77/7owcNPvmqcwqmqTOVOc+nRaItT0mTwtHYoy9zgSX72fqOffEXndWWPqa1xXsqV1YVWkgK1GH3fS8RaUtf28BaEwYJEl4gcV9p20jnZ9p1zfcLhKQjCc3JlB7qAHjqRF/Ct/3y8Bec9vIAcNh2+AAchIMn+r+Qd8ESgXQ7TEuH0ak1psrcYjXnKtwFXQTaAMRviaDH6IZaVakq0pIz0fpbxmuahL2llMHRuPNZSnckVzgK0skQ/77YW9rAfmCUUlQvPEmzZPys6WXrflnnILCWt/WVtQ/5LmzVUvJp32tYNoVUXg4rGAFWwuQcstUNFpg1AKqfDrqDW0klF4WpJMCG7/OWrYDpJX6f842R8eDC4scsesyfsGcvYATtkH9gRmzLFfkX7URq9jHfi5zGPJxepcTTUPGJ/RfzmN2SzvSc=</latexit><latexit sha1_base64="d3Pw5q+8jIlGM4xpzrnffbc+yco=">AAACgXicjVFNb9QwEHVCgRK+FjhyGbEtQiAFZy8FIVAlLhyL1KWV1svK8U52rTpOsCfQKMoP4W9x49eAd5sDtBwYydLTe/PlN3lttCfOf0bxtZ3rN27u3kpu37l77/7owcNPvmqcwqmqTOVOc+nRaItT0mTwtHYoy9zgSX72fqOffEXndWWPqa1xXsqV1YVWkgK1GH3fS8RaUtf28BaEwYJEl4gcV9p20jnZ9p1zfcLhKQjCc3JlB7qAHjqRF/Ct/3y8Bec9vIAcNh2+AAchIMn+r+Qd8ESgXQ7TEuH0ak1psrcYjXnKtwFXQTaAMRviaDH6IZaVakq0pIz0fpbxmuahL2llMHRuPNZSnckVzgK0skQ/77YW9rAfmCUUlQvPEmzZPys6WXrflnnILCWt/WVtQ/5LmzVUvJp32tYNoVUXg4rGAFWwuQcstUNFpg1AKqfDrqDW0klF4WpJMCG7/OWrYDpJX6f842R8eDC4scsesyfsGcvYATtkH9gRmzLFfkX7URq9jHfi5zGPJxepcTTUPGJ/RfzmN2SzvSc=</latexit>

x1 0 1 0 1x2 0 0 1 1

x1 OR x2 0 1 1 1<latexit sha1_base64="UM8xfazAVtlDjQbsqgXmCAzAUIo=">AAACd3icbVHJTsMwEHXCHrYCx14sWhCnKumF5YTEhRuLKCA1VeW409bCcSJ7gqhCvoF/48aHcOGE2wYBhZFGfn5vxjOeiVIpDPr+m+POzS8sLi2veKtr6xubla3tW5NkmkOLJzLR9xEzIIWCFgqUcJ9qYHEk4S56OBvrd4+gjUjUDY5S6MRsoERfcIaW6lZevHrdCyMYCJUzrdmoyPkzt1Z4T92A7lPf+vcZhl44HNeiVm6W9FfIjGyvCE+ImNOLa1rQ74TgVwKoXlna9tKt1PyGPzH6FwQlqJHSLruV17CX8CwGhVwyY9qBn2LHPoiCSyi8MDOQMv7ABtC2ULEYTCefDK6ge5bp0X6irSukE/ZnRs5iY0ZxZCNjhkMzq43J/7R2hv2jTi5UmiEoPi3UzyTFhI63QHtCA0c5soBxLWyvlA+ZZhztrjw7hGD2y39Bq9k4bvhXzdrpSTmNZVIlu+SABOSQnJJzcklahJN3p+rUnT3nw6XuvnswDXWdMmeH/DI3+ATd3bFz</latexit><latexit sha1_base64="UM8xfazAVtlDjQbsqgXmCAzAUIo=">AAACd3icbVHJTsMwEHXCHrYCx14sWhCnKumF5YTEhRuLKCA1VeW409bCcSJ7gqhCvoF/48aHcOGE2wYBhZFGfn5vxjOeiVIpDPr+m+POzS8sLi2veKtr6xubla3tW5NkmkOLJzLR9xEzIIWCFgqUcJ9qYHEk4S56OBvrd4+gjUjUDY5S6MRsoERfcIaW6lZevHrdCyMYCJUzrdmoyPkzt1Z4T92A7lPf+vcZhl44HNeiVm6W9FfIjGyvCE+ImNOLa1rQ74TgVwKoXlna9tKt1PyGPzH6FwQlqJHSLruV17CX8CwGhVwyY9qBn2LHPoiCSyi8MDOQMv7ABtC2ULEYTCefDK6ge5bp0X6irSukE/ZnRs5iY0ZxZCNjhkMzq43J/7R2hv2jTi5UmiEoPi3UzyTFhI63QHtCA0c5soBxLWyvlA+ZZhztrjw7hGD2y39Bq9k4bvhXzdrpSTmNZVIlu+SABOSQnJJzcklahJN3p+rUnT3nw6XuvnswDXWdMmeH/DI3+ATd3bFz</latexit><latexit sha1_base64="UM8xfazAVtlDjQbsqgXmCAzAUIo=">AAACd3icbVHJTsMwEHXCHrYCx14sWhCnKumF5YTEhRuLKCA1VeW409bCcSJ7gqhCvoF/48aHcOGE2wYBhZFGfn5vxjOeiVIpDPr+m+POzS8sLi2veKtr6xubla3tW5NkmkOLJzLR9xEzIIWCFgqUcJ9qYHEk4S56OBvrd4+gjUjUDY5S6MRsoERfcIaW6lZevHrdCyMYCJUzrdmoyPkzt1Z4T92A7lPf+vcZhl44HNeiVm6W9FfIjGyvCE+ImNOLa1rQ74TgVwKoXlna9tKt1PyGPzH6FwQlqJHSLruV17CX8CwGhVwyY9qBn2LHPoiCSyi8MDOQMv7ABtC2ULEYTCefDK6ge5bp0X6irSukE/ZnRs5iY0ZxZCNjhkMzq43J/7R2hv2jTi5UmiEoPi3UzyTFhI63QHtCA0c5soBxLWyvlA+ZZhztrjw7hGD2y39Bq9k4bvhXzdrpSTmNZVIlu+SABOSQnJJzcklahJN3p+rUnT3nw6XuvnswDXWdMmeH/DI3+ATd3bFz</latexit>

z2 = W 1x1 + b1

a2 = I(z2 > 0) = I(W 1x1 + b1 > 0)<latexit sha1_base64="bFJTMLtnFcASWTm+WeQFNfaTtV8=">AAACiXicdVHLTttAFB275VFDIYUlm1ETEBUisrPgJYGQ2LQ7KjUNUpxE48l1GDEeWzPXFanlj+GX2PE3jI2r8mivNNLROec+5t4ok8Kg7z847rv3C4tLyx+8ldWPa+utTxs/TZprDn2eylRfRcyAFAr6KFDCVaaBJZGEQXRzUemDX6CNSNUPnGcwSthMiVhwhpaatO46HS+MYCZUwbRm87LQXJZeEUYx/V2Oe3TndIcOxkFN3JbjgO7RGkcVDsMnJ/vjDBFuUSfFt3L3b4kz6n+hp8+1/xasrF4IatpM43U6k1bb7/p10LcgaECbNHE5ad2H05TnCSjkkhkzDPwMR7YgCi6h9MLcQMb4DZvB0ELFEjCjol5lSbctM6Vxqu1TSGv2eUbBEmPmSWSdCcNr81qryH9pwxzjo1EhVJYjKP7UKM4lxZRWd6FToYGjnFvAuBZ2VsqvmWYc7fU8u4Tg9Zffgn6ve9z1v/fa5yfNNpbJFvlMdklADsk5+UouSZ9wZ8nZdw6cQ3fV7blHbuN1nSZnk7wI9+IRCR69hg==</latexit><latexit sha1_base64="bFJTMLtnFcASWTm+WeQFNfaTtV8=">AAACiXicdVHLTttAFB275VFDIYUlm1ETEBUisrPgJYGQ2LQ7KjUNUpxE48l1GDEeWzPXFanlj+GX2PE3jI2r8mivNNLROec+5t4ok8Kg7z847rv3C4tLyx+8ldWPa+utTxs/TZprDn2eylRfRcyAFAr6KFDCVaaBJZGEQXRzUemDX6CNSNUPnGcwSthMiVhwhpaatO46HS+MYCZUwbRm87LQXJZeEUYx/V2Oe3TndIcOxkFN3JbjgO7RGkcVDsMnJ/vjDBFuUSfFt3L3b4kz6n+hp8+1/xasrF4IatpM43U6k1bb7/p10LcgaECbNHE5ad2H05TnCSjkkhkzDPwMR7YgCi6h9MLcQMb4DZvB0ELFEjCjol5lSbctM6Vxqu1TSGv2eUbBEmPmSWSdCcNr81qryH9pwxzjo1EhVJYjKP7UKM4lxZRWd6FToYGjnFvAuBZ2VsqvmWYc7fU8u4Tg9Zffgn6ve9z1v/fa5yfNNpbJFvlMdklADsk5+UouSZ9wZ8nZdw6cQ3fV7blHbuN1nSZnk7wI9+IRCR69hg==</latexit><latexit sha1_base64="bFJTMLtnFcASWTm+WeQFNfaTtV8=">AAACiXicdVHLTttAFB275VFDIYUlm1ETEBUisrPgJYGQ2LQ7KjUNUpxE48l1GDEeWzPXFanlj+GX2PE3jI2r8mivNNLROec+5t4ok8Kg7z847rv3C4tLyx+8ldWPa+utTxs/TZprDn2eylRfRcyAFAr6KFDCVaaBJZGEQXRzUemDX6CNSNUPnGcwSthMiVhwhpaatO46HS+MYCZUwbRm87LQXJZeEUYx/V2Oe3TndIcOxkFN3JbjgO7RGkcVDsMnJ/vjDBFuUSfFt3L3b4kz6n+hp8+1/xasrF4IatpM43U6k1bb7/p10LcgaECbNHE5ad2H05TnCSjkkhkzDPwMR7YgCi6h9MLcQMb4DZvB0ELFEjCjol5lSbctM6Vxqu1TSGv2eUbBEmPmSWSdCcNr81qryH9pwxzjo1EhVJYjKP7UKM4lxZRWd6FToYGjnFvAuBZ2VsqvmWYc7fU8u4Tg9Zffgn6ve9z1v/fa5yfNNpbJFvlMdklADsk5+UouSZ9wZ8nZdw6cQ3fV7blHbuN1nSZnk7wI9+IRCR69hg==</latexit>

z3 = W 2a2 + b2

y = I(z3 > 0) = I(W 2a2 + b2 > 0)<latexit sha1_base64="ASe3O1wbpXNcLSB0W+Iovcd8mnM=">AAACiHicdVFdb9MwFHUCjJEBK/DIi0XLNASqkiLYhzQ0wQu8DYnSSXVbOe5Na81xIvsGLUTZb+E38ca/wckiMTa4kqWjc8798L1xrqTFMPzl+bdu39m4u3kv2Lr/4OF279HjrzYrjICxyFRmTmNuQUkNY5So4DQ3wNNYwSQ++9Dok29grMz0FyxzmKV8pWUiBUdHLXo/BoOAxbCSuuLG8LKujFB1ULE4od/r+Wu6c7RDJ/NRS/B6PqIvaYvjBjMWsDXHqqwvWiNDOEeTVp/q3T8V3tHwBT26qv23XmMNGOhlN0wwGCx6/XAYtkFvgqgDfdLFyaL3ky0zUaSgUShu7TQKc5y5giiFgjpghYWcizO+gqmDmqdgZ1W7yZo+d8ySJplxTyNt2asZFU+tLdPYOVOOa3tda8h/adMCk/1ZJXVeIGhx2SgpFMWMNmehS2lAoCod4MJINysVa264QHe8wC0huv7lm2A8Gh4Mw8+j/vFht41N8pQ8I7skInvkmHwkJ2RMhLfhvfLeeG/9LT/y9/yDS6vvdTlPyF/hv/8NMF69sQ==</latexit><latexit sha1_base64="ASe3O1wbpXNcLSB0W+Iovcd8mnM=">AAACiHicdVFdb9MwFHUCjJEBK/DIi0XLNASqkiLYhzQ0wQu8DYnSSXVbOe5Na81xIvsGLUTZb+E38ca/wckiMTa4kqWjc8798L1xrqTFMPzl+bdu39m4u3kv2Lr/4OF279HjrzYrjICxyFRmTmNuQUkNY5So4DQ3wNNYwSQ++9Dok29grMz0FyxzmKV8pWUiBUdHLXo/BoOAxbCSuuLG8LKujFB1ULE4od/r+Wu6c7RDJ/NRS/B6PqIvaYvjBjMWsDXHqqwvWiNDOEeTVp/q3T8V3tHwBT26qv23XmMNGOhlN0wwGCx6/XAYtkFvgqgDfdLFyaL3ky0zUaSgUShu7TQKc5y5giiFgjpghYWcizO+gqmDmqdgZ1W7yZo+d8ySJplxTyNt2asZFU+tLdPYOVOOa3tda8h/adMCk/1ZJXVeIGhx2SgpFMWMNmehS2lAoCod4MJINysVa264QHe8wC0huv7lm2A8Gh4Mw8+j/vFht41N8pQ8I7skInvkmHwkJ2RMhLfhvfLeeG/9LT/y9/yDS6vvdTlPyF/hv/8NMF69sQ==</latexit><latexit sha1_base64="ASe3O1wbpXNcLSB0W+Iovcd8mnM=">AAACiHicdVFdb9MwFHUCjJEBK/DIi0XLNASqkiLYhzQ0wQu8DYnSSXVbOe5Na81xIvsGLUTZb+E38ca/wckiMTa4kqWjc8798L1xrqTFMPzl+bdu39m4u3kv2Lr/4OF279HjrzYrjICxyFRmTmNuQUkNY5So4DQ3wNNYwSQ++9Dok29grMz0FyxzmKV8pWUiBUdHLXo/BoOAxbCSuuLG8LKujFB1ULE4od/r+Wu6c7RDJ/NRS/B6PqIvaYvjBjMWsDXHqqwvWiNDOEeTVp/q3T8V3tHwBT26qv23XmMNGOhlN0wwGCx6/XAYtkFvgqgDfdLFyaL3ky0zUaSgUShu7TQKc5y5giiFgjpghYWcizO+gqmDmqdgZ1W7yZo+d8ySJplxTyNt2asZFU+tLdPYOVOOa3tda8h/adMCk/1ZJXVeIGhx2SgpFMWMNmehS2lAoCod4MJINysVa264QHe8wC0huv7lm2A8Gh4Mw8+j/vFht41N8pQ8I7skInvkmHwkJ2RMhLfhvfLeeG/9LT/y9/yDS6vvdTlPyF/hv/8NMF69sQ==</latexit>

b

b.V

Page 3: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

3

String together lots of perceptron units

Page 4: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

4

String together lots of perceptron units

Input Layer Hidden Layers Output Layer

FULLY- connect

FEED - FORWARD

MLP

=

-=

Page 5: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

5

String together lots of perceptron units

Inputs, activities, activations, and outputs all live on nodes (neurons)

z = Wattsa= gcz )

Page 6: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

6

String together lots of perceptron units

For simplicity: Think of inputs and outputs as activations too

Page 7: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

7

String together lots of perceptron units

Each transition between layers has associated set of weights an biases

W 1<latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit>

W 2<latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit>

W 3<latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit>

W 4<latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit>

b1<latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit>

b2<latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit>

b3<latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit>

b4<latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit>

Page 8: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

8

String together lots of perceptron units

Store the weights connecting layers and in the matrix

is the weight from node j in layer to node i in layer

W 1<latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit>

W 2<latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit>

W 3<latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit>

W 4<latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit>

b1<latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit>

b2<latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit>

b3<latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit>

b4<latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit>

`<latexit sha1_base64="Y0ZH/VwiSzzCk6elfmzVvy2aatg=">AAAB73icbVBNT8JAEN3iF9Yv1KOXjWDiiRQu6I3Ei0dMrGCgIdtlCht2t83u1oQ0/AovHtR49e9489+4hR4UfMkkL+/NZGZemHCmjed9O6WNza3tnfKuu7d/cHhUOT550HGqKPg05rHqhUQDZxJ8wwyHXqKAiJBDN5ze5H73CZRmsbw3swQCQcaSRYwSY6XHmjsAzt2aO6xUvbq3AF4njYJUUYHOsPI1GMU0FSAN5UTrfsNLTJARZRjlMHcHqYaE0CkZQ99SSQToIFscPMcXVhnhKFa2pMEL9fdERoTWMxHaTkHMRK96ufif109NdBVkTCapAUmXi6KUYxPj/Hs8Ygqo4TNLCFXM3orphChCjc0oD6Gx+vI68Zv167p316y2W0UaZXSGztElaqAWaqNb1EE+okigZ/SK3hzlvDjvzseyteQUM6foD5zPH8+jjpk=</latexit><latexit sha1_base64="Y0ZH/VwiSzzCk6elfmzVvy2aatg=">AAAB73icbVBNT8JAEN3iF9Yv1KOXjWDiiRQu6I3Ei0dMrGCgIdtlCht2t83u1oQ0/AovHtR49e9489+4hR4UfMkkL+/NZGZemHCmjed9O6WNza3tnfKuu7d/cHhUOT550HGqKPg05rHqhUQDZxJ8wwyHXqKAiJBDN5ze5H73CZRmsbw3swQCQcaSRYwSY6XHmjsAzt2aO6xUvbq3AF4njYJUUYHOsPI1GMU0FSAN5UTrfsNLTJARZRjlMHcHqYaE0CkZQ99SSQToIFscPMcXVhnhKFa2pMEL9fdERoTWMxHaTkHMRK96ufif109NdBVkTCapAUmXi6KUYxPj/Hs8Ygqo4TNLCFXM3orphChCjc0oD6Gx+vI68Zv167p316y2W0UaZXSGztElaqAWaqNb1EE+okigZ/SK3hzlvDjvzseyteQUM6foD5zPH8+jjpk=</latexit><latexit sha1_base64="Y0ZH/VwiSzzCk6elfmzVvy2aatg=">AAAB73icbVBNT8JAEN3iF9Yv1KOXjWDiiRQu6I3Ei0dMrGCgIdtlCht2t83u1oQ0/AovHtR49e9489+4hR4UfMkkL+/NZGZemHCmjed9O6WNza3tnfKuu7d/cHhUOT550HGqKPg05rHqhUQDZxJ8wwyHXqKAiJBDN5ze5H73CZRmsbw3swQCQcaSRYwSY6XHmjsAzt2aO6xUvbq3AF4njYJUUYHOsPI1GMU0FSAN5UTrfsNLTJARZRjlMHcHqYaE0CkZQ99SSQToIFscPMcXVhnhKFa2pMEL9fdERoTWMxHaTkHMRK96ufif109NdBVkTCapAUmXi6KUYxPj/Hs8Ygqo4TNLCFXM3orphChCjc0oD6Gx+vI68Zv167p316y2W0UaZXSGztElaqAWaqNb1EE+okigZ/SK3hzlvDjvzseyteQUM6foD5zPH8+jjpk=</latexit> `+ 1

<latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit>

W `<latexit sha1_base64="yb0RsZi969p0mBUrQpX+/snEeGM=">AAAB8XicbVBNT8JAEN3iF9Yv1KOXjWDiibRc0BuJF4+YWCEplWyXKWzYbpvdrQlp+BlePKjx6r/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMUvBowhPZD4kCzgR4mmkO/VQCiUMOvXB6U/i9J5CKJeJez1IIYjIWLGKUaCP5Dbv3OADO7YY9rNWdprMAXiduSeqoRHdY+xqMEprFIDTlRCnfdVId5ERqRjnM7UGmICV0SsbgGypIDCrIFyfP8YVRRjhKpCmh8UL9PZGTWKlZHJrOmOiJWvUK8T/Pz3R0FeRMpJkGQZeLooxjneDifzxiEqjmM0MIlczciumESEK1SakIwV19eZ14reZ107lr1TvtMo0qOkPn6BK5qI066BZ1kYcoStAzekVvlrZerHfrY9lascqZU/QH1ucPMO6PYg==</latexit><latexit sha1_base64="yb0RsZi969p0mBUrQpX+/snEeGM=">AAAB8XicbVBNT8JAEN3iF9Yv1KOXjWDiibRc0BuJF4+YWCEplWyXKWzYbpvdrQlp+BlePKjx6r/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMUvBowhPZD4kCzgR4mmkO/VQCiUMOvXB6U/i9J5CKJeJez1IIYjIWLGKUaCP5Dbv3OADO7YY9rNWdprMAXiduSeqoRHdY+xqMEprFIDTlRCnfdVId5ERqRjnM7UGmICV0SsbgGypIDCrIFyfP8YVRRjhKpCmh8UL9PZGTWKlZHJrOmOiJWvUK8T/Pz3R0FeRMpJkGQZeLooxjneDifzxiEqjmM0MIlczciumESEK1SakIwV19eZ14reZ107lr1TvtMo0qOkPn6BK5qI066BZ1kYcoStAzekVvlrZerHfrY9lascqZU/QH1ucPMO6PYg==</latexit><latexit sha1_base64="yb0RsZi969p0mBUrQpX+/snEeGM=">AAAB8XicbVBNT8JAEN3iF9Yv1KOXjWDiibRc0BuJF4+YWCEplWyXKWzYbpvdrQlp+BlePKjx6r/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMUvBowhPZD4kCzgR4mmkO/VQCiUMOvXB6U/i9J5CKJeJez1IIYjIWLGKUaCP5Dbv3OADO7YY9rNWdprMAXiduSeqoRHdY+xqMEprFIDTlRCnfdVId5ERqRjnM7UGmICV0SsbgGypIDCrIFyfP8YVRRjhKpCmh8UL9PZGTWKlZHJrOmOiJWvUK8T/Pz3R0FeRMpJkGQZeLooxjneDifzxiEqjmM0MIlczciumESEK1SakIwV19eZ14reZ107lr1TvtMo0qOkPn6BK5qI066BZ1kYcoStAzekVvlrZerHfrY9lascqZU/QH1ucPMO6PYg==</latexit>

w`ij

<latexit sha1_base64="Mij8XPNZNIXvly4jNjwN5Fmlo44=">AAAB+HicbVC7TsNAEDzzDOZloKQ5kSBRRXaaQBeJhjJImERKjHW+rJMj54fuzkGRlT+hoQBEy6fQ8TecExeQMNJKo5ld7e4EKWdS2fa3sba+sbm1Xdkxd/f2Dw6to+N7mWSCgksTnohuQCRwFoOrmOLQTQWQKODQCcbXhd+ZgJAsie/UNAUvIsOYhYwSpSXfsmrmk5+zx9lDHzg3a6ZvVe26PQdeJU5JqqhE27e++oOEZhHEinIiZc+xU+XlRChGOczMfiYhJXRMhtDTNCYRSC+fXz7D51oZ4DARumKF5+rviZxEUk6jQHdGRI3ksleI/3m9TIWXXs7iNFMQ08WiMONYJbiIAQ+YAKr4VBNCBdO3YjoiglClwypCcJZfXiVuo35Vt28b1VazTKOCTtEZukAOaqIWukFt5CKKJugZvaI3IzdejHfjY9G6ZpQzJ+gPjM8f8VaSDw==</latexit><latexit sha1_base64="Mij8XPNZNIXvly4jNjwN5Fmlo44=">AAAB+HicbVC7TsNAEDzzDOZloKQ5kSBRRXaaQBeJhjJImERKjHW+rJMj54fuzkGRlT+hoQBEy6fQ8TecExeQMNJKo5ld7e4EKWdS2fa3sba+sbm1Xdkxd/f2Dw6to+N7mWSCgksTnohuQCRwFoOrmOLQTQWQKODQCcbXhd+ZgJAsie/UNAUvIsOYhYwSpSXfsmrmk5+zx9lDHzg3a6ZvVe26PQdeJU5JqqhE27e++oOEZhHEinIiZc+xU+XlRChGOczMfiYhJXRMhtDTNCYRSC+fXz7D51oZ4DARumKF5+rviZxEUk6jQHdGRI3ksleI/3m9TIWXXs7iNFMQ08WiMONYJbiIAQ+YAKr4VBNCBdO3YjoiglClwypCcJZfXiVuo35Vt28b1VazTKOCTtEZukAOaqIWukFt5CKKJugZvaI3IzdejHfjY9G6ZpQzJ+gPjM8f8VaSDw==</latexit><latexit sha1_base64="Mij8XPNZNIXvly4jNjwN5Fmlo44=">AAAB+HicbVC7TsNAEDzzDOZloKQ5kSBRRXaaQBeJhjJImERKjHW+rJMj54fuzkGRlT+hoQBEy6fQ8TecExeQMNJKo5ld7e4EKWdS2fa3sba+sbm1Xdkxd/f2Dw6to+N7mWSCgksTnohuQCRwFoOrmOLQTQWQKODQCcbXhd+ZgJAsie/UNAUvIsOYhYwSpSXfsmrmk5+zx9lDHzg3a6ZvVe26PQdeJU5JqqhE27e++oOEZhHEinIiZc+xU+XlRChGOczMfiYhJXRMhtDTNCYRSC+fXz7D51oZ4DARumKF5+rviZxEUk6jQHdGRI3ksleI/3m9TIWXXs7iNFMQ08WiMONYJbiIAQ+YAKr4VBNCBdO3YjoiglClwypCcJZfXiVuo35Vt28b1VazTKOCTtEZukAOaqIWukFt5CKKJugZvaI3IzdejHfjY9G6ZpQzJ+gPjM8f8VaSDw==</latexit>

`<latexit sha1_base64="Y0ZH/VwiSzzCk6elfmzVvy2aatg=">AAAB73icbVBNT8JAEN3iF9Yv1KOXjWDiiRQu6I3Ei0dMrGCgIdtlCht2t83u1oQ0/AovHtR49e9489+4hR4UfMkkL+/NZGZemHCmjed9O6WNza3tnfKuu7d/cHhUOT550HGqKPg05rHqhUQDZxJ8wwyHXqKAiJBDN5ze5H73CZRmsbw3swQCQcaSRYwSY6XHmjsAzt2aO6xUvbq3AF4njYJUUYHOsPI1GMU0FSAN5UTrfsNLTJARZRjlMHcHqYaE0CkZQ99SSQToIFscPMcXVhnhKFa2pMEL9fdERoTWMxHaTkHMRK96ufif109NdBVkTCapAUmXi6KUYxPj/Hs8Ygqo4TNLCFXM3orphChCjc0oD6Gx+vI68Zv167p316y2W0UaZXSGztElaqAWaqNb1EE+okigZ/SK3hzlvDjvzseyteQUM6foD5zPH8+jjpk=</latexit><latexit sha1_base64="Y0ZH/VwiSzzCk6elfmzVvy2aatg=">AAAB73icbVBNT8JAEN3iF9Yv1KOXjWDiiRQu6I3Ei0dMrGCgIdtlCht2t83u1oQ0/AovHtR49e9489+4hR4UfMkkL+/NZGZemHCmjed9O6WNza3tnfKuu7d/cHhUOT550HGqKPg05rHqhUQDZxJ8wwyHXqKAiJBDN5ze5H73CZRmsbw3swQCQcaSRYwSY6XHmjsAzt2aO6xUvbq3AF4njYJUUYHOsPI1GMU0FSAN5UTrfsNLTJARZRjlMHcHqYaE0CkZQ99SSQToIFscPMcXVhnhKFa2pMEL9fdERoTWMxHaTkHMRK96ufif109NdBVkTCapAUmXi6KUYxPj/Hs8Ygqo4TNLCFXM3orphChCjc0oD6Gx+vI68Zv167p316y2W0UaZXSGztElaqAWaqNb1EE+okigZ/SK3hzlvDjvzseyteQUM6foD5zPH8+jjpk=</latexit><latexit sha1_base64="Y0ZH/VwiSzzCk6elfmzVvy2aatg=">AAAB73icbVBNT8JAEN3iF9Yv1KOXjWDiiRQu6I3Ei0dMrGCgIdtlCht2t83u1oQ0/AovHtR49e9489+4hR4UfMkkL+/NZGZemHCmjed9O6WNza3tnfKuu7d/cHhUOT550HGqKPg05rHqhUQDZxJ8wwyHXqKAiJBDN5ze5H73CZRmsbw3swQCQcaSRYwSY6XHmjsAzt2aO6xUvbq3AF4njYJUUYHOsPI1GMU0FSAN5UTrfsNLTJARZRjlMHcHqYaE0CkZQ99SSQToIFscPMcXVhnhKFa2pMEL9fdERoTWMxHaTkHMRK96ufif109NdBVkTCapAUmXi6KUYxPj/Hs8Ygqo4TNLCFXM3orphChCjc0oD6Gx+vI68Zv167p316y2W0UaZXSGztElaqAWaqNb1EE+okigZ/SK3hzlvDjvzseyteQUM6foD5zPH8+jjpk=</latexit> `+ 1

<latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit>

yZ3 z3=¥2§2+ bhT T# rows

# cols

"

IN± :

= =

C)= = = =

Page 9: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

9

String together lots of perceptron units

Store the biases for layer in the vector

is the bias feeding into node i in layer

W 1<latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit>

W 2<latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit>

W 3<latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit>

W 4<latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit>

b1<latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit>

b2<latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit>

b3<latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit>

b4<latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit>

`+ 1<latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit>

`+ 1<latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit><latexit sha1_base64="2STyL1FgbL604gVhhkrS5WsWITk=">AAAB8XicbVBNS8NAEN3Urxq/qh69LLaCIJSkl+qt4MVjBWMLaSib7aZdutmE3YlQQn+GFw8qXv033vw3btoctPXBwOO9GWbmhangGhzn26psbG5t71R37b39g8Oj2vHJo04yRZlHE5Gofkg0E1wyDzgI1k8VI3EoWC+c3hZ+74kpzRP5ALOUBTEZSx5xSsBIfsMeMCGuXLthD2t1p+ksgNeJW5I6KtEd1r4Go4RmMZNABdHad50Ugpwo4FSwuT3INEsJnZIx8w2VJGY6yBcnz/GFUUY4SpQpCXih/p7ISaz1LA5NZ0xgole9QvzP8zOIroOcyzQDJulyUZQJDAku/scjrhgFMTOEUMXNrZhOiCIUTEpFCO7qy+vEazVvms59q95pl2lU0Rk6R5fIRW3UQXeoizxEUYKe0St6s8B6sd6tj2VrxSpnTtEfWJ8/qXuPCQ==</latexit>

b`<latexit sha1_base64="KNyg8JYx18AjnBZEjNu+MVS3Dvs=">AAAB+XicbVBNT8JAEN36ifWr6NHLRjDxRFou6I3Ei0dMrJDQSrbLFDZst83uVkMqP8WLBzVe/Sfe/De20IOCL5nk5b2ZzMwLEs6Utu1vY219Y3Nru7Jj7u7tHxxa1aM7FaeSgktjHsteQBRwJsDVTHPoJRJIFHDoBpOrwu8+gFQsFrd6moAfkZFgIaNE59LAqtbNzAtCHMzuPeDcrJsDq2Y37DnwKnFKUkMlOgPryxvGNI1AaMqJUn3HTrSfEakZ5TAzvVRBQuiEjKCfU0EiUH42P32Gz3JliMNY5iU0nqu/JzISKTWNgrwzInqslr1C/M/rpzq88DMmklSDoItFYcqxjnGRAx4yCVTzaU4IlSy/FdMxkYTqPK0iBGf55VXiNhuXDfumWWu3yjQq6ASdonPkoBZqo2vUQS6i6BE9o1f0ZjwZL8a78bFoXTPKmWP0B8bnDxOgkhY=</latexit><latexit sha1_base64="KNyg8JYx18AjnBZEjNu+MVS3Dvs=">AAAB+XicbVBNT8JAEN36ifWr6NHLRjDxRFou6I3Ei0dMrJDQSrbLFDZst83uVkMqP8WLBzVe/Sfe/De20IOCL5nk5b2ZzMwLEs6Utu1vY219Y3Nru7Jj7u7tHxxa1aM7FaeSgktjHsteQBRwJsDVTHPoJRJIFHDoBpOrwu8+gFQsFrd6moAfkZFgIaNE59LAqtbNzAtCHMzuPeDcrJsDq2Y37DnwKnFKUkMlOgPryxvGNI1AaMqJUn3HTrSfEakZ5TAzvVRBQuiEjKCfU0EiUH42P32Gz3JliMNY5iU0nqu/JzISKTWNgrwzInqslr1C/M/rpzq88DMmklSDoItFYcqxjnGRAx4yCVTzaU4IlSy/FdMxkYTqPK0iBGf55VXiNhuXDfumWWu3yjQq6ASdonPkoBZqo2vUQS6i6BE9o1f0ZjwZL8a78bFoXTPKmWP0B8bnDxOgkhY=</latexit><latexit sha1_base64="KNyg8JYx18AjnBZEjNu+MVS3Dvs=">AAAB+XicbVBNT8JAEN36ifWr6NHLRjDxRFou6I3Ei0dMrJDQSrbLFDZst83uVkMqP8WLBzVe/Sfe/De20IOCL5nk5b2ZzMwLEs6Utu1vY219Y3Nru7Jj7u7tHxxa1aM7FaeSgktjHsteQBRwJsDVTHPoJRJIFHDoBpOrwu8+gFQsFrd6moAfkZFgIaNE59LAqtbNzAtCHMzuPeDcrJsDq2Y37DnwKnFKUkMlOgPryxvGNI1AaMqJUn3HTrSfEakZ5TAzvVRBQuiEjKCfU0EiUH42P32Gz3JliMNY5iU0nqu/JzISKTWNgrwzInqslr1C/M/rpzq88DMmklSDoItFYcqxjnGRAx4yCVTzaU4IlSy/FdMxkYTqPK0iBGf55VXiNhuXDfumWWu3yjQq6ASdonPkoBZqo2vUQS6i6BE9o1f0ZjwZL8a78bFoXTPKmWP0B8bnDxOgkhY=</latexit>

fbiases In layer 3

=

Page 10: Neural Networks Part II Feed-Forward Neural Networks

The Multilayer Perceptron

10

Question: What are the dimensions of each of the following?

: ___________ , : __________ , : ___________ , : ______________

W 1<latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit><latexit sha1_base64="NVCt+HgFWs3B0d9436ZvLk4ci98=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeXTtmj2oVJ26swBaJ25BqlCgPah89YcxSSMqNOFYqZ7rJNrPsNSMcDq3+6miCSYTPKI9QwWOqPKzxb1zdGGUIQpjaUpotFB/T2Q4UmoWBaYzwnqsVr1c/M/rpTq88jMmklRTQZaLwpQjHaP8eTRkkhLNZ4ZgIpm5FZExlphoE1Eegrv68jrxGvXrunPXqLaaRRplOINzuAQXmtCCW2iDBwQ4PMMrvFlT68V6tz6WrSWrmDmFP7A+fwCYhY3c</latexit>

W 2<latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit>

W 3<latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit><latexit sha1_base64="VLtc1PXatRWOjWYLnpklA5Yl8Nw=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYoUEKtkuW9iw3dbdrQlp+BNePKjx6u/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMeyG2BFORPU00xz2k0kxVHAaSeYXOd+54lKxWJxp6cJ9SM8EixkBGsjdWt25+HCrtmDStWpO3OgVeIWpAoF2oPKV38YkzSiQhOOleq5TqL9DEvNCKczu58qmmAywSPaM1TgiCo/m987Q2dGGaIwlqaERnP190SGI6WmUWA6I6zHatnLxf+8XqrDSz9jIkk1FWSxKEw50jHKn0dDJinRfGoIJpKZWxEZY4mJNhHlIbjLL68Sr1G/qju3jWqrWaRRhhM4hXNwoQktuIE2eECAwzO8wpv1aL1Y79bHorVkFTPH8AfW5w+bkY3e</latexit>

W 4<latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit>

b1<latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit><latexit sha1_base64="Jd5TGi0sZ3EHWxfx6vKt8PL0FzQ=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfOBa9fsYaXq1J0F0DpxC1KFAu1h5as/ikkaUaEJx0r1XCfRfoalZoTTud1PFU0wmeIx7RkqcESVny2unqMLo4xQGEtTQqOF+nsiw5FSsygwnRHWE7Xq5eJ/Xi/V4ZWfMZGkmgqyXBSmHOkY5RGgEZOUaD4zBBPJzK2ITLDERJug8hDc1ZfXideoX9edu0a11SzSKMMZnMMluNCEFtxCGzwgIOEZXuHNerJerHfrY9lasoqZU/gD6/MH/FuQXw==</latexit>

b2<latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit><latexit sha1_base64="vssOGQ5X2luGd9ktCIY4ONcEfaE=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBQrbLFjZst83uVkMa/ocXD2q8+mO8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gmmN7nfeaRSsVjc61lC/QiPBQsZwdpIg5qd9YMQBfNBw67Zw0rVqTsLoHXiFqQKBdrDyld/FJM0okITjpXquU6i/QxLzQinc7ufKppgMsVj2jNU4IgqP1tcPUcXRhmhMJamhEYL9fdEhiOlZlFgOiOsJ2rVy8X/vF6qwys/YyJJNRVkuShMOdIxyiNAIyYp0XxmCCaSmVsRmWCJiTZB5SG4qy+vE69Rv647d41qq1mkUYYzOIdLcKEJLbiFNnhAQMIzvMKb9WS9WO/Wx7K1ZBUzp/AH1ucP/eGQYA==</latexit>

b3<latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit>

b4<latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit><latexit sha1_base64="2PJM1jwya+WpIjlzbjP+B/CR+mU=">AAAB9HicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwVihTctmu2mXbjZhd6OU0P/hxYOKV3+MN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY/lQ4AV5UxQTzPN6UMiKY4CTjvB5Dr3O49UKhaLOz1NqB/hkWAhI1gbqV+zs14QomDWv7Br9qBSderOHGiVuAWpQoH2oPLVG8YkjajQhGOluq6TaD/DUjPC6czupYommEzwiHYNFTiiys/mV8/QmVGGKIylKaHRXP09keFIqWkUmM4I67Fa9nLxP6+b6vDSz5hIUk0FWSwKU450jPII0JBJSjSfGoKJZOZWRMZYYqJNUHkI7vLLq8Rr1K/qzm2j2moWaZThBE7hHFxoQgtuoA0eEJDwDK/wZj1ZL9a79bFoLVnFzDH8gfX5AwD8kGI=</latexit>

W 2<latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit><latexit sha1_base64="SoLqjIlmHsFVG/6DjfeSnKw0XA4=">AAAB7nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGBSrbLFjZst2V3a0Ia/oQXD2q8+nu8+W/cQg8KvmSSl/dmMjMvSDhT2nG+rdLG5tb2TnnX3ts/ODyqHJ88qDiVhHok5rHsBlhRzgT1NNOcdhNJcRRw2gkmN7nfeaJSsVjc61lC/QiPBAsZwdpI3ZrdeWzYNXtQqTp1ZwG0TtyCVKFAe1D56g9jkkZUaMKxUj3XSbSfYakZ4XRu91NFE0wmeER7hgocUeVni3vn6MIoQxTG0pTQaKH+nshwpNQsCkxnhPVYrXq5+J/XS3V45WdMJKmmgiwXhSlHOkb582jIJCWazwzBRDJzKyJjLDHRJqI8BHf15XXiNerXdeeuUW01izTKcAbncAkuNKEFt9AGDwhweIZXeLOm1ov1bn0sW0tWMXMKf2B9/gCaC43d</latexit> b3

<latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit><latexit sha1_base64="MYnuOl9/sROfXVJMOVctJDsqLLw=">AAAB9HicbVBNT8JAEJ3iF9Yv1KOXjWDiibR4QG8kXjxiYsUECtkuW9iw3Ta7Ww1p+B9ePKjx6o/x5r9xCz0o+JJJXt6bycy8IOFMacf5tkpr6xubW+Vte2d3b/+gcnh0r+JUEuqRmMfyIcCKciaop5nm9CGRFEcBp51gcp37nUcqFYvFnZ4m1I/wSLCQEayN1K/ZWS8IUTDrX9g1e1CpOnVnDrRK3IJUoUB7UPnqDWOSRlRowrFSXddJtJ9hqRnhdGb3UkUTTCZ4RLuGChxR5Wfzq2fozChDFMbSlNBorv6eyHCk1DQKTGeE9Vgte7n4n9dNdXjpZ0wkqaaCLBaFKUc6RnkEaMgkJZpPDcFEMnMrImMsMdEmqDwEd/nlVeI16ld157ZRbTWLNMpwAqdwDi40oQU30AYPCEh4hld4s56sF+vd+li0lqxi5hj+wPr8Af9nkGE=</latexit>

z2<latexit sha1_base64="fa3sot8eIFA+awYQPmUOJ1EHIYY=">AAAB9HicbVBNT8JAEJ36ifUL9ehlI5h4Ii0X9EbixSMmVkigkO2yhQ3bbbO71WDD//DiQY1Xf4w3/41b6EHBl0zy8t5MZuYFCWdKO863tba+sbm1Xdqxd/f2Dw7LR8f3Kk4loR6JeSw7AVaUM0E9zTSnnURSHAWctoPJde63H6hULBZ3eppQP8IjwUJGsDZSv2pnvSBET7N+3a7ag3LFqTlzoFXiFqQCBVqD8ldvGJM0okITjpXquk6i/QxLzQinM7uXKppgMsEj2jVU4IgqP5tfPUPnRhmiMJamhEZz9fdEhiOlplFgOiOsx2rZy8X/vG6qw0s/YyJJNRVksShMOdIxyiNAQyYp0XxqCCaSmVsRGWOJiTZB5SG4yy+vEq9eu6o5t/VKs1GkUYJTOIMLcKEBTbiBFnhAQMIzvMKb9Wi9WO/Wx6J1zSpmTuAPrM8fIsiQeA==</latexit><latexit sha1_base64="fa3sot8eIFA+awYQPmUOJ1EHIYY=">AAAB9HicbVBNT8JAEJ36ifUL9ehlI5h4Ii0X9EbixSMmVkigkO2yhQ3bbbO71WDD//DiQY1Xf4w3/41b6EHBl0zy8t5MZuYFCWdKO863tba+sbm1Xdqxd/f2Dw7LR8f3Kk4loR6JeSw7AVaUM0E9zTSnnURSHAWctoPJde63H6hULBZ3eppQP8IjwUJGsDZSv2pnvSBET7N+3a7ag3LFqTlzoFXiFqQCBVqD8ldvGJM0okITjpXquk6i/QxLzQinM7uXKppgMsEj2jVU4IgqP5tfPUPnRhmiMJamhEZz9fdEhiOlplFgOiOsx2rZy8X/vG6qw0s/YyJJNRVksShMOdIxyiNAQyYp0XxqCCaSmVsRGWOJiTZB5SG4yy+vEq9eu6o5t/VKs1GkUYJTOIMLcKEBTbiBFnhAQMIzvMKb9Wi9WO/Wx6J1zSpmTuAPrM8fIsiQeA==</latexit><latexit sha1_base64="fa3sot8eIFA+awYQPmUOJ1EHIYY=">AAAB9HicbVBNT8JAEJ36ifUL9ehlI5h4Ii0X9EbixSMmVkigkO2yhQ3bbbO71WDD//DiQY1Xf4w3/41b6EHBl0zy8t5MZuYFCWdKO863tba+sbm1Xdqxd/f2Dw7LR8f3Kk4loR6JeSw7AVaUM0E9zTSnnURSHAWctoPJde63H6hULBZ3eppQP8IjwUJGsDZSv2pnvSBET7N+3a7ag3LFqTlzoFXiFqQCBVqD8ldvGJM0okITjpXquk6i/QxLzQinM7uXKppgMsEj2jVU4IgqP5tfPUPnRhmiMJamhEZz9fdEhiOlplFgOiOsx2rZy8X/vG6qw0s/YyJJNRVksShMOdIxyiNAQyYp0XxqCCaSmVsRGWOJiTZB5SG4yy+vEq9eu6o5t/VKs1GkUYJTOIMLcKEBTbiBFnhAQMIzvMKb9Wi9WO/Wx6J1zSpmTuAPrM8fIsiQeA==</latexit>

W 4<latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit><latexit sha1_base64="8LPhg+W3Q0Z8mcaS8pnTzsf+l4w=">AAAB7nicbVBNS8NAEJ3Urxq/qh69LLaCp5IUoXorePFYwdhCG8tmu2mXbjZxdyOU0D/hxYOKV3+PN/+NmzYHbX0w8Hhvhpl5QcKZ0o7zbZXW1jc2t8rb9s7u3v5B5fDoXsWpJNQjMY9lN8CKciaop5nmtJtIiqOA004wuc79zhOVisXiTk8T6kd4JFjICNZG6tbszsOFXbMHlapTd+ZAq8QtSBUKtAeVr/4wJmlEhSYcK9VznUT7GZaaEU5ndj9VNMFkgke0Z6jAEVV+Nr93hs6MMkRhLE0Jjebq74kMR0pNo8B0RliP1bKXi/95vVSHl37GRJJqKshiUZhypGOUP4+GTFKi+dQQTCQztyIyxhITbSLKQ3CXX14lXqN+VXduG9VWs0ijDCdwCufgQhNacANt8IAAh2d4hTfr0Xqx3q2PRWvJKmaO4Q+szx+dF43f</latexit>

0'

=

=

4×3 3 3 1×3

Page 11: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

11

Next week we’ll see how we can learn the weights and the biases in a NN using SGD

This poses a problem with standard perceptron neurons with the Indicator activation function

w +�w<latexit sha1_base64="P5dYwG19f5Uzq8lthOL2PM8GE7Y=">AAAB+XicbVBNS8NAEN34WeNXqkcvi60gCCXppXor6MFjBWMLbSib7aRduvlgd2MpsT/FiwcVr/4Tb/4bN20O2vpg4PHeDDPz/IQzqWz721hb39jc2i7tmLt7+weHVvnoQcapoODSmMei4xMJnEXgKqY4dBIBJPQ5tP3xde63H0FIFkf3apqAF5JhxAJGidJS3ypXzQm+wL0b4IrgiVk1+1bFrtlz4FXiFKSCCrT61ldvENM0hEhRTqTsOnaivIwIxSiHmdlLJSSEjskQuppGJATpZfPTZ/hMKwMcxEJXpPBc/T2RkVDKaejrzpCokVz2cvE/r5uq4NLLWJSkCiK6WBSkHKsY5zngARNAFZ9qQqhg+lZMR0QQqnRaeQjO8surxK3Xrmr2Xb3SbBRplNAJOkXnyEEN1ES3qIVcRNEEPaNX9GY8GS/Gu/GxaF0ziplj9AfG5w/Hf5FA</latexit><latexit sha1_base64="P5dYwG19f5Uzq8lthOL2PM8GE7Y=">AAAB+XicbVBNS8NAEN34WeNXqkcvi60gCCXppXor6MFjBWMLbSib7aRduvlgd2MpsT/FiwcVr/4Tb/4bN20O2vpg4PHeDDPz/IQzqWz721hb39jc2i7tmLt7+weHVvnoQcapoODSmMei4xMJnEXgKqY4dBIBJPQ5tP3xde63H0FIFkf3apqAF5JhxAJGidJS3ypXzQm+wL0b4IrgiVk1+1bFrtlz4FXiFKSCCrT61ldvENM0hEhRTqTsOnaivIwIxSiHmdlLJSSEjskQuppGJATpZfPTZ/hMKwMcxEJXpPBc/T2RkVDKaejrzpCokVz2cvE/r5uq4NLLWJSkCiK6WBSkHKsY5zngARNAFZ9qQqhg+lZMR0QQqnRaeQjO8surxK3Xrmr2Xb3SbBRplNAJOkXnyEEN1ES3qIVcRNEEPaNX9GY8GS/Gu/GxaF0ziplj9AfG5w/Hf5FA</latexit><latexit sha1_base64="P5dYwG19f5Uzq8lthOL2PM8GE7Y=">AAAB+XicbVBNS8NAEN34WeNXqkcvi60gCCXppXor6MFjBWMLbSib7aRduvlgd2MpsT/FiwcVr/4Tb/4bN20O2vpg4PHeDDPz/IQzqWz721hb39jc2i7tmLt7+weHVvnoQcapoODSmMei4xMJnEXgKqY4dBIBJPQ5tP3xde63H0FIFkf3apqAF5JhxAJGidJS3ypXzQm+wL0b4IrgiVk1+1bFrtlz4FXiFKSCCrT61ldvENM0hEhRTqTsOnaivIwIxSiHmdlLJSSEjskQuppGJATpZfPTZ/hMKwMcxEJXpPBc/T2RkVDKaejrzpCokVz2cvE/r5uq4NLLWJSkCiK6WBSkHKsY5zngARNAFZ9qQqhg+lZMR0QQqnRaeQjO8surxK3Xrmr2Xb3SbBRplNAJOkXnyEEN1ES3qIVcRNEEPaNX9GY8GS/Gu/GxaF0ziplj9AfG5w/Hf5FA</latexit>

output +�output<latexit sha1_base64="vCDlid35x4Qyw8CtO0fEtsmBpKc=">AAACFHicbVC7SgNBFJ31GddX1NJmMBEEJeymiXYBLSwjGBNIQpid3CRDZh/M3BXDkp+w8VdsLFRsLez8G2eTLTTxwMDhnHu5c44XSaHRcb6tpeWV1bX13Ia9ubW9s5vf27/TYaw41HkoQ9X0mAYpAqijQAnNSAHzPQkNb3SZ+o17UFqEwS2OI+j4bBCIvuAMjdTNnxXtNsIDKj8JY4xinNBT2r4CiWxOt4t2N19wSs4UdJG4GSmQDLVu/qvdC3nsQ4BcMq1brhNhJ2EKBZcwsduxhojxERtAy9CA+aA7yTTVhB4bpUf7oTIvQDpVf28kzNd67Htm0mc41PNeKv7ntWLsn3cSEZhUEPDZoX4sKYY0rYj2hAKOcmwI40qYv1I+ZIpxNEWmJbjzkRdJvVy6KDk35UK1krWRI4fkiJwQl1RIlVyTGqkTTh7JM3klb9aT9WK9Wx+z0SUr2zkgf2B9/gCDRZ6S</latexit><latexit sha1_base64="vCDlid35x4Qyw8CtO0fEtsmBpKc=">AAACFHicbVC7SgNBFJ31GddX1NJmMBEEJeymiXYBLSwjGBNIQpid3CRDZh/M3BXDkp+w8VdsLFRsLez8G2eTLTTxwMDhnHu5c44XSaHRcb6tpeWV1bX13Ia9ubW9s5vf27/TYaw41HkoQ9X0mAYpAqijQAnNSAHzPQkNb3SZ+o17UFqEwS2OI+j4bBCIvuAMjdTNnxXtNsIDKj8JY4xinNBT2r4CiWxOt4t2N19wSs4UdJG4GSmQDLVu/qvdC3nsQ4BcMq1brhNhJ2EKBZcwsduxhojxERtAy9CA+aA7yTTVhB4bpUf7oTIvQDpVf28kzNd67Htm0mc41PNeKv7ntWLsn3cSEZhUEPDZoX4sKYY0rYj2hAKOcmwI40qYv1I+ZIpxNEWmJbjzkRdJvVy6KDk35UK1krWRI4fkiJwQl1RIlVyTGqkTTh7JM3klb9aT9WK9Wx+z0SUr2zkgf2B9/gCDRZ6S</latexit><latexit sha1_base64="vCDlid35x4Qyw8CtO0fEtsmBpKc=">AAACFHicbVC7SgNBFJ31GddX1NJmMBEEJeymiXYBLSwjGBNIQpid3CRDZh/M3BXDkp+w8VdsLFRsLez8G2eTLTTxwMDhnHu5c44XSaHRcb6tpeWV1bX13Ia9ubW9s5vf27/TYaw41HkoQ9X0mAYpAqijQAnNSAHzPQkNb3SZ+o17UFqEwS2OI+j4bBCIvuAMjdTNnxXtNsIDKj8JY4xinNBT2r4CiWxOt4t2N19wSs4UdJG4GSmQDLVu/qvdC3nsQ4BcMq1brhNhJ2EKBZcwsduxhojxERtAy9CA+aA7yTTVhB4bpUf7oTIvQDpVf28kzNd67Htm0mc41PNeKv7ntWLsn3cSEZhUEPDZoX4sKYY0rYj2hAKOcmwI40qYv1I+ZIpxNEWmJbjzkRdJvVy6KDk35UK1krWRI4fkiJwQl1RIlVyTGqkTTh7JM3klb9aT9WK9Wx+z0SUr2zkgf2B9/gCDRZ6S</latexit>

Page 12: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

12

Next week we’ll see how we can learn the weights and the biases in a NN using SGD

This poses a problem with standard perceptron neurons with the Indicator activation function

Q: What activation could we use instead?

w +�w<latexit sha1_base64="P5dYwG19f5Uzq8lthOL2PM8GE7Y=">AAAB+XicbVBNS8NAEN34WeNXqkcvi60gCCXppXor6MFjBWMLbSib7aRduvlgd2MpsT/FiwcVr/4Tb/4bN20O2vpg4PHeDDPz/IQzqWz721hb39jc2i7tmLt7+weHVvnoQcapoODSmMei4xMJnEXgKqY4dBIBJPQ5tP3xde63H0FIFkf3apqAF5JhxAJGidJS3ypXzQm+wL0b4IrgiVk1+1bFrtlz4FXiFKSCCrT61ldvENM0hEhRTqTsOnaivIwIxSiHmdlLJSSEjskQuppGJATpZfPTZ/hMKwMcxEJXpPBc/T2RkVDKaejrzpCokVz2cvE/r5uq4NLLWJSkCiK6WBSkHKsY5zngARNAFZ9qQqhg+lZMR0QQqnRaeQjO8surxK3Xrmr2Xb3SbBRplNAJOkXnyEEN1ES3qIVcRNEEPaNX9GY8GS/Gu/GxaF0ziplj9AfG5w/Hf5FA</latexit><latexit sha1_base64="P5dYwG19f5Uzq8lthOL2PM8GE7Y=">AAAB+XicbVBNS8NAEN34WeNXqkcvi60gCCXppXor6MFjBWMLbSib7aRduvlgd2MpsT/FiwcVr/4Tb/4bN20O2vpg4PHeDDPz/IQzqWz721hb39jc2i7tmLt7+weHVvnoQcapoODSmMei4xMJnEXgKqY4dBIBJPQ5tP3xde63H0FIFkf3apqAF5JhxAJGidJS3ypXzQm+wL0b4IrgiVk1+1bFrtlz4FXiFKSCCrT61ldvENM0hEhRTqTsOnaivIwIxSiHmdlLJSSEjskQuppGJATpZfPTZ/hMKwMcxEJXpPBc/T2RkVDKaejrzpCokVz2cvE/r5uq4NLLWJSkCiK6WBSkHKsY5zngARNAFZ9qQqhg+lZMR0QQqnRaeQjO8surxK3Xrmr2Xb3SbBRplNAJOkXnyEEN1ES3qIVcRNEEPaNX9GY8GS/Gu/GxaF0ziplj9AfG5w/Hf5FA</latexit><latexit sha1_base64="P5dYwG19f5Uzq8lthOL2PM8GE7Y=">AAAB+XicbVBNS8NAEN34WeNXqkcvi60gCCXppXor6MFjBWMLbSib7aRduvlgd2MpsT/FiwcVr/4Tb/4bN20O2vpg4PHeDDPz/IQzqWz721hb39jc2i7tmLt7+weHVvnoQcapoODSmMei4xMJnEXgKqY4dBIBJPQ5tP3xde63H0FIFkf3apqAF5JhxAJGidJS3ypXzQm+wL0b4IrgiVk1+1bFrtlz4FXiFKSCCrT61ldvENM0hEhRTqTsOnaivIwIxSiHmdlLJSSEjskQuppGJATpZfPTZ/hMKwMcxEJXpPBc/T2RkVDKaejrzpCokVz2cvE/r5uq4NLLWJSkCiK6WBSkHKsY5zngARNAFZ9qQqhg+lZMR0QQqnRaeQjO8surxK3Xrmr2Xb3SbBRplNAJOkXnyEEN1ES3qIVcRNEEPaNX9GY8GS/Gu/GxaF0ziplj9AfG5w/Hf5FA</latexit>

output +�output<latexit sha1_base64="vCDlid35x4Qyw8CtO0fEtsmBpKc=">AAACFHicbVC7SgNBFJ31GddX1NJmMBEEJeymiXYBLSwjGBNIQpid3CRDZh/M3BXDkp+w8VdsLFRsLez8G2eTLTTxwMDhnHu5c44XSaHRcb6tpeWV1bX13Ia9ubW9s5vf27/TYaw41HkoQ9X0mAYpAqijQAnNSAHzPQkNb3SZ+o17UFqEwS2OI+j4bBCIvuAMjdTNnxXtNsIDKj8JY4xinNBT2r4CiWxOt4t2N19wSs4UdJG4GSmQDLVu/qvdC3nsQ4BcMq1brhNhJ2EKBZcwsduxhojxERtAy9CA+aA7yTTVhB4bpUf7oTIvQDpVf28kzNd67Htm0mc41PNeKv7ntWLsn3cSEZhUEPDZoX4sKYY0rYj2hAKOcmwI40qYv1I+ZIpxNEWmJbjzkRdJvVy6KDk35UK1krWRI4fkiJwQl1RIlVyTGqkTTh7JM3klb9aT9WK9Wx+z0SUr2zkgf2B9/gCDRZ6S</latexit><latexit sha1_base64="vCDlid35x4Qyw8CtO0fEtsmBpKc=">AAACFHicbVC7SgNBFJ31GddX1NJmMBEEJeymiXYBLSwjGBNIQpid3CRDZh/M3BXDkp+w8VdsLFRsLez8G2eTLTTxwMDhnHu5c44XSaHRcb6tpeWV1bX13Ia9ubW9s5vf27/TYaw41HkoQ9X0mAYpAqijQAnNSAHzPQkNb3SZ+o17UFqEwS2OI+j4bBCIvuAMjdTNnxXtNsIDKj8JY4xinNBT2r4CiWxOt4t2N19wSs4UdJG4GSmQDLVu/qvdC3nsQ4BcMq1brhNhJ2EKBZcwsduxhojxERtAy9CA+aA7yTTVhB4bpUf7oTIvQDpVf28kzNd67Htm0mc41PNeKv7ntWLsn3cSEZhUEPDZoX4sKYY0rYj2hAKOcmwI40qYv1I+ZIpxNEWmJbjzkRdJvVy6KDk35UK1krWRI4fkiJwQl1RIlVyTGqkTTh7JM3klb9aT9WK9Wx+z0SUr2zkgf2B9/gCDRZ6S</latexit><latexit sha1_base64="vCDlid35x4Qyw8CtO0fEtsmBpKc=">AAACFHicbVC7SgNBFJ31GddX1NJmMBEEJeymiXYBLSwjGBNIQpid3CRDZh/M3BXDkp+w8VdsLFRsLez8G2eTLTTxwMDhnHu5c44XSaHRcb6tpeWV1bX13Ia9ubW9s5vf27/TYaw41HkoQ9X0mAYpAqijQAnNSAHzPQkNb3SZ+o17UFqEwS2OI+j4bBCIvuAMjdTNnxXtNsIDKj8JY4xinNBT2r4CiWxOt4t2N19wSs4UdJG4GSmQDLVu/qvdC3nsQ4BcMq1brhNhJ2EKBZcwsduxhojxERtAy9CA+aA7yTTVhB4bpUf7oTIvQDpVf28kzNd67Htm0mc41PNeKv7ntWLsn3cSEZhUEPDZoX4sKYY0rYj2hAKOcmwI40qYv1I+ZIpxNEWmJbjzkRdJvVy6KDk35UK1krWRI4fkiJwQl1RIlVyTGqkTTh7JM3klb9aT9WK9Wx+z0SUr2zkgf2B9/gCDRZ6S</latexit>

Page 13: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

13

Next week we’ll see how we can learn the weights and the biases in a NN using SGD

This poses a problem with standard perceptron neurons with the Indicator activation function

Q: What activation could we use instead?

A: How about a sigmoid activation?

Page 14: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

14

Is a sigmoid neuron all that different from a perceptron?

For a fixed input. ,

multiply weights and bias by a factor c

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

sigm[c(wTx+ b)]<latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit>

=

Page 15: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

15

Is a sigmoid neuron all that different from a perceptron?

For a fixed input. ,

multiply weights and bias by a factor c

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

sigm[c(wTx+ b)]<latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit>

Page 16: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

16

Is a sigmoid neuron all that different from a perceptron?

For a fixed input. ,

multiply weights and bias by a factor c

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

sigm[c(wTx+ b)]<latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit>

Page 17: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

17

Is a sigmoid neuron all that different from a perceptron?

For a fixed input. ,

multiply weights and bias by a factor c

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

sigm[c(wTx+ b)]<latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit>

Page 18: Neural Networks Part II Feed-Forward Neural Networks

Sigmoid Neurons

18

Is a sigmoid neuron all that different from a perceptron?

For a fixed input. ,

multiply weights and bias by a factor c

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

sigm[c(wTx+ b)]<latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit><latexit sha1_base64="rdONF2gN4rQC1DnCURD3VIvFJe4=">AAACDXicbVBNT8JAEN3iF9Yv1KOXjUCCMSGFC3oj8eIRExCSUsl22cKG3bbZnSqk4Rd48a948aDGq3dv/htb4KDgSyZ5eW8mM/PcUHANlvVtZNbWNza3stvmzu7e/kHu8OhWB5GirEUDEaiOSzQT3Gct4CBYJ1SMSFewtju6Sv32PVOaB34TJiFzJBn43OOUQCL1csWC2QU2BiVjzQdyatNS3HU9/DC9a47xOXbPHLNg9nJ5q2zNgFdJZUHyaIFGL/fV7Qc0kswHKojWdsUKwYmJAk4Fm5rdSLOQ0BEZMDuhPpFMO/HsnSkuJkofe4FKygc8U39PxERqPZFu0ikJDPWyl4r/eXYE3oUTcz+MgPl0vsiLBIYAp9ngPleMgpgkhFDFk1sxHRJFKCQJpiFUll9eJa1q+bJs3VTz9doijSw6QaeohCqohuroGjVQC1H0iJ7RK3oznowX4934mLdmjMXMMfoD4/MHly2aBw==</latexit>

Page 19: Neural Networks Part II Feed-Forward Neural Networks

Predictions: Forward Propagation

19

Suppose we’ve learned the weights in a NN with sigmoid neurons.

Given an input , how do we make a prediction? x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

s x. →× = y

e [ 0113

Yz →

Page 20: Neural Networks Part II Feed-Forward Neural Networks

Predictions: Forward Propagation

20

Suppose we’ve learned the weights in a NN with sigmoid neurons.

Given an input , how do we make a prediction? First, think about a simple network: x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

w'= [ Wi, Wiz) b '=[ b

'

,3

z2= W' Xtb

a2= g( wktb )

W' t

y

Page 21: Neural Networks Part II Feed-Forward Neural Networks

Predictions: Forward Propagation

21

Suppose we’ve learned the weights in a NN with sigmoid neurons.

Given an input , how do we make a prediction?

Re: where

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

a`+1 = g(z`+1)<latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit><latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit><latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit>

z`+1 = W `a` + b`<latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit><latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit><latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit>

Ia

z2=W' A' tbl

- a2=g( Z' )

LT -*EH:i:gwizazitbn

Page 22: Neural Networks Part II Feed-Forward Neural Networks

Predictions: Forward Propagation

22

Suppose we’ve learned the weights in a NN with sigmoid neurons.

Given an input , how do we make a prediction?

Re: where

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

a`+1 = g(z`+1)<latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit><latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit><latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit>

z`+1 = W `a` + b`<latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit><latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit><latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit>

Suppose the network has L layers

Intialize a1 = x<latexit sha1_base64="I5E+eit9kFybGhBmvQ+M07qRDjs=">AAACAHicbZC9TsMwFIUdfkv4C7AgsVi0SExV0qUwIFViYSwSoZXaUDmu01p1nMh2EFUUFl6FhQEQK4/BxtvgpBmg5UiWPp17r67v8WNGpbLtb2NpeWV1bb2yYW5ube/sWnv7tzJKBCYujlgkuj6ShFFOXEUVI91YEBT6jHT8yWVe79wTIWnEb9Q0Jl6IRpwGFCOlrYF1WDPTvh9AlN058AIW/JCZNXNgVe26XQguglNCFZRqD6yv/jDCSUi4wgxJ2XPsWHkpEopiRjKzn0gSIzxBI9LTyFFIpJcWF2TwRDtDGERCP65g4f6eSFEo5TT0dWeI1FjO13Lzv1ovUcGZl1IeJ4pwPFsUJAyqCOZxwCEVBCs21YCwoPqvEI+RQFjp0PIQnPmTF8Ft1M/r9nWj2mqWaVTAETgGp8ABTdACV6ANXIDBI3gGr+DNeDJejHfjY9a6ZJQzB+CPjM8f8FaUJA==</latexit><latexit sha1_base64="I5E+eit9kFybGhBmvQ+M07qRDjs=">AAACAHicbZC9TsMwFIUdfkv4C7AgsVi0SExV0qUwIFViYSwSoZXaUDmu01p1nMh2EFUUFl6FhQEQK4/BxtvgpBmg5UiWPp17r67v8WNGpbLtb2NpeWV1bb2yYW5ube/sWnv7tzJKBCYujlgkuj6ShFFOXEUVI91YEBT6jHT8yWVe79wTIWnEb9Q0Jl6IRpwGFCOlrYF1WDPTvh9AlN058AIW/JCZNXNgVe26XQguglNCFZRqD6yv/jDCSUi4wgxJ2XPsWHkpEopiRjKzn0gSIzxBI9LTyFFIpJcWF2TwRDtDGERCP65g4f6eSFEo5TT0dWeI1FjO13Lzv1ovUcGZl1IeJ4pwPFsUJAyqCOZxwCEVBCs21YCwoPqvEI+RQFjp0PIQnPmTF8Ft1M/r9nWj2mqWaVTAETgGp8ABTdACV6ANXIDBI3gGr+DNeDJejHfjY9a6ZJQzB+CPjM8f8FaUJA==</latexit><latexit sha1_base64="I5E+eit9kFybGhBmvQ+M07qRDjs=">AAACAHicbZC9TsMwFIUdfkv4C7AgsVi0SExV0qUwIFViYSwSoZXaUDmu01p1nMh2EFUUFl6FhQEQK4/BxtvgpBmg5UiWPp17r67v8WNGpbLtb2NpeWV1bb2yYW5ube/sWnv7tzJKBCYujlgkuj6ShFFOXEUVI91YEBT6jHT8yWVe79wTIWnEb9Q0Jl6IRpwGFCOlrYF1WDPTvh9AlN058AIW/JCZNXNgVe26XQguglNCFZRqD6yv/jDCSUi4wgxJ2XPsWHkpEopiRjKzn0gSIzxBI9LTyFFIpJcWF2TwRDtDGERCP65g4f6eSFEo5TT0dWeI1FjO13Lzv1ovUcGZl1IeJ4pwPFsUJAyqCOZxwCEVBCs21YCwoPqvEI+RQFjp0PIQnPmTF8Ft1M/r9nWj2mqWaVTAETgGp8ABTdACV6ANXIDBI3gGr+DNeDJejHfjY9a6ZJQzB+CPjM8f8FaUJA==</latexit>

for ` = 1, . . . , L� 1

z`+1 = W `a` + b`

a`+1 = g(z`+1)<latexit sha1_base64="U8+c+WU3EVMAaYX2L+qZSogv1T4=">AAACqnicbVFLb9QwEHbCq4TXFo5cLLZIi9qukl5KD5UqceGAUBEsu2i9XTnOZGvVsYM9QSxR/hw/gRv/BmeTA912ZFnffPPwzOe0VNJhHP8Nwjt3791/sPMwevT4ydNng93nX52prICJMMrYWcodKKlhghIVzEoLvEgVTNOrd218+gOsk0Z/wXUJi4KvtMyl4Oip5eD3XsRSWEldc2v5uqlVEzGEn2iLOjeWNpSBUqfJAWUqM+gO6IfDhDIWted7xbPNRWuW5vRXc1G32ftJQ0/p9KLFXYQ3nbPfuWnv3t6GX2uzGm33fhMx0Fk/b7QXLQfDeBxvjN4ESQ+GpLfz5eAPy4yoCtAoFHdunsQlLnxDlEKB379yUHJxxVcw91DzAtyi3mjd0NeeyWgrTW400g37f0XNC+fWReozC46XbjvWkrfF5hXmbxe11GWFoEX3UF4pioa2H0czaUGgWnvAhZV+ViouueUC/fe2IiTbK98Ek6PxyTj+dDQ8O+7V2CEvySsyIgk5JmfkPTknEyKCUfAxmAaz8DD8HH4L511qGPQ1L8g1C7N/TejNmg==</latexit><latexit sha1_base64="U8+c+WU3EVMAaYX2L+qZSogv1T4=">AAACqnicbVFLb9QwEHbCq4TXFo5cLLZIi9qukl5KD5UqceGAUBEsu2i9XTnOZGvVsYM9QSxR/hw/gRv/BmeTA912ZFnffPPwzOe0VNJhHP8Nwjt3791/sPMwevT4ydNng93nX52prICJMMrYWcodKKlhghIVzEoLvEgVTNOrd218+gOsk0Z/wXUJi4KvtMyl4Oip5eD3XsRSWEldc2v5uqlVEzGEn2iLOjeWNpSBUqfJAWUqM+gO6IfDhDIWted7xbPNRWuW5vRXc1G32ftJQ0/p9KLFXYQ3nbPfuWnv3t6GX2uzGm33fhMx0Fk/b7QXLQfDeBxvjN4ESQ+GpLfz5eAPy4yoCtAoFHdunsQlLnxDlEKB379yUHJxxVcw91DzAtyi3mjd0NeeyWgrTW400g37f0XNC+fWReozC46XbjvWkrfF5hXmbxe11GWFoEX3UF4pioa2H0czaUGgWnvAhZV+ViouueUC/fe2IiTbK98Ek6PxyTj+dDQ8O+7V2CEvySsyIgk5JmfkPTknEyKCUfAxmAaz8DD8HH4L511qGPQ1L8g1C7N/TejNmg==</latexit><latexit sha1_base64="U8+c+WU3EVMAaYX2L+qZSogv1T4=">AAACqnicbVFLb9QwEHbCq4TXFo5cLLZIi9qukl5KD5UqceGAUBEsu2i9XTnOZGvVsYM9QSxR/hw/gRv/BmeTA912ZFnffPPwzOe0VNJhHP8Nwjt3791/sPMwevT4ydNng93nX52prICJMMrYWcodKKlhghIVzEoLvEgVTNOrd218+gOsk0Z/wXUJi4KvtMyl4Oip5eD3XsRSWEldc2v5uqlVEzGEn2iLOjeWNpSBUqfJAWUqM+gO6IfDhDIWted7xbPNRWuW5vRXc1G32ftJQ0/p9KLFXYQ3nbPfuWnv3t6GX2uzGm33fhMx0Fk/b7QXLQfDeBxvjN4ESQ+GpLfz5eAPy4yoCtAoFHdunsQlLnxDlEKB379yUHJxxVcw91DzAtyi3mjd0NeeyWgrTW400g37f0XNC+fWReozC46XbjvWkrfF5hXmbxe11GWFoEX3UF4pioa2H0czaUGgWnvAhZV+ViouueUC/fe2IiTbK98Ek6PxyTj+dDQ8O+7V2CEvySsyIgk5JmfkPTknEyKCUfAxmAaz8DD8HH4L511qGPQ1L8g1C7N/TejNmg==</latexit>

Page 23: Neural Networks Part II Feed-Forward Neural Networks

Predictions: Forward Propagation

23

Suppose we’ve learned the weights in a NN with sigmoid neurons.

Given an input , how do we make a prediction?

Re: where

x<latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit><latexit sha1_base64="x2qbD6BmvRnY3pXmkJNsBDx/pfo=">AAAB8nicbVBNT8JAEJ3iF9Yv1KOXjWDiibRc0BuJF4+YWCGhDdkuW9iw3Ta7WyNp+BtePKjx6q/x5r9xCz0o+JJJXt6bycy8MOVMacf5tiobm1vbO9Vde2//4PCodnzyoJJMEuqRhCeyH2JFORPU00xz2k8lxXHIaS+c3hR+75FKxRJxr2cpDWI8FixiBGsj+Q0798MIPc3thj2s1Z2mswBaJ25J6lCiO6x9+aOEZDEVmnCs1MB1Uh3kWGpGOJ3bfqZoiskUj+nAUIFjqoJ8cfMcXRhlhKJEmhIaLdTfEzmOlZrFoemMsZ6oVa8Q//MGmY6ugpyJNNNUkOWiKONIJ6gIAI2YpETzmSGYSGZuRWSCJSbaxFSE4K6+vE68VvO66dy16p12mUYVzuAcLsGFNnTgFrrgAYEUnuEV3qzMerHerY9la8UqZ07hD6zPH/Qij9I=</latexit>

a`+1 = g(z`+1)<latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit><latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit><latexit sha1_base64="g67eZfvd/mlQV0FqveJvn/8s8Hs=">AAACFXicbZDLSsNAFIYn9VbjLerSzWArVIol6aa6EApuXFYwttDUMplO2sHJhZmJUEOewo2v4saFilvBnW/jJM1CW38Y+PnOOZw5vxsxKqRpfmulpeWV1bXyur6xubW9Y+zu3Ygw5pjYOGQh77lIEEYDYksqGelFnCDfZaTr3l1k9e494YKGwbWcRmTgo3FAPYqRVGhonFT1xHE9iNLbxCGM1a0UnsNxLYcPBYR1aKXHelUfGhWzYeaCi8YqTAUU6gyNL2cU4tgngcQMCdG3zEgOEsQlxYykuhMLEiF8h8akr2yAfCIGSX5WCo8UGUEv5OoFEub090SCfCGmvqs6fSQnYr6Wwf9q/Vh6p4OEBlEsSYBni7yYQRnCLCM4opxgyabKIMyp+ivEE8QRlirJLARr/uRFYzcbZw3zqllpt4o0yuAAHIIasEALtMEl6AAbYPAInsEreNOetBftXfuYtZa0YmYf/JH2+QOGnJv3</latexit>

z`+1 = W `a` + b`<latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit><latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit><latexit sha1_base64="fy0SaoVScwpHyfIH2HDC2W+LVfg=">AAACJHicbVDLSsNAFJ34rPEVdelmsBUEoSTdVEGh4MZlBWMLTVom00k7dDIJMxOhhvyMG3/FjQsrLtz4LSZpFtp6YeA87uXOPV7EqFSm+aWtrK6tb2xWtvTtnd29fePg8EGGscDExiELRddDkjDKia2oYqQbCYICj5GON7nJ/c4jEZKG/F5NI+IGaMSpTzFSmTQwrmp64ng+fEr7iUMYg+fQSuE17PQLVngo7ZdWQb051Wv6wKiadbMouAysElRBWe2BMXOGIY4DwhVmSMqeZUbKTZBQFDOS6k4sSYTwBI1IL4McBUS6SXFlCk8zZQj9UGSPK1iovycSFEg5DbysM0BqLBe9XPzP68XKv3ATyqNYEY7ni/yYQRXCPDI4pIJgxaYZQFjQ7K8Qj5FAWGXB5iFYiycvA7tRv6ybd41qq1mmUQHH4AScAQs0QQvcgjawAQbP4BW8g5n2or1pH9rnvHVFK2eOwJ/Svn8A1M+h7w==</latexit>

Suppose the network has L layers

Intialize

If classification and sigmoid neuron output

then predict based on

a1 = x<latexit sha1_base64="I5E+eit9kFybGhBmvQ+M07qRDjs=">AAACAHicbZC9TsMwFIUdfkv4C7AgsVi0SExV0qUwIFViYSwSoZXaUDmu01p1nMh2EFUUFl6FhQEQK4/BxtvgpBmg5UiWPp17r67v8WNGpbLtb2NpeWV1bb2yYW5ube/sWnv7tzJKBCYujlgkuj6ShFFOXEUVI91YEBT6jHT8yWVe79wTIWnEb9Q0Jl6IRpwGFCOlrYF1WDPTvh9AlN058AIW/JCZNXNgVe26XQguglNCFZRqD6yv/jDCSUi4wgxJ2XPsWHkpEopiRjKzn0gSIzxBI9LTyFFIpJcWF2TwRDtDGERCP65g4f6eSFEo5TT0dWeI1FjO13Lzv1ovUcGZl1IeJ4pwPFsUJAyqCOZxwCEVBCs21YCwoPqvEI+RQFjp0PIQnPmTF8Ft1M/r9nWj2mqWaVTAETgGp8ABTdACV6ANXIDBI3gGr+DNeDJejHfjY9a6ZJQzB+CPjM8f8FaUJA==</latexit><latexit sha1_base64="I5E+eit9kFybGhBmvQ+M07qRDjs=">AAACAHicbZC9TsMwFIUdfkv4C7AgsVi0SExV0qUwIFViYSwSoZXaUDmu01p1nMh2EFUUFl6FhQEQK4/BxtvgpBmg5UiWPp17r67v8WNGpbLtb2NpeWV1bb2yYW5ube/sWnv7tzJKBCYujlgkuj6ShFFOXEUVI91YEBT6jHT8yWVe79wTIWnEb9Q0Jl6IRpwGFCOlrYF1WDPTvh9AlN058AIW/JCZNXNgVe26XQguglNCFZRqD6yv/jDCSUi4wgxJ2XPsWHkpEopiRjKzn0gSIzxBI9LTyFFIpJcWF2TwRDtDGERCP65g4f6eSFEo5TT0dWeI1FjO13Lzv1ovUcGZl1IeJ4pwPFsUJAyqCOZxwCEVBCs21YCwoPqvEI+RQFjp0PIQnPmTF8Ft1M/r9nWj2mqWaVTAETgGp8ABTdACV6ANXIDBI3gGr+DNeDJejHfjY9a6ZJQzB+CPjM8f8FaUJA==</latexit><latexit sha1_base64="I5E+eit9kFybGhBmvQ+M07qRDjs=">AAACAHicbZC9TsMwFIUdfkv4C7AgsVi0SExV0qUwIFViYSwSoZXaUDmu01p1nMh2EFUUFl6FhQEQK4/BxtvgpBmg5UiWPp17r67v8WNGpbLtb2NpeWV1bb2yYW5ube/sWnv7tzJKBCYujlgkuj6ShFFOXEUVI91YEBT6jHT8yWVe79wTIWnEb9Q0Jl6IRpwGFCOlrYF1WDPTvh9AlN058AIW/JCZNXNgVe26XQguglNCFZRqD6yv/jDCSUi4wgxJ2XPsWHkpEopiRjKzn0gSIzxBI9LTyFFIpJcWF2TwRDtDGERCP65g4f6eSFEo5TT0dWeI1FjO13Lzv1ovUcGZl1IeJ4pwPFsUJAyqCOZxwCEVBCs21YCwoPqvEI+RQFjp0PIQnPmTF8Ft1M/r9nWj2mqWaVTAETgGp8ABTdACV6ANXIDBI3gGr+DNeDJejHfjY9a6ZJQzB+CPjM8f8FaUJA==</latexit>

for ` = 1, . . . , L� 1

z`+1 = W `a` + b`

a`+1 = g(z`+1)<latexit sha1_base64="U8+c+WU3EVMAaYX2L+qZSogv1T4=">AAACqnicbVFLb9QwEHbCq4TXFo5cLLZIi9qukl5KD5UqceGAUBEsu2i9XTnOZGvVsYM9QSxR/hw/gRv/BmeTA912ZFnffPPwzOe0VNJhHP8Nwjt3791/sPMwevT4ydNng93nX52prICJMMrYWcodKKlhghIVzEoLvEgVTNOrd218+gOsk0Z/wXUJi4KvtMyl4Oip5eD3XsRSWEldc2v5uqlVEzGEn2iLOjeWNpSBUqfJAWUqM+gO6IfDhDIWted7xbPNRWuW5vRXc1G32ftJQ0/p9KLFXYQ3nbPfuWnv3t6GX2uzGm33fhMx0Fk/b7QXLQfDeBxvjN4ESQ+GpLfz5eAPy4yoCtAoFHdunsQlLnxDlEKB379yUHJxxVcw91DzAtyi3mjd0NeeyWgrTW400g37f0XNC+fWReozC46XbjvWkrfF5hXmbxe11GWFoEX3UF4pioa2H0czaUGgWnvAhZV+ViouueUC/fe2IiTbK98Ek6PxyTj+dDQ8O+7V2CEvySsyIgk5JmfkPTknEyKCUfAxmAaz8DD8HH4L511qGPQ1L8g1C7N/TejNmg==</latexit><latexit sha1_base64="U8+c+WU3EVMAaYX2L+qZSogv1T4=">AAACqnicbVFLb9QwEHbCq4TXFo5cLLZIi9qukl5KD5UqceGAUBEsu2i9XTnOZGvVsYM9QSxR/hw/gRv/BmeTA912ZFnffPPwzOe0VNJhHP8Nwjt3791/sPMwevT4ydNng93nX52prICJMMrYWcodKKlhghIVzEoLvEgVTNOrd218+gOsk0Z/wXUJi4KvtMyl4Oip5eD3XsRSWEldc2v5uqlVEzGEn2iLOjeWNpSBUqfJAWUqM+gO6IfDhDIWted7xbPNRWuW5vRXc1G32ftJQ0/p9KLFXYQ3nbPfuWnv3t6GX2uzGm33fhMx0Fk/b7QXLQfDeBxvjN4ESQ+GpLfz5eAPy4yoCtAoFHdunsQlLnxDlEKB379yUHJxxVcw91DzAtyi3mjd0NeeyWgrTW400g37f0XNC+fWReozC46XbjvWkrfF5hXmbxe11GWFoEX3UF4pioa2H0czaUGgWnvAhZV+ViouueUC/fe2IiTbK98Ek6PxyTj+dDQ8O+7V2CEvySsyIgk5JmfkPTknEyKCUfAxmAaz8DD8HH4L511qGPQ1L8g1C7N/TejNmg==</latexit><latexit sha1_base64="U8+c+WU3EVMAaYX2L+qZSogv1T4=">AAACqnicbVFLb9QwEHbCq4TXFo5cLLZIi9qukl5KD5UqceGAUBEsu2i9XTnOZGvVsYM9QSxR/hw/gRv/BmeTA912ZFnffPPwzOe0VNJhHP8Nwjt3791/sPMwevT4ydNng93nX52prICJMMrYWcodKKlhghIVzEoLvEgVTNOrd218+gOsk0Z/wXUJi4KvtMyl4Oip5eD3XsRSWEldc2v5uqlVEzGEn2iLOjeWNpSBUqfJAWUqM+gO6IfDhDIWted7xbPNRWuW5vRXc1G32ftJQ0/p9KLFXYQ3nbPfuWnv3t6GX2uzGm33fhMx0Fk/b7QXLQfDeBxvjN4ESQ+GpLfz5eAPy4yoCtAoFHdunsQlLnxDlEKB379yUHJxxVcw91DzAtyi3mjd0NeeyWgrTW400g37f0XNC+fWReozC46XbjvWkrfF5hXmbxe11GWFoEX3UF4pioa2H0czaUGgWnvAhZV+ViouueUC/fe2IiTbK98Ek6PxyTj+dDQ8O+7V2CEvySsyIgk5JmfkPTknEyKCUfAxmAaz8DD8HH4L511qGPQ1L8g1C7N/TejNmg==</latexit>

aL<latexit sha1_base64="Ss4ZKzF5gA4Bkaib8XGhoLnTgm0=">AAAB+XicbVBNT8JAEJ3iF9avokcvG8HEE2m5oDcSLx48YCJCApVsly1s2H5kd6shtT/Fiwc1Xv0n3vw3bqEHBV8yyct7M5mZ58WcSWXb30ZpbX1jc6u8be7s7u0fWJXDOxklgtAOiXgkeh6WlLOQdhRTnPZiQXHgcdr1ppe5332gQrIovFWzmLoBHofMZwQrLQ2tSs1MB56PcHafXmfIrJlDq2rX7TnQKnEKUoUC7aH1NRhFJAloqAjHUvYdO1ZuioVihNPMHCSSxphM8Zj2NQ1xQKWbzk/P0KlWRsiPhK5Qobn6eyLFgZSzwNOdAVYTuezl4n9eP1H+uZuyME4UDclikZ9wpCKU54BGTFCi+EwTTATTtyIywQITpdPKQ3CWX14lnUb9om7fNKqtZpFGGY7hBM7AgSa04Ara0AECj/AMr/BmPBkvxrvxsWgtGcXMEfyB8fkDwYOR4A==</latexit><latexit sha1_base64="Ss4ZKzF5gA4Bkaib8XGhoLnTgm0=">AAAB+XicbVBNT8JAEJ3iF9avokcvG8HEE2m5oDcSLx48YCJCApVsly1s2H5kd6shtT/Fiwc1Xv0n3vw3bqEHBV8yyct7M5mZ58WcSWXb30ZpbX1jc6u8be7s7u0fWJXDOxklgtAOiXgkeh6WlLOQdhRTnPZiQXHgcdr1ppe5332gQrIovFWzmLoBHofMZwQrLQ2tSs1MB56PcHafXmfIrJlDq2rX7TnQKnEKUoUC7aH1NRhFJAloqAjHUvYdO1ZuioVihNPMHCSSxphM8Zj2NQ1xQKWbzk/P0KlWRsiPhK5Qobn6eyLFgZSzwNOdAVYTuezl4n9eP1H+uZuyME4UDclikZ9wpCKU54BGTFCi+EwTTATTtyIywQITpdPKQ3CWX14lnUb9om7fNKqtZpFGGY7hBM7AgSa04Ara0AECj/AMr/BmPBkvxrvxsWgtGcXMEfyB8fkDwYOR4A==</latexit><latexit sha1_base64="Ss4ZKzF5gA4Bkaib8XGhoLnTgm0=">AAAB+XicbVBNT8JAEJ3iF9avokcvG8HEE2m5oDcSLx48YCJCApVsly1s2H5kd6shtT/Fiwc1Xv0n3vw3bqEHBV8yyct7M5mZ58WcSWXb30ZpbX1jc6u8be7s7u0fWJXDOxklgtAOiXgkeh6WlLOQdhRTnPZiQXHgcdr1ppe5332gQrIovFWzmLoBHofMZwQrLQ2tSs1MB56PcHafXmfIrJlDq2rX7TnQKnEKUoUC7aH1NRhFJAloqAjHUvYdO1ZuioVihNPMHCSSxphM8Zj2NQ1xQKWbzk/P0KlWRsiPhK5Qobn6eyLFgZSzwNOdAVYTuezl4n9eP1H+uZuyME4UDclikZ9wpCKU54BGTFCi+EwTTATTtyIywQITpdPKQ3CWX14lnUb9om7fNKqtZpFGGY7hBM7AgSa04Ara0AECj/AMr/BmPBkvxrvxsWgtGcXMEfyB8fkDwYOR4A==</latexit>

Page 24: Neural Networks Part II Feed-Forward Neural Networks

Other Popular Activation Functions

24

perceptron sigmoid

tanh ReLu

Page 25: Neural Networks Part II Feed-Forward Neural Networks

Other Popular Activation Functions

25

perceptron sigmoid

tanh ReLu

Page 26: Neural Networks Part II Feed-Forward Neural Networks

Different Probs – Different Architectures

26

What if we want to do regression instead of classification?

Page 27: Neural Networks Part II Feed-Forward Neural Networks

Different Probs – Different Architectures

27

What if we want to do classification with more than two classes?

Page 28: Neural Networks Part II Feed-Forward Neural Networks

Different Probs – Different Architectures

28

What if we want to do classification with more than two classes?

←IsPEDESTRIAN

• 95 ⇐ PREDICT PED

25.00÷Then

Page 29: Neural Networks Part II Feed-Forward Neural Networks

Different Probs – Different Architectures

29

What if we want to do classification with more than two classes?

Page 30: Neural Networks Part II Feed-Forward Neural Networks

Different Probs – Different Architectures

30

What if we want to do classification with more than two classes?

Page 31: Neural Networks Part II Feed-Forward Neural Networks

Loss Functions

31

How do we know if we’re doing well?

Single Output – Squared Loss

Regression or Classification

L 1 w , b. × , y )

→ FI 1 yi - a ;)2

Page 32: Neural Networks Part II Feed-Forward Neural Networks

Loss Functions

32

How do we know if we’re doing well?

Multiple Outputs – Squared Loss

Classification

: ;§ " jcxi . a tip

° Ettenat

Page 33: Neural Networks Part II Feed-Forward Neural Networks

Loss Functions

33

How do we know if we’re doing well?

Single Output – Cross-Entropy

Regression or Classification

,§g{yi log ( at )+ Ctyi) high .and

Page 34: Neural Networks Part II Feed-Forward Neural Networks

Loss Functions

34

How do we know if we’re doing well?

Multiple Outputs – SoftMax

Classification

Page 35: Neural Networks Part II Feed-Forward Neural Networks

Training a Feed-Forward NN

35

So how do we train such a monstrosity?

Page 36: Neural Networks Part II Feed-Forward Neural Networks

Training a Feed-Forward NN

36

So how do we train such a monstrosity?

Pretty much the usual way

Define a loss function: Loss(x,y;W,b) = L(W,b)<latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit><latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit><latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit>

Page 37: Neural Networks Part II Feed-Forward Neural Networks

Training a Feed-Forward NN

37

So how do we train such a monstrosity?

Pretty much the usual way

Define a loss function:

Then we just do Stochastic Gradient Descent:

No problem right …

Loss(x,y;W,b) = L(W,b)<latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit><latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit><latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit>

w`ij w`

ij � ⌘@L

@w`ij

, b`i b`i � ⌘@L

@b`i<latexit sha1_base64="Tt/xv9DSrRGujanZVTzkBNfJboY=">AAACmHichVFdSxtBFJ1dtY2rrVGhL74MjYIPbdj4YgUfLNKq4IMWo0I2hruTuzpm9sOZu5Ww7H/qb/HNf+NsDBhNwQsDh3PP/Zhzw0xJQ77/6Lgzs3MfPtbmvYXFT5+X6ssr5ybNtcC2SFWqL0MwqGSCbZKk8DLTCHGo8CIc7Ff5i7+ojUyTMxpm2I3hOpGRFECW6tX/ra97971C3pZXASrFA4URgdbpPZ+kv/MACXgQaRBFkIEmCYofly94Ql1+48FdDn3uhZacbjzBvt/3RVx6dlevV2/4TX8UfBq0xqDBxnHSqz8E/VTkMSYkFBjTafkZdYuqvVBYekFuMAMxgGvsWJhAjKZbjJwt+YZl+jxKtX0J8RE7WVFAbMwwDq0yBroxb3MV+b9cJ6foR7eQSZYTJuJ5UJQrTimvzsT7UqMgNbQAhJZ2Vy5uwLpE9piVCa23X54G7a3mTtM/3WrsbY/dqLE19pVtshbbZnvskJ2wNhPOF2fX+eX8dtfcn+6Be/QsdZ1xzSp7Fe6fJ2MPytg=</latexit><latexit sha1_base64="Tt/xv9DSrRGujanZVTzkBNfJboY=">AAACmHichVFdSxtBFJ1dtY2rrVGhL74MjYIPbdj4YgUfLNKq4IMWo0I2hruTuzpm9sOZu5Ww7H/qb/HNf+NsDBhNwQsDh3PP/Zhzw0xJQ77/6Lgzs3MfPtbmvYXFT5+X6ssr5ybNtcC2SFWqL0MwqGSCbZKk8DLTCHGo8CIc7Ff5i7+ojUyTMxpm2I3hOpGRFECW6tX/ra97971C3pZXASrFA4URgdbpPZ+kv/MACXgQaRBFkIEmCYofly94Ql1+48FdDn3uhZacbjzBvt/3RVx6dlevV2/4TX8UfBq0xqDBxnHSqz8E/VTkMSYkFBjTafkZdYuqvVBYekFuMAMxgGvsWJhAjKZbjJwt+YZl+jxKtX0J8RE7WVFAbMwwDq0yBroxb3MV+b9cJ6foR7eQSZYTJuJ5UJQrTimvzsT7UqMgNbQAhJZ2Vy5uwLpE9piVCa23X54G7a3mTtM/3WrsbY/dqLE19pVtshbbZnvskJ2wNhPOF2fX+eX8dtfcn+6Be/QsdZ1xzSp7Fe6fJ2MPytg=</latexit><latexit sha1_base64="Tt/xv9DSrRGujanZVTzkBNfJboY=">AAACmHichVFdSxtBFJ1dtY2rrVGhL74MjYIPbdj4YgUfLNKq4IMWo0I2hruTuzpm9sOZu5Ww7H/qb/HNf+NsDBhNwQsDh3PP/Zhzw0xJQ77/6Lgzs3MfPtbmvYXFT5+X6ssr5ybNtcC2SFWqL0MwqGSCbZKk8DLTCHGo8CIc7Ff5i7+ojUyTMxpm2I3hOpGRFECW6tX/ra97971C3pZXASrFA4URgdbpPZ+kv/MACXgQaRBFkIEmCYofly94Ql1+48FdDn3uhZacbjzBvt/3RVx6dlevV2/4TX8UfBq0xqDBxnHSqz8E/VTkMSYkFBjTafkZdYuqvVBYekFuMAMxgGvsWJhAjKZbjJwt+YZl+jxKtX0J8RE7WVFAbMwwDq0yBroxb3MV+b9cJ6foR7eQSZYTJuJ5UJQrTimvzsT7UqMgNbQAhJZ2Vy5uwLpE9piVCa23X54G7a3mTtM/3WrsbY/dqLE19pVtshbbZnvskJ2wNhPOF2fX+eX8dtfcn+6Be/QsdZ1xzSp7Fe6fJ2MPytg=</latexit>

Page 38: Neural Networks Part II Feed-Forward Neural Networks

Training a Feed-Forward NN

38

So how do we train such a monstrosity?

Pretty much the usual way

Define a loss function:

Then we just do Stochastic Gradient Descent:

No problem right …

Turns out those partial derivatives are not so easy to compute

This is where Back Propagation comes in. It will require:

Calculus Linear Algebra An Iron Stomach

Loss(x,y;W,b) = L(W,b)<latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit><latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit><latexit sha1_base64="ZoZ7AfgQ8QBFd053ZFPsCDUUTXs=">AAACNXicbZBdSwJBFIZn7cu2L6vLboY0MAjZ9cYiAqGbLgwMMgUVmR1ndXD2g5mzoSz7q7rpd3TnTRcV3fYX2tUNSnth4OE953DmvJYvuALDmGqZldW19Y3spr61vbO7l9s/eFBeIClrUE94smURxQR3WQM4CNbyJSOOJVjTGl0n9eYjk4p77j1MfNZ1yMDlNqcEYquXuy3oHWBjkE5Y85SKimHHsvE4OsMzmESXc2j+OFZ0iq9wrbjs6gW9l8sbJWMmvAxmCnmUqt7LvXT6Hg0c5gIVRKm2afjQDYkETgWL9E6gmE/oiAxYO0aXOEx1w9nZET6JnT62PRk/F/DM/T0REkepiWPFnQ6BoVqsJeZ/tXYA9nk35K4fAHPpfJEdCAweTjLEfS4ZBTGJgVDJ479iOiSSUIiTTkIwF09ehka5dFEy7sr5aiVNI4uO0DEqIhNVUBXdoDpqIIqe0BS9oXftWXvVPrTPeWtGS2cO0R9pX986UagH</latexit>

w`ij w`

ij � ⌘@L

@w`ij

, b`i b`i � ⌘@L

@b`i<latexit sha1_base64="Tt/xv9DSrRGujanZVTzkBNfJboY=">AAACmHichVFdSxtBFJ1dtY2rrVGhL74MjYIPbdj4YgUfLNKq4IMWo0I2hruTuzpm9sOZu5Ww7H/qb/HNf+NsDBhNwQsDh3PP/Zhzw0xJQ77/6Lgzs3MfPtbmvYXFT5+X6ssr5ybNtcC2SFWqL0MwqGSCbZKk8DLTCHGo8CIc7Ff5i7+ojUyTMxpm2I3hOpGRFECW6tX/ra97971C3pZXASrFA4URgdbpPZ+kv/MACXgQaRBFkIEmCYofly94Ql1+48FdDn3uhZacbjzBvt/3RVx6dlevV2/4TX8UfBq0xqDBxnHSqz8E/VTkMSYkFBjTafkZdYuqvVBYekFuMAMxgGvsWJhAjKZbjJwt+YZl+jxKtX0J8RE7WVFAbMwwDq0yBroxb3MV+b9cJ6foR7eQSZYTJuJ5UJQrTimvzsT7UqMgNbQAhJZ2Vy5uwLpE9piVCa23X54G7a3mTtM/3WrsbY/dqLE19pVtshbbZnvskJ2wNhPOF2fX+eX8dtfcn+6Be/QsdZ1xzSp7Fe6fJ2MPytg=</latexit><latexit sha1_base64="Tt/xv9DSrRGujanZVTzkBNfJboY=">AAACmHichVFdSxtBFJ1dtY2rrVGhL74MjYIPbdj4YgUfLNKq4IMWo0I2hruTuzpm9sOZu5Ww7H/qb/HNf+NsDBhNwQsDh3PP/Zhzw0xJQ77/6Lgzs3MfPtbmvYXFT5+X6ssr5ybNtcC2SFWqL0MwqGSCbZKk8DLTCHGo8CIc7Ff5i7+ojUyTMxpm2I3hOpGRFECW6tX/ra97971C3pZXASrFA4URgdbpPZ+kv/MACXgQaRBFkIEmCYofly94Ql1+48FdDn3uhZacbjzBvt/3RVx6dlevV2/4TX8UfBq0xqDBxnHSqz8E/VTkMSYkFBjTafkZdYuqvVBYekFuMAMxgGvsWJhAjKZbjJwt+YZl+jxKtX0J8RE7WVFAbMwwDq0yBroxb3MV+b9cJ6foR7eQSZYTJuJ5UJQrTimvzsT7UqMgNbQAhJZ2Vy5uwLpE9piVCa23X54G7a3mTtM/3WrsbY/dqLE19pVtshbbZnvskJ2wNhPOF2fX+eX8dtfcn+6Be/QsdZ1xzSp7Fe6fJ2MPytg=</latexit><latexit sha1_base64="Tt/xv9DSrRGujanZVTzkBNfJboY=">AAACmHichVFdSxtBFJ1dtY2rrVGhL74MjYIPbdj4YgUfLNKq4IMWo0I2hruTuzpm9sOZu5Ww7H/qb/HNf+NsDBhNwQsDh3PP/Zhzw0xJQ77/6Lgzs3MfPtbmvYXFT5+X6ssr5ybNtcC2SFWqL0MwqGSCbZKk8DLTCHGo8CIc7Ff5i7+ojUyTMxpm2I3hOpGRFECW6tX/ra97971C3pZXASrFA4URgdbpPZ+kv/MACXgQaRBFkIEmCYofly94Ql1+48FdDn3uhZacbjzBvt/3RVx6dlevV2/4TX8UfBq0xqDBxnHSqz8E/VTkMSYkFBjTafkZdYuqvVBYekFuMAMxgGvsWJhAjKZbjJwt+YZl+jxKtX0J8RE7WVFAbMwwDq0yBroxb3MV+b9cJ6foR7eQSZYTJuJ5UJQrTimvzsT7UqMgNbQAhJZ2Vy5uwLpE9piVCa23X54G7a3mTtM/3WrsbY/dqLE19pVtshbbZnvskJ2wNhPOF2fX+eX8dtfcn+6Be/QsdZ1xzSp7Fe6fJ2MPytg=</latexit>

Page 39: Neural Networks Part II Feed-Forward Neural Networks

39

Page 40: Neural Networks Part II Feed-Forward Neural Networks

40

Page 41: Neural Networks Part II Feed-Forward Neural Networks

41