transformer networks - university of washington

21
Discussion Board: Available on Ed Zoom Link: Available on Canvas Instructor Contact: [email protected] Course Webpage: https://courses.cs.washington.edu/courses/cse599i/20au/ CSE 599I: Generative Models University of Washington Lecture 5 Instructor: John Thickstun Transformer Networks

Upload: others

Post on 02-Dec-2021

1 views

Category:

Documents


0 download

TRANSCRIPT

Discussion Board: Available on Ed

Zoom Link: Available on Canvas

Instructor Contact: [email protected]

Course Webpage: https://courses.cs.washington.edu/courses/cse599i/20au/

CSE 599I: Generative Models University of WashingtonLecture 5

Instructor: John Thickstun

Transformer Networks

Autoregressive Modeling

CSE 599I: Generative Models University of Washington

• Factor the joint distribution into conditionals:

• Learn the conditional distributions

• With an estimator conditioned on finite history:

p(x) =TY

t=1

p(xt|x<t).<latexit sha1_base64="Wy7TcPOwwcD10D4VPkziPBB6fbk=">AAAC+nicdVFdixMxFE3Hr7V+dfXRl2AptDIsnSIorAsL9kEEscK2W2hqyWQybdhkMiS369R0/oQ/wTfxVcRX/Rf+GzPd+rEuXgjncHPPTc69cS6FhW73Ry24dPnK1Ws71+s3bt66faexe3dk9dIwPmRaajOOqeVSZHwIAiQf54ZTFUt+HJ88q+6PT7mxQmdHsMr5VNF5JlLBKPjUrBHmbQK8gDh1RdnBB5jkRiczBwdR+eYI5+1iButi5p5C2dmbNZrdve4m8EUSbUkTbWMw2619IYlmS8UzYJJaO4m6OUwdNSCY5GWdLC3PKTuhcz7xNKOK26nb2Cpxy2cSnGrjTwZ4k/1b4aiyisIi9GhXKq4QFqqCSmBDSb01fxPGsQoL62U8nBuaLwQrwlRqCmGlB62lLfH5z0D6ZOpEli+BZ+zsL+lSYtC4GiNOhOEM5MoTyozwdjBbUEMZ+GHX66TPvV/DX/r2r3JuKGjz0BFq5ooWpfc/J2HF/lQOqO+Y9P0ilfA9xo6ILE3EaVFOelPXLl2ndK06xs2I7JOkKrPiHSdrst/step+khl/y7RSNEt+SUvX/93kf+84kmmjSkfkiBvwaDboNx39u9eLZNTbizx//ah5+GK78x10Hz1AbRShx+gQPUcDNEQMvUdf0Tf0PVgHH4KPwaez0qC21dxD5yL4/BNeTvSZ</latexit><latexit sha1_base64="Wy7TcPOwwcD10D4VPkziPBB6fbk=">AAAC+nicdVFdixMxFE3Hr7V+dfXRl2AptDIsnSIorAsL9kEEscK2W2hqyWQybdhkMiS369R0/oQ/wTfxVcRX/Rf+GzPd+rEuXgjncHPPTc69cS6FhW73Ry24dPnK1Ws71+s3bt66faexe3dk9dIwPmRaajOOqeVSZHwIAiQf54ZTFUt+HJ88q+6PT7mxQmdHsMr5VNF5JlLBKPjUrBHmbQK8gDh1RdnBB5jkRiczBwdR+eYI5+1iButi5p5C2dmbNZrdve4m8EUSbUkTbWMw2619IYlmS8UzYJJaO4m6OUwdNSCY5GWdLC3PKTuhcz7xNKOK26nb2Cpxy2cSnGrjTwZ4k/1b4aiyisIi9GhXKq4QFqqCSmBDSb01fxPGsQoL62U8nBuaLwQrwlRqCmGlB62lLfH5z0D6ZOpEli+BZ+zsL+lSYtC4GiNOhOEM5MoTyozwdjBbUEMZ+GHX66TPvV/DX/r2r3JuKGjz0BFq5ooWpfc/J2HF/lQOqO+Y9P0ilfA9xo6ILE3EaVFOelPXLl2ndK06xs2I7JOkKrPiHSdrst/step+khl/y7RSNEt+SUvX/93kf+84kmmjSkfkiBvwaDboNx39u9eLZNTbizx//ah5+GK78x10Hz1AbRShx+gQPUcDNEQMvUdf0Tf0PVgHH4KPwaez0qC21dxD5yL4/BNeTvSZ</latexit><latexit sha1_base64="Wy7TcPOwwcD10D4VPkziPBB6fbk=">AAAC+nicdVFdixMxFE3Hr7V+dfXRl2AptDIsnSIorAsL9kEEscK2W2hqyWQybdhkMiS369R0/oQ/wTfxVcRX/Rf+GzPd+rEuXgjncHPPTc69cS6FhW73Ry24dPnK1Ws71+s3bt66faexe3dk9dIwPmRaajOOqeVSZHwIAiQf54ZTFUt+HJ88q+6PT7mxQmdHsMr5VNF5JlLBKPjUrBHmbQK8gDh1RdnBB5jkRiczBwdR+eYI5+1iButi5p5C2dmbNZrdve4m8EUSbUkTbWMw2619IYlmS8UzYJJaO4m6OUwdNSCY5GWdLC3PKTuhcz7xNKOK26nb2Cpxy2cSnGrjTwZ4k/1b4aiyisIi9GhXKq4QFqqCSmBDSb01fxPGsQoL62U8nBuaLwQrwlRqCmGlB62lLfH5z0D6ZOpEli+BZ+zsL+lSYtC4GiNOhOEM5MoTyozwdjBbUEMZ+GHX66TPvV/DX/r2r3JuKGjz0BFq5ooWpfc/J2HF/lQOqO+Y9P0ilfA9xo6ILE3EaVFOelPXLl2ndK06xs2I7JOkKrPiHSdrst/step+khl/y7RSNEt+SUvX/93kf+84kmmjSkfkiBvwaDboNx39u9eLZNTbizx//ah5+GK78x10Hz1AbRShx+gQPUcDNEQMvUdf0Tf0PVgHH4KPwaez0qC21dxD5yL4/BNeTvSZ</latexit><latexit sha1_base64="Wy7TcPOwwcD10D4VPkziPBB6fbk=">AAAC+nicdVFdixMxFE3Hr7V+dfXRl2AptDIsnSIorAsL9kEEscK2W2hqyWQybdhkMiS369R0/oQ/wTfxVcRX/Rf+GzPd+rEuXgjncHPPTc69cS6FhW73Ry24dPnK1Ws71+s3bt66faexe3dk9dIwPmRaajOOqeVSZHwIAiQf54ZTFUt+HJ88q+6PT7mxQmdHsMr5VNF5JlLBKPjUrBHmbQK8gDh1RdnBB5jkRiczBwdR+eYI5+1iButi5p5C2dmbNZrdve4m8EUSbUkTbWMw2619IYlmS8UzYJJaO4m6OUwdNSCY5GWdLC3PKTuhcz7xNKOK26nb2Cpxy2cSnGrjTwZ4k/1b4aiyisIi9GhXKq4QFqqCSmBDSb01fxPGsQoL62U8nBuaLwQrwlRqCmGlB62lLfH5z0D6ZOpEli+BZ+zsL+lSYtC4GiNOhOEM5MoTyozwdjBbUEMZ+GHX66TPvV/DX/r2r3JuKGjz0BFq5ooWpfc/J2HF/lQOqO+Y9P0ilfA9xo6ILE3EaVFOelPXLl2ndK06xs2I7JOkKrPiHSdrst/step+khl/y7RSNEt+SUvX/93kf+84kmmjSkfkiBvwaDboNx39u9eLZNTbizx//ah5+GK78x10Hz1AbRShx+gQPUcDNEQMvUdf0Tf0PVgHH4KPwaez0qC21dxD5yL4/BNeTvSZ</latexit>

p(xt|x<t).<latexit sha1_base64="42MTN9bKA/2+AKfrPgEsIp00PVI=">AAADiXicdVLbbhMxEHW7XMqWSwuPvKyIIqUvUTaqREV5qEgeEBcRJJJGyobI651Nrfqysr0lwfV/8Ap/xd/gTdJCUzKS5eMzZ8Yzo0kLRrVptX5vbQd37t67v/Mg3H346PGTvf2nAy1LRaBPJJNqmGINjAroG2oYDAsFmKcMTtPzTuU/vQClqRRfzLyAMcdTQXNKsPHU16Ixm5jL2cS+Nu6gOdmrtZqthUW3QbwCNbSy3mR/G5JMkpKDMIRhrUdxqzBji5WhhIELk1JDgck5nsLIQ4E56LFdlO2iumeyKJfKH2GiBftvhMVc6zlPvZJjc6bXfRX5X1/FGCmZXivA5EdjS0VRGhBk+X9essjIqBpNlFEFxLC5B5go6luIyBlWmBg/wDBMuuB7VNDDXpd1/cg59Z6hTajIM3oxc6P22DacPXC2HkZRLU6Ok6ySafodksvkuNauh74kAd+I5ByL7CrU2e7Evv/grjPdFA2wcjapulLcVg/3t5qPnv1UgMJGKpswOaXG2eW1QaPplEuaOXsFNufKYJmsujdlk7nheOazrcAGXQpKyJIx6uyba7hB65fT2Q42MJXKbypzfjHj9TW8DQbtZuzx58PaybvViu6g5+gFaqAYvUQn6C3qoT4iSKEf6Cf6FewGcXAUvFpKt7dWMc/QDQs6fwAl3i3T</latexit><latexit sha1_base64="42MTN9bKA/2+AKfrPgEsIp00PVI=">AAADiXicdVLbbhMxEHW7XMqWSwuPvKyIIqUvUTaqREV5qEgeEBcRJJJGyobI651Nrfqysr0lwfV/8Ap/xd/gTdJCUzKS5eMzZ8Yzo0kLRrVptX5vbQd37t67v/Mg3H346PGTvf2nAy1LRaBPJJNqmGINjAroG2oYDAsFmKcMTtPzTuU/vQClqRRfzLyAMcdTQXNKsPHU16Ixm5jL2cS+Nu6gOdmrtZqthUW3QbwCNbSy3mR/G5JMkpKDMIRhrUdxqzBji5WhhIELk1JDgck5nsLIQ4E56LFdlO2iumeyKJfKH2GiBftvhMVc6zlPvZJjc6bXfRX5X1/FGCmZXivA5EdjS0VRGhBk+X9essjIqBpNlFEFxLC5B5go6luIyBlWmBg/wDBMuuB7VNDDXpd1/cg59Z6hTajIM3oxc6P22DacPXC2HkZRLU6Ok6ySafodksvkuNauh74kAd+I5ByL7CrU2e7Evv/grjPdFA2wcjapulLcVg/3t5qPnv1UgMJGKpswOaXG2eW1QaPplEuaOXsFNufKYJmsujdlk7nheOazrcAGXQpKyJIx6uyba7hB65fT2Q42MJXKbypzfjHj9TW8DQbtZuzx58PaybvViu6g5+gFaqAYvUQn6C3qoT4iSKEf6Cf6FewGcXAUvFpKt7dWMc/QDQs6fwAl3i3T</latexit><latexit sha1_base64="42MTN9bKA/2+AKfrPgEsIp00PVI=">AAADiXicdVLbbhMxEHW7XMqWSwuPvKyIIqUvUTaqREV5qEgeEBcRJJJGyobI651Nrfqysr0lwfV/8Ap/xd/gTdJCUzKS5eMzZ8Yzo0kLRrVptX5vbQd37t67v/Mg3H346PGTvf2nAy1LRaBPJJNqmGINjAroG2oYDAsFmKcMTtPzTuU/vQClqRRfzLyAMcdTQXNKsPHU16Ixm5jL2cS+Nu6gOdmrtZqthUW3QbwCNbSy3mR/G5JMkpKDMIRhrUdxqzBji5WhhIELk1JDgck5nsLIQ4E56LFdlO2iumeyKJfKH2GiBftvhMVc6zlPvZJjc6bXfRX5X1/FGCmZXivA5EdjS0VRGhBk+X9essjIqBpNlFEFxLC5B5go6luIyBlWmBg/wDBMuuB7VNDDXpd1/cg59Z6hTajIM3oxc6P22DacPXC2HkZRLU6Ok6ySafodksvkuNauh74kAd+I5ByL7CrU2e7Evv/grjPdFA2wcjapulLcVg/3t5qPnv1UgMJGKpswOaXG2eW1QaPplEuaOXsFNufKYJmsujdlk7nheOazrcAGXQpKyJIx6uyba7hB65fT2Q42MJXKbypzfjHj9TW8DQbtZuzx58PaybvViu6g5+gFaqAYvUQn6C3qoT4iSKEf6Cf6FewGcXAUvFpKt7dWMc/QDQs6fwAl3i3T</latexit><latexit sha1_base64="42MTN9bKA/2+AKfrPgEsIp00PVI=">AAADiXicdVLbbhMxEHW7XMqWSwuPvKyIIqUvUTaqREV5qEgeEBcRJJJGyobI651Nrfqysr0lwfV/8Ap/xd/gTdJCUzKS5eMzZ8Yzo0kLRrVptX5vbQd37t67v/Mg3H346PGTvf2nAy1LRaBPJJNqmGINjAroG2oYDAsFmKcMTtPzTuU/vQClqRRfzLyAMcdTQXNKsPHU16Ixm5jL2cS+Nu6gOdmrtZqthUW3QbwCNbSy3mR/G5JMkpKDMIRhrUdxqzBji5WhhIELk1JDgck5nsLIQ4E56LFdlO2iumeyKJfKH2GiBftvhMVc6zlPvZJjc6bXfRX5X1/FGCmZXivA5EdjS0VRGhBk+X9essjIqBpNlFEFxLC5B5go6luIyBlWmBg/wDBMuuB7VNDDXpd1/cg59Z6hTajIM3oxc6P22DacPXC2HkZRLU6Ok6ySafodksvkuNauh74kAd+I5ByL7CrU2e7Evv/grjPdFA2wcjapulLcVg/3t5qPnv1UgMJGKpswOaXG2eW1QaPplEuaOXsFNufKYJmsujdlk7nheOazrcAGXQpKyJIx6uyba7hB65fT2Q42MJXKbypzfjHj9TW8DQbtZuzx58PaybvViu6g5+gFaqAYvUQn6C3qoT4iSKEf6Cf6FewGcXAUvFpKt7dWMc/QDQs6fwAl3i3T</latexit>

p̂(x) =TY

t=1

p̂(xt|x<t) = p̂(x1)p̂(x2|x1)p̂(x3|x2, x1) · · ·TY

t=p+1

p̂(xt|xt�p, . . . , xt�1).<latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit><latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit><latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit><latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit>

Lecture 5

Learning the Marginals

CSE 599I: Generative Models University of Washington

• Learn an estimator conditioned on finite history :

• Option A: pad the sequence with “start” tokens.

• Option B: learn the marginals directly with NADE.

p̂(x) =TY

t=1

p̂(xt|x<t) = p̂(x1)p̂(x2|x1)p̂(x3|x2, x1) · · ·TY

t=p+1

p̂(xt|xt�p, . . . , xt�1).<latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit><latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit><latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit><latexit sha1_base64="5H9oMLcvmJm31sJzmE0MTJ3e9N4=">AAAEGnicdVPLbhMxFJ10eJTwaAtLNhZRpUSUKpMigVQqVTQLxEMUqU0r1WHk8XhSq/Z4ZN+UBMd/Aj/DDrFlw9/gyQv6iKWRzz333GN7fJ0UghtoNv9UlsIbN2/dXr5TvXvv/oOV1bWHHaP6mrJDqoTSxwkxTPCcHQIHwY4LzYhMBDtKzvbK/NE504ar/ACGBetK0st5xikBT8Wr3/EpAVTUMbABJJkduAbaQbjQKo0t7ETu8wGaSgYxjAaxfQUTyYyMGnPYGl0It3zY2hhTNFVg5q7F02t84VnhNnCp2xhHkWtsxqu15mZzPNBVEE1BLZiO/XhtiXkL2pcsByqIMSdRs4CuJRo4FcxVcd+wgtAz0mMnHuZEMtO14//o0LpnUpQp7b8c0Jj9v8ISacxQJl4pCZyay7mSvDZXMqCUMJc2ANnLruV50QeW08n6WV8gUKi8K5RyzSiIoQeEau6PgOgp0YSCv9FqFbeZP6Nm+8Tr0rbvAcl95thinmcpPx+4k1bX1p1tOLteRagW4W2cljLDvzI8wtu11nrVbylnX6iSkuTprNTZdmzfvXdzp4uiDtHO4vJUWtoycP9288GzHwumCShtsVA9Ds5OpgUaw3tS8dTZGVjslbKJWTkvclMZSDLwblOwQJcwnau+ENzZ13O4QOtfi7N7BFhPaf90hPONGV1uw6ug09qMPP70vLb7dtqiy8Hj4ElQD6LgRbAbvAn2g8OAVsJKo9KqbIXfwh/hz/DXRLpUmdY8Ci6M8PdfrdhhZg==</latexit>

p<latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit><latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit><latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit><latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit>

p<latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit><latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit><latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit><latexit sha1_base64="ry78SCCPsFmvYOrSs1pkCE1Cm78=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHUYTXZipPdZG7RVZAuQYyW1ptstyArJKk5CEMY1nqUJpUZW6wMJQxcmNUaKkxO8BRGHgrMQY/tvFIXtT1TRKVU/ggTzdl/IyzmWs947pUcm2O96mvI//oaxkjJ9EoBpnw9tlRUtQFBFv+XNYuMjJppRAVVQAybeYCJor6FiBxjhYnxMwvDrAu+RwU97HVF10+ZU+8Z2oyKsqCnZ27UGdsdZ5862w6jKE6zvaxoZJp+hexbthd32qEvScAXIjnHojgPdbY7se8/uItMl0UDrJzNmq4Ut83D/a3mo2c/VaCwkcpmTE6pcXZxrdFoOuWSFs6eg/W5Clgka+512WRpOD7z2ZZgjS4HJWTNGHX2zQVco/X76OwBNjCVyi8nc34x09U1vAoGnd3U48MX8f675YpuokfoMdpBKXqF9tFb1EN9RBCg7+gH+tn6HTwJngXPF9LWxjLmIbpkwcs/cdEpiw==</latexit>

Lecture 5

NADE Marginal Estimation

CSE 599I: Generative Models University of Washington

• Neural Autoregressive Distribution Estimation (NADE):

• Learn

• For each IMPORTANT: this helps regularize the optimization!

pt(xt|x1, . . . , xt�1) = Categorical(softmax(ft(x1, . . . , xt�1))).<latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit><latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit><latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit><latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit>

t = 1, . . . , p.<latexit sha1_base64="3kK+e2QqSqegOUrIkKzYuu8yvQY=">AAADiXicdVLbbhMxEHW7XMqWSwuPvFhEkYpURdmoEhUVUkXygLiIIJE0UjZEXu9satWXle2UBuP/4BX+ir/Bm6SFpmQky8dnzoxnRpOVnBnbbP7e2Ixu3b5zd+tevH3/wcNHO7uP+0ZNNYUeVVzpQUYMcCahZ5nlMCg1EJFxOMnO2pX/5By0YUp+trMSRoJMJCsYJTZQXyx+hZP9NFfW7JeN8U6t2WjODd8EyRLU0NK6491NCLF0KkBayokxw6RZ2pEj2jLKwcfp1EBJ6BmZwDBASQSYkZuX7XE9MDkulA5HWjxn/41wRBgzE1lQCmJPzaqvIv/rqxirFDcrBdjicOSYLKcWJF38X0w5tgpXo8E500AtnwVAqGahBUxPiSbUhgHGcdqB0KOGLgm6vBNGLljwDFzKZJGz8ws/bI3cnnfPvavHGNeS9CjNK5lh3yD9nh7VWvU4lCThK1VCEJlfhnrXGbt37/1VpuuiPtHepVVXWrjq4f9W8yGwH0vQxCrtUq4mzHq3uNZoDJsIxXLvLsH6XDksklX3umyqsIJchGxLsEaXgZZqyjnz7vUVXKMNy+ldm1iYKB02lfuwmMnqGt4E/VYjCfjTQe347XJFt9BT9AztoQS9QMfoDeqiHqJIox/oJ/oVbUdJdBi9XEg3N5YxT9A1i9p/AO2ULMw=</latexit><latexit sha1_base64="3kK+e2QqSqegOUrIkKzYuu8yvQY=">AAADiXicdVLbbhMxEHW7XMqWSwuPvFhEkYpURdmoEhUVUkXygLiIIJE0UjZEXu9satWXle2UBuP/4BX+ir/Bm6SFpmQky8dnzoxnRpOVnBnbbP7e2Ixu3b5zd+tevH3/wcNHO7uP+0ZNNYUeVVzpQUYMcCahZ5nlMCg1EJFxOMnO2pX/5By0YUp+trMSRoJMJCsYJTZQXyx+hZP9NFfW7JeN8U6t2WjODd8EyRLU0NK6491NCLF0KkBayokxw6RZ2pEj2jLKwcfp1EBJ6BmZwDBASQSYkZuX7XE9MDkulA5HWjxn/41wRBgzE1lQCmJPzaqvIv/rqxirFDcrBdjicOSYLKcWJF38X0w5tgpXo8E500AtnwVAqGahBUxPiSbUhgHGcdqB0KOGLgm6vBNGLljwDFzKZJGz8ws/bI3cnnfPvavHGNeS9CjNK5lh3yD9nh7VWvU4lCThK1VCEJlfhnrXGbt37/1VpuuiPtHepVVXWrjq4f9W8yGwH0vQxCrtUq4mzHq3uNZoDJsIxXLvLsH6XDksklX3umyqsIJchGxLsEaXgZZqyjnz7vUVXKMNy+ldm1iYKB02lfuwmMnqGt4E/VYjCfjTQe347XJFt9BT9AztoQS9QMfoDeqiHqJIox/oJ/oVbUdJdBi9XEg3N5YxT9A1i9p/AO2ULMw=</latexit><latexit sha1_base64="3kK+e2QqSqegOUrIkKzYuu8yvQY=">AAADiXicdVLbbhMxEHW7XMqWSwuPvFhEkYpURdmoEhUVUkXygLiIIJE0UjZEXu9satWXle2UBuP/4BX+ir/Bm6SFpmQky8dnzoxnRpOVnBnbbP7e2Ixu3b5zd+tevH3/wcNHO7uP+0ZNNYUeVVzpQUYMcCahZ5nlMCg1EJFxOMnO2pX/5By0YUp+trMSRoJMJCsYJTZQXyx+hZP9NFfW7JeN8U6t2WjODd8EyRLU0NK6491NCLF0KkBayokxw6RZ2pEj2jLKwcfp1EBJ6BmZwDBASQSYkZuX7XE9MDkulA5HWjxn/41wRBgzE1lQCmJPzaqvIv/rqxirFDcrBdjicOSYLKcWJF38X0w5tgpXo8E500AtnwVAqGahBUxPiSbUhgHGcdqB0KOGLgm6vBNGLljwDFzKZJGz8ws/bI3cnnfPvavHGNeS9CjNK5lh3yD9nh7VWvU4lCThK1VCEJlfhnrXGbt37/1VpuuiPtHepVVXWrjq4f9W8yGwH0vQxCrtUq4mzHq3uNZoDJsIxXLvLsH6XDksklX3umyqsIJchGxLsEaXgZZqyjnz7vUVXKMNy+ldm1iYKB02lfuwmMnqGt4E/VYjCfjTQe347XJFt9BT9AztoQS9QMfoDeqiHqJIox/oJ/oVbUdJdBi9XEg3N5YxT9A1i9p/AO2ULMw=</latexit><latexit sha1_base64="3kK+e2QqSqegOUrIkKzYuu8yvQY=">AAADiXicdVLbbhMxEHW7XMqWSwuPvFhEkYpURdmoEhUVUkXygLiIIJE0UjZEXu9satWXle2UBuP/4BX+ir/Bm6SFpmQky8dnzoxnRpOVnBnbbP7e2Ixu3b5zd+tevH3/wcNHO7uP+0ZNNYUeVVzpQUYMcCahZ5nlMCg1EJFxOMnO2pX/5By0YUp+trMSRoJMJCsYJTZQXyx+hZP9NFfW7JeN8U6t2WjODd8EyRLU0NK6491NCLF0KkBayokxw6RZ2pEj2jLKwcfp1EBJ6BmZwDBASQSYkZuX7XE9MDkulA5HWjxn/41wRBgzE1lQCmJPzaqvIv/rqxirFDcrBdjicOSYLKcWJF38X0w5tgpXo8E500AtnwVAqGahBUxPiSbUhgHGcdqB0KOGLgm6vBNGLljwDFzKZJGz8ws/bI3cnnfPvavHGNeS9CjNK5lh3yD9nh7VWvU4lCThK1VCEJlfhnrXGbt37/1VpuuiPtHepVVXWrjq4f9W8yGwH0vQxCrtUq4mzHq3uNZoDJsIxXLvLsH6XDksklX3umyqsIJchGxLsEaXgZZqyjnz7vUVXKMNy+ldm1iYKB02lfuwmMnqGt4E/VYjCfjTQe347XJFt9BT9AztoQS9QMfoDeqiHqJIox/oJ/oVbUdJdBi9XEg3N5YxT9A1i9p/AO2ULMw=</latexit>

ht = sigmoid

ct +

X

s<t

Vsxs

!, V 2 Rp⇥k⇥d, c 2 Rp⇥k,

ft(x<t) = bt +X

st

Wshs, W 2 Rp⇥d⇥k,b 2 Rp⇥d.

<latexit sha1_base64="RjtBxkEhFGvoNGQDTIs7PpKXJ4A=">AAAE0HicdVNdb9MwFE1GgRG+NnjkxaLqtIlpaickkAbSxPaA+BBjYu2kuUSO47TWnDjYN6PDWIhX/hE/hSde4V/gtOnabcRSlJN7ju89N76OcsE1tNu//IUrjavXri/eCG7eun3n7tLyva6WhaLsgEoh1WFENBM8YwfAQbDDXDGSRoL1ouOdku+dMKW5zD7Aac76KRlkPOGUgAuFy/7+CgY2gigxQxsCeo6w5oNU8hgLlsDqlKQl+ciRRRoa/QzslOjaUKNRqLHigyGsraOVGYMwzxBOCQyjyOzbjybHwFOm0TGqQGzX0axEvd6uYxysJKEBe2ZpZEPjjKyVnqtQdM6l6+ATmjntOadzveo5q7360jGaeZirU6+3G+FSs73RHi90GXQq0PSqtRcuLzAcS1qkLAMqiNZHnXYOfUMUcCqYDXChWU7oMRmwIwcz4ir1zfj0LWq5SIwSqdyTARpH53cYkmp9mjrPrdKvvsiVwf9yZQSkFPqCAUie9g3P8gJYRif1k0IgkKicMBRzxSiIUwcIVdy1gOiQKELBzWEQ4F3melRsjzhdvOsmN+WOOTTujyYxPxnZo82+WbVmzZpWgFCzg7dwXMo0/8LwV7zV3GwFzlLGPlOZpiSLp1ut2Q3N6zf2LNN5UZcoa8ZnplJTftiZm7cu+i5nioBUBgs54GDN5FWjqa6JNVNQnytmk2Tluy6bTCAlI5etAjW6iKlMFkJwa16cwRqtu+PW7BBgA6nchRfWDWbn4hheBt3NjY7D7x83t19VI7roPfAeeqtex3vibXsvvT3vwKP+T/+3/8f/29hvjBrfGt8n0gW/2nPfO7caP/4BueKrfw==</latexit><latexit sha1_base64="RjtBxkEhFGvoNGQDTIs7PpKXJ4A=">AAAE0HicdVNdb9MwFE1GgRG+NnjkxaLqtIlpaickkAbSxPaA+BBjYu2kuUSO47TWnDjYN6PDWIhX/hE/hSde4V/gtOnabcRSlJN7ju89N76OcsE1tNu//IUrjavXri/eCG7eun3n7tLyva6WhaLsgEoh1WFENBM8YwfAQbDDXDGSRoL1ouOdku+dMKW5zD7Aac76KRlkPOGUgAuFy/7+CgY2gigxQxsCeo6w5oNU8hgLlsDqlKQl+ciRRRoa/QzslOjaUKNRqLHigyGsraOVGYMwzxBOCQyjyOzbjybHwFOm0TGqQGzX0axEvd6uYxysJKEBe2ZpZEPjjKyVnqtQdM6l6+ATmjntOadzveo5q7360jGaeZirU6+3G+FSs73RHi90GXQq0PSqtRcuLzAcS1qkLAMqiNZHnXYOfUMUcCqYDXChWU7oMRmwIwcz4ir1zfj0LWq5SIwSqdyTARpH53cYkmp9mjrPrdKvvsiVwf9yZQSkFPqCAUie9g3P8gJYRif1k0IgkKicMBRzxSiIUwcIVdy1gOiQKELBzWEQ4F3melRsjzhdvOsmN+WOOTTujyYxPxnZo82+WbVmzZpWgFCzg7dwXMo0/8LwV7zV3GwFzlLGPlOZpiSLp1ut2Q3N6zf2LNN5UZcoa8ZnplJTftiZm7cu+i5nioBUBgs54GDN5FWjqa6JNVNQnytmk2Tluy6bTCAlI5etAjW6iKlMFkJwa16cwRqtu+PW7BBgA6nchRfWDWbn4hheBt3NjY7D7x83t19VI7roPfAeeqtex3vibXsvvT3vwKP+T/+3/8f/29hvjBrfGt8n0gW/2nPfO7caP/4BueKrfw==</latexit><latexit sha1_base64="RjtBxkEhFGvoNGQDTIs7PpKXJ4A=">AAAE0HicdVNdb9MwFE1GgRG+NnjkxaLqtIlpaickkAbSxPaA+BBjYu2kuUSO47TWnDjYN6PDWIhX/hE/hSde4V/gtOnabcRSlJN7ju89N76OcsE1tNu//IUrjavXri/eCG7eun3n7tLyva6WhaLsgEoh1WFENBM8YwfAQbDDXDGSRoL1ouOdku+dMKW5zD7Aac76KRlkPOGUgAuFy/7+CgY2gigxQxsCeo6w5oNU8hgLlsDqlKQl+ciRRRoa/QzslOjaUKNRqLHigyGsraOVGYMwzxBOCQyjyOzbjybHwFOm0TGqQGzX0axEvd6uYxysJKEBe2ZpZEPjjKyVnqtQdM6l6+ATmjntOadzveo5q7360jGaeZirU6+3G+FSs73RHi90GXQq0PSqtRcuLzAcS1qkLAMqiNZHnXYOfUMUcCqYDXChWU7oMRmwIwcz4ir1zfj0LWq5SIwSqdyTARpH53cYkmp9mjrPrdKvvsiVwf9yZQSkFPqCAUie9g3P8gJYRif1k0IgkKicMBRzxSiIUwcIVdy1gOiQKELBzWEQ4F3melRsjzhdvOsmN+WOOTTujyYxPxnZo82+WbVmzZpWgFCzg7dwXMo0/8LwV7zV3GwFzlLGPlOZpiSLp1ut2Q3N6zf2LNN5UZcoa8ZnplJTftiZm7cu+i5nioBUBgs54GDN5FWjqa6JNVNQnytmk2Tluy6bTCAlI5etAjW6iKlMFkJwa16cwRqtu+PW7BBgA6nchRfWDWbn4hheBt3NjY7D7x83t19VI7roPfAeeqtex3vibXsvvT3vwKP+T/+3/8f/29hvjBrfGt8n0gW/2nPfO7caP/4BueKrfw==</latexit><latexit sha1_base64="RjtBxkEhFGvoNGQDTIs7PpKXJ4A=">AAAE0HicdVNdb9MwFE1GgRG+NnjkxaLqtIlpaickkAbSxPaA+BBjYu2kuUSO47TWnDjYN6PDWIhX/hE/hSde4V/gtOnabcRSlJN7ju89N76OcsE1tNu//IUrjavXri/eCG7eun3n7tLyva6WhaLsgEoh1WFENBM8YwfAQbDDXDGSRoL1ouOdku+dMKW5zD7Aac76KRlkPOGUgAuFy/7+CgY2gigxQxsCeo6w5oNU8hgLlsDqlKQl+ciRRRoa/QzslOjaUKNRqLHigyGsraOVGYMwzxBOCQyjyOzbjybHwFOm0TGqQGzX0axEvd6uYxysJKEBe2ZpZEPjjKyVnqtQdM6l6+ATmjntOadzveo5q7360jGaeZirU6+3G+FSs73RHi90GXQq0PSqtRcuLzAcS1qkLAMqiNZHnXYOfUMUcCqYDXChWU7oMRmwIwcz4ir1zfj0LWq5SIwSqdyTARpH53cYkmp9mjrPrdKvvsiVwf9yZQSkFPqCAUie9g3P8gJYRif1k0IgkKicMBRzxSiIUwcIVdy1gOiQKELBzWEQ4F3melRsjzhdvOsmN+WOOTTujyYxPxnZo82+WbVmzZpWgFCzg7dwXMo0/8LwV7zV3GwFzlLGPlOZpiSLp1ut2Q3N6zf2LNN5UZcoa8ZnplJTftiZm7cu+i5nioBUBgs54GDN5FWjqa6JNVNQnytmk2Tluy6bTCAlI5etAjW6iKlMFkJwa16cwRqtu+PW7BBgA6nchRfWDWbn4hheBt3NjY7D7x83t19VI7roPfAeeqtex3vibXsvvT3vwKP+T/+3/8f/29hvjBrfGt8n0gW/2nPfO7caP/4BueKrfw==</latexit>

Lecture 5

AR Models in the Wild

CSE 599I: Generative Models University of Washington

• OpenAI GPT-2/GPT-3 Models are AR models:

➡ Parameterized by Transformer Networks (today’s lecture).

• DeepMind’s WaveNet models are AR models:

➡ Parameterized by ConvNets (next week).

• VQ-VAE models are a mashup of AR models and VAE’s (next week):

➡ Introducing VAE-based neural compression helps AR models scale.

Lecture 5

Transformer Networks

CSE 599I: Generative Models University of Washington

• A transformer block is a family of functions

• A transformer is a stack of blocks:

• We’ll use a transformer to parameterize a NADE.

• Prefix function is defined by masking:

pt(xt|x1, . . . , xt�1) = Categorical(softmax(ft(x1, . . . , xt�1))).<latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit><latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit><latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit><latexit sha1_base64="UlLlp9lVl0EHaLDWnVDZeHlnwKI=">AAADwXicdVJba9swFFbrXTrv0nR73ItYCCTQhTgMNiiDsuRh7MIyWNJAHIwsy6moLkaSu2Sq/tB+zV67XzM5Sbs1XQ4YfT7nO5/Pkb+0YFSbTudyZze4c/fe/b0H4cNHj5/s1w6ejrQsFSZDLJlU4xRpwqggQ0MNI+NCEcRTRk7Ss15VPzknSlMpvplFQaYczQTNKUbGp5Jav0hMc56Yi3kSHcaZNPpwnljzMnIt+BbGntaMtcwNR/NmvqRu0lqtdlKrd9qdZcDbIFqDOljHIDnYJV4El5wIgxnSehJ1CjO1SBmKGXFhXGpSIHyGZmTioUCc6Kldrutgw2cymEvlH2HgMvtvh0Vc6wVPPZMjc6o3a1Xyv7UqY6RkemMAk7+ZWiqK0hCBV9/PSwaNhNWVwowqgg1beICwon4FiE+RQtj4iw/DuE/8jooMkOdlff+rOPWVsY2pyDN6PneT7tQ2nW052wghrEfxUZxVNE1/kPgiPqp3G6EfSZDvWHKORHbV6mw/sR8/uWulm6QRUs7G1VaK2+rF/Z3ms89+KYhCRiobMzmjxtnVsYWj6YxLmjl7BbZrZWQlVp3b1Fam8mprsIWXEiVkyRh19t013ML1bnW2hwyZSeUdzpw3ZrRpw9tg1G1HHn99VT/+sLboHngOXoAmiMBrcAzegwEYAgx+gl/gEvwOegENikCtqLs7655n4EYE9g8WiUI9</latexit>

f = b(L) � · · · � b(1) : Rp⇥d ! Rp⇥d.<latexit sha1_base64="sYFFjX22gp1AB0wzgsf+Wh7c2S8=">AAAD2HicdVLLbhMxFHUzPMrwSmHJxiKKlG6iTFQJRIVU0SwQFBEQSSMyaeTxeFKrfoxspzQYS+wQW/6Iz+AL2MIf4EmmhaTkSiOfuefch69vkjOqTav1Y6MSXLl67frmjfDmrdt37la37vW1nCpMelgyqQYJ0oRRQXqGGkYGuSKIJ4wcJif7BX94SpSmUrwzs5yMOJoImlGMjHeNq8MMPoXJkW0cbDsYY6owjFNpdIkLJvLMExhzZI6TxL51RzaPDeVEw9SHGLlMwQuuOa7WWs3W3OBlEJWgBkrrjrcqxBfHU06EwQxpPYxauRlZpAzFjLgwnmqSI3yCJmTooUC+0sjOp+Bg3XtSmEnlP2Hg3PtvhEVc6xlPvLJoWK9yhfO/XOExUjK90oDJHo8sFfnUEIEX9bMpg34ixaRhShXBhs08QFhRfwWIj5FC2Pj3CMO4Q/wdFekir0s7/gU59czAxlRkKT09c8P2yDac3Xa2HkJYi+LdOC1kmn4k8ad4t9auh74lQT5gyTkS6Xmos52xfXngLjIti/pIOTt/NMVt8eP+dvPKe1/nRCEjlY2ZnFDj7OJYo9F0wiVNnT0H63OlZJGsONdlk5nh6MxnK8EaXUKUkFPGqLPPLuAard91Z/eRIROp/OIz5xczWl3Dy6DfbkYev9mp7b0oV3QTPAAPQQNE4BHYA89BF/QABt/BT/AL/A7eB5+DL8HXhbSyUcbcB0sWfPsDuHVL0g==</latexit><latexit sha1_base64="sYFFjX22gp1AB0wzgsf+Wh7c2S8=">AAAD2HicdVLLbhMxFHUzPMrwSmHJxiKKlG6iTFQJRIVU0SwQFBEQSSMyaeTxeFKrfoxspzQYS+wQW/6Iz+AL2MIf4EmmhaTkSiOfuefch69vkjOqTav1Y6MSXLl67frmjfDmrdt37la37vW1nCpMelgyqQYJ0oRRQXqGGkYGuSKIJ4wcJif7BX94SpSmUrwzs5yMOJoImlGMjHeNq8MMPoXJkW0cbDsYY6owjFNpdIkLJvLMExhzZI6TxL51RzaPDeVEw9SHGLlMwQuuOa7WWs3W3OBlEJWgBkrrjrcqxBfHU06EwQxpPYxauRlZpAzFjLgwnmqSI3yCJmTooUC+0sjOp+Bg3XtSmEnlP2Hg3PtvhEVc6xlPvLJoWK9yhfO/XOExUjK90oDJHo8sFfnUEIEX9bMpg34ixaRhShXBhs08QFhRfwWIj5FC2Pj3CMO4Q/wdFekir0s7/gU59czAxlRkKT09c8P2yDac3Xa2HkJYi+LdOC1kmn4k8ad4t9auh74lQT5gyTkS6Xmos52xfXngLjIti/pIOTt/NMVt8eP+dvPKe1/nRCEjlY2ZnFDj7OJYo9F0wiVNnT0H63OlZJGsONdlk5nh6MxnK8EaXUKUkFPGqLPPLuAard91Z/eRIROp/OIz5xczWl3Dy6DfbkYev9mp7b0oV3QTPAAPQQNE4BHYA89BF/QABt/BT/AL/A7eB5+DL8HXhbSyUcbcB0sWfPsDuHVL0g==</latexit><latexit sha1_base64="sYFFjX22gp1AB0wzgsf+Wh7c2S8=">AAAD2HicdVLLbhMxFHUzPMrwSmHJxiKKlG6iTFQJRIVU0SwQFBEQSSMyaeTxeFKrfoxspzQYS+wQW/6Iz+AL2MIf4EmmhaTkSiOfuefch69vkjOqTav1Y6MSXLl67frmjfDmrdt37la37vW1nCpMelgyqQYJ0oRRQXqGGkYGuSKIJ4wcJif7BX94SpSmUrwzs5yMOJoImlGMjHeNq8MMPoXJkW0cbDsYY6owjFNpdIkLJvLMExhzZI6TxL51RzaPDeVEw9SHGLlMwQuuOa7WWs3W3OBlEJWgBkrrjrcqxBfHU06EwQxpPYxauRlZpAzFjLgwnmqSI3yCJmTooUC+0sjOp+Bg3XtSmEnlP2Hg3PtvhEVc6xlPvLJoWK9yhfO/XOExUjK90oDJHo8sFfnUEIEX9bMpg34ixaRhShXBhs08QFhRfwWIj5FC2Pj3CMO4Q/wdFekir0s7/gU59czAxlRkKT09c8P2yDac3Xa2HkJYi+LdOC1kmn4k8ad4t9auh74lQT5gyTkS6Xmos52xfXngLjIti/pIOTt/NMVt8eP+dvPKe1/nRCEjlY2ZnFDj7OJYo9F0wiVNnT0H63OlZJGsONdlk5nh6MxnK8EaXUKUkFPGqLPPLuAard91Z/eRIROp/OIz5xczWl3Dy6DfbkYev9mp7b0oV3QTPAAPQQNE4BHYA89BF/QABt/BT/AL/A7eB5+DL8HXhbSyUcbcB0sWfPsDuHVL0g==</latexit><latexit sha1_base64="sYFFjX22gp1AB0wzgsf+Wh7c2S8=">AAAD2HicdVLLbhMxFHUzPMrwSmHJxiKKlG6iTFQJRIVU0SwQFBEQSSMyaeTxeFKrfoxspzQYS+wQW/6Iz+AL2MIf4EmmhaTkSiOfuefch69vkjOqTav1Y6MSXLl67frmjfDmrdt37la37vW1nCpMelgyqQYJ0oRRQXqGGkYGuSKIJ4wcJif7BX94SpSmUrwzs5yMOJoImlGMjHeNq8MMPoXJkW0cbDsYY6owjFNpdIkLJvLMExhzZI6TxL51RzaPDeVEw9SHGLlMwQuuOa7WWs3W3OBlEJWgBkrrjrcqxBfHU06EwQxpPYxauRlZpAzFjLgwnmqSI3yCJmTooUC+0sjOp+Bg3XtSmEnlP2Hg3PtvhEVc6xlPvLJoWK9yhfO/XOExUjK90oDJHo8sFfnUEIEX9bMpg34ixaRhShXBhs08QFhRfwWIj5FC2Pj3CMO4Q/wdFekir0s7/gU59czAxlRkKT09c8P2yDac3Xa2HkJYi+LdOC1kmn4k8ad4t9auh74lQT5gyTkS6Xmos52xfXngLjIti/pIOTt/NMVt8eP+dvPKe1/nRCEjlY2ZnFDj7OJYo9F0wiVNnT0H63OlZJGsONdlk5nh6MxnK8EaXUKUkFPGqLPPLuAard91Z/eRIROp/OIz5xczWl3Dy6DfbkYev9mp7b0oV3QTPAAPQQNE4BHYA89BF/QABt/BT/AL/A7eB5+DL8HXhbSyUcbcB0sWfPsDuHVL0g==</latexit>

b : Rp⇥d ! Rp⇥d.<latexit sha1_base64="w2vdPSObRIqDwTjDFDmVEHydS48=">AAADs3icdVLbbhMxEHWzXMpyS+GRF4soUnmJshE30ZeK5gFxEQGRNFIcIq/Xm1r1ZWV7S4Pxn/A1vMIP8Dd4k7TQlB1p5bNnzoxnxpMWnBnb7f7eakRXrl67vn0jvnnr9p27zZ17I6NKTeiQKK70OMWGcibp0DLL6bjQFIuU08P0+KDyH55QbZiSn+yioFOB55LljGAbqFnzaQpfQCSwPUpT99F/dgWyTFADMw+RVTWuzqzZ6na6S4OXQbIGLbC2wWynQVGmSCmotIRjYyZJt7BTh7VlhFMfo9LQApNjPKeTACUON03dskEP24HJYK50+KSFS/bfCIeFMQuRBmVVr9n0VeR/fRVjleJmowCbP586JovSUklW9+clh2Eg1RBhxjQlli8CwESz0AIkR1hjYsOo4xj1aehR0wEOuqwfHkew4Bk7xGSesZNTP+lN3a53j7xrxxC2ErSHskpm2FeKvqG9Vq8dh5Ik/UKUEFhmZ6He9WfuzVt/numiaIS1d8s308JVP/5vNe8C+76gGlulHeJqzqx3q6NGY9hcKJZ5dwbqc2V0law667Kp3Ap8GrKtQY0upVqqknPm3ctzWKMNa+zdAbZ0rnTYae7DYiaba3gZjHqdJOAPj1v7r9crug0egIdgFyTgGdgHr8AADAEB38EP8BP8ip5EkyiNspW0sbWOuQ8uWCT+AFtxPqA=</latexit><latexit sha1_base64="w2vdPSObRIqDwTjDFDmVEHydS48=">AAADs3icdVLbbhMxEHWzXMpyS+GRF4soUnmJshE30ZeK5gFxEQGRNFIcIq/Xm1r1ZWV7S4Pxn/A1vMIP8Dd4k7TQlB1p5bNnzoxnxpMWnBnb7f7eakRXrl67vn0jvnnr9p27zZ17I6NKTeiQKK70OMWGcibp0DLL6bjQFIuU08P0+KDyH55QbZiSn+yioFOB55LljGAbqFnzaQpfQCSwPUpT99F/dgWyTFADMw+RVTWuzqzZ6na6S4OXQbIGLbC2wWynQVGmSCmotIRjYyZJt7BTh7VlhFMfo9LQApNjPKeTACUON03dskEP24HJYK50+KSFS/bfCIeFMQuRBmVVr9n0VeR/fRVjleJmowCbP586JovSUklW9+clh2Eg1RBhxjQlli8CwESz0AIkR1hjYsOo4xj1aehR0wEOuqwfHkew4Bk7xGSesZNTP+lN3a53j7xrxxC2ErSHskpm2FeKvqG9Vq8dh5Ik/UKUEFhmZ6He9WfuzVt/numiaIS1d8s308JVP/5vNe8C+76gGlulHeJqzqx3q6NGY9hcKJZ5dwbqc2V0law667Kp3Ap8GrKtQY0upVqqknPm3ctzWKMNa+zdAbZ0rnTYae7DYiaba3gZjHqdJOAPj1v7r9crug0egIdgFyTgGdgHr8AADAEB38EP8BP8ip5EkyiNspW0sbWOuQ8uWCT+AFtxPqA=</latexit><latexit sha1_base64="w2vdPSObRIqDwTjDFDmVEHydS48=">AAADs3icdVLbbhMxEHWzXMpyS+GRF4soUnmJshE30ZeK5gFxEQGRNFIcIq/Xm1r1ZWV7S4Pxn/A1vMIP8Dd4k7TQlB1p5bNnzoxnxpMWnBnb7f7eakRXrl67vn0jvnnr9p27zZ17I6NKTeiQKK70OMWGcibp0DLL6bjQFIuU08P0+KDyH55QbZiSn+yioFOB55LljGAbqFnzaQpfQCSwPUpT99F/dgWyTFADMw+RVTWuzqzZ6na6S4OXQbIGLbC2wWynQVGmSCmotIRjYyZJt7BTh7VlhFMfo9LQApNjPKeTACUON03dskEP24HJYK50+KSFS/bfCIeFMQuRBmVVr9n0VeR/fRVjleJmowCbP586JovSUklW9+clh2Eg1RBhxjQlli8CwESz0AIkR1hjYsOo4xj1aehR0wEOuqwfHkew4Bk7xGSesZNTP+lN3a53j7xrxxC2ErSHskpm2FeKvqG9Vq8dh5Ik/UKUEFhmZ6He9WfuzVt/numiaIS1d8s308JVP/5vNe8C+76gGlulHeJqzqx3q6NGY9hcKJZ5dwbqc2V0law667Kp3Ap8GrKtQY0upVqqknPm3ctzWKMNa+zdAbZ0rnTYae7DYiaba3gZjHqdJOAPj1v7r9crug0egIdgFyTgGdgHr8AADAEB38EP8BP8ip5EkyiNspW0sbWOuQ8uWCT+AFtxPqA=</latexit><latexit sha1_base64="w2vdPSObRIqDwTjDFDmVEHydS48=">AAADs3icdVLbbhMxEHWzXMpyS+GRF4soUnmJshE30ZeK5gFxEQGRNFIcIq/Xm1r1ZWV7S4Pxn/A1vMIP8Dd4k7TQlB1p5bNnzoxnxpMWnBnb7f7eakRXrl67vn0jvnnr9p27zZ17I6NKTeiQKK70OMWGcibp0DLL6bjQFIuU08P0+KDyH55QbZiSn+yioFOB55LljGAbqFnzaQpfQCSwPUpT99F/dgWyTFADMw+RVTWuzqzZ6na6S4OXQbIGLbC2wWynQVGmSCmotIRjYyZJt7BTh7VlhFMfo9LQApNjPKeTACUON03dskEP24HJYK50+KSFS/bfCIeFMQuRBmVVr9n0VeR/fRVjleJmowCbP586JovSUklW9+clh2Eg1RBhxjQlli8CwESz0AIkR1hjYsOo4xj1aehR0wEOuqwfHkew4Bk7xGSesZNTP+lN3a53j7xrxxC2ErSHskpm2FeKvqG9Vq8dh5Ik/UKUEFhmZ6He9WfuzVt/numiaIS1d8s308JVP/5vNe8C+76gGlulHeJqzqx3q6NGY9hcKJZ5dwbqc2V0law667Kp3Ap8GrKtQY0upVqqknPm3ctzWKMNa+zdAbZ0rnTYae7DYiaba3gZjHqdJOAPj1v7r9crug0egIdgFyTgGdgHr8AADAEB38EP8BP8ip5EkyiNspW0sbWOuQ8uWCT+AFtxPqA=</latexit>

ft(x1, . . . , xt�1) = f(mt � x).<latexit sha1_base64="L2wTSgzLSx/g0ipKT14T72W9KYI=">AAADt3icdVLbbhMxEHUbLiVc2sIjLxZRpUQqUTZC4lIhVTQPiIsIEm0jZaOV1zubWLXXiz0pKcb/wtfwCq/8Dd4mLbQlI1k+PnNmPGNPWkphsdP5vbJau3b9xs21W/Xbd+7eW9/YvH9g9dRw2OdaajNImQUpCthHgRIGpQGmUgmH6dFe5T88BmOFLj7hSQkjxcaFyAVnGKhk40WeYHOWRNtxptFuzxKHjyPfoi9p3owRZpjmTvkEaayDgJ5RM99qJxuNTrtzavQqiBagQRbWTzZXIdzCpwoK5JJZO4w6JY4cMyi4BF+PpxZKxo/YGIYBFkyBHbnTJj3dCkxGc23CKpCesv9GOKasPVFpUCqGE3vZV5H/9VUMai3tpQIwfzZyoiinCAWf359PJUVNq4ekmTDAUZ4EwLgRoQXKJ8wwjuG56/W4B6FHA30WdFkvfJASwTNwsSjyTBzP/LA7ck3vWt5t1SltRPFOnFUyK75C/C3eaXS36qGkAr5wrRQrsrNQ73qJe/vOn2e6KDpgxru46sooVx3832reB/ZDCYahNi6WeizQu/m2RGPFWGmReXcGlufKYJ6s2pdl0zkqNgvZFmCJLgVT6KmUwrtX53CJNoyyd3sMYaxNmGvpw2BGl8fwKjjotqOAPz5p7L5ZjOgaeUgekSaJyFOyS16TPtknnHwnP8hP8qv2vJbU8tpkLl1dWcQ8IBes9vkP4j0/Vg==</latexit><latexit sha1_base64="L2wTSgzLSx/g0ipKT14T72W9KYI=">AAADt3icdVLbbhMxEHUbLiVc2sIjLxZRpUQqUTZC4lIhVTQPiIsIEm0jZaOV1zubWLXXiz0pKcb/wtfwCq/8Dd4mLbQlI1k+PnNmPGNPWkphsdP5vbJau3b9xs21W/Xbd+7eW9/YvH9g9dRw2OdaajNImQUpCthHgRIGpQGmUgmH6dFe5T88BmOFLj7hSQkjxcaFyAVnGKhk40WeYHOWRNtxptFuzxKHjyPfoi9p3owRZpjmTvkEaayDgJ5RM99qJxuNTrtzavQqiBagQRbWTzZXIdzCpwoK5JJZO4w6JY4cMyi4BF+PpxZKxo/YGIYBFkyBHbnTJj3dCkxGc23CKpCesv9GOKasPVFpUCqGE3vZV5H/9VUMai3tpQIwfzZyoiinCAWf359PJUVNq4ekmTDAUZ4EwLgRoQXKJ8wwjuG56/W4B6FHA30WdFkvfJASwTNwsSjyTBzP/LA7ck3vWt5t1SltRPFOnFUyK75C/C3eaXS36qGkAr5wrRQrsrNQ73qJe/vOn2e6KDpgxru46sooVx3832reB/ZDCYahNi6WeizQu/m2RGPFWGmReXcGlufKYJ6s2pdl0zkqNgvZFmCJLgVT6KmUwrtX53CJNoyyd3sMYaxNmGvpw2BGl8fwKjjotqOAPz5p7L5ZjOgaeUgekSaJyFOyS16TPtknnHwnP8hP8qv2vJbU8tpkLl1dWcQ8IBes9vkP4j0/Vg==</latexit><latexit sha1_base64="L2wTSgzLSx/g0ipKT14T72W9KYI=">AAADt3icdVLbbhMxEHUbLiVc2sIjLxZRpUQqUTZC4lIhVTQPiIsIEm0jZaOV1zubWLXXiz0pKcb/wtfwCq/8Dd4mLbQlI1k+PnNmPGNPWkphsdP5vbJau3b9xs21W/Xbd+7eW9/YvH9g9dRw2OdaajNImQUpCthHgRIGpQGmUgmH6dFe5T88BmOFLj7hSQkjxcaFyAVnGKhk40WeYHOWRNtxptFuzxKHjyPfoi9p3owRZpjmTvkEaayDgJ5RM99qJxuNTrtzavQqiBagQRbWTzZXIdzCpwoK5JJZO4w6JY4cMyi4BF+PpxZKxo/YGIYBFkyBHbnTJj3dCkxGc23CKpCesv9GOKasPVFpUCqGE3vZV5H/9VUMai3tpQIwfzZyoiinCAWf359PJUVNq4ekmTDAUZ4EwLgRoQXKJ8wwjuG56/W4B6FHA30WdFkvfJASwTNwsSjyTBzP/LA7ck3vWt5t1SltRPFOnFUyK75C/C3eaXS36qGkAr5wrRQrsrNQ73qJe/vOn2e6KDpgxru46sooVx3832reB/ZDCYahNi6WeizQu/m2RGPFWGmReXcGlufKYJ6s2pdl0zkqNgvZFmCJLgVT6KmUwrtX53CJNoyyd3sMYaxNmGvpw2BGl8fwKjjotqOAPz5p7L5ZjOgaeUgekSaJyFOyS16TPtknnHwnP8hP8qv2vJbU8tpkLl1dWcQ8IBes9vkP4j0/Vg==</latexit><latexit sha1_base64="L2wTSgzLSx/g0ipKT14T72W9KYI=">AAADt3icdVLbbhMxEHUbLiVc2sIjLxZRpUQqUTZC4lIhVTQPiIsIEm0jZaOV1zubWLXXiz0pKcb/wtfwCq/8Dd4mLbQlI1k+PnNmPGNPWkphsdP5vbJau3b9xs21W/Xbd+7eW9/YvH9g9dRw2OdaajNImQUpCthHgRIGpQGmUgmH6dFe5T88BmOFLj7hSQkjxcaFyAVnGKhk40WeYHOWRNtxptFuzxKHjyPfoi9p3owRZpjmTvkEaayDgJ5RM99qJxuNTrtzavQqiBagQRbWTzZXIdzCpwoK5JJZO4w6JY4cMyi4BF+PpxZKxo/YGIYBFkyBHbnTJj3dCkxGc23CKpCesv9GOKasPVFpUCqGE3vZV5H/9VUMai3tpQIwfzZyoiinCAWf359PJUVNq4ekmTDAUZ4EwLgRoQXKJ8wwjuG56/W4B6FHA30WdFkvfJASwTNwsSjyTBzP/LA7ck3vWt5t1SltRPFOnFUyK75C/C3eaXS36qGkAr5wrRQrsrNQ73qJe/vOn2e6KDpgxru46sooVx3832reB/ZDCYahNi6WeizQu/m2RGPFWGmReXcGlufKYJ6s2pdl0zkqNgvZFmCJLgVT6KmUwrtX53CJNoyyd3sMYaxNmGvpw2BGl8fwKjjotqOAPz5p7L5ZjOgaeUgekSaJyFOyS16TPtknnHwnP8hP8qv2vJbU8tpkLl1dWcQ8IBes9vkP4j0/Vg==</latexit>

Lecture 5

Transformer Blocks

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

ui = LayerNorm(xi + u0i; �1,�1), �1,�1 2 R,

z0i= WT

2 ReLU(WT

1 ui), W1 2 Rd⇥m,W2 2 Rm⇥d,

zi = LayerNorm(ui + z0i; �2,�2), �2,�2 2 R.

<latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit>

Multi-Headed Attention

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

ui = LayerNorm(xi + u0i; �1,�1), �1,�1 2 R,

z0i= WT

2 ReLU(WT

1 ui), W1 2 Rd⇥m,W2 2 Rm⇥d,

zi = LayerNorm(ui + z0i; �2,�2), �2,�2 2 R.

<latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit>

Multi-Headed Attention

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

• Could think of this like a learned, differentiable lookup table.

• “Queries” , “Keys” , and “Values” . “Attention” values Q<latexit sha1_base64="blvQXWcpXXhH64TEgJdhcw4TZFw=">AAADfHicdVLbbhMxEHUTLmW5tfDIy4olUhGiykaIVupLRfOAuIhEImmk7CryemdTq76sbG9pMP4CXuHj+BmEN0kLTclIlo/PnBnPjCYrGdWm3f610WjeuHnr9uad4O69+w8ebm0/GmpZKQIDIplUowxrYFTAwFDDYFQqwDxjcJydHtX+4zNQmkrx2cxKSDmeClpQgo2n+v3JVtTebc8tvA7iJYjQ0nqT7QYkuSQVB2EIw1qP43ZpUouVoYSBC5JKQ4nJKZ7C2EOBOejUzit1YcszeVhI5Y8w4Zz9N8JirvWMZ17JsTnRq76a/K+vZoyUTK8UYIr91FJRVgYEWfxfVCw0MqynEeZUATFs5gEmivoWQnKCFSbGzywIki74HhX0sNflXT9lTr1nZBMqipyenbtxJ7U7zj53thWEYRQnB0leyzT9Csm35CDqtAJfkoAvRHKORX4R6mx3Yt9/cJeZroqGWDmb1F0pbuuH+1vNR89+KkFhI5VNmJxS4+ziWqPRdMolzZ29AOtz5bBIVt/rssnCcHzusy3BGl0GSsiKMersm0u4Ruv30dkjbGAqlV9O5vxixqtreB0MO7uxx/1X0eG75YpuoifoKdpBMdpDh+gt6qEBIgjQd/QD/Wz8bj5rvmi+XEgbG8uYx+iKNV//ARHpKWw=</latexit><latexit sha1_base64="blvQXWcpXXhH64TEgJdhcw4TZFw=">AAADfHicdVLbbhMxEHUTLmW5tfDIy4olUhGiykaIVupLRfOAuIhEImmk7CryemdTq76sbG9pMP4CXuHj+BmEN0kLTclIlo/PnBnPjCYrGdWm3f610WjeuHnr9uad4O69+w8ebm0/GmpZKQIDIplUowxrYFTAwFDDYFQqwDxjcJydHtX+4zNQmkrx2cxKSDmeClpQgo2n+v3JVtTebc8tvA7iJYjQ0nqT7QYkuSQVB2EIw1qP43ZpUouVoYSBC5JKQ4nJKZ7C2EOBOejUzit1YcszeVhI5Y8w4Zz9N8JirvWMZ17JsTnRq76a/K+vZoyUTK8UYIr91FJRVgYEWfxfVCw0MqynEeZUATFs5gEmivoWQnKCFSbGzywIki74HhX0sNflXT9lTr1nZBMqipyenbtxJ7U7zj53thWEYRQnB0leyzT9Csm35CDqtAJfkoAvRHKORX4R6mx3Yt9/cJeZroqGWDmb1F0pbuuH+1vNR89+KkFhI5VNmJxS4+ziWqPRdMolzZ29AOtz5bBIVt/rssnCcHzusy3BGl0GSsiKMersm0u4Ruv30dkjbGAqlV9O5vxixqtreB0MO7uxx/1X0eG75YpuoifoKdpBMdpDh+gt6qEBIgjQd/QD/Wz8bj5rvmi+XEgbG8uYx+iKNV//ARHpKWw=</latexit><latexit sha1_base64="blvQXWcpXXhH64TEgJdhcw4TZFw=">AAADfHicdVLbbhMxEHUTLmW5tfDIy4olUhGiykaIVupLRfOAuIhEImmk7CryemdTq76sbG9pMP4CXuHj+BmEN0kLTclIlo/PnBnPjCYrGdWm3f610WjeuHnr9uad4O69+w8ebm0/GmpZKQIDIplUowxrYFTAwFDDYFQqwDxjcJydHtX+4zNQmkrx2cxKSDmeClpQgo2n+v3JVtTebc8tvA7iJYjQ0nqT7QYkuSQVB2EIw1qP43ZpUouVoYSBC5JKQ4nJKZ7C2EOBOejUzit1YcszeVhI5Y8w4Zz9N8JirvWMZ17JsTnRq76a/K+vZoyUTK8UYIr91FJRVgYEWfxfVCw0MqynEeZUATFs5gEmivoWQnKCFSbGzywIki74HhX0sNflXT9lTr1nZBMqipyenbtxJ7U7zj53thWEYRQnB0leyzT9Csm35CDqtAJfkoAvRHKORX4R6mx3Yt9/cJeZroqGWDmb1F0pbuuH+1vNR89+KkFhI5VNmJxS4+ziWqPRdMolzZ29AOtz5bBIVt/rssnCcHzusy3BGl0GSsiKMersm0u4Ruv30dkjbGAqlV9O5vxixqtreB0MO7uxx/1X0eG75YpuoifoKdpBMdpDh+gt6qEBIgjQd/QD/Wz8bj5rvmi+XEgbG8uYx+iKNV//ARHpKWw=</latexit><latexit sha1_base64="blvQXWcpXXhH64TEgJdhcw4TZFw=">AAADfHicdVLbbhMxEHUTLmW5tfDIy4olUhGiykaIVupLRfOAuIhEImmk7CryemdTq76sbG9pMP4CXuHj+BmEN0kLTclIlo/PnBnPjCYrGdWm3f610WjeuHnr9uad4O69+w8ebm0/GmpZKQIDIplUowxrYFTAwFDDYFQqwDxjcJydHtX+4zNQmkrx2cxKSDmeClpQgo2n+v3JVtTebc8tvA7iJYjQ0nqT7QYkuSQVB2EIw1qP43ZpUouVoYSBC5JKQ4nJKZ7C2EOBOejUzit1YcszeVhI5Y8w4Zz9N8JirvWMZ17JsTnRq76a/K+vZoyUTK8UYIr91FJRVgYEWfxfVCw0MqynEeZUATFs5gEmivoWQnKCFSbGzywIki74HhX0sNflXT9lTr1nZBMqipyenbtxJ7U7zj53thWEYRQnB0leyzT9Csm35CDqtAJfkoAvRHKORX4R6mx3Yt9/cJeZroqGWDmb1F0pbuuH+1vNR89+KkFhI5VNmJxS4+ziWqPRdMolzZ29AOtz5bBIVt/rssnCcHzusy3BGl0GSsiKMersm0u4Ruv30dkjbGAqlV9O5vxixqtreB0MO7uxx/1X0eG75YpuoifoKdpBMdpDh+gt6qEBIgjQd/QD/Wz8bj5rvmi+XEgbG8uYx+iKNV//ARHpKWw=</latexit>

K<latexit sha1_base64="a/IRggm9Mz868CmmGGYBsZizuU0=">AAADfHicdVLbbhMxEHUTLmW5tIVHXlYskYoQVTZCgNSXiuYB0SJSiaSRsqvI651Nrfqysr2lwfgLeIWP42dQvUlaaEpGsnx85sx4ZjRZyag27fbvtUbz1u07d9fvBfcfPHy0sbn1eKBlpQj0iWRSDTOsgVEBfUMNg2GpAPOMwXF2ul/7j89AaSrFFzMtIeV4ImhBCTaeOjoYb0btnfbMwpsgXoAILaw33mpAkktScRCGMKz1KG6XJrVYGUoYuCCpNJSYnOIJjDwUmINO7axSF7Y8k4eFVP4IE87YfyMs5lpPeeaVHJsTveyryf/6asZIyfRSAaZ4l1oqysqAIPP/i4qFRob1NMKcKiCGTT3ARFHfQkhOsMLE+JkFQdIF36OCHva6vOunzKn3DG1CRZHTs3M36qR229kXzraCMIziZDfJa5mm3yD5nuxGnVbgSxLwlUjOscgvQ53tju3BobvKdF00wMrZpO5KcVs/3N9qPnn2cwkKG6lswuSEGmfn1wqNphMuae7sJVidK4d5svpelU0WhuNzn20BVugyUEJWjFFn31/BFVq/j87uYwMTqfxyMucXM15ew5tg0NmJPT56He19XKzoOnqKnqFtFKO3aA99QD3URwQB+oF+ol+NP83nzZfNV3NpY20R8wRds+abC/9KKWY=</latexit><latexit sha1_base64="a/IRggm9Mz868CmmGGYBsZizuU0=">AAADfHicdVLbbhMxEHUTLmW5tIVHXlYskYoQVTZCgNSXiuYB0SJSiaSRsqvI651Nrfqysr2lwfgLeIWP42dQvUlaaEpGsnx85sx4ZjRZyag27fbvtUbz1u07d9fvBfcfPHy0sbn1eKBlpQj0iWRSDTOsgVEBfUMNg2GpAPOMwXF2ul/7j89AaSrFFzMtIeV4ImhBCTaeOjoYb0btnfbMwpsgXoAILaw33mpAkktScRCGMKz1KG6XJrVYGUoYuCCpNJSYnOIJjDwUmINO7axSF7Y8k4eFVP4IE87YfyMs5lpPeeaVHJsTveyryf/6asZIyfRSAaZ4l1oqysqAIPP/i4qFRob1NMKcKiCGTT3ARFHfQkhOsMLE+JkFQdIF36OCHva6vOunzKn3DG1CRZHTs3M36qR229kXzraCMIziZDfJa5mm3yD5nuxGnVbgSxLwlUjOscgvQ53tju3BobvKdF00wMrZpO5KcVs/3N9qPnn2cwkKG6lswuSEGmfn1wqNphMuae7sJVidK4d5svpelU0WhuNzn20BVugyUEJWjFFn31/BFVq/j87uYwMTqfxyMucXM15ew5tg0NmJPT56He19XKzoOnqKnqFtFKO3aA99QD3URwQB+oF+ol+NP83nzZfNV3NpY20R8wRds+abC/9KKWY=</latexit><latexit sha1_base64="a/IRggm9Mz868CmmGGYBsZizuU0=">AAADfHicdVLbbhMxEHUTLmW5tIVHXlYskYoQVTZCgNSXiuYB0SJSiaSRsqvI651Nrfqysr2lwfgLeIWP42dQvUlaaEpGsnx85sx4ZjRZyag27fbvtUbz1u07d9fvBfcfPHy0sbn1eKBlpQj0iWRSDTOsgVEBfUMNg2GpAPOMwXF2ul/7j89AaSrFFzMtIeV4ImhBCTaeOjoYb0btnfbMwpsgXoAILaw33mpAkktScRCGMKz1KG6XJrVYGUoYuCCpNJSYnOIJjDwUmINO7axSF7Y8k4eFVP4IE87YfyMs5lpPeeaVHJsTveyryf/6asZIyfRSAaZ4l1oqysqAIPP/i4qFRob1NMKcKiCGTT3ARFHfQkhOsMLE+JkFQdIF36OCHva6vOunzKn3DG1CRZHTs3M36qR229kXzraCMIziZDfJa5mm3yD5nuxGnVbgSxLwlUjOscgvQ53tju3BobvKdF00wMrZpO5KcVs/3N9qPnn2cwkKG6lswuSEGmfn1wqNphMuae7sJVidK4d5svpelU0WhuNzn20BVugyUEJWjFFn31/BFVq/j87uYwMTqfxyMucXM15ew5tg0NmJPT56He19XKzoOnqKnqFtFKO3aA99QD3URwQB+oF+ol+NP83nzZfNV3NpY20R8wRds+abC/9KKWY=</latexit><latexit sha1_base64="a/IRggm9Mz868CmmGGYBsZizuU0=">AAADfHicdVLbbhMxEHUTLmW5tIVHXlYskYoQVTZCgNSXiuYB0SJSiaSRsqvI651Nrfqysr2lwfgLeIWP42dQvUlaaEpGsnx85sx4ZjRZyag27fbvtUbz1u07d9fvBfcfPHy0sbn1eKBlpQj0iWRSDTOsgVEBfUMNg2GpAPOMwXF2ul/7j89AaSrFFzMtIeV4ImhBCTaeOjoYb0btnfbMwpsgXoAILaw33mpAkktScRCGMKz1KG6XJrVYGUoYuCCpNJSYnOIJjDwUmINO7axSF7Y8k4eFVP4IE87YfyMs5lpPeeaVHJsTveyryf/6asZIyfRSAaZ4l1oqysqAIPP/i4qFRob1NMKcKiCGTT3ARFHfQkhOsMLE+JkFQdIF36OCHva6vOunzKn3DG1CRZHTs3M36qR229kXzraCMIziZDfJa5mm3yD5nuxGnVbgSxLwlUjOscgvQ53tju3BobvKdF00wMrZpO5KcVs/3N9qPnn2cwkKG6lswuSEGmfn1wqNphMuae7sJVidK4d5svpelU0WhuNzn20BVugyUEJWjFFn31/BFVq/j87uYwMTqfxyMucXM15ew5tg0NmJPT56He19XKzoOnqKnqFtFKO3aA99QD3URwQB+oF+ol+NP83nzZfNV3NpY20R8wRds+abC/9KKWY=</latexit>

V<latexit sha1_base64="HElz8h5m73Elj5I7c4ho3EXmNt4=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHU4WCyFSe7ydyiqyBdghgtrTfZbkFWSFJzEIYwrPUoTSoztlgZShi4MKs1VJic4CmMPBSYgx7beaUuanumiEqp/BEmmrP/RljMtZ7x3Cs5Nsd61deQ//U1jJGS6ZUCTPl6bKmoagOCLP4vaxYZGTXTiAqqgBg28wATRX0LETnGChPjZxaGWRd8jwp62OuKrp8yp94ztBkVZUFPz9yoM7Y7zj51th1GUZxme1nRyDT9Ctm3bC/utENfkoAvRHKORXEe6mx3Yt9/cBeZLosGWDmbNV0pbpuH+1vNR89+qkBhI5XNmJxS4+ziWqPRdMolLZw9B+tzFbBI1tzrssnScHzmsy3BGl0OSsiaMersmwu4Ruv30dkDbGAqlV9O5vxipqtreBUMOrupx4cv4v13yxXdRI/QY7SDUvQK7aO3qIf6iCBA39EP9LP1O3gSPAueL6StjWXMQ3TJgpd/ACFhKXE=</latexit><latexit sha1_base64="HElz8h5m73Elj5I7c4ho3EXmNt4=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHU4WCyFSe7ydyiqyBdghgtrTfZbkFWSFJzEIYwrPUoTSoztlgZShi4MKs1VJic4CmMPBSYgx7beaUuanumiEqp/BEmmrP/RljMtZ7x3Cs5Nsd61deQ//U1jJGS6ZUCTPl6bKmoagOCLP4vaxYZGTXTiAqqgBg28wATRX0LETnGChPjZxaGWRd8jwp62OuKrp8yp94ztBkVZUFPz9yoM7Y7zj51th1GUZxme1nRyDT9Ctm3bC/utENfkoAvRHKORXEe6mx3Yt9/cBeZLosGWDmbNV0pbpuH+1vNR89+qkBhI5XNmJxS4+ziWqPRdMolLZw9B+tzFbBI1tzrssnScHzmsy3BGl0OSsiaMersmwu4Ruv30dkDbGAqlV9O5vxipqtreBUMOrupx4cv4v13yxXdRI/QY7SDUvQK7aO3qIf6iCBA39EP9LP1O3gSPAueL6StjWXMQ3TJgpd/ACFhKXE=</latexit><latexit sha1_base64="HElz8h5m73Elj5I7c4ho3EXmNt4=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHU4WCyFSe7ydyiqyBdghgtrTfZbkFWSFJzEIYwrPUoTSoztlgZShi4MKs1VJic4CmMPBSYgx7beaUuanumiEqp/BEmmrP/RljMtZ7x3Cs5Nsd61deQ//U1jJGS6ZUCTPl6bKmoagOCLP4vaxYZGTXTiAqqgBg28wATRX0LETnGChPjZxaGWRd8jwp62OuKrp8yp94ztBkVZUFPz9yoM7Y7zj51th1GUZxme1nRyDT9Ctm3bC/utENfkoAvRHKORXEe6mx3Yt9/cBeZLosGWDmbNV0pbpuH+1vNR89+qkBhI5XNmJxS4+ziWqPRdMolLZw9B+tzFbBI1tzrssnScHzmsy3BGl0OSsiaMersmwu4Ruv30dkDbGAqlV9O5vxipqtreBUMOrupx4cv4v13yxXdRI/QY7SDUvQK7aO3qIf6iCBA39EP9LP1O3gSPAueL6StjWXMQ3TJgpd/ACFhKXE=</latexit><latexit sha1_base64="HElz8h5m73Elj5I7c4ho3EXmNt4=">AAADfHicdVLbbhMxEHWzXMpya+GRlxVLpCJEtRshQOpLRfOAuIhUImmkbBR5vbOpVV9Wtrc0GH8Br/Bx/AzCm6SFpmQky8dnzoxnRpNXjGqTJL82WsG16zdubt4Kb9+5e+/+1vaDgZa1ItAnkkk1zLEGRgX0DTUMhpUCzHMGR/nJQeM/OgWlqRSfzayCMcdTQUtKsPHU4WCyFSe7ydyiqyBdghgtrTfZbkFWSFJzEIYwrPUoTSoztlgZShi4MKs1VJic4CmMPBSYgx7beaUuanumiEqp/BEmmrP/RljMtZ7x3Cs5Nsd61deQ//U1jJGS6ZUCTPl6bKmoagOCLP4vaxYZGTXTiAqqgBg28wATRX0LETnGChPjZxaGWRd8jwp62OuKrp8yp94ztBkVZUFPz9yoM7Y7zj51th1GUZxme1nRyDT9Ctm3bC/utENfkoAvRHKORXEe6mx3Yt9/cBeZLosGWDmbNV0pbpuH+1vNR89+qkBhI5XNmJxS4+ziWqPRdMolLZw9B+tzFbBI1tzrssnScHzmsy3BGl0OSsiaMersmwu4Ruv30dkDbGAqlV9O5vxipqtreBUMOrupx4cv4v13yxXdRI/QY7SDUvQK7aO3qIf6iCBA39EP9LP1O3gSPAueL6StjWXMQ3TJgpd/ACFhKXE=</latexit>

↵(h) 2 Rp⇥p.<latexit sha1_base64="sePnulHF1Y2NtWOfXf3mjhuVBR0=">AAADpXicdVLbbhMxEHW7XMpyS+GRF4soIn2JshESSH2paB4QUBFQk0aK08jr9SZWfVnZ3tJg/At8Da/wH/wN3iQtNCUjrXz2zJnxzHjSgjNj2+3fW9vRrdt37u7ci+8/ePjocW33ycCoUhPaJ4orPUyxoZxJ2rfMcjosNMUi5fQkPTus/CfnVBum5LGdF3Qs8FSynBFsAzWpNRHmxQyfuuZsz0PEJEQC21maus/+1BXIMkENLHxrUqu3W+2FwZsgWYE6WFlvsrtNUaZIKai0hGNjRkm7sGOHtWWEUx+j0tACkzM8paMAJQ43jd2iJQ8bgclgrnT4pIUL9t8Ih4Uxc5EGZVWvWfdV5H99FWOV4matAJu/Hjsmi9JSSZb35yWHVsFqbDBjmhLL5wFgolloAZIZ1pjYMNw4Rl0aetS0h4Mu64bnECx4hi5MNM/Y+YUfdcau6d2ed40YwnqC9lFWyQz7StE3tF/vNOJQkqRfiBICy+wy1LvuxL3/4K8yXRcNsPZu8WZauOrH/63mKLAfC6qxVdohrqbMerc8NmgMmwrFMu8uweZcGV0mq85N2VRuBb4I2VZggy6lWqqSc+bdmyu4QRsW17tDbOlU6bDF3IfFTNbX8CYYdFpJwJ9e1g/erVZ0BzwDz0ETJOAVOABvQQ/0AQHfwQ/wE/yKXkRH0XE0WEq3t1YxT8E1iyZ/APdwOM0=</latexit><latexit sha1_base64="sePnulHF1Y2NtWOfXf3mjhuVBR0=">AAADpXicdVLbbhMxEHW7XMpyS+GRF4soIn2JshESSH2paB4QUBFQk0aK08jr9SZWfVnZ3tJg/At8Da/wH/wN3iQtNCUjrXz2zJnxzHjSgjNj2+3fW9vRrdt37u7ci+8/ePjocW33ycCoUhPaJ4orPUyxoZxJ2rfMcjosNMUi5fQkPTus/CfnVBum5LGdF3Qs8FSynBFsAzWpNRHmxQyfuuZsz0PEJEQC21maus/+1BXIMkENLHxrUqu3W+2FwZsgWYE6WFlvsrtNUaZIKai0hGNjRkm7sGOHtWWEUx+j0tACkzM8paMAJQ43jd2iJQ8bgclgrnT4pIUL9t8Ih4Uxc5EGZVWvWfdV5H99FWOV4matAJu/Hjsmi9JSSZb35yWHVsFqbDBjmhLL5wFgolloAZIZ1pjYMNw4Rl0aetS0h4Mu64bnECx4hi5MNM/Y+YUfdcau6d2ed40YwnqC9lFWyQz7StE3tF/vNOJQkqRfiBICy+wy1LvuxL3/4K8yXRcNsPZu8WZauOrH/63mKLAfC6qxVdohrqbMerc8NmgMmwrFMu8uweZcGV0mq85N2VRuBb4I2VZggy6lWqqSc+bdmyu4QRsW17tDbOlU6bDF3IfFTNbX8CYYdFpJwJ9e1g/erVZ0BzwDz0ETJOAVOABvQQ/0AQHfwQ/wE/yKXkRH0XE0WEq3t1YxT8E1iyZ/APdwOM0=</latexit><latexit sha1_base64="sePnulHF1Y2NtWOfXf3mjhuVBR0=">AAADpXicdVLbbhMxEHW7XMpyS+GRF4soIn2JshESSH2paB4QUBFQk0aK08jr9SZWfVnZ3tJg/At8Da/wH/wN3iQtNCUjrXz2zJnxzHjSgjNj2+3fW9vRrdt37u7ci+8/ePjocW33ycCoUhPaJ4orPUyxoZxJ2rfMcjosNMUi5fQkPTus/CfnVBum5LGdF3Qs8FSynBFsAzWpNRHmxQyfuuZsz0PEJEQC21maus/+1BXIMkENLHxrUqu3W+2FwZsgWYE6WFlvsrtNUaZIKai0hGNjRkm7sGOHtWWEUx+j0tACkzM8paMAJQ43jd2iJQ8bgclgrnT4pIUL9t8Ih4Uxc5EGZVWvWfdV5H99FWOV4matAJu/Hjsmi9JSSZb35yWHVsFqbDBjmhLL5wFgolloAZIZ1pjYMNw4Rl0aetS0h4Mu64bnECx4hi5MNM/Y+YUfdcau6d2ed40YwnqC9lFWyQz7StE3tF/vNOJQkqRfiBICy+wy1LvuxL3/4K8yXRcNsPZu8WZauOrH/63mKLAfC6qxVdohrqbMerc8NmgMmwrFMu8uweZcGV0mq85N2VRuBb4I2VZggy6lWqqSc+bdmyu4QRsW17tDbOlU6bDF3IfFTNbX8CYYdFpJwJ9e1g/erVZ0BzwDz0ETJOAVOABvQQ/0AQHfwQ/wE/yKXkRH0XE0WEq3t1YxT8E1iyZ/APdwOM0=</latexit><latexit sha1_base64="sePnulHF1Y2NtWOfXf3mjhuVBR0=">AAADpXicdVLbbhMxEHW7XMpyS+GRF4soIn2JshESSH2paB4QUBFQk0aK08jr9SZWfVnZ3tJg/At8Da/wH/wN3iQtNCUjrXz2zJnxzHjSgjNj2+3fW9vRrdt37u7ci+8/ePjocW33ycCoUhPaJ4orPUyxoZxJ2rfMcjosNMUi5fQkPTus/CfnVBum5LGdF3Qs8FSynBFsAzWpNRHmxQyfuuZsz0PEJEQC21maus/+1BXIMkENLHxrUqu3W+2FwZsgWYE6WFlvsrtNUaZIKai0hGNjRkm7sGOHtWWEUx+j0tACkzM8paMAJQ43jd2iJQ8bgclgrnT4pIUL9t8Ih4Uxc5EGZVWvWfdV5H99FWOV4matAJu/Hjsmi9JSSZb35yWHVsFqbDBjmhLL5wFgolloAZIZ1pjYMNw4Rl0aetS0h4Mu64bnECx4hi5MNM/Y+YUfdcau6d2ed40YwnqC9lFWyQz7StE3tF/vNOJQkqRfiBICy+wy1LvuxL3/4K8yXRcNsPZu8WZauOrH/63mKLAfC6qxVdohrqbMerc8NmgMmwrFMu8uweZcGV0mq85N2VRuBb4I2VZggy6lWqqSc+bdmyu4QRsW17tDbOlU6bDF3IfFTNbX8CYYdFpJwJ9e1g/erVZ0BzwDz0ETJOAVOABvQQ/0AQHfwQ/wE/yKXkRH0XE0WEq3t1YxT8E1iyZ/APdwOM0=</latexit>

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

<latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit>

Multi-Headed Attention

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

• Could think of this like a kernel method, especially if

• Do we even need separate transformations? (Course project… ?)

Q = K.<latexit sha1_base64="Jo2r4aWgUGney/guUpQhPBJURik=">AAADgXicdVLbbhMxEHWzXMpya+GRlxVRpPYl2o0qgRQhVTQPiIJIJZJGykaR1zubmvqysr2lwfgfeIU/42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e+tRnDn7r372w/Ch48eP3m6s/tsqGWlCAyIZFKNMqyBUQEDQw2DUakA84zBaXZ+VPtPL0BpKsVnMy9hwvFM0IISbDw1PIneRMft6U4zbscLi26DZAWaaGX96W4D0lySioMwhGGtx0lcmonFylDCwIVppaHE5BzPYOyhwBz0xC7KdVHLM3lUSOWPMNGC/TfCYq71nGdeybE50+u+mvyvr2aMlEyvFWCK1xNLRVkZEGT5f1GxyMioHkmUUwXEsLkHmCjqW4jIGVaYGD+4MEx74HtU0Mdel/f8qDn1npFNqShyenHpxp2J3XN239lWGEXNJO2meS3T9Buk39Nus9MKfUkCvhLJORb5Vaizvak9/uCuM90UDbFyNq27UtzWD/e3mo+e/VSCwkYqmzI5o8bZ5bVBo+mMS5o7ewU258phmay+N2WTheH40mdbgQ26DJSQFWPU2bfXcIPWL6WzR9jATCq/ocz5xUzW1/A2GHbaiccnB83D96sV3UYv0Eu0hxL0Ch2id6iPBoigL+gH+ol+BUGwH8RBZyltbK1inqMbFnT/AKgRKZU=</latexit><latexit sha1_base64="Jo2r4aWgUGney/guUpQhPBJURik=">AAADgXicdVLbbhMxEHWzXMpya+GRlxVRpPYl2o0qgRQhVTQPiIJIJZJGykaR1zubmvqysr2lwfgfeIU/42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e+tRnDn7r372w/Ch48eP3m6s/tsqGWlCAyIZFKNMqyBUQEDQw2DUakA84zBaXZ+VPtPL0BpKsVnMy9hwvFM0IISbDw1PIneRMft6U4zbscLi26DZAWaaGX96W4D0lySioMwhGGtx0lcmonFylDCwIVppaHE5BzPYOyhwBz0xC7KdVHLM3lUSOWPMNGC/TfCYq71nGdeybE50+u+mvyvr2aMlEyvFWCK1xNLRVkZEGT5f1GxyMioHkmUUwXEsLkHmCjqW4jIGVaYGD+4MEx74HtU0Mdel/f8qDn1npFNqShyenHpxp2J3XN239lWGEXNJO2meS3T9Buk39Nus9MKfUkCvhLJORb5Vaizvak9/uCuM90UDbFyNq27UtzWD/e3mo+e/VSCwkYqmzI5o8bZ5bVBo+mMS5o7ewU258phmay+N2WTheH40mdbgQ26DJSQFWPU2bfXcIPWL6WzR9jATCq/ocz5xUzW1/A2GHbaiccnB83D96sV3UYv0Eu0hxL0Ch2id6iPBoigL+gH+ol+BUGwH8RBZyltbK1inqMbFnT/AKgRKZU=</latexit><latexit sha1_base64="Jo2r4aWgUGney/guUpQhPBJURik=">AAADgXicdVLbbhMxEHWzXMpya+GRlxVRpPYl2o0qgRQhVTQPiIJIJZJGykaR1zubmvqysr2lwfgfeIU/42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e+tRnDn7r372w/Ch48eP3m6s/tsqGWlCAyIZFKNMqyBUQEDQw2DUakA84zBaXZ+VPtPL0BpKsVnMy9hwvFM0IISbDw1PIneRMft6U4zbscLi26DZAWaaGX96W4D0lySioMwhGGtx0lcmonFylDCwIVppaHE5BzPYOyhwBz0xC7KdVHLM3lUSOWPMNGC/TfCYq71nGdeybE50+u+mvyvr2aMlEyvFWCK1xNLRVkZEGT5f1GxyMioHkmUUwXEsLkHmCjqW4jIGVaYGD+4MEx74HtU0Mdel/f8qDn1npFNqShyenHpxp2J3XN239lWGEXNJO2meS3T9Buk39Nus9MKfUkCvhLJORb5Vaizvak9/uCuM90UDbFyNq27UtzWD/e3mo+e/VSCwkYqmzI5o8bZ5bVBo+mMS5o7ewU258phmay+N2WTheH40mdbgQ26DJSQFWPU2bfXcIPWL6WzR9jATCq/ocz5xUzW1/A2GHbaiccnB83D96sV3UYv0Eu0hxL0Ch2id6iPBoigL+gH+ol+BUGwH8RBZyltbK1inqMbFnT/AKgRKZU=</latexit><latexit sha1_base64="Jo2r4aWgUGney/guUpQhPBJURik=">AAADgXicdVLbbhMxEHWzXMpya+GRlxVRpPYl2o0qgRQhVTQPiIJIJZJGykaR1zubmvqysr2lwfgfeIU/42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e+tRnDn7r372w/Ch48eP3m6s/tsqGWlCAyIZFKNMqyBUQEDQw2DUakA84zBaXZ+VPtPL0BpKsVnMy9hwvFM0IISbDw1PIneRMft6U4zbscLi26DZAWaaGX96W4D0lySioMwhGGtx0lcmonFylDCwIVppaHE5BzPYOyhwBz0xC7KdVHLM3lUSOWPMNGC/TfCYq71nGdeybE50+u+mvyvr2aMlEyvFWCK1xNLRVkZEGT5f1GxyMioHkmUUwXEsLkHmCjqW4jIGVaYGD+4MEx74HtU0Mdel/f8qDn1npFNqShyenHpxp2J3XN239lWGEXNJO2meS3T9Buk39Nus9MKfUkCvhLJORb5Vaizvak9/uCuM90UDbFyNq27UtzWD/e3mo+e/VSCwkYqmzI5o8bZ5bVBo+mMS5o7ewU258phmay+N2WTheH40mdbgQ26DJSQFWPU2bfXcIPWL6WzR9jATCq/ocz5xUzW1/A2GHbaiccnB83D96sV3UYv0Eu0hxL0Ch2id6iPBoigL+gH+ol+BUGwH8RBZyltbK1inqMbFnT/AKgRKZU=</latexit>

Q,K<latexit sha1_base64="TT295Gr6fz5IcsxBjiYrwz6trlM=">AAADf3icdVLbbhMxEHW6XMpyaQuPvKyIIoqEot0ICVBfKpoHREGkEkkjZaPI651NrPqysr2lwfgXeIVf42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e/GVnDr9p272/fC+w8ePtrZ3Xs80LJSBPpEMqmGGdbAqIC+oYbBsFSAecbgNDs7qv2n56A0leKLmZcw5ngqaEEJNjV18jI6nuw243a8sOgmSFagiVbWm+xtQZpLUnEQhjCs9SiJSzO2WBlKGLgwrTSUmJzhKYw8FJiDHttFsS5qeSaPCqn8ESZasP9GWMy1nvPMKzk2M73uq8n/+mrGSMn0WgGmeDO2VJSVAUGW/xcVi4yM6oFEOVVADJt7gImivoWIzLDCxPixhWHaBd+jgh72urzrB82p9wxtSkWR0/MLN+qM7b6zL5xthVHUTNKDNK9lmn6D9Ht60Oy0Ql+SgK9Eco5FfhnqbHdijz+6q0zXRQOsnE3rrhS39cP9reaTZz+XoLCRyqZMTqlxdnlt0Gg65ZLmzl6CzblyWCar703ZZGE4vvDZVmCDLgMlZMUYdfbdFdyg9Svp7BE2MJXK7ydzfjGT9TW8CQadduLxyavm4YfVim6jp+gZ2kcJeo0O0XvUQ31E0Az9QD/Rr6ARPA/aQbyUbjVWMU/QNQve/gEzMCki</latexit><latexit sha1_base64="TT295Gr6fz5IcsxBjiYrwz6trlM=">AAADf3icdVLbbhMxEHW6XMpyaQuPvKyIIoqEot0ICVBfKpoHREGkEkkjZaPI651NrPqysr2lwfgXeIVf42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e/GVnDr9p272/fC+w8ePtrZ3Xs80LJSBPpEMqmGGdbAqIC+oYbBsFSAecbgNDs7qv2n56A0leKLmZcw5ngqaEEJNjV18jI6nuw243a8sOgmSFagiVbWm+xtQZpLUnEQhjCs9SiJSzO2WBlKGLgwrTSUmJzhKYw8FJiDHttFsS5qeSaPCqn8ESZasP9GWMy1nvPMKzk2M73uq8n/+mrGSMn0WgGmeDO2VJSVAUGW/xcVi4yM6oFEOVVADJt7gImivoWIzLDCxPixhWHaBd+jgh72urzrB82p9wxtSkWR0/MLN+qM7b6zL5xthVHUTNKDNK9lmn6D9Ht60Oy0Ql+SgK9Eco5FfhnqbHdijz+6q0zXRQOsnE3rrhS39cP9reaTZz+XoLCRyqZMTqlxdnlt0Gg65ZLmzl6CzblyWCar703ZZGE4vvDZVmCDLgMlZMUYdfbdFdyg9Svp7BE2MJXK7ydzfjGT9TW8CQadduLxyavm4YfVim6jp+gZ2kcJeo0O0XvUQ31E0Az9QD/Rr6ARPA/aQbyUbjVWMU/QNQve/gEzMCki</latexit><latexit sha1_base64="TT295Gr6fz5IcsxBjiYrwz6trlM=">AAADf3icdVLbbhMxEHW6XMpyaQuPvKyIIoqEot0ICVBfKpoHREGkEkkjZaPI651NrPqysr2lwfgXeIVf42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e/GVnDr9p272/fC+w8ePtrZ3Xs80LJSBPpEMqmGGdbAqIC+oYbBsFSAecbgNDs7qv2n56A0leKLmZcw5ngqaEEJNjV18jI6nuw243a8sOgmSFagiVbWm+xtQZpLUnEQhjCs9SiJSzO2WBlKGLgwrTSUmJzhKYw8FJiDHttFsS5qeSaPCqn8ESZasP9GWMy1nvPMKzk2M73uq8n/+mrGSMn0WgGmeDO2VJSVAUGW/xcVi4yM6oFEOVVADJt7gImivoWIzLDCxPixhWHaBd+jgh72urzrB82p9wxtSkWR0/MLN+qM7b6zL5xthVHUTNKDNK9lmn6D9Ht60Oy0Ql+SgK9Eco5FfhnqbHdijz+6q0zXRQOsnE3rrhS39cP9reaTZz+XoLCRyqZMTqlxdnlt0Gg65ZLmzl6CzblyWCar703ZZGE4vvDZVmCDLgMlZMUYdfbdFdyg9Svp7BE2MJXK7ydzfjGT9TW8CQadduLxyavm4YfVim6jp+gZ2kcJeo0O0XvUQ31E0Az9QD/Rr6ARPA/aQbyUbjVWMU/QNQve/gEzMCki</latexit><latexit sha1_base64="TT295Gr6fz5IcsxBjiYrwz6trlM=">AAADf3icdVLbbhMxEHW6XMpyaQuPvKyIIoqEot0ICVBfKpoHREGkEkkjZaPI651NrPqysr2lwfgXeIVf42/wJmmhKRnJ8vGZM+OZ0WQlo9rE8e/GVnDr9p272/fC+w8ePtrZ3Xs80LJSBPpEMqmGGdbAqIC+oYbBsFSAecbgNDs7qv2n56A0leKLmZcw5ngqaEEJNjV18jI6nuw243a8sOgmSFagiVbWm+xtQZpLUnEQhjCs9SiJSzO2WBlKGLgwrTSUmJzhKYw8FJiDHttFsS5qeSaPCqn8ESZasP9GWMy1nvPMKzk2M73uq8n/+mrGSMn0WgGmeDO2VJSVAUGW/xcVi4yM6oFEOVVADJt7gImivoWIzLDCxPixhWHaBd+jgh72urzrB82p9wxtSkWR0/MLN+qM7b6zL5xthVHUTNKDNK9lmn6D9Ht60Oy0Ql+SgK9Eco5FfhnqbHdijz+6q0zXRQOsnE3rrhS39cP9reaTZz+XoLCRyqZMTqlxdnlt0Gg65ZLmzl6CzblyWCar703ZZGE4vvDZVmCDLgMlZMUYdfbdFdyg9Svp7BE2MJXK7ydzfjGT9TW8CQadduLxyavm4YfVim6jp+gZ2kcJeo0O0XvUQ31E0Az9QD/Rr6ARPA/aQbyUbjVWMU/QNQve/gEzMCki</latexit>

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

<latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit>

Multi-Headed Attention

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

• The weights aren’t dependent on sequence length!

• Interaction terms between positions don’t have explicit weights.

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

<latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit><latexit sha1_base64="txNLeTLEIKOr7Dc2laEE3SvWKB0=">AAAD0XicfVNda9swFFXsfXTZV7o97kUsLEvAlHgMtpdCYS9lg9FuTVKIEiMrcqxE/qgklwahMfa6f7e3/YL9jcmOC20TcsFwOfcenXuP5DDnTKp+/2/Dce/df/Bw71Hz8ZOnz5639l8MZVYIQgck45k4D7GknKV0oJji9DwXFCchp6Nw+amsjy6pkCxLz9Qqp5MEz1MWMYKVhYL9xr8OPJ3qbtwzXaTolQojfWUC1oOHcBTo2Lsw07ObBQ+iWOaYUO0niYFfdnGXu7nDXdzLDW6ncz2RV5/u1Z0QsRSiBKs4DPU3M9UzpFhCJbQtCDU7EGGexzjQzFuYtaiVqU7XMotUgq3CAnEaqS6KBCYacZzOOd1ujbdt60UPIlGRjEbyQii9NAYJNo9Vrx6ibi7M24CV+rJI7PyHvpkel5sRLy53rtBFiea35t7i1qJ37UrJ3XBhWbswK10IWu3+Qb8KuJn4ddIGdZwErT9olpEioakiHEs59vu5mmgsFCN2ySYqJLV3ucRzOrZpiq3URFcv0sA3FpnBKBP2SxWs0JsMjRMpV0loO8uB5d1aCW6rjQsVfZxoluaFoilZC0UFhyqD5fOGMyYoUXxlE0wEs7NCEmN7pcr+BE1rgn935c1k+O7At/np+/bR59qOPfAKvAZd4IMP4AgcgxMwAMT56ijHOD/c7+7K/en+Wrc6jZrzEtwK9/d/4CU94A==</latexit>

Position-wise Feedforward

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

ui = LayerNorm(xi + u0i; �1,�1), �1,�1 2 R,

z0i= WT

2 ReLU(WT

1 ui), W1 2 Rd⇥m,W2 2 Rm⇥d,

zi = LayerNorm(ui + z0i; �2,�2), �2,�2 2 R.

<latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit>

Position-wise Feedforward

CSE 599I: Generative Models University of Washington

ui = LayerNorm(xi + u0i; �1,�1), �1,�1 2 R,

z0i = WT2 ReLU(WT

1 ui), W1 2 Rd⇥m,W2 2 Rm⇥d,

zi = LayerNorm(ui + z0i; �2,�2), �2,�2 2 R.<latexit sha1_base64="Qu/hUHYzd6oK0ycZuHDuMgafpsc=">AAAE6nicfVRdb9MwFE1LgRG+NnjkxaLa6EQ1NRUSSBPSxPaAYMCY1nXS3EWO43TW7DiyndHO+E/whnjlH/HET+ENp2m3taVYinxy7/G59yQ3iTJGlW61fleqN2o3b91euuPfvXf/wcPllUeHSuQSkw4WTMijCCnCaEo6mmpGjjJJEI8Y6UZn20W+e06koiI90MOM9DjqpzShGGkXClcqbA1ATQY6SkxuQwpel7dmFw2J/Cgkt41JflDkn1+jPwvpJuwjzlEYNGFEtNvXm2DNSc5EAaQpgBzp0ygy+7YJoX9V96IQcoW7YfvkoKy+T3Y7ttENg5ODqfZK9e6c4ImJHY9yogC3zUJpjsAnhHiu/P9t59O2L67bbo8NtqdsX0ZnmtgIl+utjdZogXkQjEHdG6+9cKVKYCxwzkmqMUNKHQetTPcMkppiRqwPc0UyhM9Qnxw7mCLnsGdGc2HBqovEIBHSXakGo+j1EwZxpYY8csyiRzWbK4L/zBURLQRTMw3o5FXP0DTLNUlxWT/JGdACFLMHYioJ1mzoAMKSOgsAnyKJsHYT6vtwhziPkuwhx4t33Exz6jJHxj3FJKbnA3vc7pmGNevWrPoA1AO4CeOCpugFgV/hZr296ruWUvIFC/ci0nhy1Jqd0LzftZdK06RDJK0ZvSfJTXFjr7r54KKfMiKRFtJAJvpUW1NuCziK9rmgsTUTsFgrJqVYsS9SE4nmaODUxmABLyIyFTlj1Jo3l3AB13391mwjTfpCul8Bs24wg9kxnAeH7Y3A4c8v6lvvxiO65D3xnnoNL/BeelveW2/P63i48qvyp1p1fy1W+1b7XvtRUquV8ZnH3tSq/fwLVqKqew==</latexit><latexit sha1_base64="Qu/hUHYzd6oK0ycZuHDuMgafpsc=">AAAE6nicfVRdb9MwFE1LgRG+NnjkxaLa6EQ1NRUSSBPSxPaAYMCY1nXS3EWO43TW7DiyndHO+E/whnjlH/HET+ENp2m3taVYinxy7/G59yQ3iTJGlW61fleqN2o3b91euuPfvXf/wcPllUeHSuQSkw4WTMijCCnCaEo6mmpGjjJJEI8Y6UZn20W+e06koiI90MOM9DjqpzShGGkXClcqbA1ATQY6SkxuQwpel7dmFw2J/Cgkt41JflDkn1+jPwvpJuwjzlEYNGFEtNvXm2DNSc5EAaQpgBzp0ygy+7YJoX9V96IQcoW7YfvkoKy+T3Y7ttENg5ODqfZK9e6c4ImJHY9yogC3zUJpjsAnhHiu/P9t59O2L67bbo8NtqdsX0ZnmtgIl+utjdZogXkQjEHdG6+9cKVKYCxwzkmqMUNKHQetTPcMkppiRqwPc0UyhM9Qnxw7mCLnsGdGc2HBqovEIBHSXakGo+j1EwZxpYY8csyiRzWbK4L/zBURLQRTMw3o5FXP0DTLNUlxWT/JGdACFLMHYioJ1mzoAMKSOgsAnyKJsHYT6vtwhziPkuwhx4t33Exz6jJHxj3FJKbnA3vc7pmGNevWrPoA1AO4CeOCpugFgV/hZr296ruWUvIFC/ci0nhy1Jqd0LzftZdK06RDJK0ZvSfJTXFjr7r54KKfMiKRFtJAJvpUW1NuCziK9rmgsTUTsFgrJqVYsS9SE4nmaODUxmABLyIyFTlj1Jo3l3AB13391mwjTfpCul8Bs24wg9kxnAeH7Y3A4c8v6lvvxiO65D3xnnoNL/BeelveW2/P63i48qvyp1p1fy1W+1b7XvtRUquV8ZnH3tSq/fwLVqKqew==</latexit><latexit sha1_base64="Qu/hUHYzd6oK0ycZuHDuMgafpsc=">AAAE6nicfVRdb9MwFE1LgRG+NnjkxaLa6EQ1NRUSSBPSxPaAYMCY1nXS3EWO43TW7DiyndHO+E/whnjlH/HET+ENp2m3taVYinxy7/G59yQ3iTJGlW61fleqN2o3b91euuPfvXf/wcPllUeHSuQSkw4WTMijCCnCaEo6mmpGjjJJEI8Y6UZn20W+e06koiI90MOM9DjqpzShGGkXClcqbA1ATQY6SkxuQwpel7dmFw2J/Cgkt41JflDkn1+jPwvpJuwjzlEYNGFEtNvXm2DNSc5EAaQpgBzp0ygy+7YJoX9V96IQcoW7YfvkoKy+T3Y7ttENg5ODqfZK9e6c4ImJHY9yogC3zUJpjsAnhHiu/P9t59O2L67bbo8NtqdsX0ZnmtgIl+utjdZogXkQjEHdG6+9cKVKYCxwzkmqMUNKHQetTPcMkppiRqwPc0UyhM9Qnxw7mCLnsGdGc2HBqovEIBHSXakGo+j1EwZxpYY8csyiRzWbK4L/zBURLQRTMw3o5FXP0DTLNUlxWT/JGdACFLMHYioJ1mzoAMKSOgsAnyKJsHYT6vtwhziPkuwhx4t33Exz6jJHxj3FJKbnA3vc7pmGNevWrPoA1AO4CeOCpugFgV/hZr296ruWUvIFC/ci0nhy1Jqd0LzftZdK06RDJK0ZvSfJTXFjr7r54KKfMiKRFtJAJvpUW1NuCziK9rmgsTUTsFgrJqVYsS9SE4nmaODUxmABLyIyFTlj1Jo3l3AB13391mwjTfpCul8Bs24wg9kxnAeH7Y3A4c8v6lvvxiO65D3xnnoNL/BeelveW2/P63i48qvyp1p1fy1W+1b7XvtRUquV8ZnH3tSq/fwLVqKqew==</latexit><latexit sha1_base64="Qu/hUHYzd6oK0ycZuHDuMgafpsc=">AAAE6nicfVRdb9MwFE1LgRG+NnjkxaLa6EQ1NRUSSBPSxPaAYMCY1nXS3EWO43TW7DiyndHO+E/whnjlH/HET+ENp2m3taVYinxy7/G59yQ3iTJGlW61fleqN2o3b91euuPfvXf/wcPllUeHSuQSkw4WTMijCCnCaEo6mmpGjjJJEI8Y6UZn20W+e06koiI90MOM9DjqpzShGGkXClcqbA1ATQY6SkxuQwpel7dmFw2J/Cgkt41JflDkn1+jPwvpJuwjzlEYNGFEtNvXm2DNSc5EAaQpgBzp0ygy+7YJoX9V96IQcoW7YfvkoKy+T3Y7ttENg5ODqfZK9e6c4ImJHY9yogC3zUJpjsAnhHiu/P9t59O2L67bbo8NtqdsX0ZnmtgIl+utjdZogXkQjEHdG6+9cKVKYCxwzkmqMUNKHQetTPcMkppiRqwPc0UyhM9Qnxw7mCLnsGdGc2HBqovEIBHSXakGo+j1EwZxpYY8csyiRzWbK4L/zBURLQRTMw3o5FXP0DTLNUlxWT/JGdACFLMHYioJ1mzoAMKSOgsAnyKJsHYT6vtwhziPkuwhx4t33Exz6jJHxj3FJKbnA3vc7pmGNevWrPoA1AO4CeOCpugFgV/hZr296ruWUvIFC/ci0nhy1Jqd0LzftZdK06RDJK0ZvSfJTXFjr7r54KKfMiKRFtJAJvpUW1NuCziK9rmgsTUTsFgrJqVYsS9SE4nmaODUxmABLyIyFTlj1Jo3l3AB13391mwjTfpCul8Bs24wg9kxnAeH7Y3A4c8v6lvvxiO65D3xnnoNL/BeelveW2/P63i48qvyp1p1fy1W+1b7XvtRUquV8ZnH3tSq/fwLVqKqew==</latexit>

Transformation Weights

Lecture 5

• Forget the normalizing transformations for a moment.

• Two-layer, fully-connected ReLU network.

• Applied per-position (no interaction between terms in the sequence).

Transformer Blocks

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

ui = LayerNorm(xi + u0i; �1,�1), �1,�1 2 R,

z0i= WT

2 ReLU(WT

1 ui), W1 2 Rd⇥m,W2 2 Rm⇥d,

zi = LayerNorm(ui + z0i; �2,�2), �2,�2 2 R.

<latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit><latexit sha1_base64="Rziz/GjLUZqi+zpxBFbnZi3igUs=">AAAFPnichVTPb9MwFM7WAqP82AZHLhYTpRXR1FRIIKFJk7hMMKFtatdJcxs5rtO4jZPMdqZ1lv8yLvwN3Dhy4QBCXDnipBla2jIsRXn6/L73vu8ltpeEVMhW68vKaqV66/adtbu1e/cfPFzf2Hx0LOKUY9LFcRjzEw8JEtKIdCWVITlJOEHMC0nPm7zN9nvnhAsaRx05TUifoVFEfYqRNJC7WenUweFANYKmbkBJLqTnqwvt0ibYAT1XBfaZHnSub9gABiJBmCiHMQ3e38Sd3Mw9vol7vsCt168U2UV1u8gEkEYAMiQDz1NHeqCGUFJGBDApENbqAKIwCZCrqD3Ws6amTV5didiXDJkOYxgSXzagzxFWMETRKCTLR2Mvcz1uAshzklZQnHGpJlpDTkeBbBYiiuRUP3dp1l+kzOjfcfRgL3OG7SDznKPjDE1KupdMa9y8mkrGXZjCpJjCUM8LmPXP/e+jKeEfYs7KJsGLst43cIQYQ65jQ49I8561nkfnNJT7XhbGe267+LbqiOx3daPnOoNOSd6VsfmC5tOCwhXTdlZpIYGBf9i+/I/ttGz78rrtdmGwXbL9F50Tse1ubLW2W/kCi4FTBFtWsQ7cjc9wGOOUkUjiEAlx6rQS2VeIS4rNH1WDqSDm4EzQiJyaMELGYF/lx1+DZwYZAj/m5okkyNHrDIWYEFPmmcxMopjfy8Ble6ep9F/3FY2SVJIIzxr5aQhkDLK7BAwpJ1iGUxMgzKnRCnCAzPmR5sapmSE485YXg+P2tmPiw5dbu++KcaxZT6ynVsNyrFfWrrVnHVhdC1c+Vr5Wvld+VD9Vv1V/Vn/NUldXCs5jq7Sqv/8AT5jBIQ==</latexit>

5-Minute Break

CSE 599I: Generative Models University of WashingtonLecture 4

Layer Normalization

CSE 599I: Generative Models University of Washington

LayerNorm(z; �,�) = �(z� µz)

�z+ �, �,� 2 Rk.

µz =1

k

kX

i=1

zi, �z =

vuut1

k

kX

i=1

(zi � µz)2.

<latexit sha1_base64="kxmGc7pADX+AXvSd/bfPsOxOSTM=">AAAEs3icdVPdbtMwGE1HgRH+NrjkxqLa1IlRNYUJpGnSxHaBYMBA7Eequ8hxvnRW4zjYzlhn/CY8DbfwArwNTtutXUstRTk+Pj7+bB9HecqUbjb/VhZuVG/eur14x7977/6Dh0vLjw6VKCSFAypSIY8joiBlGRxoplM4ziUQHqVwFPV2yvGjM5CKieyr7ufQ4aSbsYRRoh0VLlc2VhHWcK7NHumD/Cgkt/UBESXmwm7iLuGcrOMINFlDW2jYRziRhJoJIXqOMC/CMbFmDVasy8kEZ9EzNHBaR6to0hlhlrn5RJ9GkfliT3oNjP3VKUdULj9YN7CmZ7EqeGjYVuDkaKwK2TrCpyonFMwLzi2aqaK0Ud+kNvPM6pNusxs7adlGuFRrNpqDhmZBMAI1b9T2w+UFwLGgBYdM05Qo1Q6aue4YIjWjKVgfFwpc0T3ShbaDGeGgOmZwwRatOCZGiZDuyzQasJMzDOFK9XnklOUhqumxkvzvWMloIVI1VYBOXncMy/JCQ0aH6ydFirRAZYhQzCRQnfYdIFQytwVET4k7Te2i5vt4F9weJewTp4t3XTg5cyPHxl1zErOzc9tudUzdGheSFR+hWoA3cVzKFLsA/ANv1lorvispg+9UuJRk8eVUa3ZD837PXjldFx0S6XJX7kpyU3bsuJoPjv2UgyRaSINT0WXamuFvjqZMjmCxNZdgvlcMQ7PyP89NJJqTc+c2AnN0EchMFGnKrHlzBedo3TO2Zodo6Arp3nRqXTCD6RjOgsNWI3D488va9rtRRBe9J95Tr+4F3itv23vr7XsHHq38rPyq/K78qW5U29WoGg+lC5XRnMfetVbl/wBFfp3P</latexit><latexit sha1_base64="kxmGc7pADX+AXvSd/bfPsOxOSTM=">AAAEs3icdVPdbtMwGE1HgRH+NrjkxqLa1IlRNYUJpGnSxHaBYMBA7Eequ8hxvnRW4zjYzlhn/CY8DbfwArwNTtutXUstRTk+Pj7+bB9HecqUbjb/VhZuVG/eur14x7977/6Dh0vLjw6VKCSFAypSIY8joiBlGRxoplM4ziUQHqVwFPV2yvGjM5CKieyr7ufQ4aSbsYRRoh0VLlc2VhHWcK7NHumD/Cgkt/UBESXmwm7iLuGcrOMINFlDW2jYRziRhJoJIXqOMC/CMbFmDVasy8kEZ9EzNHBaR6to0hlhlrn5RJ9GkfliT3oNjP3VKUdULj9YN7CmZ7EqeGjYVuDkaKwK2TrCpyonFMwLzi2aqaK0Ud+kNvPM6pNusxs7adlGuFRrNpqDhmZBMAI1b9T2w+UFwLGgBYdM05Qo1Q6aue4YIjWjKVgfFwpc0T3ShbaDGeGgOmZwwRatOCZGiZDuyzQasJMzDOFK9XnklOUhqumxkvzvWMloIVI1VYBOXncMy/JCQ0aH6ydFirRAZYhQzCRQnfYdIFQytwVET4k7Te2i5vt4F9weJewTp4t3XTg5cyPHxl1zErOzc9tudUzdGheSFR+hWoA3cVzKFLsA/ANv1lorvispg+9UuJRk8eVUa3ZD837PXjldFx0S6XJX7kpyU3bsuJoPjv2UgyRaSINT0WXamuFvjqZMjmCxNZdgvlcMQ7PyP89NJJqTc+c2AnN0EchMFGnKrHlzBedo3TO2Zodo6Arp3nRqXTCD6RjOgsNWI3D488va9rtRRBe9J95Tr+4F3itv23vr7XsHHq38rPyq/K78qW5U29WoGg+lC5XRnMfetVbl/wBFfp3P</latexit><latexit sha1_base64="kxmGc7pADX+AXvSd/bfPsOxOSTM=">AAAEs3icdVPdbtMwGE1HgRH+NrjkxqLa1IlRNYUJpGnSxHaBYMBA7Eequ8hxvnRW4zjYzlhn/CY8DbfwArwNTtutXUstRTk+Pj7+bB9HecqUbjb/VhZuVG/eur14x7977/6Dh0vLjw6VKCSFAypSIY8joiBlGRxoplM4ziUQHqVwFPV2yvGjM5CKieyr7ufQ4aSbsYRRoh0VLlc2VhHWcK7NHumD/Cgkt/UBESXmwm7iLuGcrOMINFlDW2jYRziRhJoJIXqOMC/CMbFmDVasy8kEZ9EzNHBaR6to0hlhlrn5RJ9GkfliT3oNjP3VKUdULj9YN7CmZ7EqeGjYVuDkaKwK2TrCpyonFMwLzi2aqaK0Ud+kNvPM6pNusxs7adlGuFRrNpqDhmZBMAI1b9T2w+UFwLGgBYdM05Qo1Q6aue4YIjWjKVgfFwpc0T3ShbaDGeGgOmZwwRatOCZGiZDuyzQasJMzDOFK9XnklOUhqumxkvzvWMloIVI1VYBOXncMy/JCQ0aH6ydFirRAZYhQzCRQnfYdIFQytwVET4k7Te2i5vt4F9weJewTp4t3XTg5cyPHxl1zErOzc9tudUzdGheSFR+hWoA3cVzKFLsA/ANv1lorvispg+9UuJRk8eVUa3ZD837PXjldFx0S6XJX7kpyU3bsuJoPjv2UgyRaSINT0WXamuFvjqZMjmCxNZdgvlcMQ7PyP89NJJqTc+c2AnN0EchMFGnKrHlzBedo3TO2Zodo6Arp3nRqXTCD6RjOgsNWI3D488va9rtRRBe9J95Tr+4F3itv23vr7XsHHq38rPyq/K78qW5U29WoGg+lC5XRnMfetVbl/wBFfp3P</latexit><latexit sha1_base64="kxmGc7pADX+AXvSd/bfPsOxOSTM=">AAAEs3icdVPdbtMwGE1HgRH+NrjkxqLa1IlRNYUJpGnSxHaBYMBA7Eequ8hxvnRW4zjYzlhn/CY8DbfwArwNTtutXUstRTk+Pj7+bB9HecqUbjb/VhZuVG/eur14x7977/6Dh0vLjw6VKCSFAypSIY8joiBlGRxoplM4ziUQHqVwFPV2yvGjM5CKieyr7ufQ4aSbsYRRoh0VLlc2VhHWcK7NHumD/Cgkt/UBESXmwm7iLuGcrOMINFlDW2jYRziRhJoJIXqOMC/CMbFmDVasy8kEZ9EzNHBaR6to0hlhlrn5RJ9GkfliT3oNjP3VKUdULj9YN7CmZ7EqeGjYVuDkaKwK2TrCpyonFMwLzi2aqaK0Ud+kNvPM6pNusxs7adlGuFRrNpqDhmZBMAI1b9T2w+UFwLGgBYdM05Qo1Q6aue4YIjWjKVgfFwpc0T3ShbaDGeGgOmZwwRatOCZGiZDuyzQasJMzDOFK9XnklOUhqumxkvzvWMloIVI1VYBOXncMy/JCQ0aH6ydFirRAZYhQzCRQnfYdIFQytwVET4k7Te2i5vt4F9weJewTp4t3XTg5cyPHxl1zErOzc9tudUzdGheSFR+hWoA3cVzKFLsA/ANv1lorvispg+9UuJRk8eVUa3ZD837PXjldFx0S6XJX7kpyU3bsuJoPjv2UgyRaSINT0WXamuFvjqZMjmCxNZdgvlcMQ7PyP89NJJqTc+c2AnN0EchMFGnKrHlzBedo3TO2Zodo6Arp3nRqXTCD6RjOgsNWI3D488va9rtRRBe9J95Tr+4F3itv23vr7XsHHq38rPyq/K78qW5U29WoGg+lC5XRnMfetVbl/wBFfp3P</latexit>

Transformation Weights

Lecture 5

• Normalize activations of a layer to have learned mean and variance .

• Applied per-sequence (not across batches).

�<latexit sha1_base64="LGyMe+8JUR0jPsC5BOMe6JlN1CQ=">AAADgHicdVLbbhMxEHWzQMtya+GRlxVRpPBSshESqH2paB4QFxEkkkaKo8jrnU2t+rKyvaXB+Bt4hU/jb/AmaaEpGcny8Zkz45nRZCVnxnY6v7ca0a3bd7Z37sb37j94+Gh37/HQqEpTGFDFlR5lxABnEgaWWQ6jUgMRGYeT7Oy49p+cgzZMyS92XsJEkJlkBaPEBmqAM7Bkutvs7HcWltwE6Qo00cr6070G4FzRSoC0lBNjxmmntBNHtGWUg49xZaAk9IzMYBygJALMxC2q9UkrMHlSKB2OtMmC/TfCEWHMXGRBKYg9Neu+mvyvr2asUtysFWCL1xPHZFlZkHT5f1HxxKqknkiSMw3U8nkAhGoWWkjoKdGE2jC3OMY9CD1q6JOgy3th0oIFz8hhJoucnV/4cXfi2t49964VJ0kzxYc4r2WGfQP8HR82u604lCThK1VCEJlfhnrXm7r3H/xVpuuiIdHe4borLVz98H+r+RjYTyVoYpV2mKsZs94trw0aw2ZCsdy7S7A5Vw7LZPW9KZsqrCAXIdsKbNBloKWqOGfevbmCG7RhJ707JhZmSocF5T4sZrq+hjfBsLufBvz5ZfPo3WpFd9BT9Ay1UYpeoSP0FvXRAFHE0A/0E/2KGlE7ehGlS2ljaxXzBF2z6OAPpcQqPA==</latexit><latexit sha1_base64="LGyMe+8JUR0jPsC5BOMe6JlN1CQ=">AAADgHicdVLbbhMxEHWzQMtya+GRlxVRpPBSshESqH2paB4QFxEkkkaKo8jrnU2t+rKyvaXB+Bt4hU/jb/AmaaEpGcny8Zkz45nRZCVnxnY6v7ca0a3bd7Z37sb37j94+Gh37/HQqEpTGFDFlR5lxABnEgaWWQ6jUgMRGYeT7Oy49p+cgzZMyS92XsJEkJlkBaPEBmqAM7Bkutvs7HcWltwE6Qo00cr6070G4FzRSoC0lBNjxmmntBNHtGWUg49xZaAk9IzMYBygJALMxC2q9UkrMHlSKB2OtMmC/TfCEWHMXGRBKYg9Neu+mvyvr2asUtysFWCL1xPHZFlZkHT5f1HxxKqknkiSMw3U8nkAhGoWWkjoKdGE2jC3OMY9CD1q6JOgy3th0oIFz8hhJoucnV/4cXfi2t49964VJ0kzxYc4r2WGfQP8HR82u604lCThK1VCEJlfhnrXm7r3H/xVpuuiIdHe4borLVz98H+r+RjYTyVoYpV2mKsZs94trw0aw2ZCsdy7S7A5Vw7LZPW9KZsqrCAXIdsKbNBloKWqOGfevbmCG7RhJ707JhZmSocF5T4sZrq+hjfBsLufBvz5ZfPo3WpFd9BT9Ay1UYpeoSP0FvXRAFHE0A/0E/2KGlE7ehGlS2ljaxXzBF2z6OAPpcQqPA==</latexit><latexit sha1_base64="LGyMe+8JUR0jPsC5BOMe6JlN1CQ=">AAADgHicdVLbbhMxEHWzQMtya+GRlxVRpPBSshESqH2paB4QFxEkkkaKo8jrnU2t+rKyvaXB+Bt4hU/jb/AmaaEpGcny8Zkz45nRZCVnxnY6v7ca0a3bd7Z37sb37j94+Gh37/HQqEpTGFDFlR5lxABnEgaWWQ6jUgMRGYeT7Oy49p+cgzZMyS92XsJEkJlkBaPEBmqAM7Bkutvs7HcWltwE6Qo00cr6070G4FzRSoC0lBNjxmmntBNHtGWUg49xZaAk9IzMYBygJALMxC2q9UkrMHlSKB2OtMmC/TfCEWHMXGRBKYg9Neu+mvyvr2asUtysFWCL1xPHZFlZkHT5f1HxxKqknkiSMw3U8nkAhGoWWkjoKdGE2jC3OMY9CD1q6JOgy3th0oIFz8hhJoucnV/4cXfi2t49964VJ0kzxYc4r2WGfQP8HR82u604lCThK1VCEJlfhnrXm7r3H/xVpuuiIdHe4borLVz98H+r+RjYTyVoYpV2mKsZs94trw0aw2ZCsdy7S7A5Vw7LZPW9KZsqrCAXIdsKbNBloKWqOGfevbmCG7RhJ707JhZmSocF5T4sZrq+hjfBsLufBvz5ZfPo3WpFd9BT9Ay1UYpeoSP0FvXRAFHE0A/0E/2KGlE7ehGlS2ljaxXzBF2z6OAPpcQqPA==</latexit><latexit sha1_base64="LGyMe+8JUR0jPsC5BOMe6JlN1CQ=">AAADgHicdVLbbhMxEHWzQMtya+GRlxVRpPBSshESqH2paB4QFxEkkkaKo8jrnU2t+rKyvaXB+Bt4hU/jb/AmaaEpGcny8Zkz45nRZCVnxnY6v7ca0a3bd7Z37sb37j94+Gh37/HQqEpTGFDFlR5lxABnEgaWWQ6jUgMRGYeT7Oy49p+cgzZMyS92XsJEkJlkBaPEBmqAM7Bkutvs7HcWltwE6Qo00cr6070G4FzRSoC0lBNjxmmntBNHtGWUg49xZaAk9IzMYBygJALMxC2q9UkrMHlSKB2OtMmC/TfCEWHMXGRBKYg9Neu+mvyvr2asUtysFWCL1xPHZFlZkHT5f1HxxKqknkiSMw3U8nkAhGoWWkjoKdGE2jC3OMY9CD1q6JOgy3th0oIFz8hhJoucnV/4cXfi2t49964VJ0kzxYc4r2WGfQP8HR82u604lCThK1VCEJlfhnrXm7r3H/xVpuuiIdHe4borLVz98H+r+RjYTyVoYpV2mKsZs94trw0aw2ZCsdy7S7A5Vw7LZPW9KZsqrCAXIdsKbNBloKWqOGfevbmCG7RhJ707JhZmSocF5T4sZrq+hjfBsLufBvz5ZfPo3WpFd9BT9Ay1UYpeoSP0FvXRAFHE0A/0E/2KGlE7ehGlS2ljaxXzBF2z6OAPpcQqPA==</latexit>

�2<latexit sha1_base64="jiLfIAgYf9xStZgdZvJEnbkbSOI=">AAADg3icdVJtaxNBEN7mrNbzpa1+9MthCFSEkjsKCkUoNh/EF4xg0kAuhr29uevSfTl292rjun/Cr/rH/DfuJWm1qRlY9tlnnpmdGSarGNWm2/290Qpubd6+s3U3vHf/wcPtnd1HQy1rRWBAJJNqlGENjAoYGGoYjCoFmGcMTrKz48Z/cg5KUyk+m1kFE45LQQtKsPHUKC0x5/hLMt1pd/e7c4tugngJ2mhp/eluC9JckpqDMIRhrcdxtzITi5WhhIEL01pDhckZLmHsocAc9MTOC3ZRxzN5VEjljzDRnP03wmKu9YxnXsmxOdWrvob8r69hjJRMrxRgipcTS0VVGxBk8X9Rs8jIqBlKlFMFxLCZB5go6luIyClWmBg/ujBMe+B7VNDHXpf3/LA59Z6RTakocnp+4cbJxO45+8zZThhF7Tg9TPNGpuk3SL+nh+2kE/qSBHwl0g9c5Jehzvam9t17d5XpumiIlbNp05Xitnm4v9V88OzHChQ2UtmUyZIaZxfXGo2mJZc0d/YSrM+VwyJZc6/LJgvD8YXPtgRrdBkoIWvGqLOvr+AarV9LZ4+xgVIqv6PM+cWMV9fwJhgm+7HHnw7aR2+XK7qFnqCnaA/F6AU6Qm9QHw0QQQz9QD/Rr2AzeB4kwcFC2tpYxjxG1yx49QccyCtR</latexit><latexit sha1_base64="jiLfIAgYf9xStZgdZvJEnbkbSOI=">AAADg3icdVJtaxNBEN7mrNbzpa1+9MthCFSEkjsKCkUoNh/EF4xg0kAuhr29uevSfTl292rjun/Cr/rH/DfuJWm1qRlY9tlnnpmdGSarGNWm2/290Qpubd6+s3U3vHf/wcPtnd1HQy1rRWBAJJNqlGENjAoYGGoYjCoFmGcMTrKz48Z/cg5KUyk+m1kFE45LQQtKsPHUKC0x5/hLMt1pd/e7c4tugngJ2mhp/eluC9JckpqDMIRhrcdxtzITi5WhhIEL01pDhckZLmHsocAc9MTOC3ZRxzN5VEjljzDRnP03wmKu9YxnXsmxOdWrvob8r69hjJRMrxRgipcTS0VVGxBk8X9Rs8jIqBlKlFMFxLCZB5go6luIyClWmBg/ujBMe+B7VNDHXpf3/LA59Z6RTakocnp+4cbJxO45+8zZThhF7Tg9TPNGpuk3SL+nh+2kE/qSBHwl0g9c5Jehzvam9t17d5XpumiIlbNp05Xitnm4v9V88OzHChQ2UtmUyZIaZxfXGo2mJZc0d/YSrM+VwyJZc6/LJgvD8YXPtgRrdBkoIWvGqLOvr+AarV9LZ4+xgVIqv6PM+cWMV9fwJhgm+7HHnw7aR2+XK7qFnqCnaA/F6AU6Qm9QHw0QQQz9QD/Rr2AzeB4kwcFC2tpYxjxG1yx49QccyCtR</latexit><latexit sha1_base64="jiLfIAgYf9xStZgdZvJEnbkbSOI=">AAADg3icdVJtaxNBEN7mrNbzpa1+9MthCFSEkjsKCkUoNh/EF4xg0kAuhr29uevSfTl292rjun/Cr/rH/DfuJWm1qRlY9tlnnpmdGSarGNWm2/290Qpubd6+s3U3vHf/wcPtnd1HQy1rRWBAJJNqlGENjAoYGGoYjCoFmGcMTrKz48Z/cg5KUyk+m1kFE45LQQtKsPHUKC0x5/hLMt1pd/e7c4tugngJ2mhp/eluC9JckpqDMIRhrcdxtzITi5WhhIEL01pDhckZLmHsocAc9MTOC3ZRxzN5VEjljzDRnP03wmKu9YxnXsmxOdWrvob8r69hjJRMrxRgipcTS0VVGxBk8X9Rs8jIqBlKlFMFxLCZB5go6luIyClWmBg/ujBMe+B7VNDHXpf3/LA59Z6RTakocnp+4cbJxO45+8zZThhF7Tg9TPNGpuk3SL+nh+2kE/qSBHwl0g9c5Jehzvam9t17d5XpumiIlbNp05Xitnm4v9V88OzHChQ2UtmUyZIaZxfXGo2mJZc0d/YSrM+VwyJZc6/LJgvD8YXPtgRrdBkoIWvGqLOvr+AarV9LZ4+xgVIqv6PM+cWMV9fwJhgm+7HHnw7aR2+XK7qFnqCnaA/F6AU6Qm9QHw0QQQz9QD/Rr2AzeB4kwcFC2tpYxjxG1yx49QccyCtR</latexit><latexit sha1_base64="jiLfIAgYf9xStZgdZvJEnbkbSOI=">AAADg3icdVJtaxNBEN7mrNbzpa1+9MthCFSEkjsKCkUoNh/EF4xg0kAuhr29uevSfTl292rjun/Cr/rH/DfuJWm1qRlY9tlnnpmdGSarGNWm2/290Qpubd6+s3U3vHf/wcPtnd1HQy1rRWBAJJNqlGENjAoYGGoYjCoFmGcMTrKz48Z/cg5KUyk+m1kFE45LQQtKsPHUKC0x5/hLMt1pd/e7c4tugngJ2mhp/eluC9JckpqDMIRhrcdxtzITi5WhhIEL01pDhckZLmHsocAc9MTOC3ZRxzN5VEjljzDRnP03wmKu9YxnXsmxOdWrvob8r69hjJRMrxRgipcTS0VVGxBk8X9Rs8jIqBlKlFMFxLCZB5go6luIyClWmBg/ujBMe+B7VNDHXpf3/LA59Z6RTakocnp+4cbJxO45+8zZThhF7Tg9TPNGpuk3SL+nh+2kE/qSBHwl0g9c5Jehzvam9t17d5XpumiIlbNp05Xitnm4v9V88OzHChQ2UtmUyZIaZxfXGo2mJZc0d/YSrM+VwyJZc6/LJgvD8YXPtgRrdBkoIWvGqLOvr+AarV9LZ4+xgVIqv6PM+cWMV9fwJhgm+7HHnw7aR2+XK7qFnqCnaA/F6AU6Qm9QHw0QQQz9QD/Rr2AzeB4kwcFC2tpYxjxG1yx49QccyCtR</latexit>

Positional Encoding

CSE 599I: Generative Models University of WashingtonLecture 5

Transformation Weights

zk = WTz ReLU(WT

x xk +WTe ek) 2 Rd, Wx 2 Rd⇥m,We 2 Rp⇥m,Wz 2 Rm⇥d.

<latexit sha1_base64="QbMDIgBxWkwE86mNlxtDMQ1Swkg=">AAAEPHicdVNNbxMxEN00fJTw0RaOXCyiVK2oqiRCAqlCqmgPiBZRqqap1E1XXu9satVer2ynJDH+a/A7uHNDXDkzm6QtTYhXK7997814vB7HueDG1us/SgvlO3fv3V98UHn46PGTpeWVp8dG9TSDFlNC6ZOYGhA8g5blVsBJroHKWEA7vtgp9PYlaMNVdmQHOXQk7WY85YxapKLl76sktNC3ceqGProgb0k7Gp4djTh3CPstv9aO+hMCTf3C9BJNcMMBcusk5BkJJbXncewO/VmyQVbxweBpxSWh5RIMkX4D88zIObnRi2pmDPLKkPjNaLla36yPBpkFjQmoBpNxEK0sQJgo1pOQWSaoMaeNem47jmrLmQBfCXsGcsouaBdOEWYUV+q40Z/2pIZMQlKl8c0sGbH/RjgqjRnIGJ1FwWZaK8j/agVjlRJmqgCbvuk4nuU9Cxkbr5/2BLGKFKdJEq6BWTFAQJnmuAXCzqmmzOKZVyrhLuAeNRxQ9CW72CWSo3Li8JemCb/s+9Nmx615t+5drUJItRFuhUlhM3wI4ddwq9qsVbCkDL4wJSXNkqtQ73Yjt7fvrzPdNh1T7d3o0LR0xYe/qeYjsp9y0NQq7UKhutx6N57meAzvSsUT767A/FwJjJMV87xsKrWS9jHbBMzxxaAz1ROCe/fuGs7x4n3yboda6CqNl0t4bMzGdBvOguPmZgPx51fV7Q+TFl0MngcvgrWgEbwOtoP3wUHQClipVtorHZVa5W/ln+Vf5d9j60JpEvMsuDXKf/4COplxBA==</latexit><latexit sha1_base64="QbMDIgBxWkwE86mNlxtDMQ1Swkg=">AAAEPHicdVNNbxMxEN00fJTw0RaOXCyiVK2oqiRCAqlCqmgPiBZRqqap1E1XXu9satVer2ynJDH+a/A7uHNDXDkzm6QtTYhXK7997814vB7HueDG1us/SgvlO3fv3V98UHn46PGTpeWVp8dG9TSDFlNC6ZOYGhA8g5blVsBJroHKWEA7vtgp9PYlaMNVdmQHOXQk7WY85YxapKLl76sktNC3ceqGProgb0k7Gp4djTh3CPstv9aO+hMCTf3C9BJNcMMBcusk5BkJJbXncewO/VmyQVbxweBpxSWh5RIMkX4D88zIObnRi2pmDPLKkPjNaLla36yPBpkFjQmoBpNxEK0sQJgo1pOQWSaoMaeNem47jmrLmQBfCXsGcsouaBdOEWYUV+q40Z/2pIZMQlKl8c0sGbH/RjgqjRnIGJ1FwWZaK8j/agVjlRJmqgCbvuk4nuU9Cxkbr5/2BLGKFKdJEq6BWTFAQJnmuAXCzqmmzOKZVyrhLuAeNRxQ9CW72CWSo3Li8JemCb/s+9Nmx615t+5drUJItRFuhUlhM3wI4ddwq9qsVbCkDL4wJSXNkqtQ73Yjt7fvrzPdNh1T7d3o0LR0xYe/qeYjsp9y0NQq7UKhutx6N57meAzvSsUT767A/FwJjJMV87xsKrWS9jHbBMzxxaAz1ROCe/fuGs7x4n3yboda6CqNl0t4bMzGdBvOguPmZgPx51fV7Q+TFl0MngcvgrWgEbwOtoP3wUHQClipVtorHZVa5W/ln+Vf5d9j60JpEvMsuDXKf/4COplxBA==</latexit><latexit sha1_base64="QbMDIgBxWkwE86mNlxtDMQ1Swkg=">AAAEPHicdVNNbxMxEN00fJTw0RaOXCyiVK2oqiRCAqlCqmgPiBZRqqap1E1XXu9satVer2ynJDH+a/A7uHNDXDkzm6QtTYhXK7997814vB7HueDG1us/SgvlO3fv3V98UHn46PGTpeWVp8dG9TSDFlNC6ZOYGhA8g5blVsBJroHKWEA7vtgp9PYlaMNVdmQHOXQk7WY85YxapKLl76sktNC3ceqGProgb0k7Gp4djTh3CPstv9aO+hMCTf3C9BJNcMMBcusk5BkJJbXncewO/VmyQVbxweBpxSWh5RIMkX4D88zIObnRi2pmDPLKkPjNaLla36yPBpkFjQmoBpNxEK0sQJgo1pOQWSaoMaeNem47jmrLmQBfCXsGcsouaBdOEWYUV+q40Z/2pIZMQlKl8c0sGbH/RjgqjRnIGJ1FwWZaK8j/agVjlRJmqgCbvuk4nuU9Cxkbr5/2BLGKFKdJEq6BWTFAQJnmuAXCzqmmzOKZVyrhLuAeNRxQ9CW72CWSo3Li8JemCb/s+9Nmx615t+5drUJItRFuhUlhM3wI4ddwq9qsVbCkDL4wJSXNkqtQ73Yjt7fvrzPdNh1T7d3o0LR0xYe/qeYjsp9y0NQq7UKhutx6N57meAzvSsUT767A/FwJjJMV87xsKrWS9jHbBMzxxaAz1ROCe/fuGs7x4n3yboda6CqNl0t4bMzGdBvOguPmZgPx51fV7Q+TFl0MngcvgrWgEbwOtoP3wUHQClipVtorHZVa5W/ln+Vf5d9j60JpEvMsuDXKf/4COplxBA==</latexit><latexit sha1_base64="QbMDIgBxWkwE86mNlxtDMQ1Swkg=">AAAEPHicdVNNbxMxEN00fJTw0RaOXCyiVK2oqiRCAqlCqmgPiBZRqqap1E1XXu9satVer2ynJDH+a/A7uHNDXDkzm6QtTYhXK7997814vB7HueDG1us/SgvlO3fv3V98UHn46PGTpeWVp8dG9TSDFlNC6ZOYGhA8g5blVsBJroHKWEA7vtgp9PYlaMNVdmQHOXQk7WY85YxapKLl76sktNC3ceqGProgb0k7Gp4djTh3CPstv9aO+hMCTf3C9BJNcMMBcusk5BkJJbXncewO/VmyQVbxweBpxSWh5RIMkX4D88zIObnRi2pmDPLKkPjNaLla36yPBpkFjQmoBpNxEK0sQJgo1pOQWSaoMaeNem47jmrLmQBfCXsGcsouaBdOEWYUV+q40Z/2pIZMQlKl8c0sGbH/RjgqjRnIGJ1FwWZaK8j/agVjlRJmqgCbvuk4nuU9Cxkbr5/2BLGKFKdJEq6BWTFAQJnmuAXCzqmmzOKZVyrhLuAeNRxQ9CW72CWSo3Li8JemCb/s+9Nmx615t+5drUJItRFuhUlhM3wI4ddwq9qsVbCkDL4wJSXNkqtQ73Yjt7fvrzPdNh1T7d3o0LR0xYe/qeYjsp9y0NQq7UKhutx6N57meAzvSsUT767A/FwJjJMV87xsKrWS9jHbBMzxxaAz1ROCe/fuGs7x4n3yboda6CqNl0t4bMzGdBvOguPmZgPx51fV7Q+TFl0MngcvgrWgEbwOtoP3wUHQClipVtorHZVa5W/ln+Vf5d9j60JpEvMsuDXKf/4COplxBA==</latexit>

• Transformer doesn’t account for sequence position.

• Need to encode sequence position as data.

• Learn positional embeddings, or joint token-position embeddings.

Dropout

CSE 599I: Generative Models University of Washington

Transformation Weights

Lecture 5

Q(h)(xi) = WT

h,qxi, K

(h)(xi) = WT

h,kxi, V

(h)(xi) = WT

h,vxi, Wh,q,Wh,k,Wh,v 2 Rd⇥k,

↵(h)i,j

= Dropout

✓softmaxj

✓hQ(h)(xi),K(h)(xj)ip

k

◆◆,

u0i=

HX

h=1

WT

c,h

pX

j=1

↵i,jV(h)(xj), Wc,h 2 Rk⇥d,

ui = LayerNorm(xi +Dropout(u0i); �1,�1), �1,�1 2 R,

z0i= WT

2 Dropout�ReLU(WT

1 ui)�, W1 2 Rd⇥m,W2 2 Rm⇥d,

zi = LayerNorm(ui +Dropout(z0i); �2,�2), �2,�2 2 R.

<latexit sha1_base64="VNdWzpVG7+ufDr+wJ8jXq/6gD4Q=">AAAFyXicjZRda9swFIbdNdm67KvdLncjVtYlzJQ4DDYYhcLGKGsZbUmaQpUY2ZFjJZbtSnJpInS1f7i7Xe2vTP5oqZ0sq8D4cHRen/c8tuzEAeGi3f699mC9Vn/4aONx48nTZ89fbG69PONRwlzcc6MgYucO4jggIe4JIgJ8HjOMqBPgvjP9ku73rzDjJAq7YhbjAUXjkHjERUKn7K31P40dcDKUTb+lmlDga+F48lrZpAX2QN+Wvnmpht27GyaAPo+Ri6VFqQKHq7TT1dqzVdqrBe3Ozo0js3i6WVQCSEIAKRK+48hTNZQjKAjFHOgSCPWEEAWxj2xJzInKm+o2Emb8JMMjlXWSX1kUR4lQCgbYE7kvySNPUKQtTIqsx5ArYYDCcYCXszOXYZm0AGSZSEnIL5mQU92IkbEvWsWtMFtoEvXOJton5AnVc+5ZaniQEnBNP2WTZSdpNi7Nt4TqpHVDL9Uu0JoWtEaqaiDvn2E4QjPMfkSMlmcF71dxbJZmaX2GY0Qpsi0TOljoe+6rmq0YLJuaF1T6dmfYvecrPMVHPdXs29awWxrulntOp9pYf0egQEOVmXZcKKDgH+zm/2GX3J/dvMSuU1DqlNjdZisOd+3N7fZuO1tgMbCKYNso1rG9+QuOIjehOBRugDi/sNqxGEjEBHH1p9uACcf6CE/RGF/oMER6+oHMplDgrc6MgBcxfYUCZNm7Coko5zPq6MrUIq/upcllexeJ8D4NJAnjRODQzRt5SQBEBNLfGhgRhl0RzHSAXEa0V+D6SB9UoX9+DQ3Bqo68GJx1di0dn3zY3v9e4NgwXhtvjKZhGR+NfePAODZ6hlv7VgtqSe2qfli/rF/X53npg7VC88oorfrPv+Zv9pg=</latexit><latexit sha1_base64="VNdWzpVG7+ufDr+wJ8jXq/6gD4Q=">AAAFyXicjZRda9swFIbdNdm67KvdLncjVtYlzJQ4DDYYhcLGKGsZbUmaQpUY2ZFjJZbtSnJpInS1f7i7Xe2vTP5oqZ0sq8D4cHRen/c8tuzEAeGi3f699mC9Vn/4aONx48nTZ89fbG69PONRwlzcc6MgYucO4jggIe4JIgJ8HjOMqBPgvjP9ku73rzDjJAq7YhbjAUXjkHjERUKn7K31P40dcDKUTb+lmlDga+F48lrZpAX2QN+Wvnmpht27GyaAPo+Ri6VFqQKHq7TT1dqzVdqrBe3Ozo0js3i6WVQCSEIAKRK+48hTNZQjKAjFHOgSCPWEEAWxj2xJzInKm+o2Emb8JMMjlXWSX1kUR4lQCgbYE7kvySNPUKQtTIqsx5ArYYDCcYCXszOXYZm0AGSZSEnIL5mQU92IkbEvWsWtMFtoEvXOJton5AnVc+5ZaniQEnBNP2WTZSdpNi7Nt4TqpHVDL9Uu0JoWtEaqaiDvn2E4QjPMfkSMlmcF71dxbJZmaX2GY0Qpsi0TOljoe+6rmq0YLJuaF1T6dmfYvecrPMVHPdXs29awWxrulntOp9pYf0egQEOVmXZcKKDgH+zm/2GX3J/dvMSuU1DqlNjdZisOd+3N7fZuO1tgMbCKYNso1rG9+QuOIjehOBRugDi/sNqxGEjEBHH1p9uACcf6CE/RGF/oMER6+oHMplDgrc6MgBcxfYUCZNm7Coko5zPq6MrUIq/upcllexeJ8D4NJAnjRODQzRt5SQBEBNLfGhgRhl0RzHSAXEa0V+D6SB9UoX9+DQ3Bqo68GJx1di0dn3zY3v9e4NgwXhtvjKZhGR+NfePAODZ6hlv7VgtqSe2qfli/rF/X53npg7VC88oorfrPv+Zv9pg=</latexit><latexit sha1_base64="VNdWzpVG7+ufDr+wJ8jXq/6gD4Q=">AAAFyXicjZRda9swFIbdNdm67KvdLncjVtYlzJQ4DDYYhcLGKGsZbUmaQpUY2ZFjJZbtSnJpInS1f7i7Xe2vTP5oqZ0sq8D4cHRen/c8tuzEAeGi3f699mC9Vn/4aONx48nTZ89fbG69PONRwlzcc6MgYucO4jggIe4JIgJ8HjOMqBPgvjP9ku73rzDjJAq7YhbjAUXjkHjERUKn7K31P40dcDKUTb+lmlDga+F48lrZpAX2QN+Wvnmpht27GyaAPo+Ri6VFqQKHq7TT1dqzVdqrBe3Ozo0js3i6WVQCSEIAKRK+48hTNZQjKAjFHOgSCPWEEAWxj2xJzInKm+o2Emb8JMMjlXWSX1kUR4lQCgbYE7kvySNPUKQtTIqsx5ArYYDCcYCXszOXYZm0AGSZSEnIL5mQU92IkbEvWsWtMFtoEvXOJton5AnVc+5ZaniQEnBNP2WTZSdpNi7Nt4TqpHVDL9Uu0JoWtEaqaiDvn2E4QjPMfkSMlmcF71dxbJZmaX2GY0Qpsi0TOljoe+6rmq0YLJuaF1T6dmfYvecrPMVHPdXs29awWxrulntOp9pYf0egQEOVmXZcKKDgH+zm/2GX3J/dvMSuU1DqlNjdZisOd+3N7fZuO1tgMbCKYNso1rG9+QuOIjehOBRugDi/sNqxGEjEBHH1p9uACcf6CE/RGF/oMER6+oHMplDgrc6MgBcxfYUCZNm7Coko5zPq6MrUIq/upcllexeJ8D4NJAnjRODQzRt5SQBEBNLfGhgRhl0RzHSAXEa0V+D6SB9UoX9+DQ3Bqo68GJx1di0dn3zY3v9e4NgwXhtvjKZhGR+NfePAODZ6hlv7VgtqSe2qfli/rF/X53npg7VC88oorfrPv+Zv9pg=</latexit><latexit sha1_base64="VNdWzpVG7+ufDr+wJ8jXq/6gD4Q=">AAAFyXicjZRda9swFIbdNdm67KvdLncjVtYlzJQ4DDYYhcLGKGsZbUmaQpUY2ZFjJZbtSnJpInS1f7i7Xe2vTP5oqZ0sq8D4cHRen/c8tuzEAeGi3f699mC9Vn/4aONx48nTZ89fbG69PONRwlzcc6MgYucO4jggIe4JIgJ8HjOMqBPgvjP9ku73rzDjJAq7YhbjAUXjkHjERUKn7K31P40dcDKUTb+lmlDga+F48lrZpAX2QN+Wvnmpht27GyaAPo+Ri6VFqQKHq7TT1dqzVdqrBe3Ozo0js3i6WVQCSEIAKRK+48hTNZQjKAjFHOgSCPWEEAWxj2xJzInKm+o2Emb8JMMjlXWSX1kUR4lQCgbYE7kvySNPUKQtTIqsx5ArYYDCcYCXszOXYZm0AGSZSEnIL5mQU92IkbEvWsWtMFtoEvXOJton5AnVc+5ZaniQEnBNP2WTZSdpNi7Nt4TqpHVDL9Uu0JoWtEaqaiDvn2E4QjPMfkSMlmcF71dxbJZmaX2GY0Qpsi0TOljoe+6rmq0YLJuaF1T6dmfYvecrPMVHPdXs29awWxrulntOp9pYf0egQEOVmXZcKKDgH+zm/2GX3J/dvMSuU1DqlNjdZisOd+3N7fZuO1tgMbCKYNso1rG9+QuOIjehOBRugDi/sNqxGEjEBHH1p9uACcf6CE/RGF/oMER6+oHMplDgrc6MgBcxfYUCZNm7Coko5zPq6MrUIq/upcllexeJ8D4NJAnjRODQzRt5SQBEBNLfGhgRhl0RzHSAXEa0V+D6SB9UoX9+DQ3Bqo68GJx1di0dn3zY3v9e4NgwXhtvjKZhGR+NfePAODZ6hlv7VgtqSe2qfli/rF/X53npg7VC88oorfrPv+Zv9pg=</latexit>

Dataset Augmentation

CSE 599I: Generative Models University of WashingtonLecture 5

• Input dropout: randomly mask word in the history (uncertain history).

• Embedding dropout: randomly mask dimensions in the input embeddings.

• Makes a big difference! At increased computational cost….

• Wikitext-2 without augmentation: ~100ppl, with augmentation: ~60ppl

https://twitter.com/Tim_Dettmers/status/1247998807494684672

Hyper-Parameter Selection

CSE 599I: Generative Models University of WashingtonLecture 5

• Width of the transformer block

• Width of the internal position-wise feedforward network

• Width of each transformer head

• Number of transformer heads

• Depth of the stack of transformer blocks

• Context length

• All the usual hyper-parameters: dropout, l2, learning rates, initialization…

d.<latexit sha1_base64="Msfrff3ffEmHcSyNy5EsbOIDTVc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp6w732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/AAHsKbc=</latexit><latexit sha1_base64="Msfrff3ffEmHcSyNy5EsbOIDTVc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp6w732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/AAHsKbc=</latexit><latexit sha1_base64="Msfrff3ffEmHcSyNy5EsbOIDTVc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp6w732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/AAHsKbc=</latexit><latexit sha1_base64="Msfrff3ffEmHcSyNy5EsbOIDTVc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp6w732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/AAHsKbc=</latexit>

m.<latexit sha1_base64="hccvVRi2NldkZLucnd0b7w2oszM=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUJ96d77R73d7SousgbkAbNTac77YgySSpOAhDGNZ6GvdKM7NYGUoYuDCpNJSYnOICph4KzEHP7LJUF3U8k0W5VP4IEy3ZfyMs5loveOqVHJsTve6ryf/6asZIyfRaASZ/ObNUlJUBQVb/5xWLjIzqcUQZVUAMW3iAiaK+hYicYIWJ8UMLw2QAvkcFQ+x12cCPmVPvmdiEijyjZ+du2p/ZPWefOdsJo6gdJwdJVss0/QrJt+Sg3e+EviQBX4jkHIvsItTZwdy+fecuM10VjbFyNqm7UtzWD/e3mvee/VCCwkYqmzBZUOPs6tqg0bTgkmbOXoDNuTJYJavvTdlkbjg+99kasEGXghKyYow6++oSbtD6hXT2CBsopPLbyZxfzHh9Da+Dcb8be/zxRfvwTbOi2+gJeor2UIz20SF6jYZohAjK0Xf0A/1s/Q46wfOgu5K2tpqYx+iKBft/AB3NKcA=</latexit><latexit sha1_base64="hccvVRi2NldkZLucnd0b7w2oszM=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUJ96d77R73d7SousgbkAbNTac77YgySSpOAhDGNZ6GvdKM7NYGUoYuDCpNJSYnOICph4KzEHP7LJUF3U8k0W5VP4IEy3ZfyMs5loveOqVHJsTve6ryf/6asZIyfRaASZ/ObNUlJUBQVb/5xWLjIzqcUQZVUAMW3iAiaK+hYicYIWJ8UMLw2QAvkcFQ+x12cCPmVPvmdiEijyjZ+du2p/ZPWefOdsJo6gdJwdJVss0/QrJt+Sg3e+EviQBX4jkHIvsItTZwdy+fecuM10VjbFyNqm7UtzWD/e3mvee/VCCwkYqmzBZUOPs6tqg0bTgkmbOXoDNuTJYJavvTdlkbjg+99kasEGXghKyYow6++oSbtD6hXT2CBsopPLbyZxfzHh9Da+Dcb8be/zxRfvwTbOi2+gJeor2UIz20SF6jYZohAjK0Xf0A/1s/Q46wfOgu5K2tpqYx+iKBft/AB3NKcA=</latexit><latexit sha1_base64="hccvVRi2NldkZLucnd0b7w2oszM=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUJ96d77R73d7SousgbkAbNTac77YgySSpOAhDGNZ6GvdKM7NYGUoYuDCpNJSYnOICph4KzEHP7LJUF3U8k0W5VP4IEy3ZfyMs5loveOqVHJsTve6ryf/6asZIyfRaASZ/ObNUlJUBQVb/5xWLjIzqcUQZVUAMW3iAiaK+hYicYIWJ8UMLw2QAvkcFQ+x12cCPmVPvmdiEijyjZ+du2p/ZPWefOdsJo6gdJwdJVss0/QrJt+Sg3e+EviQBX4jkHIvsItTZwdy+fecuM10VjbFyNqm7UtzWD/e3mvee/VCCwkYqmzBZUOPs6tqg0bTgkmbOXoDNuTJYJavvTdlkbjg+99kasEGXghKyYow6++oSbtD6hXT2CBsopPLbyZxfzHh9Da+Dcb8be/zxRfvwTbOi2+gJeor2UIz20SF6jYZohAjK0Xf0A/1s/Q46wfOgu5K2tpqYx+iKBft/AB3NKcA=</latexit><latexit sha1_base64="hccvVRi2NldkZLucnd0b7w2oszM=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUJ96d77R73d7SousgbkAbNTac77YgySSpOAhDGNZ6GvdKM7NYGUoYuDCpNJSYnOICph4KzEHP7LJUF3U8k0W5VP4IEy3ZfyMs5loveOqVHJsTve6ryf/6asZIyfRaASZ/ObNUlJUBQVb/5xWLjIzqcUQZVUAMW3iAiaK+hYicYIWJ8UMLw2QAvkcFQ+x12cCPmVPvmdiEijyjZ+du2p/ZPWefOdsJo6gdJwdJVss0/QrJt+Sg3e+EviQBX4jkHIvsItTZwdy+fecuM10VjbFyNqm7UtzWD/e3mvee/VCCwkYqmzBZUOPs6tqg0bTgkmbOXoDNuTJYJavvTdlkbjg+99kasEGXghKyYow6++oSbtD6hXT2CBsopPLbyZxfzHh9Da+Dcb8be/zxRfvwTbOi2+gJeor2UIz20SF6jYZohAjK0Xf0A/1s/Q46wfOgu5K2tpqYx+iKBft/AB3NKcA=</latexit>

k.<latexit sha1_base64="Fzezv+hC7Rrx6+mZ6VEVgQI+dZU=">AAADfXicdVLbbhMxEHUTCmUp0MIjLytWkYqEomyEVKS+VDQPiIsIiKSRslHk9c6mVnxZ2d7SYPwHvMK/8TXgTbaFpmQky8dnzoxnRpMWjGrT6fzaajRvbd++s3M3uLd7/8HDvf1HQy1LRWBAJJNqlGINjAoYGGoYjAoFmKcMTtP5SeU/PQelqRSfzaKACcczQXNKsPHUp3l7uhd12p2lhTdBXIMI1daf7jcgySQpOQhDGNZ6HHcKM7FYGUoYuCApNRSYzPEMxh4KzEFP7LJUF7Y8k4W5VP4IEy7ZfyMs5loveOqVHJszve6ryP/6KsZIyfRaASZ/ObFUFKUBQVb/5yULjQyrcYQZVUAMW3iAiaK+hZCcYYWJ8UMLgqQHvkcFfex1Wc+PmVPvGdmEijyj5xdu3J3YA2efOdsKwjCKk6Mkq2SafoXkW3IUdVuBL0nAFyI5xyK7DHW2N7Vv37mrTNdFQ6ycTaquFLfVw/2t5r1nPxSgsJHKJkzOqHF2dW3QaDrjkmbOXoLNuTJYJavuTdlkbji+8NlqsEGXghKyZIw6++oKbtD6hXT2BBuYSeW3kzm/mPH6Gt4Ew2479vjji+j4Tb2iO+gJeooOUIwO0TF6jfpogAjK0Xf0A/1s/G62ms+b7ZW0sVXHPEbXrHn4BxebKb4=</latexit><latexit sha1_base64="Fzezv+hC7Rrx6+mZ6VEVgQI+dZU=">AAADfXicdVLbbhMxEHUTCmUp0MIjLytWkYqEomyEVKS+VDQPiIsIiKSRslHk9c6mVnxZ2d7SYPwHvMK/8TXgTbaFpmQky8dnzoxnRpMWjGrT6fzaajRvbd++s3M3uLd7/8HDvf1HQy1LRWBAJJNqlGINjAoYGGoYjAoFmKcMTtP5SeU/PQelqRSfzaKACcczQXNKsPHUp3l7uhd12p2lhTdBXIMI1daf7jcgySQpOQhDGNZ6HHcKM7FYGUoYuCApNRSYzPEMxh4KzEFP7LJUF7Y8k4W5VP4IEy7ZfyMs5loveOqVHJszve6ryP/6KsZIyfRaASZ/ObFUFKUBQVb/5yULjQyrcYQZVUAMW3iAiaK+hZCcYYWJ8UMLgqQHvkcFfex1Wc+PmVPvGdmEijyj5xdu3J3YA2efOdsKwjCKk6Mkq2SafoXkW3IUdVuBL0nAFyI5xyK7DHW2N7Vv37mrTNdFQ6ycTaquFLfVw/2t5r1nPxSgsJHKJkzOqHF2dW3QaDrjkmbOXoLNuTJYJavuTdlkbji+8NlqsEGXghKyZIw6++oKbtD6hXT2BBuYSeW3kzm/mPH6Gt4Ew2479vjji+j4Tb2iO+gJeooOUIwO0TF6jfpogAjK0Xf0A/1s/G62ms+b7ZW0sVXHPEbXrHn4BxebKb4=</latexit><latexit sha1_base64="Fzezv+hC7Rrx6+mZ6VEVgQI+dZU=">AAADfXicdVLbbhMxEHUTCmUp0MIjLytWkYqEomyEVKS+VDQPiIsIiKSRslHk9c6mVnxZ2d7SYPwHvMK/8TXgTbaFpmQky8dnzoxnRpMWjGrT6fzaajRvbd++s3M3uLd7/8HDvf1HQy1LRWBAJJNqlGINjAoYGGoYjAoFmKcMTtP5SeU/PQelqRSfzaKACcczQXNKsPHUp3l7uhd12p2lhTdBXIMI1daf7jcgySQpOQhDGNZ6HHcKM7FYGUoYuCApNRSYzPEMxh4KzEFP7LJUF7Y8k4W5VP4IEy7ZfyMs5loveOqVHJszve6ryP/6KsZIyfRaASZ/ObFUFKUBQVb/5yULjQyrcYQZVUAMW3iAiaK+hZCcYYWJ8UMLgqQHvkcFfex1Wc+PmVPvGdmEijyj5xdu3J3YA2efOdsKwjCKk6Mkq2SafoXkW3IUdVuBL0nAFyI5xyK7DHW2N7Vv37mrTNdFQ6ycTaquFLfVw/2t5r1nPxSgsJHKJkzOqHF2dW3QaDrjkmbOXoLNuTJYJavuTdlkbji+8NlqsEGXghKyZIw6++oKbtD6hXT2BBuYSeW3kzm/mPH6Gt4Ew2479vjji+j4Tb2iO+gJeooOUIwO0TF6jfpogAjK0Xf0A/1s/G62ms+b7ZW0sVXHPEbXrHn4BxebKb4=</latexit><latexit sha1_base64="Fzezv+hC7Rrx6+mZ6VEVgQI+dZU=">AAADfXicdVLbbhMxEHUTCmUp0MIjLytWkYqEomyEVKS+VDQPiIsIiKSRslHk9c6mVnxZ2d7SYPwHvMK/8TXgTbaFpmQky8dnzoxnRpMWjGrT6fzaajRvbd++s3M3uLd7/8HDvf1HQy1LRWBAJJNqlGINjAoYGGoYjAoFmKcMTtP5SeU/PQelqRSfzaKACcczQXNKsPHUp3l7uhd12p2lhTdBXIMI1daf7jcgySQpOQhDGNZ6HHcKM7FYGUoYuCApNRSYzPEMxh4KzEFP7LJUF7Y8k4W5VP4IEy7ZfyMs5loveOqVHJszve6ryP/6KsZIyfRaASZ/ObFUFKUBQVb/5yULjQyrcYQZVUAMW3iAiaK+hZCcYYWJ8UMLgqQHvkcFfex1Wc+PmVPvGdmEijyj5xdu3J3YA2efOdsKwjCKk6Mkq2SafoXkW3IUdVuBL0nAFyI5xyK7DHW2N7Vv37mrTNdFQ6ycTaquFLfVw/2t5r1nPxSgsJHKJkzOqHF2dW3QaDrjkmbOXoLNuTJYJavuTdlkbji+8NlqsEGXghKyZIw6++oKbtD6hXT2BBuYSeW3kzm/mPH6Gt4Ew2479vjji+j4Tb2iO+gJeooOUIwO0TF6jfpogAjK0Xf0A/1s/G62ms+b7ZW0sVXHPEbXrHn4BxebKb4=</latexit>

H.<latexit sha1_base64="60D3Z5qlANbIHPgo02NLBO3fjiM=">AAADfXicdVLbahsxEFW8vaTbW5I+9mXpYkihGK8JpJCX0Pgh9ELdUjsGrzFa7awjossiaZO4qv6gr+2/9Wtare2kjVMPCB2dOTOaGSYrGdWm3f610Qju3L13f/NB+PDR4ydPt7Z3BlpWikCfSCbVMMMaGBXQN9QwGJYKMM8YnGRnR7X/5ByUplJ8MbMSxhxPBS0owcZTn49bk6243WrPLboNkiWI0dJ6k+0GpLkkFQdhCMNaj5J2acYWK0MJAxemlYYSkzM8hZGHAnPQYzsv1UVNz+RRIZU/wkRz9t8Ii7nWM555JcfmVK/6avK/vpoxUjK9UoApXo8tFWVlQJDF/0XFIiOjehxRThUQw2YeYKKobyEip1hhYvzQwjDtgu9RQQ97Xd71Y+bUe4Y2paLI6fmlG3XGdtfZl842wyiKk/QgzWuZpl8h/ZYexJ1m6EsScEEk51jkV6HOdif23Xt3nemmaICVs2ndleK2fri/1Xzw7McSFDZS2ZTJKTXOLq41Gk2nXNLc2SuwPlcOi2T1vS6bLAzHlz7bEqzRZaCErBijzr65hmu0fiGdPcIGplL57WTOL2ayuoa3waDTSjz+tBcfvl2u6CZ6jl6gXZSgfXSIjlEP9RFBBfqOfqCfjd9BM3gVtBbSxsYy5hm6YcH+H6shKZs=</latexit><latexit sha1_base64="60D3Z5qlANbIHPgo02NLBO3fjiM=">AAADfXicdVLbahsxEFW8vaTbW5I+9mXpYkihGK8JpJCX0Pgh9ELdUjsGrzFa7awjossiaZO4qv6gr+2/9Wtare2kjVMPCB2dOTOaGSYrGdWm3f610Qju3L13f/NB+PDR4ydPt7Z3BlpWikCfSCbVMMMaGBXQN9QwGJYKMM8YnGRnR7X/5ByUplJ8MbMSxhxPBS0owcZTn49bk6243WrPLboNkiWI0dJ6k+0GpLkkFQdhCMNaj5J2acYWK0MJAxemlYYSkzM8hZGHAnPQYzsv1UVNz+RRIZU/wkRz9t8Ii7nWM555JcfmVK/6avK/vpoxUjK9UoApXo8tFWVlQJDF/0XFIiOjehxRThUQw2YeYKKobyEip1hhYvzQwjDtgu9RQQ97Xd71Y+bUe4Y2paLI6fmlG3XGdtfZl842wyiKk/QgzWuZpl8h/ZYexJ1m6EsScEEk51jkV6HOdif23Xt3nemmaICVs2ndleK2fri/1Xzw7McSFDZS2ZTJKTXOLq41Gk2nXNLc2SuwPlcOi2T1vS6bLAzHlz7bEqzRZaCErBijzr65hmu0fiGdPcIGplL57WTOL2ayuoa3waDTSjz+tBcfvl2u6CZ6jl6gXZSgfXSIjlEP9RFBBfqOfqCfjd9BM3gVtBbSxsYy5hm6YcH+H6shKZs=</latexit><latexit sha1_base64="60D3Z5qlANbIHPgo02NLBO3fjiM=">AAADfXicdVLbahsxEFW8vaTbW5I+9mXpYkihGK8JpJCX0Pgh9ELdUjsGrzFa7awjossiaZO4qv6gr+2/9Wtare2kjVMPCB2dOTOaGSYrGdWm3f610Qju3L13f/NB+PDR4ydPt7Z3BlpWikCfSCbVMMMaGBXQN9QwGJYKMM8YnGRnR7X/5ByUplJ8MbMSxhxPBS0owcZTn49bk6243WrPLboNkiWI0dJ6k+0GpLkkFQdhCMNaj5J2acYWK0MJAxemlYYSkzM8hZGHAnPQYzsv1UVNz+RRIZU/wkRz9t8Ii7nWM555JcfmVK/6avK/vpoxUjK9UoApXo8tFWVlQJDF/0XFIiOjehxRThUQw2YeYKKobyEip1hhYvzQwjDtgu9RQQ97Xd71Y+bUe4Y2paLI6fmlG3XGdtfZl842wyiKk/QgzWuZpl8h/ZYexJ1m6EsScEEk51jkV6HOdif23Xt3nemmaICVs2ndleK2fri/1Xzw7McSFDZS2ZTJKTXOLq41Gk2nXNLc2SuwPlcOi2T1vS6bLAzHlz7bEqzRZaCErBijzr65hmu0fiGdPcIGplL57WTOL2ayuoa3waDTSjz+tBcfvl2u6CZ6jl6gXZSgfXSIjlEP9RFBBfqOfqCfjd9BM3gVtBbSxsYy5hm6YcH+H6shKZs=</latexit><latexit sha1_base64="60D3Z5qlANbIHPgo02NLBO3fjiM=">AAADfXicdVLbahsxEFW8vaTbW5I+9mXpYkihGK8JpJCX0Pgh9ELdUjsGrzFa7awjossiaZO4qv6gr+2/9Wtare2kjVMPCB2dOTOaGSYrGdWm3f610Qju3L13f/NB+PDR4ydPt7Z3BlpWikCfSCbVMMMaGBXQN9QwGJYKMM8YnGRnR7X/5ByUplJ8MbMSxhxPBS0owcZTn49bk6243WrPLboNkiWI0dJ6k+0GpLkkFQdhCMNaj5J2acYWK0MJAxemlYYSkzM8hZGHAnPQYzsv1UVNz+RRIZU/wkRz9t8Ii7nWM555JcfmVK/6avK/vpoxUjK9UoApXo8tFWVlQJDF/0XFIiOjehxRThUQw2YeYKKobyEip1hhYvzQwjDtgu9RQQ97Xd71Y+bUe4Y2paLI6fmlG3XGdtfZl842wyiKk/QgzWuZpl8h/ZYexJ1m6EsScEEk51jkV6HOdif23Xt3nemmaICVs2ndleK2fri/1Xzw7McSFDZS2ZTJKTXOLq41Gk2nXNLc2SuwPlcOi2T1vS6bLAzHlz7bEqzRZaCErBijzr65hmu0fiGdPcIGplL57WTOL2ayuoa3waDTSjz+tBcfvl2u6CZ6jl6gXZSgfXSIjlEP9RFBBfqOfqCfjd9BM3gVtBbSxsYy5hm6YcH+H6shKZs=</latexit>

L.<latexit sha1_base64="24ZzP1RqPOzOe+9xwnM+tsQkOuA=">AAADfXicdVLbbhMxEHUTLmW5teWRlxWrSEVCUTaqVKS+VDQPCIoIiKSRslHk9c6mVn1Z2d62wfgPeIV/42vAm2wLTclIlo/PnBnPjCYtGNWm0/m10WjeuXvv/uaD4OGjx0+ebm3vDLUsFYEBkUyqUYo1MCpgYKhhMCoUYJ4yOEnPjir/yTkoTaX4YuYFTDieCZpTgo2nPh+3p1tRp91ZWHgbxDWIUG396XYDkkySkoMwhGGtx3GnMBOLlaGEgQuSUkOByRmewdhDgTnoiV2U6sKWZ7Iwl8ofYcIF+2+ExVzrOU+9kmNzqld9FflfX8UYKZleKcDkryeWiqI0IMjy/7xkoZFhNY4wowqIYXMPMFHUtxCSU6wwMX5oQZD0wPeooI+9Luv5MXPqPSObUJFn9PzSjbsTu+vsS2dbQRhGcXKQZJVM06+QfEsOom4r8CUJuCCScyyyq1Bne1P7/thdZ7opGmLlbFJ1pbitHu5vNR88+7EAhY1UNmFyRo2zy2uNRtMZlzRz9gqsz5XBMll1r8smc8Pxpc9WgzW6FJSQJWPU2TfXcI3WL6SzR9jATCq/ncz5xYxX1/A2GHbbscef9qLDd/WKbqLn6AXaRTHaR4foLeqjASIoR9/RD/Sz8bvZar5qtpfSxkYd8wzdsOb+H7eFKZ8=</latexit><latexit sha1_base64="24ZzP1RqPOzOe+9xwnM+tsQkOuA=">AAADfXicdVLbbhMxEHUTLmW5teWRlxWrSEVCUTaqVKS+VDQPCIoIiKSRslHk9c6mVn1Z2d62wfgPeIV/42vAm2wLTclIlo/PnBnPjCYtGNWm0/m10WjeuXvv/uaD4OGjx0+ebm3vDLUsFYEBkUyqUYo1MCpgYKhhMCoUYJ4yOEnPjir/yTkoTaX4YuYFTDieCZpTgo2nPh+3p1tRp91ZWHgbxDWIUG396XYDkkySkoMwhGGtx3GnMBOLlaGEgQuSUkOByRmewdhDgTnoiV2U6sKWZ7Iwl8ofYcIF+2+ExVzrOU+9kmNzqld9FflfX8UYKZleKcDkryeWiqI0IMjy/7xkoZFhNY4wowqIYXMPMFHUtxCSU6wwMX5oQZD0wPeooI+9Luv5MXPqPSObUJFn9PzSjbsTu+vsS2dbQRhGcXKQZJVM06+QfEsOom4r8CUJuCCScyyyq1Bne1P7/thdZ7opGmLlbFJ1pbitHu5vNR88+7EAhY1UNmFyRo2zy2uNRtMZlzRz9gqsz5XBMll1r8smc8Pxpc9WgzW6FJSQJWPU2TfXcI3WL6SzR9jATCq/ncz5xYxX1/A2GHbbscef9qLDd/WKbqLn6AXaRTHaR4foLeqjASIoR9/RD/Sz8bvZar5qtpfSxkYd8wzdsOb+H7eFKZ8=</latexit><latexit sha1_base64="24ZzP1RqPOzOe+9xwnM+tsQkOuA=">AAADfXicdVLbbhMxEHUTLmW5teWRlxWrSEVCUTaqVKS+VDQPCIoIiKSRslHk9c6mVn1Z2d62wfgPeIV/42vAm2wLTclIlo/PnBnPjCYtGNWm0/m10WjeuXvv/uaD4OGjx0+ebm3vDLUsFYEBkUyqUYo1MCpgYKhhMCoUYJ4yOEnPjir/yTkoTaX4YuYFTDieCZpTgo2nPh+3p1tRp91ZWHgbxDWIUG396XYDkkySkoMwhGGtx3GnMBOLlaGEgQuSUkOByRmewdhDgTnoiV2U6sKWZ7Iwl8ofYcIF+2+ExVzrOU+9kmNzqld9FflfX8UYKZleKcDkryeWiqI0IMjy/7xkoZFhNY4wowqIYXMPMFHUtxCSU6wwMX5oQZD0wPeooI+9Luv5MXPqPSObUJFn9PzSjbsTu+vsS2dbQRhGcXKQZJVM06+QfEsOom4r8CUJuCCScyyyq1Bne1P7/thdZ7opGmLlbFJ1pbitHu5vNR88+7EAhY1UNmFyRo2zy2uNRtMZlzRz9gqsz5XBMll1r8smc8Pxpc9WgzW6FJSQJWPU2TfXcI3WL6SzR9jATCq/ncz5xYxX1/A2GHbbscef9qLDd/WKbqLn6AXaRTHaR4foLeqjASIoR9/RD/Sz8bvZar5qtpfSxkYd8wzdsOb+H7eFKZ8=</latexit><latexit sha1_base64="24ZzP1RqPOzOe+9xwnM+tsQkOuA=">AAADfXicdVLbbhMxEHUTLmW5teWRlxWrSEVCUTaqVKS+VDQPCIoIiKSRslHk9c6mVn1Z2d62wfgPeIV/42vAm2wLTclIlo/PnBnPjCYtGNWm0/m10WjeuXvv/uaD4OGjx0+ebm3vDLUsFYEBkUyqUYo1MCpgYKhhMCoUYJ4yOEnPjir/yTkoTaX4YuYFTDieCZpTgo2nPh+3p1tRp91ZWHgbxDWIUG396XYDkkySkoMwhGGtx3GnMBOLlaGEgQuSUkOByRmewdhDgTnoiV2U6sKWZ7Iwl8ofYcIF+2+ExVzrOU+9kmNzqld9FflfX8UYKZleKcDkryeWiqI0IMjy/7xkoZFhNY4wowqIYXMPMFHUtxCSU6wwMX5oQZD0wPeooI+9Luv5MXPqPSObUJFn9PzSjbsTu+vsS2dbQRhGcXKQZJVM06+QfEsOom4r8CUJuCCScyyyq1Bne1P7/thdZ7opGmLlbFJ1pbitHu5vNR88+7EAhY1UNmFyRo2zy2uNRtMZlzRz9gqsz5XBMll1r8smc8Pxpc9WgzW6FJSQJWPU2TfXcI3WL6SzR9jATCq/ncz5xYxX1/A2GHbbscef9qLDd/WKbqLn6AXaRTHaR4foLeqjASIoR9/RD/Sz8bvZar5qtpfSxkYd8wzdsOb+H7eFKZ8=</latexit>

p.<latexit sha1_base64="Z8xeDOVoYhe9uk5l6wT4/nFnaDc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp7I732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/ACcYKcM=</latexit><latexit sha1_base64="Z8xeDOVoYhe9uk5l6wT4/nFnaDc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp7I732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/ACcYKcM=</latexit><latexit sha1_base64="Z8xeDOVoYhe9uk5l6wT4/nFnaDc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp7I732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/ACcYKcM=</latexit><latexit sha1_base64="Z8xeDOVoYhe9uk5l6wT4/nFnaDc=">AAADfXicdVLbbhMxEHWzXMpya+GRlxVRpCKhKBshFakvFc0D4iICImmkbBR5vbNbq76sbG9pMP4DXuHf+BrwJttCUzKS5eMzZ8Yzo0lLRrXp9X5ttYIbN2/d3r4T3r13/8HDnd1HYy0rRWBEJJNqkmINjAoYGWoYTEoFmKcMjtPTo9p/fAZKUyk+m0UJM44LQXNKsPHUp7I732n3ur2lRddB3IA2amw4321BkklScRCGMKz1NO6VZmaxMpQwcGFSaSgxOcUFTD0UmIOe2WWpLup4JotyqfwRJlqy/0ZYzLVe8NQrOTYnet1Xk//11YyRkum1Akz+cmapKCsDgqz+zysWGRnV44gyqoAYtvAAE0V9CxE5wQoT44cWhskAfI8KhtjrsoEfM6feM7EJFXlGz87dtD+ze84+c7YTRlE7Tg6SrJZp+hWSb8lBu98JfUkCvhDJORbZRaizg7l9+85dZroqGmPlbFJ3pbitH+5vNe89+6EEhY1UNmGyoMbZ1bVBo2nBJc2cvQCbc2WwSlbfm7LJ3HB87rM1YIMuBSVkxRh19tUl3KD1C+nsETZQSOW3kzm/mPH6Gl4H43439vjji/bhm2ZFt9ET9BTtoRjto0P0Gg3RCBGUo+/oB/rZ+h10gudBdyVtbTUxj9EVC/b/ACcYKcM=</latexit>

Comparison to RNN/LSTM

CSE 599I: Generative Models University of WashingtonLecture 5

• RNN/LSTM maintain a running state:

• Relatively fast to sample from RNN/LSTM.

• Relatively slow to train RNN/LSTM: more serial computation.

• Easier to learn long-term dependencies in transformer networks?

h0 = Winit, Winit 2 Rk,

ht = sigmoid(c+Whht�1 +Wxxt), Wh 2 Rk⇥k,Wx 2 Rk⇥d, c 2 Rk,

ft(x<t) = b+Woht�1, Wo 2 Rd⇥k,b 2 Rd.<latexit sha1_base64="YJd+MGsL4MHdWTtKYroY+0V7epc=">AAAFPnicdVRdb9MwFM1CgRG+NnjkxaLq6MSYmgkJpIE0sT0gYGIg1k6aS+Q4TmfVjiPbHR3G/21/gz/AG+KVR5x+bFlKLEU5uffck3OTa8c5o0p3Oj+X/GuN6zduLt8Kbt+5e+/+yuqDrhIjickhFkzIoxgpwmhGDjXVjBzlkiAeM9KLh7tFvndKpKIi+6LPctLnaJDRlGKkXShaXTpfg5qMdZyaExt1wGswf+zZaAINzai2G2BtrSYFIM0A5EifxLH5bL8ONyAMyqq6UFV0wAVN2vM4tuBpWdARSyVGPwurhPEFYew016uOXFnViRlCTTlRYGg3rirVMhPXacniAs1Omksj3S6ZMa+0XS99urhiXVQ7q3oXCy9KLp2XZRdodjNaaXY2O5MFFkE4A01vtg6iVZ/AROARJ5nGDCl1HHZy3TdIaooZsQEcKZIjPEQDcuxghpyRvpnMmgUtF0lAKqS7Mg0m0XKFQVypM+68tgqfqporgv/NFREtBFMVAzp92XeDlo80yfD0/emIAS1AMc8goZJgzc4cQFhS1wLAJ0girN3UBwHcI65HSQ6Q4yV7bp9w6jJHxn3JNKGnY3u81Tdta9ataQUANEO4DZOCpuh3An/A7eZWK3CWMvINC85RlsxLrdmLzPsP9kLpKqmLpDWTfyW5KR7spZt9F/2YE4m0kAYyMXDbyExvNZzZ9rFmDuq1EjIVK+51aiLVHI2d2gzU8GIiMzFijFrz5gLWcN2JYs0u0mQgpDtemHWDGVbHcBF0tzZDhz89b+68m43osvfIe+y1vdB74e14b70D79DD/hN/3+/6vcZ541fjd+PPlOovzWoeeldW4+8/O/nVHg==</latexit><latexit sha1_base64="YJd+MGsL4MHdWTtKYroY+0V7epc=">AAAFPnicdVRdb9MwFM1CgRG+NnjkxaLq6MSYmgkJpIE0sT0gYGIg1k6aS+Q4TmfVjiPbHR3G/21/gz/AG+KVR5x+bFlKLEU5uffck3OTa8c5o0p3Oj+X/GuN6zduLt8Kbt+5e+/+yuqDrhIjickhFkzIoxgpwmhGDjXVjBzlkiAeM9KLh7tFvndKpKIi+6LPctLnaJDRlGKkXShaXTpfg5qMdZyaExt1wGswf+zZaAINzai2G2BtrSYFIM0A5EifxLH5bL8ONyAMyqq6UFV0wAVN2vM4tuBpWdARSyVGPwurhPEFYew016uOXFnViRlCTTlRYGg3rirVMhPXacniAs1Omksj3S6ZMa+0XS99urhiXVQ7q3oXCy9KLp2XZRdodjNaaXY2O5MFFkE4A01vtg6iVZ/AROARJ5nGDCl1HHZy3TdIaooZsQEcKZIjPEQDcuxghpyRvpnMmgUtF0lAKqS7Mg0m0XKFQVypM+68tgqfqporgv/NFREtBFMVAzp92XeDlo80yfD0/emIAS1AMc8goZJgzc4cQFhS1wLAJ0girN3UBwHcI65HSQ6Q4yV7bp9w6jJHxn3JNKGnY3u81Tdta9ataQUANEO4DZOCpuh3An/A7eZWK3CWMvINC85RlsxLrdmLzPsP9kLpKqmLpDWTfyW5KR7spZt9F/2YE4m0kAYyMXDbyExvNZzZ9rFmDuq1EjIVK+51aiLVHI2d2gzU8GIiMzFijFrz5gLWcN2JYs0u0mQgpDtemHWDGVbHcBF0tzZDhz89b+68m43osvfIe+y1vdB74e14b70D79DD/hN/3+/6vcZ541fjd+PPlOovzWoeeldW4+8/O/nVHg==</latexit><latexit sha1_base64="YJd+MGsL4MHdWTtKYroY+0V7epc=">AAAFPnicdVRdb9MwFM1CgRG+NnjkxaLq6MSYmgkJpIE0sT0gYGIg1k6aS+Q4TmfVjiPbHR3G/21/gz/AG+KVR5x+bFlKLEU5uffck3OTa8c5o0p3Oj+X/GuN6zduLt8Kbt+5e+/+yuqDrhIjickhFkzIoxgpwmhGDjXVjBzlkiAeM9KLh7tFvndKpKIi+6LPctLnaJDRlGKkXShaXTpfg5qMdZyaExt1wGswf+zZaAINzai2G2BtrSYFIM0A5EifxLH5bL8ONyAMyqq6UFV0wAVN2vM4tuBpWdARSyVGPwurhPEFYew016uOXFnViRlCTTlRYGg3rirVMhPXacniAs1Omksj3S6ZMa+0XS99urhiXVQ7q3oXCy9KLp2XZRdodjNaaXY2O5MFFkE4A01vtg6iVZ/AROARJ5nGDCl1HHZy3TdIaooZsQEcKZIjPEQDcuxghpyRvpnMmgUtF0lAKqS7Mg0m0XKFQVypM+68tgqfqporgv/NFREtBFMVAzp92XeDlo80yfD0/emIAS1AMc8goZJgzc4cQFhS1wLAJ0girN3UBwHcI65HSQ6Q4yV7bp9w6jJHxn3JNKGnY3u81Tdta9ataQUANEO4DZOCpuh3An/A7eZWK3CWMvINC85RlsxLrdmLzPsP9kLpKqmLpDWTfyW5KR7spZt9F/2YE4m0kAYyMXDbyExvNZzZ9rFmDuq1EjIVK+51aiLVHI2d2gzU8GIiMzFijFrz5gLWcN2JYs0u0mQgpDtemHWDGVbHcBF0tzZDhz89b+68m43osvfIe+y1vdB74e14b70D79DD/hN/3+/6vcZ541fjd+PPlOovzWoeeldW4+8/O/nVHg==</latexit><latexit sha1_base64="YJd+MGsL4MHdWTtKYroY+0V7epc=">AAAFPnicdVRdb9MwFM1CgRG+NnjkxaLq6MSYmgkJpIE0sT0gYGIg1k6aS+Q4TmfVjiPbHR3G/21/gz/AG+KVR5x+bFlKLEU5uffck3OTa8c5o0p3Oj+X/GuN6zduLt8Kbt+5e+/+yuqDrhIjickhFkzIoxgpwmhGDjXVjBzlkiAeM9KLh7tFvndKpKIi+6LPctLnaJDRlGKkXShaXTpfg5qMdZyaExt1wGswf+zZaAINzai2G2BtrSYFIM0A5EifxLH5bL8ONyAMyqq6UFV0wAVN2vM4tuBpWdARSyVGPwurhPEFYew016uOXFnViRlCTTlRYGg3rirVMhPXacniAs1Omksj3S6ZMa+0XS99urhiXVQ7q3oXCy9KLp2XZRdodjNaaXY2O5MFFkE4A01vtg6iVZ/AROARJ5nGDCl1HHZy3TdIaooZsQEcKZIjPEQDcuxghpyRvpnMmgUtF0lAKqS7Mg0m0XKFQVypM+68tgqfqporgv/NFREtBFMVAzp92XeDlo80yfD0/emIAS1AMc8goZJgzc4cQFhS1wLAJ0girN3UBwHcI65HSQ6Q4yV7bp9w6jJHxn3JNKGnY3u81Tdta9ataQUANEO4DZOCpuh3An/A7eZWK3CWMvINC85RlsxLrdmLzPsP9kLpKqmLpDWTfyW5KR7spZt9F/2YE4m0kAYyMXDbyExvNZzZ9rFmDuq1EjIVK+51aiLVHI2d2gzU8GIiMzFijFrz5gLWcN2JYs0u0mQgpDtemHWDGVbHcBF0tzZDhz89b+68m43osvfIe+y1vdB74e14b70D79DD/hN/3+/6vcZ541fjd+PPlOovzWoeeldW4+8/O/nVHg==</latexit>