{"id":275,"date":"2020-09-25T23:24:51","date_gmt":"2020-09-25T15:24:51","guid":{"rendered":"http:\/\/blog.cyasylum.top\/?p=275"},"modified":"2020-09-25T23:32:33","modified_gmt":"2020-09-25T15:32:33","slug":"%e7%ae%80%e5%8d%95%e5%a4%8d%e7%8e%b0%e5%8f%98%e5%bd%a2%e9%87%91%e5%88%9a%ef%bc%88%ef%bc%9f","status":"publish","type":"post","link":"http:\/\/blog.cyasylum.top\/index.php\/2020\/09\/25\/%e7%ae%80%e5%8d%95%e5%a4%8d%e7%8e%b0%e5%8f%98%e5%bd%a2%e9%87%91%e5%88%9a%ef%bc%88%ef%bc%9f\/","title":{"rendered":"\u7b80\u5355\u590d\u73b0\u53d8\u5f62\u91d1\u521a\uff08\uff1f"},"content":{"rendered":"\n<p>\u6211\u4eec\u90fd\u662f\u8bfb\u8fc7\u9ad8\u4e2d\u82f1\u8bed\u7684\u4eba\uff0c\u4f17\u6240\u5468\u77e5\u53d8\u5f62\u91d1\u521a=transformer\uff08\u554a\u8fd9<\/p>\n\n\n\n<p>\u6700\u8fd1\u88ab\u5bfc\u5e08\u5b89\u6392\u8bfe\u9898\u4e86\uff0c\u5b89\u6392\u5f97\u660e\u660e\u767d\u767d\uff0c\u8001CV\u88ab\u8feb\u63a5\u89e6NLP<\/p>\n\n\n\n<p>\u6211\u8fd8\u662f\u8001\u54b8\u9c7c\u4e86\u3002\u4e3a\u4e86\u4e0d\u88ab\u5bfc\u5e08\u5355\u6740\uff0c\u59d1\u4e14\u4f5c\u4e3a\u5165\u95e8\u548c\u590d\u5efa\uff0c\u5c1d\u8bd5\u6027\u5730\u505a\u4e00\u4e0b\u6700\u70ed\u95e8\u4e14\u6700\u5b9e\u7528\u7684Transformer\u7ed3\u6784\u590d\u73b0\uff0c\u8fc7\u7a0b\u4e2d\u4e3b\u8981\u53c2\u8003\u4e86\u4ee5\u4e0b\u6587\u7ae0<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li><a href=\"https:\/\/zhuanlan.zhihu.com\/p\/97448796\" target=\"_blank\"  rel=\"nofollow\" >SangrealGao\uff1a\u4ece 0 \u5f00\u59cb\u5b66\u4e60 Transformer<\/a><\/li><li><a href=\"https:\/\/blog.csdn.net\/stupid_3\/article\/details\/83184691\" target=\"_blank\"  rel=\"nofollow\" >\u7f57\u5468\u6768\uff1aTransformer\u7684PyTorch\u5b9e\u73b0<\/a><\/li><li><a href=\"https:\/\/jalammar.github.io\/illustrated-transformer\/\" target=\"_blank\"  rel=\"nofollow\" >Jay Alammar\uff1aillustrated transformer<\/a><\/li><li><a href=\"https:\/\/tensorflow.google.cn\/tutorials\/text\/transformer\" target=\"_blank\"  rel=\"nofollow\" >TensorFlow:\u7406\u89e3\u8bed\u8a00\u7684 Transformer \u6a21\u578b<\/a><\/li><\/ul>\n\n\n\n<p>\u5728\u8fdb\u884c\u5177\u4f53\u6784\u5efa\u7f51\u7edc\u7684\u5de5\u4f5c\u524d\uff0c\u6211\u4eec\u53ef\u4ee5\u5bf9\u6574\u4e2aTransformer\u7f51\u7edc\u7684\u7ed3\u6784\u8fdb\u884c\u8003\u8651\u3002<\/p>\n\n\n\n<figure class=\"wp-block-image\"><img decoding=\"async\" src=\"https:\/\/jalammar.github.io\/images\/t\/transformer_resideual_layer_norm_3.png\" alt=\"img\"\/><\/figure>\n\n\n\n<p>\u5982\u4e0a\u56fe\u6240\u793a\u7684\u662f\u7f51\u7edc\u7684\u6982\u8ff0\u6982\u89c8\uff0c\u611f\u8c22Jay Alammar\u7684\u5de5\u4f5c\uff0c\u6211\u4eec\u81ea\u9876\u5411\u4e0b\u63cf\u8ff0Transformer\u7684\u7ed3\u6784\u3002<\/p>\n\n\n\n<p>\u5176\u7ea6\u83ab\u662f\u7531\u4e5d\u4e2a\u4e0d\u540c\u5c42\u6b21\u7684\u7ed3\u6784\u7ec4\u6210\u7684\u3002<\/p>\n\n\n\n<ul class=\"wp-block-list\"><li>Transformer<ul><li>Decoder(DecoderLayer*N)<ul><li>PositionalEncoding<\/li><li>MultiHeadAttention<ul><li>ScaledDotProductAttention<\/li><\/ul><\/li><li>PositionalWiseFeedForward<\/li><\/ul><\/li><li>Encoder(EncoderLayer*N)<ul><li>PositionalEncoding<\/li><li>MultiHeadAttention<ul><li>ScaledDotProductAttention<\/li><\/ul><\/li><li>PositionalWiseFeedForward<\/li><\/ul><\/li><\/ul><\/li><\/ul>\n\n\n\n<p>\u5177\u4f53\u5b9e\u73b0\u54b1\u5148\u522b\u8c08\uff0c\u6211\u4eec\u6765\u8bf4\u8bf4\u5176\u4e2d\u7684\u6bcf\u4e2a\u673a\u5236\u662f\u505a\u4ec0\u4e48\u7684<\/p>\n\n\n\n<p>\u7531\u8f93\u5165\u4fe1\u606f\u7ecf\u8fc7\u7684\u7ed3\u6784\u5f00\u59cb\u63cf\u8ff0\uff0c\u53ef\u80fd\u4f1a\u6709\u4e00\u70b9\u62bd\u8c61<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u4f4d\u7f6e\u7f16\u7801\u7cfb\u7edfPositionalEncoding<\/h4>\n\n\n\n<p>\u63cf\u8ff0\u8f93\u5165\u5355\u8bcd\u4e4b\u95f4\u7684\u8ddd\u79bb\u7ed3\u6784\uff0c\u7528\u4e8e\u548c\u8bcd\u5d4c\u5165\u7f16\u7801\u8fdb\u884c\u7ec4\u5408\uff0c\u4f7f\u8ddd\u79bb\u8f83\u8fdc\u8bcd\u6c47\u76f8\u5173\u6027\u5f31<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u70b9\u79ef\u6ce8\u610f\u529b\u673a\u5236ScaledDotProductAttention<\/h4>\n\n\n\n<p>\u4f7f\u7528\u4e58\u6cd5\u52a0\u6743\u5468\u56f4\u5355\u8bcd\u7279\u5f81\u53d6\u5f97\u81ea\u6ce8\u610f\u529b\uff0c\u83b7\u53d6\u7f16\u7801\u6743\u91cd<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236MultiHeadAttention<\/h4>\n\n\n\n<p>\u540c\u65f6\u4f7f\u7528\u591a\u7ec4\u6743\u91cd\u7684\u70b9\u79ef\u6ce8\u610f\u529b\u673a\u5236\u8fdb\u884c\u5224\u65ad\uff0c\u7ec4\u5408\u7279\u5f81<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u4f4d\u7f6e\u5f0f\u524d\u9988\u7f51\u7edcPositionalWiseFeedForward<\/h4>\n\n\n\n<p>\u53cc\u5c42\u524d\u9988\u5168\u8fde\u63a5\u795e\u7ecf\u7f51\u7edc\uff0c\u91cd\u7f16\u7801\u6743\u91cd\u6ce8\u610f\u529b\u53ca\u4f4d\u7f6e\u4fe1\u606f<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u7f16\u7801\u5668Encoder<\/h4>\n\n\n\n<p>\u591a\u5c42\uff08\u591a\u5934\u6ce8\u610f\u529b+\u524d\u9988\u7f51\u7edc\uff09\u7ec4\u5408\uff0c\u53d6\u5f97\u6df1\u5ea6\u8f93\u51fa<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u89e3\u7801\u5668Decoder<\/h4>\n\n\n\n<p>\u591a\u5c42\u5f15\u5165\u7f16\u7801\u4fe1\u606f\u7684\u6ce8\u610f\u529b+\u524d\u9988\u7f51\u7edc<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">\u53d8\u5f62\u91d1\u521a\uff08\uff1f<\/h4>\n\n\n\n<p>\u7f16\u7801\u5668\u4e0e\u89e3\u7801\u5668\u7684\u7ec4\u5408\uff0c\u4ece\u8bcd\u7279\u5f81\u89e3\u7801\u81f3\u8bed\u6599\u5e93\uff0c\u751f\u6210\u65b0\u8bcd<\/p>\n\n\n\n<!doctype html>\n<html>\n<head>\n<meta charset='UTF-8'><meta name='viewport' content='width=device-width initial-scale=1'>\n<title>Transformer (copy)<\/title><\/head>\n<body><h1>\u7f51\u7edc\u7684\u6784\u5efa<\/h1>\n<p>\u6211\u4eec\u4ece\u5934\u5f00\u59cb\u53bb\u8003\u8651transformer\u7f51\u7edc\u7684\u7ec6\u8282\uff0c\u5e76\u9010\u6b65\u8003\u8651\u5982\u4f55\u5b8c\u5584\u8fd9\u4e2a\u7cfb\u7edf\u3002<\/p>\n<h2>\u53e5\u5b50\u7f16\u7801<\/h2>\n<p>transformer\u4f5c\u4e3a\u4e00\u4e2a\u57fa\u4e8e\u81ea\u7136\u8bed\u8a00\u5904\u7406\u7684\u6a21\u578b\uff0c\u5176\u7b2c\u4e00\u8981\u70b9\u4fbf\u662f\u8bcd\u6c47\u542b\u4e49\u7684\u8868\u8fbe\uff0c\u8fd9\u95e8\u5b66\u95ee\u4fbf\u662f\u5355\u8bcd\u8bed\u6599\u7684\u5411\u91cf\u5316\uff0c\u5373embedding<\/p>\n<p>\u5728\u8fd9\u9879\u5de5\u4f5c\u91cc\uff0c\u6211\u4eec\u8981\u51b3\u5b9a\u6a21\u578b\u603b\u7684\u6587\u5b57\u7f16\u7801\u5411\u91cf\u957f\u5ea6<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"5.792ex\" height=\"2.344ex\" viewBox=\"0 -755.9 2493.8 1009.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E9-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E9-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">d_{model}<\/script>\uff0c\u540c\u65f6\u8981\u628a\u4e00\u6bb5\u53e5\u5b50\u8f6c\u5316\u4e3a\u4e00\u4e2a\u5927\u5c0f\u4e3a<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"13.796ex\" height=\"2.577ex\" viewBox=\"0 -806.1 5940 1109.7\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.705ex;\"><defs><path stroke-width=\"0\" id=\"E99-MJMAIN-5B\" d=\"M118 -250V750H255V710H158V-210H255V-250H118Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-71\" d=\"M33 157Q33 258 109 349T280 441Q340 441 372 389Q373 390 377 395T388 406T404 418Q438 442 450 442Q454 442 457 439T460 434Q460 425 391 149Q320 -135 320 -139Q320 -147 365 -148H390Q396 -156 396 -157T393 -175Q389 -188 383 -194H370Q339 -192 262 -192Q234 -192 211 -192T174 -192T157 -193Q143 -193 143 -185Q143 -182 145 -170Q149 -154 152 -151T172 -148Q220 -148 230 -141Q238 -136 258 -53T279 32Q279 33 272 29Q224 -10 172 -10Q117 -10 75 30T33 157ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-6E\" d=\"M21 287Q22 293 24 303T36 341T56 388T89 425T135 442Q171 442 195 424T225 390T231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336T465 179T427 52Q427 26 444 26Q450 26 453 27Q482 32 505 65T540 145Q542 153 560 153Q580 153 580 145Q580 144 576 130Q568 101 554 73T508 17T439 -10Q392 -10 371 17T350 73Q350 92 386 193T423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 180T152 343Q153 348 153 366Q153 405 129 405Q91 405 66 305Q60 285 60 284Q58 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E99-MJMAIN-5D\" d=\"M22 710V750H159V-250H22V-210H119V710H22Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E99-MJMAIN-5B\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E99-MJMATHI-73\" x=\"278\" y=\"0\"><\/use><use xlink:href=\"#E99-MJMATHI-65\" x=\"747\" y=\"0\"><\/use><g transform=\"translate(1213,0)\"><use xlink:href=\"#E99-MJMATHI-71\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(446,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-6C\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-65\" x=\"298\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-6E\" x=\"764\" y=\"0\"><\/use><\/g><\/g><use xlink:href=\"#E99-MJMAIN-2C\" x=\"2723\" y=\"0\"><\/use><g transform=\"translate(3168,0)\"><use xlink:href=\"#E99-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E99-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><use xlink:href=\"#E99-MJMAIN-5D\" x=\"5661\" y=\"0\"><\/use><\/g><\/svg><\/span><script type=\"math\/tex\">[seq_{len},d_{model}]<\/script>\u7684\u77e9\u9635\u3002\u8fd9\u5176\u4e2d\u5305\u542b\u4e86\u4e24\u4e2a\u64cd\u4f5c\uff0c\u5176\u4e00\u662f\u5bf9\u53e5\u5b50\u4e2d\u7684\u5355\u8bcd\u8fdb\u884c\u5411\u91cf\u5316\uff0c\u957f\u5ea6\u4e3a<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"5.792ex\" height=\"2.344ex\" viewBox=\"0 -755.9 2493.8 1009.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E9-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E9-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">d_{model}<\/script>\u3002\u5176\u4e8c\u662f\u4e00\u7ec4\u5173\u4e8e<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"5.68ex\" height=\"1.76ex\" viewBox=\"0 -504.6 2445.5 757.9\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E123-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E123-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E123-MJMATHI-71\" d=\"M33 157Q33 258 109 349T280 441Q340 441 372 389Q373 390 377 395T388 406T404 418Q438 442 450 442Q454 442 457 439T460 434Q460 425 391 149Q320 -135 320 -139Q320 -147 365 -148H390Q396 -156 396 -157T393 -175Q389 -188 383 -194H370Q339 -192 262 -192Q234 -192 211 -192T174 -192T157 -193Q143 -193 143 -185Q143 -182 145 -170Q149 -154 152 -151T172 -148Q220 -148 230 -141Q238 -136 258 -53T279 32Q279 33 272 29Q224 -10 172 -10Q117 -10 75 30T33 157ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E123-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><path stroke-width=\"0\" id=\"E123-MJMATHI-6E\" d=\"M21 287Q22 293 24 303T36 341T56 388T89 425T135 442Q171 442 195 424T225 390T231 369Q231 367 232 367L243 378Q304 442 382 442Q436 442 469 415T503 336T465 179T427 52Q427 26 444 26Q450 26 453 27Q482 32 505 65T540 145Q542 153 560 153Q580 153 580 145Q580 144 576 130Q568 101 554 73T508 17T439 -10Q392 -10 371 17T350 73Q350 92 386 193T423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 180T152 343Q153 348 153 366Q153 405 129 405Q91 405 66 305Q60 285 60 284Q58 278 41 278H27Q21 284 21 287Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E123-MJMATHI-73\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E123-MJMATHI-65\" x=\"469\" y=\"0\"><\/use><g transform=\"translate(935,0)\"><use xlink:href=\"#E123-MJMATHI-71\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(446,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E123-MJMATHI-6C\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E123-MJMATHI-65\" x=\"298\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E123-MJMATHI-6E\" x=\"764\" y=\"0\"><\/use><\/g><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">seq_{len}<\/script>\u5f15\u5bfc\u7684\u4f4d\u7f6e\u7f16\u7801\u5411\u91cf\uff0c\u7531\u4ee5\u4e0b\u65b9\u6cd5\u6784\u5efa\u3002<\/p>\n<h3>\u4f4d\u7f6e\u7f16\u7801<\/h3>\n<p>\u8fd9\u4e00\u6b65\uff0c\u6211\u4eec\u9700\u8981\u5bf9\u8f93\u5165\u5355\u8bcd\u4e4b\u95f4\u7684\u8ddd\u79bb\u8fdb\u884c\u8868\u793a<\/p>\n<p>\u5728Transformer\u4e2d\u6240\u4f7f\u7528\u7684\u4f4d\u7f6e\u7f16\u7801(positional encoding)\u6709\u5982\u4e0b\u4e09\u89d2\u51fd\u6570\u5f62\u5f0f\u5b9a\u4e49<\/p>\n<div contenteditable=\"false\" spellcheck=\"false\" class=\"mathjax-block md-end-block md-math-block md-rawblock\" id=\"mathjax-n778\" cid=\"n778\" mdtype=\"math_block\"><div class=\"md-rawblock-container md-math-container\" tabindex=\"-1\"><div class=\"MathJax_SVG_Display\" style=\"text-align: center;\"><span class=\"MathJax_SVG\" id=\"MathJax-Element-218-Frame\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"58.799ex\" height=\"11.916ex\" viewBox=\"0 -2816.5 25316.2 5130.4\" role=\"img\" focusable=\"false\" style=\"vertical-align: -5.374ex; max-width: 100%;\"><defs><path stroke-width=\"0\" id=\"E257-MJMATHI-50\" d=\"M287 628Q287 635 230 637Q206 637 199 638T192 648Q192 649 194 659Q200 679 203 681T397 683Q587 682 600 680Q664 669 707 631T751 530Q751 453 685 389Q616 321 507 303Q500 302 402 301H307L277 182Q247 66 247 59Q247 55 248 54T255 50T272 48T305 46H336Q342 37 342 35Q342 19 335 5Q330 0 319 0Q316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11Q33 13 36 25Q40 41 44 43T67 46Q94 46 127 49Q141 52 146 61Q149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597T609 619T560 635Q553 636 480 637Q463 637 445 637T416 636T404 636Q391 635 386 627Q384 621 367 550T332 412T314 344Q314 342 395 342H407H430Q542 342 590 392Q617 419 631 471T645 554Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-45\" d=\"M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-70\" d=\"M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-3D\" d=\"M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-7B\" d=\"M434 -231Q434 -244 428 -250H410Q281 -250 230 -184Q225 -177 222 -172T217 -161T213 -148T211 -133T210 -111T209 -84T209 -47T209 0Q209 21 209 53Q208 142 204 153Q203 154 203 155Q189 191 153 211T82 231Q71 231 68 234T65 250T68 266T82 269Q116 269 152 289T203 345Q208 356 208 377T209 529V579Q209 634 215 656T244 698Q270 724 324 740Q361 748 377 749Q379 749 390 749T408 750H428Q434 744 434 732Q434 719 431 716Q429 713 415 713Q362 710 332 689T296 647Q291 634 291 499V417Q291 370 288 353T271 314Q240 271 184 255L170 250L184 245Q202 239 220 230T262 196T290 137Q291 131 291 1Q291 -134 296 -147Q306 -174 339 -192T415 -213Q429 -213 431 -216Q434 -219 434 -231Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-32\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-69\" d=\"M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-73\" d=\"M295 316Q295 356 268 385T190 414Q154 414 128 401Q98 382 98 349Q97 344 98 336T114 312T157 287Q175 282 201 278T245 269T277 256Q294 248 310 236T342 195T359 133Q359 71 321 31T198 -10H190Q138 -10 94 26L86 19L77 10Q71 4 65 -1L54 -11H46H42Q39 -11 33 -5V74V132Q33 153 35 157T45 162H54Q66 162 70 158T75 146T82 119T101 77Q136 26 198 26Q295 26 295 104Q295 133 277 151Q257 175 194 187T111 210Q75 227 54 256T33 318Q33 357 50 384T93 424T143 442T187 447H198Q238 447 268 432L283 424L292 431Q302 440 314 448H322H326Q329 448 335 442V310L329 304H301Q295 310 295 316Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-69\" d=\"M69 609Q69 637 87 653T131 669Q154 667 171 652T188 609Q188 579 171 564T129 549Q104 549 87 564T69 609ZM247 0Q232 3 143 3Q132 3 106 3T56 1L34 0H26V46H42Q70 46 91 49Q100 53 102 60T104 102V205V293Q104 345 102 359T88 378Q74 385 41 385H30V408Q30 431 32 431L42 432Q52 433 70 434T106 436Q123 437 142 438T171 441T182 442H185V62Q190 52 197 50T232 46H255V0H247Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-6E\" d=\"M41 46H55Q94 46 102 60V68Q102 77 102 91T102 122T103 161T103 203Q103 234 103 269T102 328V351Q99 370 88 376T43 385H25V408Q25 431 27 431L37 432Q47 433 65 434T102 436Q119 437 138 438T167 441T178 442H181V402Q181 364 182 364T187 369T199 384T218 402T247 421T285 437Q305 442 336 442Q450 438 463 329Q464 322 464 190V104Q464 66 466 59T477 49Q498 46 526 46H542V0H534L510 1Q487 2 460 2T422 3Q319 3 310 0H302V46H318Q379 46 379 62Q380 64 380 200Q379 335 378 343Q372 371 358 385T334 402T308 404Q263 404 229 370Q202 343 195 315T187 232V168V108Q187 78 188 68T191 55T200 49Q221 46 249 46H265V0H257L234 1Q210 2 183 2T145 3Q42 3 33 0H25V46H41Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-31\" d=\"M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-30\" d=\"M96 585Q152 666 249 666Q297 666 345 640T423 548Q460 465 460 320Q460 165 417 83Q397 41 362 16T301 -15T250 -22Q224 -22 198 -16T137 16T82 83Q39 165 39 320Q39 494 96 585ZM321 597Q291 629 250 629Q208 629 178 597Q153 571 145 525T137 333Q137 175 145 125T181 46Q209 16 250 16Q290 16 318 46Q347 76 354 130T362 333Q362 478 354 524T321 597Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-2F\" d=\"M423 750Q432 750 438 744T444 730Q444 725 271 248T92 -240Q85 -250 75 -250Q68 -250 62 -245T56 -231Q56 -221 230 257T407 740Q411 750 423 750Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJSZ3-28\" d=\"M701 -940Q701 -943 695 -949H664Q662 -947 636 -922T591 -879T537 -818T475 -737T412 -636T350 -511T295 -362T250 -186T221 17T209 251Q209 962 573 1361Q596 1386 616 1405T649 1437T664 1450H695Q701 1444 701 1441Q701 1436 681 1415T629 1356T557 1261T476 1118T400 927T340 675T308 359Q306 321 306 250Q306 -139 400 -430T690 -924Q701 -936 701 -940Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJSZ3-29\" d=\"M34 1438Q34 1446 37 1448T50 1450H56H71Q73 1448 99 1423T144 1380T198 1319T260 1238T323 1137T385 1013T440 864T485 688T514 485T526 251Q526 134 519 53Q472 -519 162 -860Q139 -885 119 -904T86 -936T71 -949H56Q43 -949 39 -947T34 -937Q88 -883 140 -813Q428 -430 428 251Q428 453 402 628T338 922T245 1146T145 1309T46 1425Q44 1427 42 1429T39 1433T36 1436L34 1438Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-2B\" d=\"M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-63\" d=\"M370 305T349 305T313 320T297 358Q297 381 312 396Q317 401 317 402T307 404Q281 408 258 408Q209 408 178 376Q131 329 131 219Q131 137 162 90Q203 29 272 29Q313 29 338 55T374 117Q376 125 379 127T395 129H409Q415 123 415 120Q415 116 411 104T395 71T366 33T318 2T249 -11Q163 -11 99 53T34 214Q34 318 99 383T250 448T370 421T404 357Q404 334 387 320Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJMAIN-6F\" d=\"M28 214Q28 309 93 378T250 448Q340 448 405 380T471 215Q471 120 407 55T250 -10Q153 -10 91 57T28 214ZM250 30Q372 30 372 193V225V250Q372 272 371 288T364 326T348 362T317 390T268 410Q263 411 252 411Q222 411 195 399Q152 377 139 338T126 246V226Q126 130 145 91Q177 30 250 30Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJSZ4-23A7\" d=\"M712 899L718 893V876V865Q718 854 704 846Q627 793 577 710T510 525Q510 524 509 521Q505 493 504 349Q504 345 504 334Q504 277 504 240Q504 -2 503 -4Q502 -8 494 -9T444 -10Q392 -10 390 -9Q387 -8 386 -5Q384 5 384 230Q384 262 384 312T383 382Q383 481 392 535T434 656Q510 806 664 892L677 899H712Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJSZ4-23A9\" d=\"M718 -893L712 -899H677L666 -893Q542 -825 468 -714T385 -476Q384 -466 384 -282Q384 3 385 5L389 9Q392 10 444 10Q486 10 494 9T503 4Q504 2 504 -239V-310V-366Q504 -470 508 -513T530 -609Q546 -657 569 -698T617 -767T661 -812T699 -843T717 -856T718 -876V-893Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJSZ4-23A8\" d=\"M389 1159Q391 1160 455 1160Q496 1160 498 1159Q501 1158 502 1155Q504 1145 504 924Q504 691 503 682Q494 549 425 439T243 259L229 250L243 241Q349 175 421 66T503 -182Q504 -191 504 -424Q504 -600 504 -629T499 -659H498Q496 -660 444 -660T390 -659Q387 -658 386 -655Q384 -645 384 -425V-282Q384 -176 377 -116T342 10Q325 54 301 92T255 155T214 196T183 222T171 232Q170 233 170 250T171 268Q171 269 191 284T240 331T300 407T354 524T383 679Q384 691 384 925Q384 1152 385 1155L389 1159Z\"><\/path><path stroke-width=\"0\" id=\"E257-MJSZ4-23AA\" d=\"M384 150V266Q384 304 389 309Q391 310 455 310Q496 310 498 309Q502 308 503 298Q504 283 504 150Q504 32 504 12T499 -9H498Q496 -10 444 -10T390 -9Q386 -8 385 2Q384 17 384 150Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E257-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-29\" x=\"3361\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-3D\" x=\"4027\" y=\"0\"><\/use><g transform=\"translate(5083,0)\"><g transform=\"translate(0,2749)\"><use xlink:href=\"#E257-MJSZ4-23A7\" x=\"0\" y=\"-899\"><\/use><g transform=\"translate(0,-1614.625) scale(1,2.4375)\"><use xlink:href=\"#E257-MJSZ4-23AA\"><\/use><\/g><use xlink:href=\"#E257-MJSZ4-23A8\" x=\"0\" y=\"-2749\"><\/use><g transform=\"translate(0,-4114.625) scale(1,2.4375)\"><use xlink:href=\"#E257-MJSZ4-23AA\"><\/use><\/g><use xlink:href=\"#E257-MJSZ4-23A9\" x=\"0\" y=\"-4099\"><\/use><\/g><g transform=\"translate(1056,0)\"><g transform=\"translate(-15,0)\"><g transform=\"translate(0,1299)\"><use xlink:href=\"#E257-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-2C\" x=\"3361\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-32\" x=\"3805\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-69\" x=\"4305\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-29\" x=\"4650\" y=\"0\"><\/use><\/g><g transform=\"translate(0,-1300)\"><use xlink:href=\"#E257-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-2C\" x=\"3361\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-32\" x=\"3805\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-69\" x=\"4305\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-2B\" x=\"4872\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-31\" x=\"5873\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-29\" x=\"6373\" y=\"0\"><\/use><\/g><\/g><g transform=\"translate(7747,0)\"><use xlink:href=\"#E257-MJMAIN-3D\" x=\"0\" y=\"1299\"><\/use><use xlink:href=\"#E257-MJMAIN-3D\" x=\"0\" y=\"-1300\"><\/use><\/g><g transform=\"translate(9525,0)\"><g transform=\"translate(0,1299)\"><use xlink:href=\"#E257-MJMAIN-73\"><\/use><use xlink:href=\"#E257-MJMAIN-69\" x=\"394\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-6E\" x=\"672\" y=\"0\"><\/use><g transform=\"translate(1228,0)\"><use xlink:href=\"#E257-MJSZ3-28\"><\/use><g transform=\"translate(736,0)\"><g transform=\"translate(120,0)\"><rect stroke=\"none\" width=\"5434\" height=\"60\" x=\"0\" y=\"220\"><\/rect><g transform=\"translate(1988,676)\"><use xlink:href=\"#E257-MJMATHI-70\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-6F\" x=\"503\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-73\" x=\"988\" y=\"0\"><\/use><\/g><g transform=\"translate(60,-889)\"><use xlink:href=\"#E257-MJMAIN-31\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"500\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"1000\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"1500\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"2000\" y=\"0\"><\/use><g transform=\"translate(2500,392)\"><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMAIN-32\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMATHI-69\" x=\"500\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMAIN-2F\" x=\"845\" y=\"0\"><\/use><g transform=\"translate(951,0)\"><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(367,-107)\"><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/g><\/g><\/g><\/g><use xlink:href=\"#E257-MJSZ3-29\" x=\"6410\" y=\"-1\"><\/use><\/g><\/g><g transform=\"translate(0,-1300)\"><use xlink:href=\"#E257-MJMAIN-63\"><\/use><use xlink:href=\"#E257-MJMAIN-6F\" x=\"444\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-73\" x=\"944\" y=\"0\"><\/use><g transform=\"translate(1338,0)\"><use xlink:href=\"#E257-MJSZ3-28\"><\/use><g transform=\"translate(736,0)\"><g transform=\"translate(120,0)\"><rect stroke=\"none\" width=\"5434\" height=\"60\" x=\"0\" y=\"220\"><\/rect><g transform=\"translate(1988,676)\"><use xlink:href=\"#E257-MJMATHI-70\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-6F\" x=\"503\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMATHI-73\" x=\"988\" y=\"0\"><\/use><\/g><g transform=\"translate(60,-889)\"><use xlink:href=\"#E257-MJMAIN-31\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"500\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"1000\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"1500\" y=\"0\"><\/use><use xlink:href=\"#E257-MJMAIN-30\" x=\"2000\" y=\"0\"><\/use><g transform=\"translate(2500,392)\"><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMAIN-32\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMATHI-69\" x=\"500\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMAIN-2F\" x=\"845\" y=\"0\"><\/use><g transform=\"translate(951,0)\"><use transform=\"scale(0.707)\" xlink:href=\"#E257-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(367,-107)\"><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.5)\" xlink:href=\"#E257-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/g><\/g><\/g><\/g><use xlink:href=\"#E257-MJSZ3-29\" x=\"6410\" y=\"-1\"><\/use><\/g><\/g><\/g><\/g><\/g><\/g><\/svg><\/span><\/div><script type=\"math\/tex; mode=display\" id=\"MathJax-Element-218\">PE(pos)=\\begin{cases}\nPE(pos,2i)&=&\\sin\\left(\\dfrac{pos}{10000^{2i\/d_{model}}}\\right)&\\\\\nPE(pos,2i+1)&=&\\cos\\left(\\dfrac{pos}{10000^{2i\/d_{model}}}\\right)&\n\\end{cases}<\/script><\/div><\/div>\n<p>\u5176\u4e2d<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"5.792ex\" height=\"2.344ex\" viewBox=\"0 -755.9 2493.8 1009.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E9-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E9-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">d_{model}<\/script>\u4e3a\u8bcd\u5d4c\u5165\u7684\u7ef4\u6570\uff0c\u53e6\u5916\u6709<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"15.913ex\" height=\"2.344ex\" viewBox=\"0 -755.9 6851.2 1009.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E6-MJMATHI-69\" d=\"M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-3D\" d=\"M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-30\" d=\"M96 585Q152 666 249 666Q297 666 345 640T423 548Q460 465 460 320Q460 165 417 83Q397 41 362 16T301 -15T250 -22Q224 -22 198 -16T137 16T82 83Q39 165 39 320Q39 494 96 585ZM321 597Q291 629 250 629Q208 629 178 597Q153 571 145 525T137 333Q137 175 145 125T181 46Q209 16 250 16Q290 16 318 46Q347 76 354 130T362 333Q362 478 354 524T321 597Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-31\" d=\"M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-22EF\" d=\"M78 250Q78 274 95 292T138 310Q162 310 180 294T199 251Q199 226 182 208T139 190T96 207T78 250ZM525 250Q525 274 542 292T585 310Q609 310 627 294T646 251Q646 226 629 208T586 190T543 207T525 250ZM972 250Q972 274 989 292T1032 310Q1056 310 1074 294T1093 251Q1093 226 1076 208T1033 190T990 207T972 250Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-32\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"><\/path><path stroke-width=\"0\" id=\"E6-MJMAIN-35\" d=\"M164 157Q164 133 148 117T109 101H102Q148 22 224 22Q294 22 326 82Q345 115 345 210Q345 313 318 349Q292 382 260 382H254Q176 382 136 314Q132 307 129 306T114 304Q97 304 95 310Q93 314 93 485V614Q93 664 98 664Q100 666 102 666Q103 666 123 658T178 642T253 634Q324 634 389 662Q397 666 402 666Q410 666 410 648V635Q328 538 205 538Q174 538 149 544L139 546V374Q158 388 169 396T205 412T256 420Q337 420 393 355T449 201Q449 109 385 44T229 -22Q148 -22 99 32T50 154Q50 178 61 192T84 210T107 214Q132 214 148 197T164 157Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E6-MJMATHI-69\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-3D\" x=\"622\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-30\" x=\"1678\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-2C\" x=\"2178\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-31\" x=\"2623\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-2C\" x=\"3123\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-22EF\" x=\"3567\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-2C\" x=\"4906\" y=\"0\"><\/use><g transform=\"translate(5351,0)\"><use xlink:href=\"#E6-MJMAIN-32\"><\/use><use xlink:href=\"#E6-MJMAIN-35\" x=\"500\" y=\"0\"><\/use><use xlink:href=\"#E6-MJMAIN-35\" x=\"1000\" y=\"0\"><\/use><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">i=0,1,\\cdots,255<\/script>\u8f85\u52a9\u8ba1\u7b97\u5404\u7ef4\u5ea6\u5411\u91cf\u503c\uff0c\u6709\u5947\u6570\u7ef4\u7531cos\u7f16\u7801\uff0c\u5076\u6570\u7ef4\u7531sin\u7f16\u7801\uff0c\u53e6\u5916pos\u8868\u793a\u5404\u4e2a\u4f4d\u7f6e\u5411\u91cf\uff0c\u6709<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"8.71ex\" height=\"2.577ex\" viewBox=\"0 -806.1 3750 1109.7\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.705ex;\"><defs><path stroke-width=\"0\" id=\"E7-MJMATHI-50\" d=\"M287 628Q287 635 230 637Q206 637 199 638T192 648Q192 649 194 659Q200 679 203 681T397 683Q587 682 600 680Q664 669 707 631T751 530Q751 453 685 389Q616 321 507 303Q500 302 402 301H307L277 182Q247 66 247 59Q247 55 248 54T255 50T272 48T305 46H336Q342 37 342 35Q342 19 335 5Q330 0 319 0Q316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11Q33 13 36 25Q40 41 44 43T67 46Q94 46 127 49Q141 52 146 61Q149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597T609 619T560 635Q553 636 480 637Q463 637 445 637T416 636T404 636Q391 635 386 627Q384 621 367 550T332 412T314 344Q314 342 395 342H407H430Q542 342 590 392Q617 419 631 471T645 554Z\"><\/path><path stroke-width=\"0\" id=\"E7-MJMATHI-45\" d=\"M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z\"><\/path><path stroke-width=\"0\" id=\"E7-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E7-MJMATHI-70\" d=\"M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z\"><\/path><path stroke-width=\"0\" id=\"E7-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E7-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E7-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E7-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E7-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E7-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E7-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E7-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E7-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E7-MJMAIN-29\" x=\"3361\" y=\"0\"><\/use><\/g><\/svg><\/span><script type=\"math\/tex\">PE(pos)<\/script>\u8868\u793apos\u4f4d\u7f6e\u5411\u91cf\u7684<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"5.792ex\" height=\"2.344ex\" viewBox=\"0 -755.9 2493.8 1009.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E9-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E9-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">d_{model}<\/script>\u7ef4\u4f4d\u7f6e\u7f16\u7801\u5411\u91cf\u3002\u5728\u901a\u7528\u6a21\u578b\u4e2d\uff0c\u4e00\u822c\u628a<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"5.792ex\" height=\"2.344ex\" viewBox=\"0 -755.9 2493.8 1009.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.588ex;\"><defs><path stroke-width=\"0\" id=\"E9-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E9-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E9-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E9-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">d_{model}<\/script>\u8bbe\u7f6e\u4e3a512<\/p>\n<p>\u6709\u610f\u601d\u7684\u662f\uff0c\u5f53\u6c42<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"15.754ex\" height=\"2.577ex\" viewBox=\"0 -806.1 6783.1 1109.7\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.705ex;\"><defs><path stroke-width=\"0\" id=\"E10-MJMATHI-50\" d=\"M287 628Q287 635 230 637Q206 637 199 638T192 648Q192 649 194 659Q200 679 203 681T397 683Q587 682 600 680Q664 669 707 631T751 530Q751 453 685 389Q616 321 507 303Q500 302 402 301H307L277 182Q247 66 247 59Q247 55 248 54T255 50T272 48T305 46H336Q342 37 342 35Q342 19 335 5Q330 0 319 0Q316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11Q33 13 36 25Q40 41 44 43T67 46Q94 46 127 49Q141 52 146 61Q149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597T609 619T560 635Q553 636 480 637Q463 637 445 637T416 636T404 636Q391 635 386 627Q384 621 367 550T332 412T314 344Q314 342 395 342H407H430Q542 342 590 392Q617 419 631 471T645 554Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMATHI-45\" d=\"M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMATHI-70\" d=\"M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMAIN-2B\" d=\"M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMATHI-6B\" d=\"M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686Q294 679 244 477Q194 279 194 272Q213 282 223 291Q247 309 292 354T362 415Q402 442 438 442Q468 442 485 423T503 369Q503 344 496 327T477 302T456 291T438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316T228 255Q230 254 243 252T267 246T293 238T320 224T342 206T359 180T365 147Q365 130 360 106T354 66Q354 26 381 26Q429 26 459 145Q461 153 479 153H483Q499 153 499 144Q499 139 496 130Q455 -11 378 -11Q333 -11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206T200 217T182 220H180Q168 178 159 139T145 81T136 44T129 20T122 7T111 -2Q98 -11 83 -11Q66 -11 57 -1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640T121 647Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMAIN-32\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMATHI-69\" d=\"M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E10-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E10-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMAIN-2B\" x=\"3583\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMATHI-6B\" x=\"4583\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMAIN-2C\" x=\"5104\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMAIN-32\" x=\"5549\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMATHI-69\" x=\"6049\" y=\"0\"><\/use><use xlink:href=\"#E10-MJMAIN-29\" x=\"6394\" y=\"0\"><\/use><\/g><\/svg><\/span><script type=\"math\/tex\">PE(pos+k,2i)<\/script>\u4f4d\u7f6e\u7684\u5411\u91cf\u65f6\uff0c\u53ef\u4ee5\u8f6c\u5316\u4e3a\u5bf9<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"11.705ex\" height=\"2.577ex\" viewBox=\"0 -806.1 5039.7 1109.7\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.705ex;\"><defs><path stroke-width=\"0\" id=\"E11-MJMATHI-50\" d=\"M287 628Q287 635 230 637Q206 637 199 638T192 648Q192 649 194 659Q200 679 203 681T397 683Q587 682 600 680Q664 669 707 631T751 530Q751 453 685 389Q616 321 507 303Q500 302 402 301H307L277 182Q247 66 247 59Q247 55 248 54T255 50T272 48T305 46H336Q342 37 342 35Q342 19 335 5Q330 0 319 0Q316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11Q33 13 36 25Q40 41 44 43T67 46Q94 46 127 49Q141 52 146 61Q149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597T609 619T560 635Q553 636 480 637Q463 637 445 637T416 636T404 636Q391 635 386 627Q384 621 367 550T332 412T314 344Q314 342 395 342H407H430Q542 342 590 392Q617 419 631 471T645 554Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMATHI-45\" d=\"M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMATHI-70\" d=\"M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMAIN-32\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMATHI-69\" d=\"M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E11-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E11-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMAIN-2C\" x=\"3361\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMAIN-32\" x=\"3805\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMATHI-69\" x=\"4305\" y=\"0\"><\/use><use xlink:href=\"#E11-MJMAIN-29\" x=\"4650\" y=\"0\"><\/use><\/g><\/svg><\/span><script type=\"math\/tex\">PE(pos,2i)<\/script>\u7684\u7ebf\u6027\u8868\u793a\uff0c\u7531\u4e09\u89d2\u51fd\u6570\u7ec4\u5408\u516c\u5f0f\u6709<\/p>\n<div contenteditable=\"false\" spellcheck=\"false\" class=\"mathjax-block md-end-block md-math-block md-rawblock\" id=\"mathjax-n781\" cid=\"n781\" mdtype=\"math_block\"><div class=\"md-rawblock-container md-math-container\" tabindex=\"-1\"><div class=\"MathJax_SVG_Display\" style=\"text-align: center;\"><span class=\"MathJax_SVG\" id=\"MathJax-Element-219-Frame\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block; zoom: 0.955638;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"84.326ex\" height=\"5.846ex\" viewBox=\"0 -1509.8 36307.1 2517\" role=\"img\" focusable=\"false\" style=\"vertical-align: -2.339ex; max-width: 100%;\"><defs><path stroke-width=\"0\" id=\"E258-MJMAIN-7B\" d=\"M434 -231Q434 -244 428 -250H410Q281 -250 230 -184Q225 -177 222 -172T217 -161T213 -148T211 -133T210 -111T209 -84T209 -47T209 0Q209 21 209 53Q208 142 204 153Q203 154 203 155Q189 191 153 211T82 231Q71 231 68 234T65 250T68 266T82 269Q116 269 152 289T203 345Q208 356 208 377T209 529V579Q209 634 215 656T244 698Q270 724 324 740Q361 748 377 749Q379 749 390 749T408 750H428Q434 744 434 732Q434 719 431 716Q429 713 415 713Q362 710 332 689T296 647Q291 634 291 499V417Q291 370 288 353T271 314Q240 271 184 255L170 250L184 245Q202 239 220 230T262 196T290 137Q291 131 291 1Q291 -134 296 -147Q306 -174 339 -192T415 -213Q429 -213 431 -216Q434 -219 434 -231Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-50\" d=\"M287 628Q287 635 230 637Q206 637 199 638T192 648Q192 649 194 659Q200 679 203 681T397 683Q587 682 600 680Q664 669 707 631T751 530Q751 453 685 389Q616 321 507 303Q500 302 402 301H307L277 182Q247 66 247 59Q247 55 248 54T255 50T272 48T305 46H336Q342 37 342 35Q342 19 335 5Q330 0 319 0Q316 0 282 1T182 2Q120 2 87 2T51 1Q33 1 33 11Q33 13 36 25Q40 41 44 43T67 46Q94 46 127 49Q141 52 146 61Q149 65 218 339T287 628ZM645 554Q645 567 643 575T634 597T609 619T560 635Q553 636 480 637Q463 637 445 637T416 636T404 636Q391 635 386 627Q384 621 367 550T332 412T314 344Q314 342 395 342H407H430Q542 342 590 392Q617 419 631 471T645 554Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-45\" d=\"M492 213Q472 213 472 226Q472 230 477 250T482 285Q482 316 461 323T364 330H312Q311 328 277 192T243 52Q243 48 254 48T334 46Q428 46 458 48T518 61Q567 77 599 117T670 248Q680 270 683 272Q690 274 698 274Q718 274 718 261Q613 7 608 2Q605 0 322 0H133Q31 0 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H757Q764 676 764 669Q764 664 751 557T737 447Q735 440 717 440H705Q698 445 698 453L701 476Q704 500 704 528Q704 558 697 578T678 609T643 625T596 632T532 634H485Q397 633 392 631Q388 629 386 622Q385 619 355 499T324 377Q347 376 372 376H398Q464 376 489 391T534 472Q538 488 540 490T557 493Q562 493 565 493T570 492T572 491T574 487T577 483L544 351Q511 218 508 216Q505 213 492 213Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-70\" d=\"M23 287Q24 290 25 295T30 317T40 348T55 381T75 411T101 433T134 442Q209 442 230 378L240 387Q302 442 358 442Q423 442 460 395T497 281Q497 173 421 82T249 -10Q227 -10 210 -4Q199 1 187 11T168 28L161 36Q160 35 139 -51T118 -138Q118 -144 126 -145T163 -148H188Q194 -155 194 -157T191 -175Q188 -187 185 -190T172 -194Q170 -194 161 -194T127 -193T65 -192Q-5 -192 -24 -194H-32Q-39 -187 -39 -183Q-37 -156 -26 -148H-6Q28 -147 33 -136Q36 -130 94 103T155 350Q156 355 156 364Q156 405 131 405Q109 405 94 377T71 316T59 280Q57 278 43 278H29Q23 284 23 287ZM178 102Q200 26 252 26Q282 26 310 49T356 107Q374 141 392 215T411 325V331Q411 405 350 405Q339 405 328 402T306 393T286 380T269 365T254 350T243 336T235 326L232 322Q232 321 229 308T218 264T204 212Q178 106 178 102Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-73\" d=\"M131 289Q131 321 147 354T203 415T300 442Q362 442 390 415T419 355Q419 323 402 308T364 292Q351 292 340 300T328 326Q328 342 337 354T354 372T367 378Q368 378 368 379Q368 382 361 388T336 399T297 405Q249 405 227 379T204 326Q204 301 223 291T278 274T330 259Q396 230 396 163Q396 135 385 107T352 51T289 7T195 -10Q118 -10 86 19T53 87Q53 126 74 143T118 160Q133 160 146 151T160 120Q160 94 142 76T111 58Q109 57 108 57T107 55Q108 52 115 47T146 34T201 27Q237 27 263 38T301 66T318 97T323 122Q323 150 302 164T254 181T195 196T148 231Q131 256 131 289Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-2B\" d=\"M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-6B\" d=\"M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686Q294 679 244 477Q194 279 194 272Q213 282 223 291Q247 309 292 354T362 415Q402 442 438 442Q468 442 485 423T503 369Q503 344 496 327T477 302T456 291T438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316T228 255Q230 254 243 252T267 246T293 238T320 224T342 206T359 180T365 147Q365 130 360 106T354 66Q354 26 381 26Q429 26 459 145Q461 153 479 153H483Q499 153 499 144Q499 139 496 130Q455 -11 378 -11Q333 -11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206T200 217T182 220H180Q168 178 159 139T145 81T136 44T129 20T122 7T111 -2Q98 -11 83 -11Q66 -11 57 -1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640T121 647Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-32\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMATHI-69\" d=\"M184 600Q184 624 203 642T247 661Q265 661 277 649T290 619Q290 596 270 577T226 557Q211 557 198 567T184 600ZM21 287Q21 295 30 318T54 369T98 420T158 442Q197 442 223 419T250 357Q250 340 236 301T196 196T154 83Q149 61 149 51Q149 26 166 26Q175 26 185 29T208 43T235 78T260 137Q263 149 265 151T282 153Q302 153 302 143Q302 135 293 112T268 61T223 11T161 -11Q129 -11 102 10T74 74Q74 91 79 106T122 220Q160 321 166 341T173 380Q173 404 156 404H154Q124 404 99 371T61 287Q60 286 59 284T58 281T56 279T53 278T49 278T41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-3D\" d=\"M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-D7\" d=\"M630 29Q630 9 609 9Q604 9 587 25T493 118L389 222L284 117Q178 13 175 11Q171 9 168 9Q160 9 154 15T147 29Q147 36 161 51T255 146L359 250L255 354Q174 435 161 449T147 471Q147 480 153 485T168 490Q173 490 175 489Q178 487 284 383L389 278L493 382Q570 459 587 475T609 491Q630 491 630 471Q630 464 620 453T522 355L418 250L522 145Q606 61 618 48T630 29Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJMAIN-31\" d=\"M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z\"><\/path><path stroke-width=\"0\" id=\"E258-MJSZ3-7B\" d=\"M618 -943L612 -949H582L568 -943Q472 -903 411 -841T332 -703Q327 -682 327 -653T325 -350Q324 -28 323 -18Q317 24 301 61T264 124T221 171T179 205T147 225T132 234Q130 238 130 250Q130 255 130 258T131 264T132 267T134 269T139 272T144 275Q207 308 256 367Q310 436 323 519Q324 529 325 851Q326 1124 326 1154T332 1205Q369 1358 566 1443L582 1450H612L618 1444V1429Q618 1413 616 1411L608 1406Q599 1402 585 1393T552 1372T515 1343T479 1305T449 1257T429 1200Q425 1180 425 1152T423 851Q422 579 422 549T416 498Q407 459 388 424T346 364T297 318T250 284T214 264T197 254L188 251L205 242Q290 200 345 138T416 3Q421 -18 421 -48T423 -349Q423 -397 423 -472Q424 -677 428 -694Q429 -697 429 -699Q434 -722 443 -743T465 -782T491 -816T519 -845T548 -868T574 -886T595 -899T610 -908L616 -910Q618 -912 618 -928V-943Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E258-MJSZ3-7B\"><\/use><g transform=\"translate(917,0)\"><g transform=\"translate(-15,0)\"><g transform=\"translate(0,600)\"><use xlink:href=\"#E258-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"3583\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6B\" x=\"4583\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"5104\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"5549\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"6049\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"6394\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-3D\" x=\"7060\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"8116\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"8867\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"9631\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-70\" x=\"10020\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6F\" x=\"10523\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-73\" x=\"11008\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"11477\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"11922\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"12422\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"12767\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-D7\" x=\"13378\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"14378\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"15129\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"15893\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6B\" x=\"16282\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"16803\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"17248\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"17748\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"18315\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-31\" x=\"19315\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"19815\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"20427\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"21427\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"22178\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"22942\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-70\" x=\"23331\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6F\" x=\"23834\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-73\" x=\"24319\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"24788\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"25233\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"25733\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"26300\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-31\" x=\"27300\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"27800\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-D7\" x=\"28411\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"29411\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"30162\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"30926\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6B\" x=\"31315\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"31836\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"32281\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"32781\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"33126\" y=\"0\"><\/use><\/g><g transform=\"translate(0,-650)\"><use xlink:href=\"#E258-MJMATHI-50\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"751\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"1515\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-70\" x=\"1904\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6F\" x=\"2407\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-73\" x=\"2892\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"3583\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6B\" x=\"4583\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"5104\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"5549\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"6049\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"6616\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-31\" x=\"7616\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"8116\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-3D\" x=\"8783\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"9839\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"10590\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"11354\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-70\" x=\"11743\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6F\" x=\"12246\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-73\" x=\"12731\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"13200\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"13644\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"14144\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"14712\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-31\" x=\"15712\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"16212\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-D7\" x=\"16823\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"17823\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"18574\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"19338\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6B\" x=\"19727\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"20248\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"20693\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"21193\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"21760\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-31\" x=\"22760\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"23260\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2B\" x=\"23872\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"24872\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"25623\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"26387\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-70\" x=\"26776\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6F\" x=\"27279\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-73\" x=\"27764\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"28233\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"28677\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"29177\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"29522\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-D7\" x=\"30134\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-50\" x=\"31134\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-45\" x=\"31885\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-28\" x=\"32649\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-6B\" x=\"33038\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-2C\" x=\"33559\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-32\" x=\"34004\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMATHI-69\" x=\"34504\" y=\"0\"><\/use><use xlink:href=\"#E258-MJMAIN-29\" x=\"34849\" y=\"0\"><\/use><\/g><\/g><\/g><\/g><\/svg><\/span><\/div><script type=\"math\/tex; mode=display\" id=\"MathJax-Element-219\">\\begin{cases}\nPE(pos+k,2i)=PE(pos,2i)\\times PE(k,2i+1)+PE(pos,2i+1)\\times PE(k,2i)\\\\\nPE(pos+k,2i+1)=PE(pos,2i+1)\\times PE(k,2i+1)+PE(pos,2i)\\times PE(k,2i)\\\\\n\\end{cases}<\/script><\/div><\/div>\n<p><em>\u597d\u5427\u4f46\u662f\u6211\u8fd8\u662f\u4e0d\u6e05\u695a\u4e3a\u5565\u8981\u8fd9\u6837\u8868\u793a<\/em><\/p>\n<pre><code class='language-python' lang='python'>def get_angles(pos, i, d_model):\n    return pos \/ np.power(10000, (2 * (i\/\/2)) \/ np.float64(d_model))\ndef positional_encoding(position, d_model):\n    # np.newaxis\u7684\u4f5c\u7528\u4e3a\u5c06\u6570\u7ec4\u5316\u6210\u77e9\u9635\u5f62\u5f0f\n    angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n                            np.arange(d_model)[np.newaxis, :],\n                            d_model)\n    # \u4fee\u6539\u5076\u6570\u7d22\u5f15\n    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n\n    # \u518d\u6b21\u589e\u52a0\u7ef4\u5ea6\n    pos_encoding = angle_rads[np.newaxis, ...]\n\n    return torch.from_numpy(pos_encoding)\n\n# print(positional_encoding(10,12))\n<\/code><\/pre>\n<p><strong>\u53e6\u5916\uff0c\u6b64\u5904\u7684\u8ddd\u79bb\u663e\u7136\u662f\u53ef\u4ee5\u66ff\u6362\u7684\uff0c\u672a\u6765\u6211\u4f1a\u5c06\u5176\u4f7f\u7528\u4e8e\u56fe\u7ed3\u6784\u4fe1\u606f\u7684\u7f16\u7801\u4e2d\uff0c\u91cd\u6784\u8ddd\u79bb\u7f16\u7801\uff0c\u6216\u8bb8\u80fd\u53c2\u8003<a href='https:\/\/arxiv.org\/pdf\/2009.00142.pdf' target=\"_blank\"  rel=\"nofollow\" >Distance Encoding \u2013 Design Provably More Powerful GNNs for Structural Representation Learning<\/a>\u4e00\u6587\u6240\u63d0\u51fa\u7684\u8ddd\u79bb\u7f16\u7801<\/strong><\/p>\n<p>\u4e3a\u4e86\u65b9\u4fbfpytorch\u7f51\u7edc\u6a21\u578b\u7684\u6784\u9020\uff0c\u53c2\u8003\u4e86\u7f57\u5468\u6768\u4e00\u6587\u4e2d\u7684\u5c42\u5f0f\u5b9e\u73b0<\/p>\n<pre><code class='language-python' lang='python'># \u4f4d\u7f6e\u7f16\u7801\u6a21\u5757\nclass PositionalEncoding(nn.Module):\n    def __init__(self, d_model, max_l):\n        super(PositionalEncoding, self).__init__()\n        &quot;&quot;&quot;\n        d_model:\u6a21\u578b\u7ef4\u5ea6\uff0c\u5373\u7f16\u7801\u603b\u957f\u5ea6\n        max_l:\u6700\u5927\u6587\u672c\u5e8f\u5217\u957f\u5ea6\uff0c\u7528\u4e8e\u63a7\u5236\u7f13\u5b58\n        &quot;&quot;&quot;\n\n        # \u6784\u9020Pe\u77e9\u9635\n        position_encoding = np.array([[pos \/ np.power(10000, 2.0 * (j \/\/ 2) \/ d_model) for j in range(d_model)] for pos in range(max_l)]) \n\n        # \u5076\u6570\u5217\u4f7f\u7528sin\uff0c\u5947\u6570\u5217\u4f7f\u7528cos\n        position_encoding[:, 0::2] = np.sin(position_encoding[:, 0::2])\n        position_encoding[:, 1::2] = np.cos(position_encoding[:, 1::2])\n\n        # \u8fdb\u884cPAD\u64cd\u4f5c\uff0c\u96be\u4ee5\u7406\u89e3\u7684\u90e8\u5206\uff0c\u53ef\u80fd\u662f\u7531\u4e8e\u53e5\u5b50\u521d\u59cb\u5316\u5e76\u975e\u4ece0\u5f00\u59cb\n        pad_row = torch.zeros([1, d_model])\n        position_encoding = torch.cat((pad_row, torch.from_numpy(position_encoding)))\n\n        # \u4f7f\u7528\u8bcd\u5d4c\u5165\u65b9\u6cd5\u8fdb\u884c\u5c01\u88c5\uff0c\u8bbe\u5b9a\n        self.Pe = nn.Embedding(max_l + 1, d_model)\n        self.Pe.weight = nn.Parameter(position_encoding, requires_grad=False)\n\n    def forward(self, input_len):\n        # \u901a\u5e38input_len\u7684\u6700\u5927\u503c\u88ab\u56fa\u5b9a\u4e3a\u6700\u5927\u53ef\u611f\u957f\u5ea6\uff0c128\u3001256\u6216512\uff1f\n        max_l = torch.max(input_len)\n        tensor = torch.cuda.LongTensor if input_len.is_cuda else torch.LongTensor\n\n        # \u5bf9\u6bcf\u4e00\u4e2a\u5e8f\u5217\u7684\u4f4d\u7f6e\u8fdb\u884c\u5bf9\u9f50\uff0c\u5728\u539f\u5e8f\u5217\u4f4d\u7f6e\u7684\u540e\u9762\u8865\u4e0a0\n        # \u8fd9\u91ccrange\u4ece1\u5f00\u59cb\u4e5f\u662f\u56e0\u4e3a\u8981\u907f\u5f00PAD(0)\u7684\u4f4d\u7f6e\n\n        input_pos = tensor(\n          [list(range(1, l + 1)) + [0] * (max_l - l) for l in input_len])\n\n        # \u8fd4\u56de\u5355\u53e5\u4f4d\u7f6e\u7f16\u7801\u5411\u91cf\uff0c\u540e\u4e0e\u8bcd\u5d4c\u5165\u5411\u91cf\u62fc\u8d34\n        return self.Pe(input_pos)\n<\/code><\/pre>\n<p>\u5728\u8be5\u5b9e\u73b0\u4e2d\uff0c\u6a21\u5757\u7531\u5c06\u901a\u8fc7\u8f93\u5165\u6a21\u578b\u6700\u5927\u6587\u672c\u5e8f\u5217\u957f\u5ea6\u53ca\u6a21\u578b\u7f16\u7801\u7ef4\u5ea6\u751f\u6210\u7279\u5b9a\u7684\u8bcd\u5d4c\u5165\u504f\u79fb\u5411\u91cf\uff0c\u4ee5\u5e94\u7528\u4e8e\u6bcf\u4e2a\u53e5\u5b50\uff08\u5728Encoder\u90e8\u5206\u7ec4\u5408\u4e2d\u53ef\u4ee5\u4fdd\u5b58\u5355\u6b21\u524d\u5411\u4f20\u64ad\u7ed3\u679c\uff0c\u51cf\u5c11\u91cd\u590d\u8fd0\u7b97\uff09<\/p>\n<h3>\u8bcd\u6c47\u5d4c\u5165<\/h3>\n<p>\u8bcd\u5d4c\u5165\u90e8\u5206\u76f8\u5bf9\u6bd4\u8f83\u7b80\u5355\uff0c\u7531\u5355\u884c\u4ee3\u7801\u5b8c\u6210\u3002<\/p>\n<pre><code class='language-python' lang='python'>self.Em = nn.Embedding(vocab_size + 1, d_model, padding_idx=0)\nout = self.Em(x) * (self.d_model ** -0.5)\n<\/code><\/pre>\n<p>\u5176\u4e2d\u7531\u8bba\u6587\u8981\u6c42\u5f15\u5165\u56e0\u5b50<span class=\"MathJax_SVG\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"7.727ex\" height=\"2.811ex\" viewBox=\"0 -906.7 3326.8 1210.2\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.705ex;\"><defs><path stroke-width=\"0\" id=\"E152-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E152-MJMATHI-6D\" d=\"M21 287Q22 293 24 303T36 341T56 388T88 425T132 442T175 435T205 417T221 395T229 376L231 369Q231 367 232 367L243 378Q303 442 384 442Q401 442 415 440T441 433T460 423T475 411T485 398T493 385T497 373T500 364T502 357L510 367Q573 442 659 442Q713 442 746 415T780 336Q780 285 742 178T704 50Q705 36 709 31T724 26Q752 26 776 56T815 138Q818 149 821 151T837 153Q857 153 857 145Q857 144 853 130Q845 101 831 73T785 17T716 -10Q669 -10 648 17T627 73Q627 92 663 193T700 345Q700 404 656 404H651Q565 404 506 303L499 291L466 157Q433 26 428 16Q415 -11 385 -11Q372 -11 364 -4T353 8T350 18Q350 29 384 161L420 307Q423 322 423 345Q423 404 379 404H374Q288 404 229 303L222 291L189 157Q156 26 151 16Q138 -11 108 -11Q95 -11 87 -5T76 7T74 17Q74 30 112 181Q151 335 151 342Q154 357 154 369Q154 405 129 405Q107 405 92 377T69 316T57 280Q55 278 41 278H27Q21 284 21 287Z\"><\/path><path stroke-width=\"0\" id=\"E152-MJMATHI-6F\" d=\"M201 -11Q126 -11 80 38T34 156Q34 221 64 279T146 380Q222 441 301 441Q333 441 341 440Q354 437 367 433T402 417T438 387T464 338T476 268Q476 161 390 75T201 -11ZM121 120Q121 70 147 48T206 26Q250 26 289 58T351 142Q360 163 374 216T388 308Q388 352 370 375Q346 405 306 405Q243 405 195 347Q158 303 140 230T121 120Z\"><\/path><path stroke-width=\"0\" id=\"E152-MJMATHI-65\" d=\"M39 168Q39 225 58 272T107 350T174 402T244 433T307 442H310Q355 442 388 420T421 355Q421 265 310 237Q261 224 176 223Q139 223 138 221Q138 219 132 186T125 128Q125 81 146 54T209 26T302 45T394 111Q403 121 406 121Q410 121 419 112T429 98T420 82T390 55T344 24T281 -1T205 -11Q126 -11 83 42T39 168ZM373 353Q367 405 305 405Q272 405 244 391T199 357T170 316T154 280T149 261Q149 260 169 260Q282 260 327 284T373 353Z\"><\/path><path stroke-width=\"0\" id=\"E152-MJMATHI-6C\" d=\"M117 59Q117 26 142 26Q179 26 205 131Q211 151 215 152Q217 153 225 153H229Q238 153 241 153T246 151T248 144Q247 138 245 128T234 90T214 43T183 6T137 -11Q101 -11 70 11T38 85Q38 97 39 102L104 360Q167 615 167 623Q167 626 166 628T162 632T157 634T149 635T141 636T132 637T122 637Q112 637 109 637T101 638T95 641T94 647Q94 649 96 661Q101 680 107 682T179 688Q194 689 213 690T243 693T254 694Q266 694 266 686Q266 675 193 386T118 83Q118 81 118 75T117 65V59Z\"><\/path><path stroke-width=\"0\" id=\"E152-MJMAIN-221A\" d=\"M95 178Q89 178 81 186T72 200T103 230T169 280T207 309Q209 311 212 311H213Q219 311 227 294T281 177Q300 134 312 108L397 -77Q398 -77 501 136T707 565T814 786Q820 800 834 800Q841 800 846 794T853 782V776L620 293L385 -193Q381 -200 366 -200Q357 -200 354 -197Q352 -195 256 15L160 225L144 214Q129 202 113 190T95 178Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E152-MJMAIN-221A\" x=\"0\" y=\"-10\"><\/use><rect stroke=\"none\" width=\"2493\" height=\"60\" x=\"833\" y=\"730\"><\/rect><g transform=\"translate(833,0)\"><use xlink:href=\"#E152-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(520,-150)\"><use transform=\"scale(0.707)\" xlink:href=\"#E152-MJMATHI-6D\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E152-MJMATHI-6F\" x=\"878\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E152-MJMATHI-64\" x=\"1363\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E152-MJMATHI-65\" x=\"1886\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E152-MJMATHI-6C\" x=\"2352\" y=\"0\"><\/use><\/g><\/g><\/g><\/svg><\/span><script type=\"math\/tex\">\\sqrt{d_{model}}<\/script>\u8fdb\u884c\u5f52\u7ea6\u5316\uff0c\u7136\u800c\u83ab\u5f97\u7406\u7531\u3002<\/p>\n<h2>*Mask<\/h2>\n<p>\u53e6\u5916\uff0c\u4f5c\u4e3a\u8865\u5145\u6211\u4eec\u53ef\u4ee5\u8c08\u8c08\u906e\u7f69\uff0c\u5728transformer\u6a21\u578b\u4e2d\u6211\u4eec\u4f1a\u4f7f\u7528\u5230\u4e24\u79cd\u906e\u7f69\uff0c\u5206\u522b\u4e3apadding_mask\u4ee5\u53casequence_mask\uff0c\u5176\u4e2dpadding mask\u591a\u7528\u4e8e\u70b9\u79ef\u6ce8\u610f\u529b\u7684\u83b7\u53d6\u4e2d\uff0c\u800csequence mask\u7528\u4e8edecoder\u4e2d\u7684\u524d\u81ea\u6ce8\u610f\u90e8\u5206<\/p>\n<h3>padding mask<\/h3>\n<p>\u7528\u4e8e\u906e\u6321\u9762\u5bf9\u7a7a\u4f59\u4f4d\u7f6e\u6ce8\u610f\u529b\u7684<\/p>\n<pre><code class='language-python' lang='python'>def padding_mask(seq_k, seq_q):\n\t# seq_k\u548cseq_q\u7684\u5f62\u72b6\u90fd\u662f[B,L]\n    len_q = seq_q.size(1)\n    # `PAD` is 0\n    pad_mask = seq_k.eq(0)\n    # shape [batch_size, L_q, L_k]\n    pad_mask = pad_mask.unsqueeze(1).expand(-1, len_q, -1)  \n    return pad_mask\n<\/code><\/pre>\n<h3>sequence mask<\/h3>\n<p>\u8be5\u5e8f\u5217\u906e\u7f69\u7528\u4e8e\u4e3adecoder\u906e\u6321\u672a\u6765\u4fe1\u606f\uff0c\u56e0\u6b64\u6784\u9020\u4e09\u89d2\u77e9\u9635\u81f3\u4e8e\u65f6\u95f4\u6b65\u906e\u6321\u5e8f\u5217<\/p>\n<pre><code class='language-python' lang='python'>def sequence_mask(seq):\n    batch_size, seq_len = seq.size()\n    mask = torch.triu(torch.ones((seq_len, seq_len), dtype=torch.uint8),\n                    diagonal=1)\n    mask = mask.unsqueeze(0).expand(batch_size, -1, -1)  # [B, L, L]\n    return mask\n<\/code><\/pre>\n<h2>\u70b9\u79ef\u81ea\u6ce8\u610f\u673a\u5236<\/h2>\n<p>\u81ea\u6ce8\u610f\u529b\u673a\u5236\u662ftransformer\u7f51\u7edc\u7684\u6838\u5fc3\u673a\u5236\uff0c\u901a\u8fc7\u6587\u732e\u7684\u9605\u8bfb\u6211\u4eec\u53ef\u4ee5\u4e86\u89e3\u5230\uff0ctransformer\u6a21\u578b\u4e2d\u5143\u7d20\u4e4b\u95f4\u7684\u6ce8\u610f\u529b\u7531\u4ee5\u4e0b\u8981\u7d20\u6784\u6210\uff1a\u8bcd\u6c47\u8868\u793a\u7684\u76f8\u8fd1\u7a0b\u5ea6\u53ca\u8bcd\u8bed\u4f4d\u7f6e\u7684\u76f8\u90bb\u7a0b\u5ea6\u3002\u5e76\u901a\u8fc7\u6ce8\u610f\u529b\u7684\u53e0\u52a0\u83b7\u53d6\u8bcd\u6c47\u7684\u6df1\u5c42\u8bed\u4e49<\/p>\n<p>\u6ce8\u610f\u529b\u7684\u8ba1\u7b97\u5728transformer\u4e2d\u88ab\u8868\u73b0\u4e3a\u4ee5\u4e0b\u5f62\u5f0f<\/p>\n<div contenteditable=\"false\" spellcheck=\"false\" class=\"mathjax-block md-end-block md-math-block md-rawblock\" id=\"mathjax-n803\" cid=\"n803\" mdtype=\"math_block\"><div class=\"md-rawblock-container md-math-container\" tabindex=\"-1\"><div class=\"MathJax_SVG_Display\" style=\"text-align: center;\"><span class=\"MathJax_SVG\" id=\"MathJax-Element-220-Frame\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"42.678ex\" height=\"6.313ex\" viewBox=\"0 -1610.3 18375.1 2718\" role=\"img\" focusable=\"false\" style=\"vertical-align: -2.573ex; max-width: 100%;\"><defs><path stroke-width=\"0\" id=\"E259-MJMAIN-41\" d=\"M255 0Q240 3 140 3Q48 3 39 0H32V46H47Q119 49 139 88Q140 91 192 245T295 553T348 708Q351 716 366 716H376Q396 715 400 709Q402 707 508 390L617 67Q624 54 636 51T687 46H717V0H708Q699 3 581 3Q458 3 437 0H427V46H440Q510 46 510 64Q510 66 486 138L462 209H229L209 150Q189 91 189 85Q189 72 209 59T259 46H264V0H255ZM447 255L345 557L244 256Q244 255 345 255H447Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-74\" d=\"M27 422Q80 426 109 478T141 600V615H181V431H316V385H181V241Q182 116 182 100T189 68Q203 29 238 29Q282 29 292 100Q293 108 293 146V181H333V146V134Q333 57 291 17Q264 -10 221 -10Q187 -10 162 2T124 33T105 68T98 100Q97 107 97 248V385H18V422H27Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-65\" d=\"M28 218Q28 273 48 318T98 391T163 433T229 448Q282 448 320 430T378 380T406 316T415 245Q415 238 408 231H126V216Q126 68 226 36Q246 30 270 30Q312 30 342 62Q359 79 369 104L379 128Q382 131 395 131H398Q415 131 415 121Q415 117 412 108Q393 53 349 21T250 -11Q155 -11 92 58T28 218ZM333 275Q322 403 238 411H236Q228 411 220 410T195 402T166 381T143 340T127 274V267H333V275Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-6E\" d=\"M41 46H55Q94 46 102 60V68Q102 77 102 91T102 122T103 161T103 203Q103 234 103 269T102 328V351Q99 370 88 376T43 385H25V408Q25 431 27 431L37 432Q47 433 65 434T102 436Q119 437 138 438T167 441T178 442H181V402Q181 364 182 364T187 369T199 384T218 402T247 421T285 437Q305 442 336 442Q450 438 463 329Q464 322 464 190V104Q464 66 466 59T477 49Q498 46 526 46H542V0H534L510 1Q487 2 460 2T422 3Q319 3 310 0H302V46H318Q379 46 379 62Q380 64 380 200Q379 335 378 343Q372 371 358 385T334 402T308 404Q263 404 229 370Q202 343 195 315T187 232V168V108Q187 78 188 68T191 55T200 49Q221 46 249 46H265V0H257L234 1Q210 2 183 2T145 3Q42 3 33 0H25V46H41Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-69\" d=\"M69 609Q69 637 87 653T131 669Q154 667 171 652T188 609Q188 579 171 564T129 549Q104 549 87 564T69 609ZM247 0Q232 3 143 3Q132 3 106 3T56 1L34 0H26V46H42Q70 46 91 49Q100 53 102 60T104 102V205V293Q104 345 102 359T88 378Q74 385 41 385H30V408Q30 431 32 431L42 432Q52 433 70 434T106 436Q123 437 142 438T171 441T182 442H185V62Q190 52 197 50T232 46H255V0H247Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-6F\" d=\"M28 214Q28 309 93 378T250 448Q340 448 405 380T471 215Q471 120 407 55T250 -10Q153 -10 91 57T28 214ZM250 30Q372 30 372 193V225V250Q372 272 371 288T364 326T348 362T317 390T268 410Q263 411 252 411Q222 411 195 399Q152 377 139 338T126 246V226Q126 130 145 91Q177 30 250 30Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMATHI-51\" d=\"M399 -80Q399 -47 400 -30T402 -11V-7L387 -11Q341 -22 303 -22Q208 -22 138 35T51 201Q50 209 50 244Q50 346 98 438T227 601Q351 704 476 704Q514 704 524 703Q621 689 680 617T740 435Q740 255 592 107Q529 47 461 16L444 8V3Q444 2 449 -24T470 -66T516 -82Q551 -82 583 -60T625 -3Q631 11 638 11Q647 11 649 2Q649 -6 639 -34T611 -100T557 -165T481 -194Q399 -194 399 -87V-80ZM636 468Q636 523 621 564T580 625T530 655T477 665Q429 665 379 640Q277 591 215 464T153 216Q153 110 207 59Q231 38 236 38V46Q236 86 269 120T347 155Q372 155 390 144T417 114T429 82T435 55L448 64Q512 108 557 185T619 334T636 468ZM314 18Q362 18 404 39L403 49Q399 104 366 115Q354 117 347 117Q344 117 341 117T337 118Q317 118 296 98T274 52Q274 18 314 18Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMATHI-4B\" d=\"M285 628Q285 635 228 637Q205 637 198 638T191 647Q191 649 193 661Q199 681 203 682Q205 683 214 683H219Q260 681 355 681Q389 681 418 681T463 682T483 682Q500 682 500 674Q500 669 497 660Q496 658 496 654T495 648T493 644T490 641T486 639T479 638T470 637T456 637Q416 636 405 634T387 623L306 305Q307 305 490 449T678 597Q692 611 692 620Q692 635 667 637Q651 637 651 648Q651 650 654 662T659 677Q662 682 676 682Q680 682 711 681T791 680Q814 680 839 681T869 682Q889 682 889 672Q889 650 881 642Q878 637 862 637Q787 632 726 586Q710 576 656 534T556 455L509 418L518 396Q527 374 546 329T581 244Q656 67 661 61Q663 59 666 57Q680 47 717 46H738Q744 38 744 37T741 19Q737 6 731 0H720Q680 3 625 3Q503 3 488 0H478Q472 6 472 9T474 27Q478 40 480 43T491 46H494Q544 46 544 71Q544 75 517 141T485 216L427 354L359 301L291 248L268 155Q245 63 245 58Q245 51 253 49T303 46H334Q340 37 340 35Q340 19 333 5Q328 0 317 0Q314 0 280 1T180 2Q118 2 85 2T49 1Q31 1 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q147 65 216 339T285 628Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMATHI-56\" d=\"M52 648Q52 670 65 683H76Q118 680 181 680Q299 680 320 683H330Q336 677 336 674T334 656Q329 641 325 637H304Q282 635 274 635Q245 630 242 620Q242 618 271 369T301 118L374 235Q447 352 520 471T595 594Q599 601 599 609Q599 633 555 637Q537 637 537 648Q537 649 539 661Q542 675 545 679T558 683Q560 683 570 683T604 682T668 681Q737 681 755 683H762Q769 676 769 672Q769 655 760 640Q757 637 743 637Q730 636 719 635T698 630T682 623T670 615T660 608T652 599T645 592L452 282Q272 -9 266 -16Q263 -18 259 -21L241 -22H234Q216 -22 216 -15Q213 -9 177 305Q139 623 138 626Q133 637 76 637H59Q52 642 52 648Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-3D\" d=\"M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-73\" d=\"M295 316Q295 356 268 385T190 414Q154 414 128 401Q98 382 98 349Q97 344 98 336T114 312T157 287Q175 282 201 278T245 269T277 256Q294 248 310 236T342 195T359 133Q359 71 321 31T198 -10H190Q138 -10 94 26L86 19L77 10Q71 4 65 -1L54 -11H46H42Q39 -11 33 -5V74V132Q33 153 35 157T45 162H54Q66 162 70 158T75 146T82 119T101 77Q136 26 198 26Q295 26 295 104Q295 133 277 151Q257 175 194 187T111 210Q75 227 54 256T33 318Q33 357 50 384T93 424T143 442T187 447H198Q238 447 268 432L283 424L292 431Q302 440 314 448H322H326Q329 448 335 442V310L329 304H301Q295 310 295 316Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-66\" d=\"M273 0Q255 3 146 3Q43 3 34 0H26V46H42Q70 46 91 49Q99 52 103 60Q104 62 104 224V385H33V431H104V497L105 564L107 574Q126 639 171 668T266 704Q267 704 275 704T289 705Q330 702 351 679T372 627Q372 604 358 590T321 576T284 590T270 627Q270 647 288 667H284Q280 668 273 668Q245 668 223 647T189 592Q183 572 182 497V431H293V385H185V225Q185 63 186 61T189 57T194 54T199 51T206 49T213 48T222 47T231 47T241 46T251 46H282V0H273Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-6D\" d=\"M41 46H55Q94 46 102 60V68Q102 77 102 91T102 122T103 161T103 203Q103 234 103 269T102 328V351Q99 370 88 376T43 385H25V408Q25 431 27 431L37 432Q47 433 65 434T102 436Q119 437 138 438T167 441T178 442H181V402Q181 364 182 364T187 369T199 384T218 402T247 421T285 437Q305 442 336 442Q351 442 364 440T387 434T406 426T421 417T432 406T441 395T448 384T452 374T455 366L457 361L460 365Q463 369 466 373T475 384T488 397T503 410T523 422T546 432T572 439T603 442Q729 442 740 329Q741 322 741 190V104Q741 66 743 59T754 49Q775 46 803 46H819V0H811L788 1Q764 2 737 2T699 3Q596 3 587 0H579V46H595Q656 46 656 62Q657 64 657 200Q656 335 655 343Q649 371 635 385T611 402T585 404Q540 404 506 370Q479 343 472 315T464 232V168V108Q464 78 465 68T468 55T477 49Q498 46 526 46H542V0H534L510 1Q487 2 460 2T422 3Q319 3 310 0H302V46H318Q379 46 379 62Q380 64 380 200Q379 335 378 343Q372 371 358 385T334 402T308 404Q263 404 229 370Q202 343 195 315T187 232V168V108Q187 78 188 68T191 55T200 49Q221 46 249 46H265V0H257L234 1Q210 2 183 2T145 3Q42 3 33 0H25V46H41Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-61\" d=\"M137 305T115 305T78 320T63 359Q63 394 97 421T218 448Q291 448 336 416T396 340Q401 326 401 309T402 194V124Q402 76 407 58T428 40Q443 40 448 56T453 109V145H493V106Q492 66 490 59Q481 29 455 12T400 -6T353 12T329 54V58L327 55Q325 52 322 49T314 40T302 29T287 17T269 6T247 -2T221 -8T190 -11Q130 -11 82 20T34 107Q34 128 41 147T68 188T116 225T194 253T304 268H318V290Q318 324 312 340Q290 411 215 411Q197 411 181 410T156 406T148 403Q170 388 170 359Q170 334 154 320ZM126 106Q126 75 150 51T209 26Q247 26 276 49T315 109Q317 116 318 175Q318 233 317 233Q309 233 296 232T251 223T193 203T147 166T126 106Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-78\" d=\"M201 0Q189 3 102 3Q26 3 17 0H11V46H25Q48 47 67 52T96 61T121 78T139 96T160 122T180 150L226 210L168 288Q159 301 149 315T133 336T122 351T113 363T107 370T100 376T94 379T88 381T80 383Q74 383 44 385H16V431H23Q59 429 126 429Q219 429 229 431H237V385Q201 381 201 369Q201 367 211 353T239 315T268 274L272 270L297 304Q329 345 329 358Q329 364 327 369T322 376T317 380T310 384L307 385H302V431H309Q324 428 408 428Q487 428 493 431H499V385H492Q443 385 411 368Q394 360 377 341T312 257L296 236L358 151Q424 61 429 57T446 50Q464 46 499 46H516V0H510H502Q494 1 482 1T457 2T432 2T414 3Q403 3 377 3T327 1L304 0H295V46H298Q309 46 320 51T331 63Q331 65 291 120L250 175Q249 174 219 133T185 88Q181 83 181 74Q181 63 188 55T206 46Q208 46 208 23V0H201Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMATHI-54\" d=\"M40 437Q21 437 21 445Q21 450 37 501T71 602L88 651Q93 669 101 677H569H659Q691 677 697 676T704 667Q704 661 687 553T668 444Q668 437 649 437Q640 437 637 437T631 442L629 445Q629 451 635 490T641 551Q641 586 628 604T573 629Q568 630 515 631Q469 631 457 630T439 622Q438 621 368 343T298 60Q298 48 386 46Q418 46 427 45T436 36Q436 31 433 22Q429 4 424 1L422 0Q419 0 415 0Q410 0 363 1T228 2Q99 2 64 0H49Q43 6 43 9T45 27Q49 40 55 46H83H94Q174 46 189 55Q190 56 191 56Q196 59 201 76T241 233Q258 301 269 344Q339 619 339 625Q339 630 310 630H279Q212 630 191 624Q146 614 121 583T67 467Q60 445 57 441T43 437H40Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMATHI-64\" d=\"M366 683Q367 683 438 688T511 694Q523 694 523 686Q523 679 450 384T375 83T374 68Q374 26 402 26Q411 27 422 35Q443 55 463 131Q469 151 473 152Q475 153 483 153H487H491Q506 153 506 145Q506 140 503 129Q490 79 473 48T445 8T417 -8Q409 -10 393 -10Q359 -10 336 5T306 36L300 51Q299 52 296 50Q294 48 292 46Q233 -10 172 -10Q117 -10 75 30T33 157Q33 205 53 255T101 341Q148 398 195 420T280 442Q336 442 364 400Q369 394 369 396Q370 400 396 505T424 616Q424 629 417 632T378 637H357Q351 643 351 645T353 664Q358 683 366 683ZM352 326Q329 405 277 405Q242 405 210 374T160 293Q131 214 119 129Q119 126 119 118T118 106Q118 61 136 44T179 26Q233 26 290 98L298 109L352 326Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMATHI-6B\" d=\"M121 647Q121 657 125 670T137 683Q138 683 209 688T282 694Q294 694 294 686Q294 679 244 477Q194 279 194 272Q213 282 223 291Q247 309 292 354T362 415Q402 442 438 442Q468 442 485 423T503 369Q503 344 496 327T477 302T456 291T438 288Q418 288 406 299T394 328Q394 353 410 369T442 390L458 393Q446 405 434 405H430Q398 402 367 380T294 316T228 255Q230 254 243 252T267 246T293 238T320 224T342 206T359 180T365 147Q365 130 360 106T354 66Q354 26 381 26Q429 26 459 145Q461 153 479 153H483Q499 153 499 144Q499 139 496 130Q455 -11 378 -11Q333 -11 305 15T277 90Q277 108 280 121T283 145Q283 167 269 183T234 206T200 217T182 220H180Q168 178 159 139T145 81T136 44T129 20T122 7T111 -2Q98 -11 83 -11Q66 -11 57 -1T48 16Q48 26 85 176T158 471L195 616Q196 629 188 632T149 637H144Q134 637 131 637T124 640T121 647Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJMAIN-221A\" d=\"M95 178Q89 178 81 186T72 200T103 230T169 280T207 309Q209 311 212 311H213Q219 311 227 294T281 177Q300 134 312 108L397 -77Q398 -77 501 136T707 565T814 786Q820 800 834 800Q841 800 846 794T853 782V776L620 293L385 -193Q381 -200 366 -200Q357 -200 354 -197Q352 -195 256 15L160 225L144 214Q129 202 113 190T95 178Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJSZ3-28\" d=\"M701 -940Q701 -943 695 -949H664Q662 -947 636 -922T591 -879T537 -818T475 -737T412 -636T350 -511T295 -362T250 -186T221 17T209 251Q209 962 573 1361Q596 1386 616 1405T649 1437T664 1450H695Q701 1444 701 1441Q701 1436 681 1415T629 1356T557 1261T476 1118T400 927T340 675T308 359Q306 321 306 250Q306 -139 400 -430T690 -924Q701 -936 701 -940Z\"><\/path><path stroke-width=\"0\" id=\"E259-MJSZ3-29\" d=\"M34 1438Q34 1446 37 1448T50 1450H56H71Q73 1448 99 1423T144 1380T198 1319T260 1238T323 1137T385 1013T440 864T485 688T514 485T526 251Q526 134 519 53Q472 -519 162 -860Q139 -885 119 -904T86 -936T71 -949H56Q43 -949 39 -947T34 -937Q88 -883 140 -813Q428 -430 428 251Q428 453 402 628T338 922T245 1146T145 1309T46 1425Q44 1427 42 1429T39 1433T36 1436L34 1438Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E259-MJMAIN-41\"><\/use><use xlink:href=\"#E259-MJMAIN-74\" x=\"750\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-74\" x=\"1139\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-65\" x=\"1528\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-6E\" x=\"1972\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-74\" x=\"2528\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-69\" x=\"2917\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-6F\" x=\"3195\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-6E\" x=\"3695\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-28\" x=\"4251\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMATHI-51\" x=\"4640\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-2C\" x=\"5431\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMATHI-4B\" x=\"5875\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-2C\" x=\"6764\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMATHI-56\" x=\"7209\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-29\" x=\"7978\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-3D\" x=\"8645\" y=\"0\"><\/use><g transform=\"translate(9700,0)\"><use xlink:href=\"#E259-MJMAIN-73\"><\/use><use xlink:href=\"#E259-MJMAIN-6F\" x=\"394\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-66\" x=\"894\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-74\" x=\"1200\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-6D\" x=\"1589\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-61\" x=\"2422\" y=\"0\"><\/use><use xlink:href=\"#E259-MJMAIN-78\" x=\"2922\" y=\"0\"><\/use><\/g><g transform=\"translate(13317,0)\"><use xlink:href=\"#E259-MJSZ3-28\"><\/use><g transform=\"translate(736,0)\"><g transform=\"translate(120,0)\"><rect stroke=\"none\" width=\"2409\" height=\"60\" x=\"0\" y=\"220\"><\/rect><g transform=\"translate(60,676)\"><use xlink:href=\"#E259-MJMATHI-51\" x=\"0\" y=\"0\"><\/use><g transform=\"translate(791,0)\"><use xlink:href=\"#E259-MJMATHI-4B\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E259-MJMATHI-54\" x=\"1274\" y=\"513\"><\/use><\/g><\/g><g transform=\"translate(294,-817)\"><use xlink:href=\"#E259-MJMAIN-221A\" x=\"0\" y=\"-10\"><\/use><rect stroke=\"none\" width=\"988\" height=\"60\" x=\"833\" y=\"730\"><\/rect><g transform=\"translate(833,0)\"><use xlink:href=\"#E259-MJMATHI-64\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E259-MJMATHI-6B\" x=\"735\" y=\"-213\"><\/use><\/g><\/g><\/g><\/g><use xlink:href=\"#E259-MJSZ3-29\" x=\"3385\" y=\"-1\"><\/use><\/g><use xlink:href=\"#E259-MJMATHI-56\" x=\"17606\" y=\"0\"><\/use><\/g><\/svg><\/span><\/div><script type=\"math\/tex; mode=display\" id=\"MathJax-Element-220\">\\text{Attention}(Q,K,V)=\\text{softmax}\\left( \\frac{QK^T}{\\sqrt{d_k}} \\right)V<\/script><\/div><\/div>\n<p>\u6211\u4eec\u5448\u73b0\u51fa\u7684\u6ce8\u610f\u529b\u8868\u73b0\u4e3a\u8f93\u5165\u8bcd\u6c47\u5bf9\u5e94\u4e09\u7ec4\u5411\u91cf\u7684\u51fd\u6570\uff0c\u5206\u522b\u4e3a<\/p>\n<ul>\n<li>Q\uff1a\u8bf7\u6c42\u5411\u91cf query<\/li>\n<li>K\uff1a\u952e\u5411\u91cf key<\/li>\n<li>V\uff1a\u503c\u5411\u91cf value<\/li>\n\n<\/ul>\n<p>\u53ef\u4ee5\u8fd9\u6837\u5bf9\u6b64\u4e09\u7ec4\u5411\u91cf\u8fdb\u884c\u7406\u89e3\uff1a\u5bf9\u4e8e\u8f93\u5165\u5e8f\u5217\u4e2d\u7684\u4e24\u4e2a\u5143\u7d20\uff0c\u53ef\u4ee5\u901a\u8fc7\u8bf7\u6c42\u5411\u91cf\u4e0e\u952e\u5411\u91cf\u7684\u5339\u914d\u786e\u5b9a\u5176\u8bed\u4e49\u95f4\u5173\u8054\uff0c\u5728softmax\u538b\u7f29\u83b7\u53d6\u5bf9\u5e94\u6743\u91cd\u540e\u4e0e\u8bcd\u5411\u91cf\u8fdb\u884c\u52a0\u6743\u6c42\u548c\uff0c\u4ece\u800c\u83b7\u5f97\u6700\u7ec8\u6ce8\u610f\u529b\u7684\u503c\u3002\u5bf9\u4e8e\u6574\u7ec4\u5e8f\u5217\u6570\u636e\uff0c\u4f7f\u7528\u77e9\u9635\u76f8\u4e58\u8fdb\u884c\u8ba1\u7b97\uff0c\u6709\u5982\u4e0b\u8ba1\u7b97\u56fe<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/tensorflow.google.cn\/images\/tutorials\/transformer\/scaled_attention.png\" referrerpolicy=\"no-referrer\" alt=\"scaled_dot_product_attention\"><\/p>\n<p>\u5176\u4e2d\u7b49\u6bd4\u7f29\u653eScale\u8fc7\u7a0b\u4e3a\u9632\u6b62\u68af\u5ea6\u6d88\u5931\uff0c\u53c2\u8003\u5f15\u6587\u8ba8\u8bba\u90e8\u5206\uff1a<\/p>\n<p><code>\u4e3a\u4ec0\u4e48\u8981\u6309\u6bd4\u7f29\u653eScale\uff1a\u8fd9\u662f\u539f\u8bba\u6587\u4e2d\u7684\u4e00\u4e2a\u63a8\u6d4b\u2014\u2014\u5982\u679c\u8f93\u5165\u7684Q\uff0cK\u7ef4\u5ea6\u8fc7\u5927\uff0c\u5219\u4f1a\u5bfc\u81f4\u70b9\u79ef\u540e\u7684\u7ed3\u679c\u5f88\u5927softmax\u51fd\u6570\u6709\u4e00\u4e2a\u7279\u70b9\uff0c\u5f53\u8f93\u5165\u7684x\u8d8a\u5927\uff0c\u5176\u68af\u5ea6\u4f1a\u8d8b\u8fd1\u4e8e0\u3002\u8fd9\u5bf9\u4e8e\u57fa\u4e8e\u68af\u5ea6\u4e0b\u964d\u6cd5\u7684\u4f18\u5316\u975e\u5e38\u4e0d\u5229\u3002\uff08\u8fd9\u4e2a\u662f\u4e00\u4e2a\u6709\u6839\u636e\u7684\u63a8\u6d4b\uff1a\u5047\u8bbeq\u548ck\u90fd\u662f\u72ec\u7acb\u7684\u968f\u673a\u53d8\u91cf\uff0c\u90a3\u4e48q\u4e58\u4e0ak\u662f\u5747\u503c\u76840\u65b9\u5dee\u4e3a$d_k$\u7684\u3002\u9664\u4ee5\u6df1\u5ea6\u7684\u5e73\u65b9\u6839\uff0c\u53ef\u4ee5\u8ba9\u65b9\u5dee\u4e3a 1\uff09<\/code><\/p>\n<p>\u53e6\u5916\u6b64\u5904\u7684mask\u64cd\u4f5c\u662f\u53ef\u9009\u7684\uff0c\u7528\u4e8e\u8bbe\u8ba1\u5ffd\u7565\u90e8\u5206\u8f93\u5165\uff08\u7136\u800c\u5728\u56fe\u50cf\u7528transformer\u4e2d\u5e94\u8be5\u6ca1\u5565\u7528\uff09<\/p>\n<p>\u6682\u4e14\u4e0d\u8c08\u4e09\u7ec4\u5411\u91cf\u7684\u83b7\u5f97\u65b9\u5f0f\uff0c\u8003\u8651\u8f93\u5165\u96c6\u7fa4\uff0c\u6211\u4eec\u6784\u9020\u5411\u91cf\u5316\u7684\u6ce8\u610f\u529b\u6279\u91cf\u8ba1\u7b97\u5f62\u5f0f<\/p>\n<pre><code class='language-python' lang='python'>def attention(q, k, v, mask = None):\n\n    &quot;&quot;&quot;\n    \u8981\u6c42\u53c2\u6570\u7b26\u5408\u4e0b\u5217\u5f62\u6001\u6807\u51c6\uff1a\n        q: \u8bf7\u6c42 == (..., seq_len_q, depth)\n        k: \u952e   == (..., seq_len_k, depth)\n        v: \u503c   == (..., seq_len_v, depth_v)\n        mask:   == (..., seq_len_q, seq_len_k)\n    &quot;&quot;&quot;\n    # \u8f6c\u7f6ek\u77e9\u9635\u76f8\u4e58\uff0cQK^T == (..., seq_len_q, seq_len_k)\n    matmul_qk = torch.matmul(q, k.transpose(k.dim()-2,k.dim()-1))\n\n    # \u7f16\u7801\u957f\u5ea6\n    sqrt_dk = math.sqrt(q.shape[-1])\n\n    scaled_qk = matmul_qk \/ sqrt_dk\n\n    if mask is not None:\n        scaled_qk += (mask * -1e9)\n\n    # \u5728seq_len_k\u8f74\u4e0a\u8fdb\u884c\u5f52\u4e00\u5316\uff0c\u83b7\u53d6\u6ce8\u610f\u529b\u6743\u503c\u8868\n    attention_weights = torch.softmax(scaled_qk, dim=-1)\n\n    # \u8f93\u51fa\u5bf9\u4e8e\u6bcf\u4e2a\u8f93\u5165\u8bcd\u81ea\u6ce8\u610f\u540e\u7684\u503cdepth_v\u7ef4\u8bcd\u5d4c\u5165\u5411\u91cf\n    # (..., seq_len_q, depth_v)\n    out = torch.matmul(attention_weights, v)\n\n    return out, attention_weights\n<\/code><\/pre>\n<p>\u540c\u6837\u7684\uff0c\u6211\u4eec\u53ef\u4ee5\u5c06\u8be5\u6ce8\u610f\u529b\u5de5\u4f5c\u6539\u5199\u6210pytorch\u4e2d\u7684\u5c42\u8fd0\u7b97\uff1a<\/p>\n<pre><code class='language-python' lang='python'>class ScaledDotProductAttention(nn.Module):\n    def __init__(self, dropout=0):\n        super(ScaledDotProductAttention, self).__init__()\n\n        self.dropout = nn.Dropout(dropout)\n        self.softmax = nn.Softmax(dim=-1)\n    def forward(self, q, k, v, mask=None):\n        &quot;&quot;&quot;\n        \u8981\u6c42\u53c2\u6570\u7b26\u5408\u4e0b\u5217\u5f62\u6001\u6807\u51c6\uff1a\n            q: \u8bf7\u6c42 == (..., seq_len_q, depth)\n            k: \u952e   == (..., seq_len_k, depth)\n            v: \u503c   == (..., seq_len_v, depth_v)\n            mask:   == (..., seq_len_q, seq_len_k)\n        &quot;&quot;&quot;\n        # \u83b7\u53d6\u6df1\u5ea6\n        depth = torch.tensor([q.shape[-1]], dtype=torch.float32)\n\n        # \u8f6c\u7f6ek\u77e9\u9635\u76f8\u4e58\uff0cQK^T == (..., seq_len_q, seq_len_k)\n        # \u9ed8\u8ba4\u7f29\u653e$\\sqrt{d_k}$\n        attention_weights = torch.matmul(q, k.transpose(-2,-1)) \/ torch.sqrt(depth)\n\n        if mask is not None:\n            attention_weights = attention_weights.masked_fill_(mask, -np.inf)\n            \n        attention_weights = self.dropout(self.softmax(attention_weights))\n        \n        return attention_weights.matmul(v), attention_weights\n<\/code><\/pre>\n<h2>\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236<\/h2>\n<p>\u6211\u4eec\u91cd\u65b0\u8003\u8651transformer\u4e2d\u4e09\u7ec4\u5411\u91cf\u7684\u6784\u5efa\u3002\u901a\u8fc7\u4e0b\u5217\u56fe\u793a\uff0c\u6211\u4eec\u53ef\u4ee5\u66f4\u597d\u5730\u7406\u89e3transformer\u7684\u673a\u5236\uff0c\u5373\u5bf9\u4e8e\u67d0\u4e2a\u8f93\u5165\u7684\u8bcd\u6c47\uff0c\u6b64\u4e09\u7ec4\u5411\u91cf\u5747\u7531\u4e00\u7ec4\u77e9\u9635\u540c\u539f\u59cb\u8bcd\u5d4c\u5165\u76f8\u4e58\u6784\u5efa\u3002\u5176\u673a\u7406\u540c\u6837\u53ef\u4ee5\u8868\u793a\u4e3a\u4e09\u7ec4\u5168\u8fde\u63a5\u795e\u7ecf\u7f51\u7edc\uff0c\u4ece\u800c\u65b9\u4fbf\u6211\u4eec\u6a21\u578b\u7684\u6784\u5efa\u3002\u4f7f\u7528\u8be5\u65b9\u6cd5\u5c06\u8bcd\u6c47\u539f\u59cb\u7684\u7a7a\u95f4\u5d4c\u5165\u7f16\u7801\u4e3a\u53ef\u901a\u8fc7\u5b66\u4e60\u6539\u53d8\u7684\u8bf7\u6c42\u3001\u952e\u3001\u503c\u3002<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/jalammar.github.io\/images\/t\/transformer_self_attention_vectors.png\" referrerpolicy=\"no-referrer\" alt=\"img\"><\/p>\n<p>\u5bf9\u5e94\u524d\u4e00\u8282\u7684\u9010\u6b65\u8ba1\u7b97\u65b9\u5f0f\uff0c\u6709\u4e0b\u56fe\u8fdb\u884c\u6897\u6982\uff1a<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/jalammar.github.io\/images\/t\/self-attention-output.png\" referrerpolicy=\"no-referrer\" alt=\"img\"><\/p>\n<p>\u6211\u4eec\u53ef\u4ee5\u7b80\u5355\u7684\u7406\u89e3\u4e3a\u5728\u5f53\u524d\u6ce8\u610f\u529b\u5934\u4e2d\uff0c\u5355\u8bcd\u7684\u503c\u7531\u4e0e\u5355\u8bcd\u542b\u4e49\u76f8\u5173\u7684\u8bcd\u6c47\u7684value\u76f8\u53e0\u52a0\u3002<\/p>\n<p>\u591a\u5934\u6ce8\u610f\u529b\u7684\u5b9e\u8d28\u662f\uff0c\u901a\u8fc7\u6784\u5efa\u591a\u4e2a\u81ea\u6ce8\u610f\u529b\u673a\u5236\u53ca\u5176\u6743\u91cd\u6765\u5b66\u4e60\u66f4\u52a0\u590d\u6742\u7684\u5173\u8054\u4fe1\u606f\u3002\u800c\u6211\u4eec\u5e0c\u671b\u5c06\u6784\u5efa\u6240\u6709\u6ce8\u610f\u529b\u5934\u7684\u8f93\u51fa\uff0c\u901a\u8fc7\u518d\u4e00\u6b21\u7684\u7ebf\u6027\u53d8\u6362\uff0c\u878d\u5408\u4e3a\u7edf\u4e00\u7684\u8f93\u51fa<\/p>\n<p>\u6211\u4eec\u901a\u8fc7\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\u7edf\u62ec\u6574\u4e2a\u6ce8\u610f\u529b\u6a21\u578b\u7684\u7ed3\u6784<\/p>\n<pre><code class='language-python' lang='python'>def __init__(self, d_model=512, head_num=8, dropout=0):\n        &quot;&quot;&quot;\n        head_num: \u8868\u793a\u6ce8\u610f\u529b\u673a\u5236\u7684\u603b\u5934\u6570\uff0c\u9ed8\u8ba4\u4e3a8\n        d_model\uff1a\u8868\u793a\u8bcd\u5d4c\u5165\u7684\u7ef4\u6570\uff0c\u9ed8\u8ba4512\u7ef4\u6620\u5c04\u7ed9\u4e868\u4e2a\u5934\u6bcf\u4e2a\u593464\u7ef4\n        &quot;&quot;&quot;\n        assert d_model % head_num == 0\n\n        super(MultiHeadAttention, self).__init__()\n\n        self.head_num = head_num\n        self.d_model = d_model\n\n        # \u5bf9\u4e8e\u6bcf\u4e2a\u5934\u7684\u7f16\u7801\u7ef4\u6570\n        self.depth = d_model \/\/ head_num\n\n        self.Wq = nn.Linear(self.d_model, self.depth * self.head_num)\n        self.Wk = nn.Linear(self.d_model, self.depth * self.head_num)\n        self.Wv = nn.Linear(self.d_model, self.depth * self.head_num)\n\n        self.Fc = nn.Linear(self.d_model, self.d_model)\n\n        self.Attention = ScaledDotProductAttention(dropout)\n\n        self.Dropout = nn.Dropout(dropout)\n\n        self.Norm = nn.LayerNorm(self.d_model)\n    def forward(self, q, k, v, mask=None):\n        # \u5b9e\u9645\u8f93\u5165\u4e09\u7ec4\u5411\u91cf\u53ef\u80fd\u76f8\u540c\n        # \u5f15\u5165\u6b8b\u5dee\u8fde\u63a5\n        res = q\n\n        batch_size = q.shape[0]\n\n        # \u7ebf\u6027\u7f16\u7801\u8f93\u5165x\u540e\u6309\u5934\u62c6\u5206\n        q = self.Wq(q).view(batch_size * self.head_num, -1, self.depth)\n        k = self.Wk(k).view(batch_size * self.head_num, -1, self.depth)\n        v = self.Wv(v).view(batch_size * self.head_num, -1, self.depth)\n\n        if mask is not None:\n            mask = mask.repeat(self.head_num, 1, 1)\n\n        out, attention = self.Attention(q, k, v, mask)\n\n        # \u5c06\u6309\u5934\u62c6\u5206\u7684\u6ce8\u610f\u529b\u5408\u5e76\n        out = out.view(batch_size, -1, self.depth * self.head_num)\n        out = self.Norm(res + self.Dropout(self.Fc(out)))\n\n        return out, attention\n<\/code><\/pre>\n<p>\u5176\u4e2d\u5c06\u6bcf\u4e2a\u6ce8\u610f\u529b\u5934\u7edf\u4e00\u521d\u59cb\u5316\u4e3a\u4e00\u4e2a\u5927\u77e9\u9635\uff0c\u7136\u540e\u901a\u8fc7\u5411\u91cfreshape\u7684\u65b9\u5f0f\u5206\u522b\u8ba1\u7b97\uff0c\u5e76\u8fdb\u884c\u9006\u64cd\u4f5c\u62fc\u5408\u6ce8\u610f\u529b\uff0c\u4ece\u800c\u5b8c\u6210\u6574\u4e2a\u591a\u5934\u6ce8\u610f\u529b\u7684\u6784\u5efa<\/p>\n<h2>\u4f4d\u7f6e\u5f0f\u524d\u9988\u795e\u7ecf\u7f51\u7edc<\/h2>\n<p>\u8be5\u795e\u7ecf\u7f51\u7edc\u8868\u73b0\u4e3a\u4e00\u7ec4\u5168\u8fde\u63a5\u5c42\uff0c\u4f4d\u4e8e\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\u4e0a\u65b9\uff0c\u7528\u4e8e\u91cd\u65b0\u7f16\u7801\u591a\u5934\u6ce8\u610f\u529b\uff0c\u5b8c\u6210encoding<\/p>\n<p>\u5176\u6570\u5b66\u8868\u73b0\u5982\u4e0b<\/p>\n<p>&nbsp;<\/p>\n<div contenteditable=\"false\" spellcheck=\"false\" class=\"mathjax-block md-end-block md-math-block md-rawblock\" id=\"mathjax-n836\" cid=\"n836\" mdtype=\"math_block\"><div class=\"md-rawblock-container md-math-container\" tabindex=\"-1\"><div class=\"MathJax_SVG_Display\" style=\"text-align: center;\"><span class=\"MathJax_SVG\" id=\"MathJax-Element-221-Frame\" tabindex=\"-1\" style=\"font-size: 100%; display: inline-block;\"><svg xmlns:xlink=\"http:\/\/www.w3.org\/1999\/xlink\" width=\"37.696ex\" height=\"2.577ex\" viewBox=\"0 -806.1 16230.3 1109.7\" role=\"img\" focusable=\"false\" style=\"vertical-align: -0.705ex; max-width: 100%;\"><defs><path stroke-width=\"0\" id=\"E260-MJMATHI-46\" d=\"M48 1Q31 1 31 11Q31 13 34 25Q38 41 42 43T65 46Q92 46 125 49Q139 52 144 61Q146 66 215 342T285 622Q285 629 281 629Q273 632 228 634H197Q191 640 191 642T193 659Q197 676 203 680H742Q749 676 749 669Q749 664 736 557T722 447Q720 440 702 440H690Q683 445 683 453Q683 454 686 477T689 530Q689 560 682 579T663 610T626 626T575 633T503 634H480Q398 633 393 631Q388 629 386 623Q385 622 352 492L320 363H375Q378 363 398 363T426 364T448 367T472 374T489 386Q502 398 511 419T524 457T529 475Q532 480 548 480H560Q567 475 567 470Q567 467 536 339T502 207Q500 200 482 200H470Q463 206 463 212Q463 215 468 234T473 274Q473 303 453 310T364 317H309L277 190Q245 66 245 60Q245 46 334 46H359Q365 40 365 39T363 19Q359 6 353 0H336Q295 2 185 2Q120 2 86 2T48 1Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMATHI-4E\" d=\"M234 637Q231 637 226 637Q201 637 196 638T191 649Q191 676 202 682Q204 683 299 683Q376 683 387 683T401 677Q612 181 616 168L670 381Q723 592 723 606Q723 633 659 637Q635 637 635 648Q635 650 637 660Q641 676 643 679T653 683Q656 683 684 682T767 680Q817 680 843 681T873 682Q888 682 888 672Q888 650 880 642Q878 637 858 637Q787 633 769 597L620 7Q618 0 599 0Q585 0 582 2Q579 5 453 305L326 604L261 344Q196 88 196 79Q201 46 268 46H278Q284 41 284 38T282 19Q278 6 272 0H259Q228 2 151 2Q123 2 100 2T63 2T46 1Q31 1 31 10Q31 14 34 26T39 40Q41 46 62 46Q130 49 150 85Q154 91 221 362L289 634Q287 635 234 637Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-28\" d=\"M94 250Q94 319 104 381T127 488T164 576T202 643T244 695T277 729T302 750H315H319Q333 750 333 741Q333 738 316 720T275 667T226 581T184 443T167 250T184 58T225 -81T274 -167T316 -220T333 -241Q333 -250 318 -250H315H302L274 -226Q180 -141 137 -14T94 250Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMATHI-78\" d=\"M52 289Q59 331 106 386T222 442Q257 442 286 424T329 379Q371 442 430 442Q467 442 494 420T522 361Q522 332 508 314T481 292T458 288Q439 288 427 299T415 328Q415 374 465 391Q454 404 425 404Q412 404 406 402Q368 386 350 336Q290 115 290 78Q290 50 306 38T341 26Q378 26 414 59T463 140Q466 150 469 151T485 153H489Q504 153 504 145Q504 144 502 134Q486 77 440 33T333 -11Q263 -11 227 52Q186 -10 133 -10H127Q78 -10 57 16T35 71Q35 103 54 123T99 143Q142 143 142 101Q142 81 130 66T107 46T94 41L91 40Q91 39 97 36T113 29T132 26Q168 26 194 71Q203 87 217 139T245 247T261 313Q266 340 266 352Q266 380 251 392T217 404Q177 404 142 372T93 290Q91 281 88 280T72 278H58Q52 284 52 289Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-29\" d=\"M60 749L64 750Q69 750 74 750H86L114 726Q208 641 251 514T294 250Q294 182 284 119T261 12T224 -76T186 -143T145 -194T113 -227T90 -246Q87 -249 86 -250H74Q66 -250 63 -250T58 -247T55 -238Q56 -237 66 -225Q221 -64 221 250T66 725Q56 737 55 738Q55 746 60 749Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-3D\" d=\"M56 347Q56 360 70 367H707Q722 359 722 347Q722 336 708 328L390 327H72Q56 332 56 347ZM56 153Q56 168 72 173H708Q722 163 722 153Q722 140 707 133H70Q56 140 56 153Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-6D\" d=\"M41 46H55Q94 46 102 60V68Q102 77 102 91T102 122T103 161T103 203Q103 234 103 269T102 328V351Q99 370 88 376T43 385H25V408Q25 431 27 431L37 432Q47 433 65 434T102 436Q119 437 138 438T167 441T178 442H181V402Q181 364 182 364T187 369T199 384T218 402T247 421T285 437Q305 442 336 442Q351 442 364 440T387 434T406 426T421 417T432 406T441 395T448 384T452 374T455 366L457 361L460 365Q463 369 466 373T475 384T488 397T503 410T523 422T546 432T572 439T603 442Q729 442 740 329Q741 322 741 190V104Q741 66 743 59T754 49Q775 46 803 46H819V0H811L788 1Q764 2 737 2T699 3Q596 3 587 0H579V46H595Q656 46 656 62Q657 64 657 200Q656 335 655 343Q649 371 635 385T611 402T585 404Q540 404 506 370Q479 343 472 315T464 232V168V108Q464 78 465 68T468 55T477 49Q498 46 526 46H542V0H534L510 1Q487 2 460 2T422 3Q319 3 310 0H302V46H318Q379 46 379 62Q380 64 380 200Q379 335 378 343Q372 371 358 385T334 402T308 404Q263 404 229 370Q202 343 195 315T187 232V168V108Q187 78 188 68T191 55T200 49Q221 46 249 46H265V0H257L234 1Q210 2 183 2T145 3Q42 3 33 0H25V46H41Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-61\" d=\"M137 305T115 305T78 320T63 359Q63 394 97 421T218 448Q291 448 336 416T396 340Q401 326 401 309T402 194V124Q402 76 407 58T428 40Q443 40 448 56T453 109V145H493V106Q492 66 490 59Q481 29 455 12T400 -6T353 12T329 54V58L327 55Q325 52 322 49T314 40T302 29T287 17T269 6T247 -2T221 -8T190 -11Q130 -11 82 20T34 107Q34 128 41 147T68 188T116 225T194 253T304 268H318V290Q318 324 312 340Q290 411 215 411Q197 411 181 410T156 406T148 403Q170 388 170 359Q170 334 154 320ZM126 106Q126 75 150 51T209 26Q247 26 276 49T315 109Q317 116 318 175Q318 233 317 233Q309 233 296 232T251 223T193 203T147 166T126 106Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-78\" d=\"M201 0Q189 3 102 3Q26 3 17 0H11V46H25Q48 47 67 52T96 61T121 78T139 96T160 122T180 150L226 210L168 288Q159 301 149 315T133 336T122 351T113 363T107 370T100 376T94 379T88 381T80 383Q74 383 44 385H16V431H23Q59 429 126 429Q219 429 229 431H237V385Q201 381 201 369Q201 367 211 353T239 315T268 274L272 270L297 304Q329 345 329 358Q329 364 327 369T322 376T317 380T310 384L307 385H302V431H309Q324 428 408 428Q487 428 493 431H499V385H492Q443 385 411 368Q394 360 377 341T312 257L296 236L358 151Q424 61 429 57T446 50Q464 46 499 46H516V0H510H502Q494 1 482 1T457 2T432 2T414 3Q403 3 377 3T327 1L304 0H295V46H298Q309 46 320 51T331 63Q331 65 291 120L250 175Q249 174 219 133T185 88Q181 83 181 74Q181 63 188 55T206 46Q208 46 208 23V0H201Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-30\" d=\"M96 585Q152 666 249 666Q297 666 345 640T423 548Q460 465 460 320Q460 165 417 83Q397 41 362 16T301 -15T250 -22Q224 -22 198 -16T137 16T82 83Q39 165 39 320Q39 494 96 585ZM321 597Q291 629 250 629Q208 629 178 597Q153 571 145 525T137 333Q137 175 145 125T181 46Q209 16 250 16Q290 16 318 46Q347 76 354 130T362 333Q362 478 354 524T321 597Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-2C\" d=\"M78 35T78 60T94 103T137 121Q165 121 187 96T210 8Q210 -27 201 -60T180 -117T154 -158T130 -185T117 -194Q113 -194 104 -185T95 -172Q95 -168 106 -156T131 -126T157 -76T173 -3V9L172 8Q170 7 167 6T161 3T152 1T140 0Q113 0 96 17Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMATHI-57\" d=\"M436 683Q450 683 486 682T553 680Q604 680 638 681T677 682Q695 682 695 674Q695 670 692 659Q687 641 683 639T661 637Q636 636 621 632T600 624T597 615Q597 603 613 377T629 138L631 141Q633 144 637 151T649 170T666 200T690 241T720 295T759 362Q863 546 877 572T892 604Q892 619 873 628T831 637Q817 637 817 647Q817 650 819 660Q823 676 825 679T839 682Q842 682 856 682T895 682T949 681Q1015 681 1034 683Q1048 683 1048 672Q1048 666 1045 655T1038 640T1028 637Q1006 637 988 631T958 617T939 600T927 584L923 578L754 282Q586 -14 585 -15Q579 -22 561 -22Q546 -22 542 -17Q539 -14 523 229T506 480L494 462Q472 425 366 239Q222 -13 220 -15T215 -19Q210 -22 197 -22Q178 -22 176 -15Q176 -12 154 304T131 622Q129 631 121 633T82 637H58Q51 644 51 648Q52 671 64 683H76Q118 680 176 680Q301 680 313 683H323Q329 677 329 674T327 656Q322 641 318 637H297Q236 634 232 620Q262 160 266 136L501 550L499 587Q496 629 489 632Q483 636 447 637Q428 637 422 639T416 648Q416 650 418 660Q419 664 420 669T421 676T424 680T428 682T436 683Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-31\" d=\"M213 578L200 573Q186 568 160 563T102 556H83V602H102Q149 604 189 617T245 641T273 663Q275 666 285 666Q294 666 302 660V361L303 61Q310 54 315 52T339 48T401 46H427V0H416Q395 3 257 3Q121 3 100 0H88V46H114Q136 46 152 46T177 47T193 50T201 52T207 57T213 61V578Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-2B\" d=\"M56 237T56 250T70 270H369V420L370 570Q380 583 389 583Q402 583 409 568V270H707Q722 262 722 250T707 230H409V-68Q401 -82 391 -82H389H387Q375 -82 369 -68V230H70Q56 237 56 250Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMATHI-62\" d=\"M73 647Q73 657 77 670T89 683Q90 683 161 688T234 694Q246 694 246 685T212 542Q204 508 195 472T180 418L176 399Q176 396 182 402Q231 442 283 442Q345 442 383 396T422 280Q422 169 343 79T173 -11Q123 -11 82 27T40 150V159Q40 180 48 217T97 414Q147 611 147 623T109 637Q104 637 101 637H96Q86 637 83 637T76 640T73 647ZM336 325V331Q336 405 275 405Q258 405 240 397T207 376T181 352T163 330L157 322L136 236Q114 150 114 114Q114 66 138 42Q154 26 178 26Q211 26 245 58Q270 81 285 114T318 219Q336 291 336 325Z\"><\/path><path stroke-width=\"0\" id=\"E260-MJMAIN-32\" d=\"M109 429Q82 429 66 447T50 491Q50 562 103 614T235 666Q326 666 387 610T449 465Q449 422 429 383T381 315T301 241Q265 210 201 149L142 93L218 92Q375 92 385 97Q392 99 409 186V189H449V186Q448 183 436 95T421 3V0H50V19V31Q50 38 56 46T86 81Q115 113 136 137Q145 147 170 174T204 211T233 244T261 278T284 308T305 340T320 369T333 401T340 431T343 464Q343 527 309 573T212 619Q179 619 154 602T119 569T109 550Q109 549 114 549Q132 549 151 535T170 489Q170 464 154 447T109 429Z\"><\/path><\/defs><g stroke=\"currentColor\" fill=\"currentColor\" stroke-width=\"0\" transform=\"matrix(1 0 0 -1 0 0)\"><use xlink:href=\"#E260-MJMATHI-46\" x=\"0\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMATHI-46\" x=\"749\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMATHI-4E\" x=\"1498\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMAIN-28\" x=\"2386\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMATHI-78\" x=\"2775\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMAIN-29\" x=\"3347\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMAIN-3D\" x=\"4013\" y=\"0\"><\/use><g transform=\"translate(5069,0)\"><use xlink:href=\"#E260-MJMAIN-6D\"><\/use><use xlink:href=\"#E260-MJMAIN-61\" x=\"833\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMAIN-78\" x=\"1333\" y=\"0\"><\/use><\/g><use xlink:href=\"#E260-MJMAIN-28\" x=\"6930\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMAIN-30\" x=\"7319\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMAIN-2C\" x=\"7819\" y=\"0\"><\/use><use xlink:href=\"#E260-MJMATHI-78\" x=\"8264\" y=\"0\"><\/use><g transform=\"translate(8836,0)\"><use xlink:href=\"#E260-MJMATHI-57\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E260-MJMAIN-31\" x=\"1335\" y=\"-213\"><\/use><\/g><use xlink:href=\"#E260-MJMAIN-2B\" x=\"10455\" y=\"0\"><\/use><g transform=\"translate(11456,0)\"><use xlink:href=\"#E260-MJMATHI-62\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E260-MJMAIN-31\" x=\"606\" y=\"-213\"><\/use><\/g><use xlink:href=\"#E260-MJMAIN-29\" x=\"12338\" y=\"0\"><\/use><g transform=\"translate(12727,0)\"><use xlink:href=\"#E260-MJMATHI-57\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E260-MJMAIN-32\" x=\"1335\" y=\"-213\"><\/use><\/g><use xlink:href=\"#E260-MJMAIN-2B\" x=\"14347\" y=\"0\"><\/use><g transform=\"translate(15347,0)\"><use xlink:href=\"#E260-MJMATHI-62\" x=\"0\" y=\"0\"><\/use><use transform=\"scale(0.707)\" xlink:href=\"#E260-MJMAIN-32\" x=\"606\" y=\"-213\"><\/use><\/g><\/g><\/svg><\/span><\/div><script type=\"math\/tex; mode=display\" id=\"MathJax-Element-221\">FFN(x)=\\max(0,xW_1+b_1)W_2+b_2<\/script><\/div><\/div>\n<p>\u53ef\u4ee5\u4f7f\u75281*1\u5377\u79ef\u6838\u5bf9\u5168\u8fde\u63a5\u5c42\u8fdb\u884c\u6a21\u62df\uff0c\u4ece\u800c\u65b9\u4fbf\u64cd\u4f5c\u3002<\/p>\n<p>\u5176\u7ed3\u6784\u4e5f\u76f8\u5f53\u7b80\u5355<\/p>\n<pre><code class='language-python' lang='python'>class PositionalWiseFeedForward(nn.Module):\n    def __init__(self, d_model=512, dff=2048, dropout=0):\n        super(PositionalWiseFeedForward, self).__init__()\n        self.W1 = nn.Conv1d(d_model, dff, 1)\n        self.W2 = nn.Conv1d(dff, d_model, 1)\n        self.Dropout = nn.Dropout(dropout)\n        self.Norm = nn.LayerNorm(d_model)\n        self.Relu = nn.ReLU()\n    def forward(self, x):\n        out = self.Relu(self.W1(x.transpose(-1,-2)))\n        out = self.W2(out).transpose(-1,-2)\n        out = self.Norm(x+self.Dropout(out))\n        return out\n<\/code><\/pre>\n<p>\u6709\u4e86\u4e0a\u8ff0\u7ed3\u6784\uff0c\u6211\u4eec\u5b8c\u6210\u4e86transformer\u4e2d\u6240\u9700\u7684\u6240\u6709\u5b50\u7ed3\u6784<\/p>\n<p>\u968f\u540e\uff0c\u6211\u4eec\u53ef\u4ee5\u57fa\u4e8e\u4e0a\u8ff0\u5b9e\u73b0\u7684\u5b50\u7ed3\u6784\u5bf9\u7f16\u7801\u5668\u3001\u89e3\u7801\u5668\uff0c\u4e43\u81f3\u5b8c\u6574\u7684transformer\u7ed3\u6784\u8fdb\u884c\u6784\u9020<\/p>\n<h2>\u7f16\u7801\u5668<\/h2>\n<p><img decoding=\"async\" src=\"https:\/\/jalammar.github.io\/images\/t\/transformer_resideual_layer_norm.png\" referrerpolicy=\"no-referrer\" alt=\"img\"><\/p>\n<p>\u7f16\u7801\u5668\u76f8\u5bf9\u89e3\u7801\u5668\u7ed3\u6784\u4f1a\u66f4\u7b80\u5355\u4e00\u70b9\uff0c\u5728LayerNorm\u5df2\u7ecf\u88ab\u96c6\u6210\u5728\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\u53ca\u524d\u9988\u795e\u7ecf\u7f51\u7edc\u4e2d\u7684\u524d\u63d0\u4e0b\uff0c\u6211\u4eec\u80fd\u7528\u5c11\u91cf\u7684\u4ee3\u7801\u5b8c\u6210\u7f16\u7801\u5668\u5c42\u7684\u6784\u5efa<\/p>\n<pre><code class='language-python' lang='python'>class EncoderLayer(nn.Module):\n    def __init__(self, d_model, head_num, dff, dropout=0):\n        super(EncoderLayer, self).__init__()\n\n        self.Att = MultiHeadAttention(d_model, head_num, dropout)\n        self.ffn = PositionalWiseFeedForward(d_model, dff, dropout)\n\n    def forward(self, x, mask=None):\n        out, attention = self.Att(x, x, x, mask)\n        out = self.ffn(out)\n        return out, attention\n<\/code><\/pre>\n<p>\u5e76\u901a\u8fc7\u7f16\u7801\u5668\u5c42\u7684\u53e0\u52a0\u53ca\u521d\u59cb\u5316\u7684\u8f93\u5165\u8bcd\u5d4c\u5165\u5b8c\u6210\u8f93\u5165\u7aef\u7684\u7f16\u7801\u5668\u7ed3\u6784<\/p>\n<pre><code class='language-python' lang='python'># \u7f16\u7801\u5668\nclass Encoder(nn.Module):\n    def __init__(self, vocab_size, max_l, layer_num=6, d_model=512, head_num=8, dff=2048, dropout=.0):\n        &quot;&quot;&quot;\n        vocab_size: \u8bed\u6599\u5e93\u5927\u5c0f\n        max_l: \u6700\u5927\u5355\u53e5\u957f\u5ea6\n        layer_num: \u7f16\u7801\u5668\u603b\u5c42\u6570 \n        d_model: \u8bcd\u5d4c\u5165\u7ef4\u6570\n        head_num: \u591a\u5934\u6ce8\u610f\u529b\u7ef4\u6570\n        dff: \u524d\u9988\u9690\u85cf\u5c42\u6570\n        dropout: \u8df3\u8dc3\u8fde\u63a5\u6982\u7387\n        &quot;&quot;&quot;\n        super(Encoder, self).__init__()\n        self.d_model = d_model\n        self.layer_num = layer_num\n        self.head_num = head_num\n        self.dff = dff\n\n        self.Em = nn.Embedding(vocab_size + 1, d_model, padding_idx=0)\n        self.Pe = PositionalEncoding(d_model, max_l)\n\n        self.Encoderlayers = nn.ModuleList(\n            [EncoderLayer(d_model, head_num, dff, dropout) for _ in range(layer_num)])\n    def forward(self, x, seq_len):\n\n        # \u5b8c\u6210\u8bcd\u5d4c\u5165\n        out = self.Em(x)\n        out *= torch.sqrt(torch.FloatTensor([self.d_model]))\n        out += self.Pe(seq_len)\n\n\n        # \u6784\u9020mask\n        mask = padding_mask(x, x)\n        \n        # \u9010\u5c42\u8ba1\u7b97\u6ce8\u610f\u529b\n        attentions = []\n        for encoder in self.Encoderlayers:\n            out, attention = encoder(out, mask)\n            attentions.append(attention)\n\n        return out, attentions\n<\/code><\/pre>\n<h2>\u89e3\u7801\u5668<\/h2>\n<p>\u89e3\u7801\u5668\u7684\u7ed3\u6784\u76f8\u5bf9\u4e8e\u7f16\u7801\u5668\u4f1a\u7a0d\u5fae\u590d\u6742\u4e00\u4e9b<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/tensorflow.google.cn\/images\/tutorials\/transformer\/transformer.png\" referrerpolicy=\"no-referrer\" alt=\"transformer\"><\/p>\n<p>\u5176\u4e2d\u7531\u4e24\u7ec4\u591a\u5934\u6ce8\u610f\u529b\u6784\u6210\uff0c\u5e76\u4e14\u6ce8\u610f\u529b\u540c\u65f6\u53d7\u5230\u4e86\u8f93\u5165\u53ca\u5148\u524d\u8f93\u51fa\u7684\u5f71\u54cd\uff0c\u4ece\u800c\u5b8c\u6210\u6574\u4e2a\u6ce8\u610f\u529b\u8fc7\u7a0b\uff0c\u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0c\u89e3\u7801\u5668\u63a5\u53d7\u6765\u81eaencoder\u7684\u952e\u503c\uff0c\u5e76\u4f7f\u7528\u6765\u81ea\u524d\u6bb5\u7f16\u7801\u7684\u8bf7\u6c42\uff0c\u4ece\u800c\u8fbe\u6210\u901a\u8fc7\u5df2\u77e5\u4fe1\u606f\u8bf7\u6c42\u672a\u77e5\u4fe1\u606f\u7684\u6548\u679c<\/p>\n<pre><code class='language-python' lang='python'>class DecoderLayer(nn.Module):\n    def __init__(self, d_model, head_num, dff, dropout=0):\n        super(DecoderLayer, self).__init__()\n\n        self.self_Att = MultiHeadAttention(d_model, head_num, dropout)\n        self.ende_Att = MultiHeadAttention(d_model, head_num, dropout)\n\n        self.ffn = PositionalWiseFeedForward(d_model, dff, dropout)\n    \n    def forward(self, x, encoded, self_attn_mask, mask=None):\n        out, self_att = self.self_Att(x, x, x, self_attn_mask)\n        out, ende_att = self.ende_Att(out, encoded, encoded, mask)\n        out = self.ffn(out)\n        return out, self_att, ende_att\n<\/code><\/pre>\n<p>\u5176\u4e2d\u8f93\u5165x\u662f\u524d\u7f6e\u5c42\/\u524d\u7f6e\u8f93\u51fa\u7684\u7f16\u7801\u5411\u91cf\uff0c\u800cencoded\u662f\u5b8c\u6210\u8f93\u5165\u7f16\u7801\u540e\u7684\u9ad8\u5c42\u8bed\u4e49\u4fe1\u606f\uff0c\u6574\u4e2a\u7f51\u7edc\u6bcf\u6b21\u4ec5\u901a\u8fc7\u5b8c\u6574\u8f93\u5165\u548c\u524d\u4e00\u6b65\u8f93\u51fa\u7684\u77ed\u53e5\u9884\u6d4b\u4e0b\u4e00\u4e2a\u8bcd\u6c47\uff0c\u4ec5\u5f53\u5f00\u59cb\u7b2c\u4e00\u6b65\u9884\u6d4b\u65f6\u5b58\u5728\u65e0\u4e2d\u751f\u6709\uff0c\u5e76\u4e14\u89e3\u7801\u5668\u5bf9\u5e94\u7684\u8bcd\u5d4c\u5165\u4e3a\u76ee\u6807\u57df\u4e0a\u7684\u5d4c\u5165\uff0c\u4e0e\u8f93\u5165\u5b58\u5728\u4e0d\u540c\u7684\u5d4c\u5165\u65b9\u5f0f\uff0c\u4ee5\u4e0b\u4e3a\u89e3\u7801\u5668\u7ed3\u6784\u7684\u6784\u9020<\/p>\n<pre><code class='language-python' lang='python'>class Decoder(nn.Module):\n    def __init__(self, vocab_size, max_l, layer_num=6, d_model=512, head_num=8, dff=2048, dropout=.0):\n        super(Decoder, self).__init__()\n        self.d_model = d_model\n        self.layer_num = layer_num\n        self.head_num = head_num\n        self.dff = dff\n\n        self.Em = nn.Embedding(vocab_size + 1, d_model, padding_idx=0)\n        self.Pe = PositionalEncoding(d_model, max_l)\n\n        self.Decoderlayers = nn.ModuleList(\n            [DecoderLayer(d_model, head_num, dff, dropout) for _ in range(layer_num)])\n\n    def forward(self, x, encoded, seq_len, mask=None):\n        out = self.Em(x)\n        out *= torch.sqrt(torch.FloatTensor([self.d_model]))\n        out += self.Pe(seq_len)\n\n        self_attention_padding_mask = padding_mask(x, x)\n        seq_mask = sequence_mask(x)\n        self_attn_mask = torch.gt((self_attention_padding_mask + seq_mask), 0)\n        \n        for decoder in self.Decoderlayers:\n            out, _, _ = decoder(out, encoded, self_attn_mask, mask)\n\n        return out\n<\/code><\/pre>\n<p>\u5176\u4e2dTODO\u7684\u90e8\u5206\u4e3a\u89e3\u7801\u5668\u7279\u5316\u7684\u906e\u7f69\uff0c\u6709\u65f6\u6211\u4eec\u5e0c\u671b\u89e3\u7801\u5668\u4ec5\u5173\u6ce8\u524d\u90e8\u4fe1\u606f\u6240\u4ee5\u4f1a\u6784\u5efa\u7279\u6b8a\u7684\u906e\u7f69\u4ee5\u6539\u5584\u6548\u679c\uff0c\u4f46\u662f\u6211\u5728\u6b64\u5904\u5e76\u6ca1\u6709\u8fdb\u884c\u5b9e\u73b0\u3002<\/p>\n<h2>Transformer<\/h2>\n<p>\u6700\u540e\uff0c\u6211\u4eec\u80fd\u5c06encoder\u548cdecoder\u7ec4\u88c5\uff0c\u53d6\u5f97\u5b8c\u6210\u7684transformer<\/p>\n<pre><code class='language-python' lang='python'>class Transformer(nn.Module):\n    def __init__(self, i_vocab_size, i_len, o_vocab_size, o_len, layer_num=6, d_model=512, head_num=8, dff=2048, dropout=.0):\n        super(Transformer, self).__init__()\n\n        self.Encoder = Encoder(i_vocab_size, i_len, layer_num, d_model, head_num, dff, dropout)\n        self.Decoder = Decoder(o_vocab_size, o_len, layer_num, d_model, head_num, dff, dropout)\n\n        self.Fc = nn.Linear(d_model, o_vocab_size)\n        self.Softmax = nn.Softmax(dim=-1)\n\n    def forward(self, raw, len_r, tar, len_t, mask=None):\n        if mask is None:\n            mask = padding_mask(raw, tar)\n            \n        out, _ = self.Encoder(raw, len_r)\n        out = self.Decoder(tar, out, len_t, mask)\n        out = self.Softmax(self.Fc(out))\n        return out\n<\/code><\/pre>\n<p>\u53ef\u4ee5\u4f7f\u7528\u4ee5\u4e0b\u4ee3\u7801\u8fdb\u884c\u6d4b\u8bd5\u221a<\/p>\n<pre><code class='language-python' lang='python'>trans = Transformer(8500,62,8000,26)\n\nraw = (torch.rand(64,62)*5000).long()\ntar = (torch.rand(64,26)*5000).long()\nlen_r = (torch.ones(raw.shape[0]) * raw.shape[1]).int()\nlen_t = (torch.ones(tar.shape[0]) * tar.shape[1]).int()\nprint(raw)\nprint(tar)\n\no = trans(raw,len_r,tar,len_t)\nprint(o.shape)\nprint(o)\n<\/code><\/pre>\n<p>&nbsp;<\/p>\n<\/body>\n<\/html>\n","protected":false},"excerpt":{"rendered":"<p>\u6211\u4eec\u90fd\u662f\u8bfb\u8fc7\u9ad8\u4e2d\u82f1\u8bed\u7684\u4eba\uff0c\u4f17\u6240\u5468\u77e5\u53d8\u5f62\u91d1\u521a=transformer\uff08\u554a\u8fd9<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"emotion":"","emotion_color":"","title_style":"","license":"","footnotes":""},"categories":[41,10],"tags":[55,54,57,56],"class_list":["post-275","post","type-post","status-publish","format-standard","hentry","category-ai","category-10","tag-dl","tag-nlp","tag-pytorch","tag-transformer"],"_links":{"self":[{"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/posts\/275","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/comments?post=275"}],"version-history":[{"count":10,"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/posts\/275\/revisions"}],"predecessor-version":[{"id":287,"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/posts\/275\/revisions\/287"}],"wp:attachment":[{"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/media?parent=275"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/categories?post=275"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/blog.cyasylum.top\/index.php\/wp-json\/wp\/v2\/tags?post=275"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}