[{"data":1,"prerenderedAt":69},["ShallowReactive",2],{"rendered-post-a8cf276f":3},{"abbrlink":4,"title":5,"date":6,"categories":7,"tags":9,"excerpt":11,"htmlChunkEnds":12,"toc":14,"htmlGzip":68},"a8cf276f","论文阅读: PyTorch Distributed: Experiences on Accelerating Data Parallel Training","2023-09-17",[8],"懵逼的深度学习",[10],"分布式训练框架","摘要 本文提出的 DistributedDataParallel 在优化器运行之前进行梯度平均，用相同的梯度集更新所有模型副本，这样在数学上和本地训练完全等价，而且可以实现异步，比参数平均更加高效。",[13],6241,[15,19,22,26,29,32,35,38,41,44,47,50,53,56,59,62,65],{"id":16,"text":17,"html":17,"level":18},"post-heading-摘要","摘要",2,{"id":20,"text":21,"html":21,"level":18},"post-heading-背景","背景",{"id":23,"text":24,"html":24,"level":25},"post-heading-pytorch-训练流程","PyTorch 训练流程",3,{"id":27,"text":28,"html":28,"level":18},"post-heading-allreduce","AllReduce",{"id":30,"text":31,"html":31,"level":18},"post-heading-系统设计","系统设计",{"id":33,"text":34,"html":34,"level":18},"post-heading-梯度规约","梯度规约",{"id":36,"text":37,"html":37,"level":25},"post-heading-次级实现-all-reduce","次级实现（All Reduce）",{"id":39,"text":40,"html":40,"level":25},"post-heading-梯度桶","梯度桶",{"id":42,"text":43,"html":43,"level":25},"post-heading-通信和计算重叠","通信和计算重叠",{"id":45,"text":46,"html":46,"level":25},"post-heading-梯度累积","梯度累积",{"id":48,"text":49,"html":49,"level":18},"post-heading-集合通信","集合通信",{"id":51,"text":52,"html":52,"level":18},"post-heading-工程实现","工程实现",{"id":54,"text":55,"html":55,"level":25},"post-heading-python-前后端","python 前后端",{"id":57,"text":58,"html":58,"level":25},"post-heading-梯度规约的重点实现","梯度规约的重点实现",{"id":60,"text":61,"html":61,"level":18},"post-heading-实验","实验",{"id":63,"text":64,"html":64,"level":18},"post-heading-讨论","讨论",{"id":66,"text":67,"html":67,"level":18},"post-heading-改进","改进","H4sIAAAAAAAAA+VaS1MbWZbe+1dkaDHRvcC4XF3dPT1YEa7yTHVH1IPo6Fk7BKRBYSFpElHV7lUCQg\u002F05C1E8hAWSAUoUxgsUlIK\u002FZd23puZK\u002F2FiXNPKpWAcE\u002F3gs1EOAJLefM+zuM73\u002Fmuxmaec\u002F6pF55waC4yMsP7pvzB6RG6WjBPFjxe\u002FDs2OvPc+2Qs7KXSOd2K01yexFtGMcq98s9FBP\u002FEfISfeuWL+MZ9gi8Q4AMckaq6ViDpLbJTNbt5s5TWmymSzJjdXbOUpkcKaZ2Q5iXZi\u002Fe0tLFRNXZVkk8bxSg+snZjdPeKbtVpUqRSklZLZD9FkgqVznta2uzu0MNrIlXpZp3UTnR1haylqXROpLop14z2EpHTZLlq1JJ6+xrGi2ld3SA5RW8fE3nfyNaJtkhrxz0tTZUNkluEedhm6O4VWTm0zgp0M\u002FFJXBwbDbNTh31hXvgDN+bjZgT+zQvPTCQSnvvD6KhP+Kv\u002Fp6chYXrUNzE3+vzZs98+\u002FeKr3z37jYeL+IRpPvLC83oi4Au+9XACH3jhCYZCYT7IC1wwJPBveEHgBY\u002F3H801Nurz9vcxGZrih2xj2h+ZmZ94OhmaHQ2\u002Fi4SEyRnn77+yk89MN9jMsJgxl9J0R\u002FF48a8dMzNf3h9oTzeC3qIfF4xqyuMdf\u002FcX+JZzfzs2OvOl98nYfMD7ZCzg9\u002F5XSPjZJ0xxYd\u002FcXE8rmnLJkLdp5oCUL8ZGA34c9LVv8u2QUSywBqN+DEf8s\u002F6\u002F8QI3F+HDPa2IAYfRMBhGN+s0I5Nmwyyl7W9H2W7Czn6plNTbx7qaoltNouUwxEkiRtQl+MhO09PS5MSOp\u002F5h3AnT04oks0mlFikXSaZkz5HZNLu7RjVFykWj1TWqKfducOLBTh9IRZi5XLw7c7mIM39uwj+Pf9PTEvzT6accWoVKGbJSIjvVnpaEaftHtNPz\u002FiR9S5Hcia6KRiVDt5q0lIB3XUln+8eV6XRvyYpnjGKU7MWJqPW0tN6J0YJCOusOquhqDQx\u002Fs2d+3DK7cZJtkwvRWqqin+FdeDFBTha5l1O+WU5Xa\u002FDlStWK59j+085URjFqrDSouEByirnU0bWiJeapmicLOeOkDSghiUY7QRTNjF\u002FZ0bmZoPuSrqasncwncZHkTkh5C5ycXyX5rK4dWuIaWUu7j6mrGXOpY8UzJHeIkAgzfzw1r5p6K0bFirnU6Sf6\u002FyOYfRhOfIGAwE\u002FNT\u002FIe78tA4M\u002Fsv4NCdF4yWhVcxrGaktPVUztl6jGy3DBv1snyMdEOrXiONLt0rWa0d\u002FCwRGkatSQOBuNEGySfNVLnxlmKSHXS2gBzydtUUsxu3pC3cYwppulmwsiCN62zQn9lvZWlBYX74Sf\u002FlN8HXuF++Oab7ziY8UyBSGWe1FsxctDSW1kjq4BXnVNxECrsCT1cvfWkv8BgKF3P6B3JkkTzZMGoJcnNMp6nf+wLo31K8lmahGgbxFlBIfk0rR3j6+itO1mtqzW90zU2qtz483HOEot6t\u002FSwd4zLttE+MOUbUy55vO5Pjo9ejv+Jc3vIPzvNzQmTLzyj\u002FlnfND9qyi26FbcKy6bSHrHRdMQV6iP\u002F+dcwL\u002Fj54CQ\u002FNxIKjrycnOQDvOCLwPqQByP9RBj5i+DzB\u002F3B6dEvnv7MT4Q93M\u002F+qcjMC8+Xz555uFEEoduITLcbJHeKdtS7Mt1okvqqtVQliRgOMKNFKp0\u002F3s6\u002FvL3zrwY7p8UrS\u002FoASVMu0uwxyb13MoymTmjiVO+WSO6arKXNygnJraKPIculFkk0EAwQJx72JyKCWYkarROP1\u002F3pM4XcnYWQryP9hHU\u002F6GmJl4EAh\u002FHb05J2TQ97X70ap9kKSTQwgo1i1LY8C+V+6AMAITAl9kglRdJb4DpM5s6q0ZYQuyGk5V\u002FcQKa3j228Q3DXRFJJ4QpPqJKj5yWzW9Pb72E6qYrVg5uwycOvfs0BlOazZG1QKBhVsJ8kMzRZcUAWyomzXB9ebWxlxax\u002FmlevxjmESHpZJbE055uPhKYF3xQ3Ewq9nWNYCVtzb4Tks+hYm2Yw9+KEIZtLsJc5mjyjuZyNb1gN0XJ3SI3eziJW4gMAqPeicXVkFwwGl4PRJJaxcWGAQtZuTO+sI0a46ze6TldrODNUXET8O+yL1GNGe53uS2Z3g+zuD1l4NITcASs\u002FlHq1rKunWCetbdk6KtymU+DDeo6L8MG5kMDp6opRjFq7MZJP4DYRt629faKqeifrongrK3a4s4MAFjOoBMDQRIwC8C+r36b8C5GX9PY5Torl\u002FBbdGZonGA+lRj+zaKnhpIGupsjuDS3cGGUoEbqawSBCE5qVKDhRqmK4cYNU4uh2A2KNnQu4BwsrQ0qRRL1vBrOxQsoViCo2ymysOFXrMTDt+cOY1jeDrsKhzcp7Ert0H26YO93OhwPXY6RchDAeDKQbDYhNtaWrIoRLqQEDpap5c0MSdV0ViVy0TZrPIjJybnQC1kPqMfdOYGGWcmbnnOQKuGGkNo9nyd88YMlh0YahCeWABTTGqMc79GsnCiHCPpQgz1iuO\u002F7BGoKI19PSziS3UoZFHk7oapthyjvTqMC1BuRFqpJ82vEUiS2DrRHCkaBCwrEV5DRN5DER+9QGfIfb+SQuoldwCz0t3e\u002FNiLZJlo8ZtnK4BQwHksuQ\u002FKquHdK12v0VzPqSsVE146dELurNJIl3GHw+lq+\u002FejhrlBz0QGgwtwNcmydKE9hrMYqnJvWYrhXNozMsITCgs+42\u002FMT85Fs+gu2R0zzBq2D8o7M3foG3J65foGGcMkCUpqkc4wRufzi0mGG32d0xKhmjtmVsHutaEdsoRPNhOH6XyVtLHVpQkJkbxaiDxHp3z1QWbpHfctTIx2gJUNEqtUgrp6uiGb\u002FC5hthlvuV79ccTYpGudXfJWIPtKaXm9Ca1mPuqsmAOANzdSRdbblnZ5gLcYQfMQvM7qquaWapapRbTpRZ4prbwHYCsrdwF04x6ufNQV5v7VqFBpU\u002FQufBiA5OAV5r7ZKLRWc6XBg6EmSzLIHx0NwrIRQOzUc4o5Z0WjqzuwOgebGIoeI43U6Tump2JV3N3A2bgQkn\u002FpEJpSpJDhIMKv3eAfTF7Sx0ZPEWcjFrIUOyMTuCd29Ym894xoQ\u002FMusL97SEsVHVW1k0pt5Km91dYItMs2CVj\u002F0HeES0pyU5U66TziZZPwX7YLuZW9TVLK4AR3A9QrC7O0BpQhN275FTjQF+DuOmXNfVFifwvql3bv8hAXgMiPjtP1EObP9eKUZF6fMP\u002FDSgINqh0T7AR+gz62jVyMahbZSqNlnu9\u002FyINDjMBbWWWAQdRqrCv2Xg03bHibO6dAkuyNHzEhLUPuMHIg4OiOdIfdUGf4Yl4OfthrV9heELIpK4Z14rZrdgJxyLPVKucBO+yOQMN+f\u002FG8+RzCEg+McobeVhV+UKZK1DHFCjqMdsBIT3SCJOEkAckFLM+ieF0BP2BCsGlUQ4NXuBPcTl7liFbXKgvWAH8H+AQ8x3NAPQ8WER2O80Eq687mlJqMW190RV7wyk19ckn7C5q7OOlMQe10aOPtI4K5tHZ8aVQlYObYkAfNnQ21vuAVRdtnYbA9frzQO6fUgvN0miTLdqmGVDyxOIK5lLW4Nk0cEMVgCAryWf4Mpm6pg0PzD8Z2tiGxbLuF97AHgcnYX7t\u002F+ZD0X+Ixh6PfcuOIkfdHWFibRxEOw6624\u002FYTqQZsMGoM+9TbcbaFHWk4gDrLlnSyqdmkdnDhzrndhQc+IAh8MYJRlKW3cHjCid4lOnFtnZxNAZ1D81paui08rihKai9Fdkp\u002Fm7KCGi\u002Fl3ce0L3jhEnH1oBWqUFBWwkVQFwl48\u002FZw2nPXJ6eA5aXFZJkEAYJwu4Gyg+0RxZvTT2xDvnIfUYBp5RUZiMaB\u002FPVI4HK6m1O5wPQoE5A+HmoaPozQO7lGF9kG5bBJThT+IilkbMdV1NPUGz2q3jtjvateIQu3J075im40TR3OBlywLMo872nGLi1FhdOzRv1tngLBy\u002Fu0fTC05ZprX3RkmmYsWh1o8ipH3xUIkZIh65+2yP1\u002F3JEQP7ZQKig24oNL2gq0mjkoGyISuktd7TiiCa9rT0t4FQCCRR7vvxP6FnIMla6+bRGUkvm++XSaLu0FduXAhN8nNz3wqh+TAHkqOu1uyej6mfiOe3hzk5jB4xO7KpHJn1JQxFt7gBUHl1Y7R3SD56bxoi7+s30NGY2Wtys013r6yzArhXPSFyhyR+AW9n4w+LbuT6GHgrk8k8Xvenz9+ezYSCI4xhgb7s8eI3nPPNoKnLKdZyxujIfQXoLtG2j8Ox8wCApuPQIDM6bbSjaMG7cvQtsors\u002F\u002FWkL\u002Fx6dgJu2+pLdPMKurlyhdRzqJ1hlbTEAyYC3ZvjjT849Xo+OD\u002FHT70O+wTfLB\u002FhBbi6Q3kQYwYU7BPS2jCVY6Qad+gjJOB7kX5M3U+0IUt+H5riA9zX82\u002Fe4EoOepOLRQwg\u002FSZFTha\u002FhiL\u002FQ0iYJReLg9a1fUVXygj+jvoIdxwni8J8ENKH+8kn+H1BUBQSaES4HLyWsV73B83yvqAzoH\u002FXlXRCHqfGJSGcbaoUR4WNpheMds2+tVxLD8qMAzjYxdx7Ea67qpdELlKxAuSBiSpO+UG0wVbqlpHAmqiLraVRfv0kLr56NY7OwMDhBF\u002FwLfeMsy8X945xHiwmJA9lKRjioIzA5rQc5iHIpq0NuFZisWNo6yT2gaRbTD3sIsu0j+Tu9lh5AsvaV1IQBwOdYqlDU8cgJEgi3YJicO8wonafuz\u002FMn1ELBxYZzxiLzX7OfuahKwsR7bPYPNLCIalHe1oRz4RIxmEe2deUnJMFUEShZMdPHcSb4n\u002Fyu26G+vrxH0Ohtz24ugR3QKfKemiQXq7bZOWQiBp8idRws27Fc7BdcY3Ec6ZcAv6yUwWHIYXrrJNkxtGgOZDOWjHj\u002FJzEDzERsFIxryYscRFZOhF3BljhlFKspO6GEN0OturIzuLOLV6pgeaEw7QLplzu24Yxe2i8nn\u002F1Pcwu75vXH3W1ZmeIvG+dpsnNNknUqSTqnUtEIEeNGkYSMPO5b8b\u002FGyRGktgl7ZbdfkJFB\u002FfUChhgbnVsUMIZw4DNNBt9y4OPrKM9Ut4yilHnEhM3xk0HQhO+gL3E41Xz3z1wLTa0KDFDerz41yngeDzuO1+ED06+474WeN\u002FbqdDPAGCgA93GGBSxUb8AY7UbZvcA09Vox0BZX4OfSND9fE9LUmXDvFlypG\u002FErU+iLWTglywW+eDUJ3EBknr70KVmwsqQNKGfeCEAuoEdBW6Rj90hiKa4TbcbtKA4yorrDrZ+ge0uk0dEHGGXgnbGiDZsdYUVNPeVgtPeMXUEreYcxh27j+fuf7\u002Ft7t8PFEQHsO0sYORICM0Hp0aE0ISflSMXG+ppSYcM2Cm\u002FvAzuY9wGeQ4rHYOjD7zpch8ijjMVUZpcwA4kCI\u002FOBVnPPCK3ffaAfYb+qkmumnLL48W\u002Fdjq4OZQdiYx52SSW07WC3spy337344+Pd6zfP5DkQ\u002Fgai0f20yCASKuYM\u002FZEzDtwx9E+KVfA1+w\u002FQ9gTCiskEbOWM1DINuwfPEANaLdMWaYrK1ivSK4Oujv8umDFEfcHwu7GjZvLD0JNrQ2pzcNurTeaZnfX48W\u002Fd5xjd2JMZrWOovRjquf0breuXW3lcFDvbkm8zYZDUJBt9XN61hcOA42zf1rExNS7V63f+d7xAjclhNhQLDdv8FdscN\u002Fdx0urmKNSi43jaDaJZRZVX9jCyqElHjhAh32vtdAlyxmgXqkDkk8wXTxLa2V8ilvGHzYM2MRIJDRye\u002FNAXBNbejvbPxS5WLT2jnAnuAFysei6LMV+PZsytF\u002FY77hO7YtQBo7Q\u002Fny4sdtW95Uv8+D\u002FAku7c+1pKgAA",1794370566340]