[{"data":1,"prerenderedAt":94},["ShallowReactive",2],{"rendered-post-82ab1321":3},{"abbrlink":4,"title":5,"date":6,"categories":7,"tags":9,"excerpt":11,"htmlChunkEnds":12,"toc":14,"htmlGzip":93},"82ab1321","论文阅读: ZeRO++: Extremely Eficient Collective Communication for Giant Model Training","2023-11-25",[8],"懵逼的深度学习",[10],"分布式训练框架","摘要 原文链接：https:\u002F\u002Farxiv.org\u002Fpdf\u002F2306.10209.pdf 开源代码：https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002Fdeepspeed",[13],8163,[15,19,22,25,29,32,36,39,42,45,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90],{"id":16,"text":17,"html":17,"level":18},"post-heading-摘要","摘要",2,{"id":20,"text":21,"html":21,"level":18},"post-heading-背景","背景",{"id":23,"text":24,"html":24,"level":18},"post-heading-介绍","介绍",{"id":26,"text":27,"html":27,"level":28},"post-heading-limitations-of-zero","Limitations of ZeRO",3,{"id":30,"text":31,"html":31,"level":28},"post-heading-zero","ZeRO++",{"id":33,"text":34,"html":34,"level":35},"post-heading-quantized-weight-communication-qwz","Quantized Weight Communication (qwZ)",4,{"id":37,"text":38,"html":38,"level":35},"post-heading-hierarchical-weight-partition-hpz","Hierarchical Weight Partition (hpZ)",{"id":40,"text":41,"html":41,"level":35},"post-heading-quantized-gradient-communication-qgz","Quantized Gradient Communication (qgZ)",{"id":43,"text":44,"html":44,"level":35},"post-heading-communication-volume-reduction","Communication Volume Reduction",{"id":46,"text":47,"html":47,"level":18},"post-heading-先前工作","先前工作",{"id":49,"text":50,"html":50,"level":28},"post-heading-3d-并行","3D 并行",{"id":52,"text":53,"html":53,"level":35},"post-heading-zero-优化器","ZeRO 优化器",{"id":55,"text":56,"html":56,"level":35},"post-heading-communication-reduction-techniques","Communication Reduction Techniques",{"id":58,"text":59,"html":59,"level":35},"post-heading-zero-communication-reduction","ZeRO Communication Reduction",{"id":61,"text":62,"html":62,"level":18},"post-heading-设计","设计",{"id":64,"text":65,"html":65,"level":28},"post-heading-qwz","qwZ",{"id":67,"text":68,"html":68,"level":28},"post-heading-hpz","hpZ",{"id":70,"text":71,"html":71,"level":28},"post-heading-qgz","qgZ",{"id":73,"text":74,"html":74,"level":18},"post-heading-优化实现","优化实现",{"id":76,"text":77,"html":77,"level":28},"post-heading-overlap-compute-and-communication","Overlap Compute and Communication",{"id":79,"text":80,"html":80,"level":28},"post-heading-cuda-kernels","CUDA Kernels",{"id":82,"text":83,"html":83,"level":18},"post-heading-实验","实验",{"id":85,"text":86,"html":86,"level":28},"post-heading-配置","配置",{"id":88,"text":89,"html":89,"level":28},"post-heading-结果","结果",{"id":91,"text":92,"html":92,"level":18},"post-heading-结论","结论","H4sIAAAAAAAAA8VbW3cTR7Z+z6+opSdyZgn5QjIky2itgCcM6wSGE0iYxctZwm7bWpElIQuYyZN808XWDXxFlq+RsWKji2MjWmrJ+i9DV3XryX\u002FhrF27utUSFsmcB8+TcFfVrl37+u1dxdDEAHGP3rD5fVNB+4TkGnV7x+3s5br+ZtrmxN8hx8SA87Mhv5Mmt9lqpLV0xpL75\u002FXMkItMBKSxG7aJYNA\u002F9bXD4Qr8w\u002F38qi8w7vCPjjkGBvu+vNrfN9D31VX\u002F6JiNBF2BcSl4w\u002Fa\u002FTzwu7082EpA8N2xen88veaUA8foC0pgUCEgBm\u002FOP0BtyuJxDDj\u002FyVQ+xWlpVftF2pi\u002Fia9wdnHj25OqIb9Ix6R4J+KZ8Y0HHqCT5p\u002FySNPr\u002F4ezTFNu8XSRcfTbOXpdsTvw1hdva3KOlKvnuu7vn9fjgMK1W9N04fb+v5RdpcUtLlmkuwTZnetNVlUVNSdic+NumG8rozYiq7KvyIhkk2kGCrVbZyQqNpGj5ZSuUUZu7SPSZx\u002FnZkMftpNs1tZakm2utSIrGV7XMHHF5PPZxV3BCCgw5PG6choTZSpkliq1Igq3v0PKctpxHkkiehudpYZ3WQ63lUHslbsCJBn12l8dDtMwcbsb2SrT2hlZP6GYE+Tyvx9lGU1V+IQFp9NmIZJ8acQWDJiMOzrXf+Vj6\u002Fm9\u002F+hMthwn8ixPkfABZzopaC5NrhIYS5\u002FU4rVZoNq82ktpvZ7T2RpUX1eYmi0\u002FTSFjbLbLQwXk9rjaa9GyNZvNk8Po1Qus75Pb9H1R5EZant2g6DZRTJVXZZ6l062idDFzt\u002FxLoCw0Nfqwhj3vSHXQF3T7vlN03Zv9ZCvhszu\u002FaH4lvjHM\u002F5JgYRMPmTFL5DS02Whth7SynygVgILFCE7taZs7CySHLxlpH69bJWmZOlUM0scCXnMCJsvmuOUhQbTS15TxbKdPErCofwlGJ3tzQd+N6saAps9Y9yRNXcGSCTLl\u002Floha3aapNRotk3GP74nLYx0DFbxO02jlvB79eFSVE\u002Fpsg63twJy9LbqwwwX7hkZfq\u002FVM63VCbSRZfpduLbLlClvZYCtRLRk5r8dQvO7JcTIVGLlhc7gnXeOSQy\u002FWIDStz+slxQ4ytP\u002FlH8GANCl5\u002Fmn\u002Fy5h7xC15g\u002FZbPo9HGgm6n0v2W77JyWde9wgXvH3MF7Dfdru8Qftd36jksT8MuNxet3fc0X\u002F1hfTEbyMv3KPBiRu2L\u002Fr6bMTBVcNiIVXZp9k8TczSxC6Kia1VekmKNudbu0prc4vKMi2nzutxvdk+qV7c1YprqpxEo9Uyc6y0rCUjRoRDm14+Br8sVVEu2kKFhaZRSzQapvEauMraDjtZ0TZOWXK\u002FaybNJbgy4si6KifV6iKNJbTMnL4wo2WnheO+31cbWVVO0Pk6LVZ7WzOaL\u002Fpd22KnI3rxDLelqRm2Um5FUqpcI3eNo3z7aPi8nsExCAHEiCs4fPPiYWCbhwYRBYgZBfxGlENnZyFFL+6SQbHfxLWPGX\u002F6zOUNun+WRu0vJPf4RNA+0mEMT1\u002F8bHP+jzGHPOJzSIfBkCtPXzz+fMgxcc2yOY3x0FIOI\u002FOqkiRj\u002Fv4vCQY2kIHbG7wOzibX1FpYbW7qpWm0G4w73L6FK5InHt\u002FIT\u002FYnrilplJhxmIU22ewJBjAx79YPw9+Qn6SAV\u002FIQJNk6WmehA3220VsCE24p4AqMTLhHXB5DCH5XIOjmApjw\u002F2xz\u002FtUyxZDBfWMKuTLht5yfe8GKKh+CHc1UVbmgNjdpYZ2tnKryodUYIGTFSiz79rwex\u002FyBS2h4XqQZoW+iKr+wjSaOttZOUcTn9bgWq2mx39j6GS2sgxNZLPePaHw84BrlsaBL5+MdOr8tZnVpnVx5Om45tvCybL7LJgn6ldB7PUMbx3QpgakGdMeTBkRizrXejGj5RZDZx4q\u002Fc+\u002FhNaF9mlzU6r+iD5gZjKZetkLTNJ1k03s0l7CaFG5nuIcq51hh3xQinY+acm9\u002FDCe6xG2s7vosksW7aVY+1WpNrbCqreyDLrYSNB1Fi\u002FxvbpG9NdIp\u002Fuc+z7NJyc7FCB9szk7B\u002F8jHyffGuKmCpy8ef03675J\u002FhV+RvqtfGAxP+C2fzcjzdNw6eeALc8Dl8XxNBo2BPxsDF4EsOh+lsQTGSJvT+pcAXBdFysFRO8I5m3NwmOA\u002FzYhpfhEp5N20ll9EBnwCkFncAt2oDaW+fTTc\u002FsN04fYna4A1vt38nTXdEdb4rtbXwRBf59nGKVs1mHBwLi\u002FSMSQIu7kIUwUx\u002FzZ1yLP1IGHZEESulSiN\u002Fqot5xE58ngYbmVD+ptpmsuwt7vCaQ5+oeETzHqttWJrb\u002F2PmpppY\u002FagNDLhdT99Jk11m5tpZ+ShOacd7Lk74qbn9cyYf3DAAcCvVGWlZR7iCcvGEBbr8Vm6cUpfxbXfXtHaG4h+74u0PmMYHluu6s0NA+dmTH9rRpiSo\u002FUZwAqhOkS9+q+0nAJ8FqoLou+L+vtNlI8JW\u002FW9I6bk2MmeVlwDiM8DD884y8DTWqW1dopg\u002FDOQnYHwTZ0gULAKXW0W2bIBI5QltjVn8o5InYel83qm3\u002F7EHSSuUdekw+OafEJElOOVC+rVhOSCaQ6IMGSxUspAnaq80JptALdI38gWCH6IMBaQJFe+GXwQT9Lasrac720L3CR7GIQw0B6mYOr\u002FrvvWA3JFlDDDkuR\u002FACWf\u002FXnf1WtXvyL0VZzc\u002F+dDX2Bkwv68\u002F2p\u002F\u002F+eAZkpVEW6jYU2ZAyzDD6wpcyJRWmAaHK0YZyunePQPoRmazZvzaXgellsB4N6Rif7URhPcaO+lVuV0ulEUW0qojayVniovIIJEKxFVZy5jJvPW2ikobiOsNpbaqSKbR14Ak3A96WdnNFrWz6raCpCi0ddIVlPmrNmbFXI0t8rB73orEtGW82otrClh9A+ajrPC\u002FgfIaFFareAq8s39OyBWDAjaXlKLRdjKqZYOa4UYWq4l7BNVThKuIu1YUet1YWupQ\u002FDPzdlWJGHqwBQDGfc\u002FA7tTm5vayusu1GxaGtjghgw+zskj8FBrYeTrE+V+8Uwv7tqc+PuJLMFx59MXj83MgDgRQ51ZqEK9IO+3IgDbjSAkMKO1Ym8jxXIYTylOX99pRVLcZl7RaFiVa2zjlJZTcMzNNTjjXIWmk0hQW3yrHS2atTngj+Uz7URhK2VRdpSq2kG5DXDv3HsIALddcA9CQYzjeqlE3xcvtXob6FG9XSR9Dnon\u002FG3pC\u002FOUc1rtwBprunol83l6HFJlxTCSDIuFIHpZC69sHj\u002FqxTOai6jyAq4ylkTpmxmMa7y8NdIte7trbg1BJJtnb3dxJf9+dF6PqmeL9M0M+BMvINFlz+sxGt2gSo18ex8yk4G8BUEjogoIyX3dupcqF7BU7OiwXJLKBnsW3ICzv300TFrrFVZ8B\u002FybHp3NmwpQ5QVLvPsQmjGn8MU0nYQotFNliaL1zNYq+kNoRiSbbx8NO24+GgY831o77a5jCuugkrUKraVUOaRHTrFvBFvxRXKBhhOAWHgiRT5Eg4AnD3N\u002FTZnD\u002FS8oh2ljqYPP4pnWKKpyzapvJNqabtL5BDidBS6axQUU+e\u002FzIqR3gHuazevNAjRPlSW2uS04zObbaK+w3zoUfaKPjRuSfGqhy9JAHdzMIJVzYZsKMrCjAYBKLD6tKoraWLGek+YOsElCqxWWPKDRSpeFapm52\u002Fd\u002FwPbCpcaUa\u002F9GTOFV5dPxdkz53canaP8YoAbyllEnWotKswyE3JbNo\u002FyMKhSTPE6HbkRxVlXe0pRYSqNlaONnY5hRCOyysK3PNlR5QSvEaDoOPSaO6qAi7+y5CtomTuWpCXoZck6VD01M3Fm9aMUQdMSyZfAMqGgdkCa4fxS32Gqjew+omEt18Kf1M\u002F3lGvLJEts0d2xpIYOB7mjKNgkGJImw6CoJgMixbu6maBTXArsjspVDrZWmKv8m5PIqbsoILR5bdEYrDRjCFKYdlLTTEn0VF\u002Fxl5qhS0ZvbHfUQrzahyYkbA5MsutqTRcTrsGMoztZLaCeqHOLd+rLAIMczVsvBquFSbf+LnsEZD1tg2Zgqx1T5UJUViHlCjl0GIZB9NKLFIhATV8ss+YrWUiLucGMWLbXUS215G01ZRHxEmyJcAwVIY6Uq4poLaUKvExtj6FxGmOssOiA\u002FZkOaEsUd1EZCaxTZeokVfoHWXGaubXtdrROA\u002FKhNiMJoYRYOTIiMzFmBp0moHf4N7bYrJaUjXoDXlKosuc+WKzRa7iIu5GLYMcZP8fE41Jo1OlXdZzEhesdZRPpC9+DLcYimk+a+3En4PYGlWmjza20uAXq5iPWLmeb5DTUCMfJoXcQBaxO4w+MuyQe+7OEDl7T9n3u6oKa8psWM1bDIPSIM\u002FENo+i6kGgFheHIFVT+25B9+33Bej9+7des7q8XRclhb3mbRNLn7X\u002Fccj4kJI9i76ctOvtc7T9\u002FfZzk+r83U6iJe0WFzmTwdf2ytgbizW6EQngF87K7jMRF3ltEyueu48nhImnTe+\u002FxDaIbQjR1WgHzAQop50cAXDTlgUudSvg7W87HPyQ2gbQIeowBGBgGapnbMOEBfxbt7rRxeWTurNDzPdmS2ua83N1Q5xAr7iNTgrHA27agE3nVUMrJSR7nSI0ySKw8l75QvQB543CPQaPUFRiXIVp9fFPa4wZArd7zBgMvu9Y1KRDTYufouWmJ6d8eqYen31qG7QzMkHe1YavZlPloFohNbub1BKdCbQSxsLrxKR+zLm1c2p\u002FWvT9TvvudSwOPyQ3vJ\u002Fywo2V3e0c5Wk835N5wCPSaYQlze0c5+k0CKhrpEoYJeezwDHhw70Hfjrd0araUsnVnMbxip6zOssI\u002Fzuu7I9eR7mlrlrhBCcojYReKVEzQdB9DH7Yy9m4ZKNR2HEiddogt52lgCSMkXmukC7s4NasgAdKwaS+Z3RFsiyPCuKJYY4sKet3zE6DuoJrTdotrctM6nkTDNRWj5WFMO2y3S1+Y9ysd8YadTf58XPZNMWl9LcZzazSyLpsVtlDHHkCI+WbjE4PZV79LXLBtI0EdE2WAtGLCTI1C4qI8zH8OUHtc72GfCtnvXFDQjVFgXPQg1v9Zaa7+B9XDfgGbV2RKd3xeQhMcagV031yDf+N1+yeP2SkSt7+hnSwYD5mfkXa8s0FyGg6l2hGKxUCsbgk3NbqElltHyMa4y42wqTSP8mpWDaFptqtVYr+hnom9WeadtrEIN\u002FcLBW7kvHL4202iNyDooYHaeht9ZDLIV4d4zFXSNS1MEsGM8whb3odbKzLXJ8HGuQV69ntfjEKEewCMBWGNJ4LAMMlbHRE7ZHP8PVMD9ff9GCTzybNRlx+tpuGZpXw1OdbU5WTYEuCS+io9TxCXQqzh8L3OP5HkP\u002FZI3vOBwWETC46hqBcM0GGHkkBYzajUG28Ga5iy\u002FQMhDZYsXEuj7HcUsrYdo6iVajCof0lQJWtzpqN5Ma8U17S3c3iCxjwtGWlviTW\u002Fg2bzsoOmXYpR3iPVis7VWZJv7TNnVKycI16CHsVVhW3PYCMEihOYO0MKhfWXgBSTO4hFV4R1J3qhnoQNEENjFw43U2iJdaDs3toZzGSglF0IsWxIPvyJtofJ+sYKrERF9CM20W4vlWbZyinYpVm3yqtfs0ZTD6FnAy+5L+kpUQYhaoOGQjkPY5XthH0k0q5Sk6IQi43IBKOQSdGFHbzTQ3T4CTrQc1ptbmlLg6Sth5JxpgcCMzVG5bRbgfYJgQUQruWCq3uxXfwX9aiBskcUnLoOLW63DuM2Jv5+ABq15qCZtTvztSvLa7ltVqeDQeT0jFBafp+mj\u002Fi9V+fDH\u002Fr4+8uDvdwfJ4AC5fRMigpaZG7imyof3frwzfOcbMnz77\u002FYBDAkfQjPwrqi2Ybb69GaEzr9n2ZiY\u002FOCv33x\u002F3zS3O94xt9d90+UdPa9H79w8r8f05hZ\u002F3imCul6Z15uR63195PYT\u002F5R4K8QzBDqqpV\u002F9blEvrWjJEt2bhXPg7UVpTq2XRa43MUbjpaZkcaYqL3LzgwcrsOnxipaDhYLzdElbzvfTaPm6Kh\u002FeuWkwF6WJk37BEwyKf57XIdTibnozjR7CX8JYnh7SSBjbqKawsfnOXz3VcAJwsrFD5yvqWROiRu5AP4DqSTzXeZ+Hd2kclgHaSYsMaT2yAI1t\u002FxIH1JvruKlaC4uLQjOetDa3hMFa7NQ8EPLHKlF9K6HtLGu\u002FVsBQt2tarWlRAmfKNCZEDnelcVcw4PPaHz7jHah7393Gg3wx2HdToMtsfqCvr0+VD\u002FkrxwXxxqOUgvwDIyTo+0nygv6F7KAbYzDGL4s2ZISP7dk4VfQHuFS4ea4LCdWgKuMJrv\u002FCDeBpJbcp03ygebw1rR2VUBtw\u002FmyNvS5ZjQi24BGLHgO2BbFyaEePZzBiAdmtOarU0Bjxo1kcd9YGF\u002FgyvwXP2pz4K3z5slJvr\u002FeIl7V\u002Frxu1y9q\u002F1\u002FXQZe3fq\u002Ft\u002FWfv36sBe1v7\u002F4e5Xf6\u002F212Xtf70X9L0AHWjKkl6s8UChF2vt\u002F0TB3zbwdxdlVY61dquYKtpPP+G9OU\u002F67VfYZgLCIGW+NtZOFE3ZNl+BX\u002FQS\u002Ff8AxE4rAeExAAA=",1793389736721]