[{"data":1,"prerenderedAt":145},["ShallowReactive",2],{"rendered-post-fc5b020":3},{"abbrlink":4,"title":5,"date":6,"categories":7,"tags":9,"excerpt":11,"htmlChunkEnds":12,"toc":14,"htmlGzip":144},"fc5b020","论文阅读: ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","2023-10-26",[8],"懵逼的深度学习",[10],"分布式训练框架","摘要 ZeRO-Infinity 是基于 ZeRO 的扩展，Infinity 离线引擎可以同时利用 GPU、CPU 和 NVMe 内存，还提出了其他的优化技术。",[13],51358,[15,19,22,25,28,31,34,38,41,44,48,51,54,57,60,63,66,69,72,75,78,81,84,87,90,93,96,99,102,105,108,111,114,117,120,123,126,129,132,135,138,141],{"id":16,"text":17,"html":17,"level":18},"post-heading-摘要","摘要",2,{"id":20,"text":21,"html":21,"level":18},"post-heading-背景","背景",{"id":23,"text":24,"html":24,"level":18},"post-heading-介绍","介绍",{"id":26,"text":27,"html":27,"level":18},"post-heading-先前工作","先前工作",{"id":29,"text":30,"html":30,"level":18},"post-heading-显存需求","显存需求",{"id":32,"text":33,"html":33,"level":18},"post-heading-模型状态内存","模型状态内存",{"id":35,"text":36,"html":36,"level":37},"post-heading-模型参数量","模型参数量",3,{"id":39,"text":40,"html":40,"level":18},"post-heading-剩余状态内存","剩余状态内存",{"id":42,"text":43,"html":43,"level":37},"post-heading-中间激活值","中间激活值",{"id":45,"text":46,"html":46,"level":47},"post-heading-self-attention","self-attention",4,{"id":49,"text":50,"html":50,"level":47},"post-heading-mlp","MLP",{"id":52,"text":53,"html":53,"level":37},"post-heading-模型状态工作内存-mswm","模型状态工作内存 (MSWM)",{"id":55,"text":56,"html":56,"level":37},"post-heading-激活值工作内存-awm","激活值工作内存 (AWM)",{"id":58,"text":59,"html":59,"level":37},"post-heading-带宽需求","带宽需求",{"id":61,"text":62,"html":62,"level":37},"post-heading-quantifying-ait-in-dl-training","Quantifying AIT in DL training",{"id":64,"text":65,"html":65,"level":37},"post-heading-bandwidth-requirements","Bandwidth Requirements",{"id":67,"text":68,"html":68,"level":18},"post-heading-zero-infinity-设计概览","ZeRO-Infinity 设计概览",{"id":70,"text":71,"html":71,"level":37},"post-heading-design-for-unprecedented-scale","Design for Unprecedented Scale",{"id":73,"text":74,"html":74,"level":37},"post-heading-design-for-excellent-training-efficiency","Design for Excellent Training Efficiency",{"id":76,"text":77,"html":77,"level":47},"post-heading-优化参数和梯度效率","优化参数和梯度效率",{"id":79,"text":80,"html":80,"level":47},"post-heading-提高优化器状态效率","提高优化器状态效率",{"id":82,"text":83,"html":83,"level":47},"post-heading-优化激活值效率","优化激活值效率",{"id":85,"text":86,"html":86,"level":37},"post-heading-design-for-ease-of-use","Design for Ease of Use",{"id":88,"text":89,"html":89,"level":18},"post-heading-效率优化","效率优化",{"id":91,"text":92,"html":92,"level":37},"post-heading-bandwidth-centric-partitioning","Bandwidth-Centric Partitioning",{"id":94,"text":95,"html":95,"level":37},"post-heading-overlap-centric-design","Overlap Centric Design",{"id":97,"text":98,"html":98,"level":37},"post-heading-infinity-offload-engine","Infinity Offload Engine",{"id":100,"text":101,"html":101,"level":47},"post-heading-deepnvme","DeepNVMe",{"id":103,"text":104,"html":104,"level":47},"post-heading-pinned-memory-management-layer","Pinned memory management layer",{"id":106,"text":107,"html":107,"level":18},"post-heading-实现细节","实现细节",{"id":109,"text":110,"html":110,"level":37},"post-heading-automating-data-movement","Automating Data Movement",{"id":112,"text":113,"html":113,"level":37},"post-heading-auto-registration-of-external-parameters","Auto Registration of External Parameters",{"id":115,"text":116,"html":116,"level":37},"post-heading-automatic-model-partitioning-during-initialization","Automatic Model Partitioning during Initialization",{"id":118,"text":119,"html":119,"level":18},"post-heading-实验","实验",{"id":121,"text":122,"html":122,"level":37},"post-heading-配置","配置",{"id":124,"text":125,"html":125,"level":37},"post-heading-model-size","Model Size",{"id":127,"text":128,"html":128,"level":37},"post-heading-model-speed","Model Speed",{"id":130,"text":131,"html":131,"level":37},"post-heading-superlinear-scalability","Superlinear Scalability",{"id":133,"text":134,"html":134,"level":37},"post-heading-democratizing-large-model-training","Democratizing Large Model Training",{"id":136,"text":137,"html":137,"level":37},"post-heading-impact-of-system-features-on-model-scale","Impact of System Features on Model Scale",{"id":139,"text":140,"html":140,"level":37},"post-heading-impact-of-system-features-on-performance","Impact of System Features on Performance",{"id":142,"text":143,"html":143,"level":18},"post-heading-结论","结论","H4sIAAAAAAAAA+19aVNbV7bod\u002F+KXarqW0kHoZGpG6sq7qR9U4lv3N1Oct97ue06iIOlsqZIwkNSqRKjBEhItjGDGIwJU2wzOHZjIQn4cP\u002FJ9dnnSJ\u002F4C6\u002FWXvtMQmCwHU+XfIjROXtca+01n7XbfU7i7zxriYRjcatPFDr9oStW+dZkZbnH4sF\u002F220+p+dMe8Tzf8W\u002Ff239ItTlD\u002FnjN4k8uUHvFaXiKIHnRMn3y0O\u002F0sd398tprY2yXFKKe7R8V74zSrMbUmmJ5tLyxBZN\u002FaqMrZLzF795nuj5y8VvCL2dJv\u002Fx7QWR0MEBuja5X05X9iblbI4mi1JxkA5sSaVxJd8vlSdpelweTsgzG88Tve22CFsYHb0njyerd3bl0aX9cr5dIL6o2HXW4ovHI7E\u002F2WxC9Ib\u002FWmM4esUW6eyyOR12d6O9pbWppTHS2WUhcSF6RYyftVzuCAihqxYSFQNnLaFwOCKGxCgJhaNilxiNilGL5zjjtdsEj7auckIu5qTSL8p8T711XfHHfd0djd5w0Bb0e6PhWLgrbusUxUgsIoqdL7Oyo0fU11YP65W+tDy1YfHgvxrWzwN6GFbowtR+OS+v3qdzI5WVfnn1vpzKVe\u002FuScVB4rDb7YQmMvvltLQzSAydsg\u002Fowhxv1QRNDl+BVBpRShmLB\u002F99+3QnlZakwohUGJIKD5DocO3dAc+Z9oDfc0EMhqM3rX8RQ\u002FGo30su+QP+0JX9cp4ms3TzFl1c4VC6k5F2ZpR8\u002F\u002FmL3+A81ZmE\u002FLgXZkuk5Yn56kyCXLjYbgv4cdxzQqjzur8z7tOGvihE4\u002F64PxzCCXAxuBt5KCHPDNHtrcr9dGV9ly4mlXx\u002FpSdPcymOgcIyXd\u002FRR\u002F9ciInWL0KxiD8qdpIvgpGAGBRDcQGG3y\u002Fnqz17dCCDWEbSrSYz8lw\u002FH8DGNl8Pe3QgRYcy9NmStDNj8Rh\u002FmTB59AEVOmI2R5vD3mh32pvcr3Y4zWMZDyajqK+7ugJhofNYDMPRaG9uddtfG8MwjHf0oZQndzV6sXiMvzhQOwJh79UfusNxke2ssjclFRKV4V75t\u002FvyzJA8vy1n1ivrRXkcyAIIYn2bDk4B6bGHlfXf4PniA2U9odxdkoqD++W0MjOi5PuVJ8PVZFp5MixvcU5rq5lMPeFwwKTSqFRYhIMyvl2dXUBObT4uSFPK8JacAHYoL2zQ4vJ+OU2zvfLdTeAcrBOdWsU2OsnSoV+lnSmtp1RYq048lfcS8tMSTZSfJ3qkwlN5Yquju6tLjD5P9CCclIVRZSiprA09T\u002FRKhZHq7IKcuiWVHtHsg8r6amW9WDMOsBLWkWbmlbHVFxK8cT8ICovn4DOOJ5\u002FrsAFw+9VkVu2tPWi3+VwM0FIB2JE8vi3t7tHUoDyXA7Ct56XtIRKPCqFYVzgaFKOEz57vp49hCKlQJIH9crqaz1UmsvRxr1J6SovL8Ni3X07LT1bl\u002FiwdnqaLT3lrwSBVKxuZyv1VurhCN7Pw7hojmjWl1Cff3ZQz64zzbsuP7utNOmBZxSxNTVTv7vGZYup4hoXydY49JgEiFR4o0wWaS+OylVK\u002FnILtyRtZqfCAPu6lqUE2jhjosgrxuBgCRkXo7AQw7wtfXYQ\u002FzYQGs5mbA1NmUJVnhsgPDeRqA7nGRMdsXzWZUe79Wp38Fwx3\u002FYcGcv1qA7nO3tKerLKzTjp+aCAdVxtIB4PA7h2aLBr7kethkCNq4\u002FB+Oe2GXRnbwGA7C8rwFuzk\u002F\u002FkaiO+\u002FoBlskHWref9fzxO98lBCKi3V2Ta0VSkEmhO3759O8glx+xAK\u002FuAVEot6z1ps\u002FqBwRbThQa9ODlQ2SlaTLLWei4rCVaDEuE+0nr\u002F4jZVLtO+EQMDaFY5aP78Rj4pB0foPrxAQrZ+JYsT6lShEQQzZHI3XxY6IhTBJddbSardbiA0OjHpqAQ0Xvrq4X84Dpp0M08U9ObFiRDNwq9Qj5dEjqZAwNgAxP5Cim4OcZkujxEfkyXm62U9Tm8TtI8CoHj2SimljNzqYMfVxGzv5nid6D85UQwaE1CLL7fsvmKwepuAVH9S8jsOR7\u002FY1GMczoZ3oeOeEXYvsVt8\u002FnZ80cUybIB0QbqL0CQoB\u002F4+qND\u002Fy1NBcvzwzhOAgdfunaXqA5h7S7AaefBAL5V\u002FlsV1cFJD99hNlpaSxcNxOtW9HntxQd+SUCg\u002FqjF67NbdpV8h2Ix4EBzIDE69DzJlGcDgBOg4XY25ri3RxHLS5mSEzdxCDHZ3AfmEEoE1GK9VEnhYKytqQVBytYZeMV5rmuQaEJM8kaG4UdC3GEQ4SEg4pFUaJcUaga4TdwGOp+FDJ9+tolHYyQGHlcbQX6HKvPDdTTSaVsVUNA7ScBUPI1DItlfMckcVpraVRmABA2PLljbHKbh\u002FdvPU80WscH1jwxnbtsLsjdLmX\u002FD188XOmKX\u002F6lf+cH4CRmpIKmYOkYZzxeaK3sjdVTabp3o5ydwn+7lulqUF8W4NkpVSWCsP75XQAQHSISKu3L46QwEca6j\u002F+hGGH7txBNKo4JLDtxRV5Yothk9kEbGVSISE\u002Ful+9v82U\u002F1tSuYykFPD906mKLmV6nQ5lpEJJ\u002FlcPU5dWuBWECxrflp\u002FclSc3pJ09MC8+7RSCRNNlKnvTlftp+dkzmkspv+3S4jL20oiK3k6jKgSaR+8qzP7Xi47mg\u002FoQTQ\u002FII4\u002FIXy+6nIxHDa9Wk1nQdsa36bP154memvFghul5eW2Ri1KE2EyistxDnPaO\u002FXLK2S4GPc4OEB7tNjHocXfsl4eIpgcB1SRKRp2Ibj6Gg+102wnAh9C1icpw7+Hqq1FxU1Wkg8+OUJFqVDSLp+aBpiIhSpXcYGXlF6lQBNsjd0sqz1cTtyt7SWV1RCqsVdbvK+sTdHeCpjaVfD\u002FAZ3uLzqzS3OjBxggo2DNrjGYWiYuhWDhqZih1eBvnLTghYkOe2DIOifwCeMfAEumI+TgouYYxsAQInk3SRBnQyRBMOmJ6I2XsMZjAPoLyE7hKpk\u002FaAfubC5B6q0JU6lSuz6szoc5oOBLujvMd0JlVJFepsKbTZ1CIXSXI5XRNbaBPeToP5n5mnjjMw2rMvC4bR0YC9jZKoiNlFkdaapCmHtKNbVocO1x+kcrenDy6xMnTfZC6zDNZapTGdpvPjZow0s3AI1oGdnOmPSZ6oYGnXfwh5GmPRYQQ8QaEWOys5aoQF29YO\u002F2xSEC4aanzrt4za1CI+4IBi6cd\u002FiA3goFQDI3RP9ls169fb7zuYsajo62t1XZBiPvY\u002Fy58ZSF8orMWZpnB2GJQCMX93pinPRgNX\u002Fe0B\u002F2ev7Xbgn5PezDsOdtuC4bZsxv8Way7g\u002F3+jv\u002F2e37Av2z8VZjExIgQFeLh6FlLPNotWjwN2ihfnmjkqycY+dsTjXzNPHIsInhFEvCHxA5QcM9aQuJ1+GVhTeClYRR1VBVaYW3Ubu2vuDo+a6RtwLC2mNZW799V2z\u002Fo9wS1vwTtL3UFYRKLR8W413fzrKVLCMREi+cjPkFXVPDWwWisO2LCg99zSYdERFtwMPZDNM5e+7TX7IlNHbjO1B\u002FzqcOe\u002FxkZqIMX\u002FelBfIRrMO35pBaJfGVCKBRGzxMRQ94wHMqzFiESCfi97KnthpWdmTN\u002FO3vju8s\u002FNHwJ\u002F1xt+Bb+uUa+\u002F\u002F7Mjcs\u002FhbvjP5OzBFydQeHGR9\u002FDln7625f\u002FvPTzT9\u002FDNn\u002Fy\u002Ffzzx997O8Nx8i3+893lMPnkxpl2mz69p91mODo2OIfwKCLUPd++eDBgIULUL1h9\u002Fs5OMaQSsKlthxCrfRSLR7vjFhKL3wyIZy0+0X\u002FFF\u002F+TvbG1paVVDP75mhiN+71CwCoE\u002FFdCf7LaGx1tbrcY\u002FLOl7lqC4WgngZUi07MAWdRpxQhemzMoRK\u002F4Q9Yon9mJMx8yQVQMWIDIX3nUOiMcHzptzW3uutBxtjY7jg+dG4e1shzd8ZA9OlytbWzy7+rDpzsS6+6oGfpawB+LW+OE\u002F+us+zpa7+lBqDiaHJw0WGP+Oh6O\u002FMnqbGxqEoN\u002F5gsOiF1xRku44Jpt2JsMg\u002FAZI3UR4WxsORTYMf+PYN9ExZgYt8b8P4rNBP7vIsE49D0awmqjQwBtdzUhoH+ooabDaQsBGbN4\u002Ficx8sJmx4S3TmxHLOKFKwtGukPeOJN0Jz1UjubmwxFwTKK1tzgQ319+mKziNTHSD4xVuFxG0v2gWQUn76tvlVU4mt57RuF0Op1sjG8\u002FUEbhcrnqM4qm\u002F7Vs4n+fRnHtvWQThx4dYrBzX+VwNLXam1\u002FmcLyA7A85L787XTtxO8eka\u002Fu7RNEnoHUwul\u002FIs7TG3SdpHD8pOb5Lp+VDklvORre7tb7canMdX27FjkMqh0opewuepq5jzRU\u002FVqug8CoCNyKGmKvsmFwpIoZIqDsQ6BQD\u002FqA\u002FDok6dfuCI+l3Y0uOxiZHq+tQtuTQqObFTOdozNfjHuAW+x0ZbpuLK49mir52BcnVYtqr6zVtU20dEVislTNzUPQOTnCAsHxH8A4zYlrb6u3tpCv2+TtFa1zwG06ZP2RlmRSw5iboqRGKXaWEa1cOjw047Xa7LXbtipaP4bbbxaCF4ChnLTiMhVzzi9fPhW+ctdiJnbjt8B9x2FvtFhIByRS9Jn4ai4je+N\u002FBLXrWcuOCP\u002FR\u002FLvhDJBbwe4FHRSBC0XnWcqGtqaHF7jzjtTobWxrsDdaWRkdLA\u002FthdbgamxqsrV5rU2Nrg7Wp0dVgbYMnDrv6h\u002FuM195gdTbY4Z2r0dXgaLC6vQ74BSM4XY2t8Ke9saWhuQV6NLnPeN3uRmeD1QXNm3Fk6N7czN47sDf8D3o2NDVYnV43W1prY0sDm8PZ4LCfiTlaXA2ultYG\u002Fq\u002FXzhq52DJbHA0Ou7vB6nS4vM2tbC9uZ4PDxdbgbG1qcNrZdG5n2xlvcxu8dkMP9trR0tjS4MAGTqfjTMBOrPYzXr5\u002FNr\u002FV4W5w2hkA\u002Fh2Bf81t\u002F\u002FdWd1Ojs8XpPhOzOp3szwZ3MwBT\u002FxGzOl1NDe7W5gb1DwQ87NDa1gALbWtoOeO1NgMuANCOBitM6IpZHW1uWDHMzv+KWZubGtwtDfjPj2cutLrcpNXu05bks+KfP1psQL\u002FXrrwtoW83ssGXVZJN7KbR+Ro4DogGK1O11S4d4WinGLV2hOPxcFA7yXb3oaLfvKjmlpbfj92\u002FOEbw0jbrixyKR+j2ryJhWt2Ow2Snq9He7HqXdPjjWqWa+X\u002Fp5GT+NhXzNtcJT6gJRN5AOCYepo4d2oGFJ0+sias+rbpdO\u002FwhCwY1X3ncVzK+mzWN5S15794LKL2s\u002F+7UM\u002Fdu8EDdW\u002FGhOhNefIw+eduHyO1SHWMnMr6P\u002FIdlRNm0BClbbV7+3xrIlw3kW\u002FjQiGXDSXuzkFc28JjmeOYbucEyMacw19CY77dfTt8gMZ8QEQlLmO5oIDGeSo\u002FZZ5B1jx8ydMRY1i5LWsSsN3jqNCa8Rep+PQOfvOT7tVnW8DsaIrDvzqbnIbUQUuJYXh\u002FLJeU5bZBSRjAhDfpjWgg7GKSy8LA6tQjfyBQHWTtMe5e2J6VCojq1yNKsx5X1hFQcrPehjSnNGxJbaoBHAKb75TSuWc29NgLHAAb9Yw13R8x3Rs2a+bgWIckd+e4mRwjOmf4bhwoMLeijf\u002FnPS+YXvgYSgxd\u002FO\u002FAmxt\u002FUwcs\u002FnUKdpHaejKilUtL1tJroQyBrk6Vykpg3HBXhOxe2AWxJtK5pDiVz0qKK4FG2O4KfoNRZ2mEr47NcI1JhGBCu5yvC4LAg8kf2yYwJU3yledwwNL2Gv3yYa0tzKQ6MT+DhofPyr1Dwy4bbaZ4CqW\u002FZ+N58zrYQp\u002FvltNOUdaqU7sDkavanq2Z2c6a2PJSo7EFS+cFUTVN+bg0sHQ6Y8hPShLs3JQ\u002FXyc4MBiIW+IDkQ8nDvHEgp1HNnOuqyUG8omXSidpfgdq8xNpsxMOzB1891bE2xS\u002FkcbTbgqEjZ\u002F\u002F4kPTAkMdZ01dLD3zlhZ40rfAGOUu6Lv90RQx0\u002F0w+4smE3112fPzdZfikij94hzIFX1Jkn0DDeOeDII5GO+azvERC4Euq\u002F0eHPN7B\u002FB1c8Duj\u002Fr\u002FGkP2VkwQuxRM0PmxiRxvCIvBS4dW3aeS8pozFk4bXToP+b+60nAb9j0PA76vbx2V3vN95m\u002FyN4\u002F2noSO9zqcE9pYJzPn+E9h770w8TRp8xw7Fqf7wyim2J3dxQ06AEL3s4F493XfHPWLMrbpfzreaXvJezgMvdP+n\u002FvgQVxz\u002FPvpI\u002F1tbrf8bvptfpovjdV15xkohA4\u002FVj60XjygWonoXaS5VWb+vetyZu1tfFXM8q9PX\u002FyL8qE243Ic6ESMe+PD9\u002FMVLLqujpekcrx0BxZN2R8C\u002Fyn5WB6DGBV2GklB1vuPfL6cC++Wh\u002FXK+rVnHg8Gdv19O+bCBw+lsbdXb6LBDH39KODhOTPyhWwzBx8li6Erct19OxbCR0+5urcEv7UlW1nexPEfd8hHVvh18Szq6HM1ETo2TroijmeB3+rUf5mfmoQ5Lx824GMMSHsY6GgeA3GQn588heOJCRwDCEHHYFvwbZT88HULc61NBAoUj2m1xH74y4g8HJR+Z8fbxHwNHNbcZF4ePeHMbm96mLaYj3HlTW1QnqJrxTvzT2dLE9qD+bmn7A\u002F+BY6hdmt16G0dLYzO5ZOjUZLfb63dzOFv1ZpA9ZurnsDtqO9rUtdpUiGpVK2ZGaGrzMKLX6pVUE\u002FnKXlIrfUG8PtF7NRL2h+IECwLKk7uVWxNYfZBDcyhRnUnwym9sKGlnUCr9IpWeQTxrYY4Oz0NUCnxMUJuxmsygk11jD1hkzAsBN17frd780vZIdeIpxEEuGU\u002Fy7IRayIyPtrhC50Y4bpd7yaXuKDj6\u002F+Or88TR0ug8x6rMnL94yeoi7ABjLAvLqlQ2lrDQChZU0YrOQLEXRvD1AWMGAC99Ud40sCfyRxKweTUOUhih07tEIMqjR6SF0NQEAOrYE2g7ZYcXCqLlXU7grsAv7E73fjnthT8hRjO1o\u002FRvKWP35FQOCozUHX1jjK6n5bsAWhN9bGZpkoc7q8msUlxB1EJscmugspeUCkmpuMfPD4tT6iU5pVJG6d+SJ+blJ3f5zlhxkwOlVd5kNTHnIdXEDq+bxyvIsAqTuDFrMHY9WFODz\u002FCafHThH99d+FgrGQOIneF7N\u002FapLDwkYSwOSSrJJwQKhQJzxUKhuVE4kDOrqnBZk4dWKvfTWHAGmp377jOdKqBK1GbWWA+V1xZU5dWoVBgGgvtI+JjQxG357iYQnhMIj+5OKPeWcDbEKnFcJVJxTyvmg4sxVoWU9xJ0YQ77VHuSEB0FES4V1uBwop5BlLEnlY1niHq1ohjRBoWyNHtzSmkNF8yj3CwMXtl6QhfvaQ2UX4vViadMvI\u002FI40m9bmohoaxkoCpp6iFUqblwkdUUwtO78gsdfMLKVD6oTqxXF\u002FiRqYtnlRRNSBYYjuu9Ih99ehC\u002FyC4YVqTtETqUQeDgg8KaxvHqMkysYkWz43TwiVTkFS5x80fzQf6WBdu1KlIsxGrAN5YQg2I5Kahr8+l3F4gy9pgm52mmRyo8MJ3Hx70cO2rhPKhiudwjJ0qAWOjKZUbpfmXriSY5pEJGgpquQAlGpEmFTGXriZLvfxEOsHqsWoDU+EsvvTQ8X9nZIXXq6dKBJ9WxdXlyQ54Zqk4Bl8MBADO5ZamcpzuP6Z2MWlUypYwmobLaeBKOVTZb2duE6s+sC93YNjZjNfWgL7B\u002FJqOgQCZ7RcSuLr\u002FXL4a8N83cmNMww3Y1kZf27leTGZqdZzWT2BlmtFR5eq+ys8P4Kcgd+mSTJso89Jybo7lcNZklETF09XI8gnXVoBjmSokOr+JSScd1CKivT0CN3nIRyo8J\u002FjipbPQzwaNvQl0evAUgJUp8cVDVbJT\u002FjTWj2PBAV2w2ncw2xpTRJKxChZa8kZUf3TcuCo6rOiy0nH5afTjJUhZgUfdXlcUi6PZs6dB2uQf4yPRTaWdU09JrAUvk0VW6UUYt+n2JwzO1i0TD18ErwkLC9kZnE3wU4Q0HuoMh5rE4a2EGPAGzXn2htxeDsKh4FP7XCQFrsDdJzBv1R+IB8RoUArZrq1B9Bhjv1YLZ3jqlgvSyQJHDQ93GdABmcV+DuHIoDsWH0VdyuU4Xf51JxNrYOawU\u002FmCbesmd8bHwh57jUFO6KF5n9\u002Foz4WC6w4t2+irgFI6Elz5eyAQvbS+Rg3jRhlRTGa7WpDLEa5dWN4FBLcxUH0M2ToGvRIZCnR0fyKk4pYuXoYvXvpfOI1cmHHeYYJ116dsX67QTa7dah0h+d0o9GpPBOlsJ1cGa92Wxe8ps3wtC1DbdoTW4\u002FkYp9YNA8FvkmkcIpGDY898TWgnBt4VgExl21fmrHrPRn9UjYb3dzbfHJd45hfQdXJmphuUHJpA+PLb4yqzkNYxkopi3aynY0PQ9QV709x3iFX\u002FoJ2YRi50\u002Fn\u002FGGg5HuuPj95bg\u002FKJJ\u002FE4KRP5OzBEutxsNxIfD9ZWzChvn5p4goXL38Uzzy889QpBUcDS\u002Fuw591CnHh+8vBcDe7lYj194aDwe4QX+NRa6jp+1PHddb\u002FxXPDCr+HcWOE9zH4OszdTaD42fyTfELqrLXOGszzmX+ST4gOvjPfi6FOHQ\u002FvSiK6o6nRabfX+wKzpdHVZD9ZEjYjzpqH3jAfsX6k\u002F7UkebQ0tmpLPZDk0Vb78sgUDlej60UFJl+U5+I9VlEeYOPHaBU5Vqvj5HMct4LPoanWhyZWO\u002FknAZdf4yr8xwOPWJvSYehjJIKWRkerq\u002FkNEoHgPxkwXrB+d6O7zfUuEvHx0HQ8+gwdD2beY7U6Xh2qYxLjsVodD2KhD\u002FF4ORsdDufx654cizyPntLe6HI0v8kDcZwvUF5PpbXXU67tdZ6l4411LAgdTv\u002FHqVGjfjsEAcHjEKazse21s\u002F23kP5oVMXqzf7CJelKWODDVcIOafwufCT5wtW\u002FI5UMdbDXYfEuU+mSoxH4Sqz2eDrv8djNoWrAS1bJOlCX3EwV71Yyu1Vb8HuZ0h6PnJTPvVsfX75AaXpHC9i9ofJ1x9NTj6dTH2+sY1kEL\u002F3x8GtS399z58E7b3W9FRbS1uJ8Fe3tWKX13gEHyKkC9oEoYKe8ue4iO984Czxce309OzqeGDkeoo8nRo6ntZ\u002FQO3U81n88fJwqbaeM4VRpO1XaGtvamt+yzvYmgj6nOtsHorN1vAqjdDajX\u002F36qRQ8lYKn6vGpenxM9fgNieLm1rctin+PAPepKP5ARfFJc39OgJgXl4+CTNbXO+yppnGqabxbmsapvf0B2tstzb\u002F\u002F\u002FUNHn+s3kVN2KuRfXcg3tTW9fSH\u002FnvOg9zPL+7VqSq+j0OZrIZHTHOrTHOr\u002F7ZruKTt96xT5hvS8tx5X+T2Swk\u002FVvNev5rUccVXiqS\u002FnXfTlvF\u002Fq2Wma+Wma+Wma+dvSK9\u002FYPemnHPylvfFvSiN84+nRb2RjL\u002FXV3gn+qXd7Qb3Crz90Q2mJrpvwt+CPW\u002F0ha2fAGo8KfqjXbPH8TX9PPv3iEvGHyGdfEfW9XoTXWIz5drp+lWuoPZ2hubRalZQmylCplNWB5VXPWVlTLJ5a2VuDEuaspLGxZirU1MZSvqzu\u002FWEza1VUsTxqNZnlJYJh5moyS9Pj5nsBzHPk+VLYImpWIBVKleUerVywqRg6rheL+BIoHEuLWZqaqN7dgwKxMVY611hSP00zT6A+OTSFWsXGMq50cUUpLsM1v3\u002FFN1JhkSagFq68tojXOdQFtFr3WJ65V514yq\u002FLZZuvLszSxXHYztovNHcLZ2KFcRNSaenILRdJK+mIRfRLDnht6ttpeWGDFpf3y3n+ZGaVrRd2qq+iknxCN2\u002FxS2pLo1hGmFUUhuLVUDhbKizKj+4DRNYmYQxnhBzEIVZ1xpK99YmM1bHmK8luVPp2cE4zTGqggQuBfmqpfHxFHPXWYII\u002F2zvdG6jeL+EWYXOsDjiru01Tm3RgS5lel+eT0g5cSSEVEnyb6lxYH7ruVNCMFTWuATYffGYVLzRW8v3GKfbL6YMIQITWjM8uiI5wKmyN8Gs2YIOHkwK0np6X1xbZSZIKozVrA3gOPFGelPBeZZ1gpPIkTY\u002FTqVU8rPvlfM0TI41UNkpwwG6n6eAUHVhCqAFg8QblOvtwhuHGC6W4zH60i0GPozmi3Xyc0spyy+NJxoA6hSC0fzRWSQxU1helQhFOmrPdJgY9Trvec0jbMJ1Z1cr0G3lUZS+prEKN+P1ymsOkZmdGgNjcZ+pRLpwffnJwpPpHm1Eav+U5tVmDdbq9ZaZz+ZeE8nQB8FkHZKN1ZzChz0n+SLS7E\u002FTdHaOj2+c1XXRST\u002FZ0CKFOphpao+IP3f4oK+cUs3jOqc\u002FJ3w3PNVmjjD0GrgtrYawWIK3d+1IjCGrkT3V2jhYKmhTCathYChuAZBhBKg3oDL4jpuHDhAytgVe76QYWVxiuzi4wcMAV1lp9f0OtKW1urLFe2ZvUx2JVoVjNcahSnhhAZkf+49svPvviU\u002FKtw24nn53\u002FT6uT\u002FKM7IkYvfv0ZqU4PKruLyNOgkjor103X56oP0voFH\u002FALeQkr3Z9hxMweZicZOTC5NbAllcaxajqv4T4xX53KsRLzY+rSoIq5khusrBexavqbvULCdcgVEqocF3RJZGRJiGp2v0IL3HpjiyHbk3b2WOV6uMAB+IJJOEvb9\u002FBSFro+p4xuNtn\u002FIJWW8Cp4LJbOr21g9xzgFFJhRCvbTzp+JHiZAl1cqS0iz8bVCsDT9TQdWNUq1WtTwv0y2mw6L4UbR5TpgrwxRgRobmY3yEKrDydpsljZu0XccOmLXpZ\u002FKFPp26ns7oLkbbL\u002Fgegl+VWxVsu+akS6Urojz96TJ7bk6afy+CZyHqkwBtcN9O3oRe3ZbvRFw5UoNLdxmOjmd24wlBC4qaJInFfhBntAln5kpO17lb4dpfRUTvew5deg5HmiF6TFxDLxEX6fRisMYzxwdHOwOpWBWz8cjBRwjBEzx3Ie5Fg\u002FitGw1a+ScWV9t7J+X17uraykLB4TiRPju3abz8lOIegFeFjh9KEMn1qVx7erswt04BkwqIeTIN7LvfJcv5xYeZ7orU4tSsXB8xe\u002F4XctMPxU9iblmSFUovhlL0s0O4yHldFaSl3JfnkIyJpd16D80qNMT+LVEM8TveYVowZOU7\u002FCLTx7U1JxGpehrROk5YMeZf4pXCVkuOrklx46d5tBHe7UkTfGKrt90MT1GaHbW3Dhwu00Mc8F15mpJMivT1lcQfZbWemXV+\u002FjNWPy2Iac7uGXhx1kgJXhXqV3mzgIXoOjXW0CC8WzoS9BuwOpyQ6H4c2yLLeZZTWpLCviURaLcn+WTu++2QU1HbKgelK6U4xBbRgYsjsUiYpesVMMxcVOawwGtng+Y+9JVzhKvjG+J2xiLrVV5qxRgHrzjhi64g+JrHcw3CkGSCwusLvL8rBdorWn94ogHBkMXPx0AZFsbNe52YdJQpoapOkicN2tVHVqkRPr4IS0M8VMSBgBCOKQJcHgm4M4OA5lnEJrDDzUeHMQzJcaryQG5PltObOOsyLTkfZmlbtTwEvR3BmGa2c2nslPVuX+LJOwoDfArTFw4d3tNPurg2uDbc1wdQ9S\u002FH455WhyNUuFB0wfZucbTwQyF1TYjTe0cCGEVwHBASKmm6NwY\u002Fl+4w51tg3b+5pvF\u002FAkeOP+a6w2JaAJgKTd0oP389TChrOowpo+JidWrxiKR\u002F1eEvcHwMEAo18PR4GsSZC14HSgcw4Um8gYpFIJDg+7fQdMoPUJupbjt+Hszlb+NU4uXEQ04o1k2ICmBisrv0iFIlzCM3qPrvDnmtlOU0llKIk38KBJRteWQcKsDUmlZ6D77\u002FbBjMU9ObHCx2RGPlzWl8gBmkafofFSTQ7LY7v8gj4cVptFVU5eqCYbDqB4wysGAmIornlorLpWaTqKn6stwTvBWpLPDRfmcFUarwUCLHLdJUPT29WBzH45DzT6b1fif2YYZH8APWn65N4AGNWs1345b\u002FakHPQMtNhByh5lCjoamy6ZmhxiIzmsbmIYi0PNfRBqfArzUlBDsHiOeNlu87nRpaXdc8UaS4VFuO1qfFu+n6rZLmNN2s06NHWbpgaVtXHl7tJ+OQXXEU2OKlPgXlBmRuRkL5jU8z00N0oHM\u002FLskjz9lN2qqG4ctSbj\u002FUUv3KuczVUfTtYAVd3rES+1vRq5q3rUtQsJuUrQk4e7KfP9KPpvpw1Hm1MP00H0dvCee+pml7h1oq5CUxzRFgODIt\u002FPr+xifEvlIejP4T6czUF+MVJpFFtJ5Xm4itTEZ7g2Chcp9tJcyvAGkDM7gfzi8PWAo0EqJODEzk7QmU1syZSuKbjLKjsB87Hp\u002F2fwNg6v4h60r9SWDrnFvMa0ES6cKbP2+l2EcEkXs1rAXwVzbJQryafGlS+MKkNJZuemLt68FI56fcTZaCcajcrjm9ptZtgY3BQ5jQzTNDdanV0AQTmxBd6gmRFlZoTRXeTIA6QdQ\u002FPRqXmsHxrU1Ziiy\u002FRJcvEvX4igXhOXqsZXE\u002FdAx2V6Gbh5BqcQqxyNGvRU5d3+B2Kwt1J0Y1sqjlbzucoE5\u002F10cUUqjrZ+yX01hosxayyM6sItZTSpGXV4MoyXdL2I\u002FQox0RrusnbHzHrP50IMKJZ8ExM11oqEjffiwdEfW5VTz6RCRnNHcguBiVx5YquS6KtM3ar07dAch412LZd+4R7cS4cIqEEsYH59G1QVZjUqpUHtulzNLu6Oh4MCKGZQP5sEw9eYX2W\u002FnOeaf+4WUWmLruVgYbMT8pNVOrAExp8NDD9fOHw1RioryzR7S7tGUB7bqk4PgsKCatKdjLQzAxe2odVoM93JpzrhA4ErQtwnRgn2RgoAhSiRru3HzEJOTQcXYDLyVZWPZjeqiSF55FeQ\u002Fet9UulRZecR2N012prhWl8NOqiGRoRo3B8PM8mpgyg9UPllQFc4ZyeUx3AxnzzXX03Mq7c+w41yXOSrYNSl\u002FswqTc3SlRGaHtfchkZ9ld5Oo3\u002FghbYo0i1Sg8Vj\u002FKXZm1JhpDq7oMqvYamQqAz3IjkxvG1X9qaV3u3DaV\u002F30HFNzYpw8YcxOqR56qx\u002F4ZrcRcN77SzUGH9mpoWolyfn6WY\u002FIJO5LDXyBj7A7pLknLOO3EGmW3PJIWweBRrRzE9Vsmn+P3lyY7+crlEwNwc58h73GlRCkOk0tYm4Zwr3RQLQU\u002F2+6ABhx3qvOrGuXcnKvXc6vVcSIPPA1dwRDQudXiEWh+6j9+j0PHruNH0q4qE7d7jE28kAK+GGAwpFfQC2RX0K1BrkVE6eeYQBgv2yxk3g+tiaURlk4Uiwo8HGrtFwlnvluRm+lY6oEFaXja5YDi+EFPNjMS8oiFAdTPLIkjwzpMdmmCObDqTMYmIUrP5ijodKNOl+3ijdlf4tmhtFL5YZiDOr8tqidqxo9gELObJ1sPFVqdwPi37Kbk3k+me6kkhX9ubk0SWM2EilcUQ1mxrwzQao3tmtPNuo9u3gGNWJ35Sd9Vp7yQQtDSsYZYMRa0bKpeWJrbprQrhiS4QFcdg6I0QjS1RK5PlngDkbakQFTiXq3ZFGZy7cEZrPKrMJAqMw+4UuzFXv7tXeST6zShdX0BPDb7JkVKFdzqzJT2l3WjegYV3oMjKcTjjDu3ANNLD1ZjdhRGFw30iF4YNnEOX\u002F6DO6O8G1ANclrj0cOP70dlq9NhdtCKL5u4x6Vj32Fr4mRgNCRGNuKOotnq\u002FxOVGZGop6jZ0hQwIgZe6q2o6cGsdf6q7WNJcjXsSMyp2cyoHleWerupDSONbzhHazveaQNiPxwOGGyTezBmc1Ac1oete1Xx6SCiPS9j2mRm5zsVBcUcbugWGxuIL3qWq6herxRbZl9BkY48ZSYQhC7WtL1QeLZtZg1IPNLgN4eN748Lz+kHykbejjWtpjp6d6v0iLWbyvFf0iODlzoqOky9PdFH0GOkOtc1GVL4Tj1+DRSQ\u002FII4\u002FUzXfeDAlBv5dEomKXGPeyQ5o3ah+c6pd7+B2x7NDp1wszLY3fKasqQyi82XW1c6DVPbtXKTxQB\u002FwOlj\u002FXT0vFmviq0YuAeny1bxVF4n6Z2R5snJpmxrgfrAHD6ZknzG8Fq6pxbEBLdg5r3nIjDSPnG9umEfL9xAVnFhtLu7Pyo\u002FvGjZsMof1y2oB5fs279huAsnNHAy7N3lLG7mH4DufWFXNmjRl3i7BQEWe6fYUIoU7NXOQYj6AHyesTQv5YELBqXDLeCT3KrX6cS0Wb8rgEV8czkuPBHQYdjHuwFemAevabUsrqgHIeBSjjbCqTVu1HfIcs\u002FnBupcUd+GatSNYW3amqKjifsxecX9Wz7TpFMRK6FmRWjBhBTZhbcH\u002F55BPEaWWjZIPge\u002FEO7LbcK68tQfQbfWJD25BQgy3Ah\u002Fm4F+mUpp7J45uQXaJeyQ30he2YwrULEdFcWl5bwm5AFAM9lfUC3RzEqAM+N4KI81cTyJANmuNMWqhTGWV3JI8uVfZuYYhSEwnIWNDs1AQUsgW6sU2+sH1NcAFwavBYYXR5qgi2GV4dvdnH3Frpas8eHchw63UxQ1NbLGGByabpIl3PH25dR\u002FyhkNhpRS+nNSiEhCvMHLMGhJuQoXaRvedeUKK\u002FJ+y9hi3l\u002Frq0N2v2RnBwE7BtEysQK1OVbC0EaUojmFnFxXLvQfkOHfyNpotSYY05qJfo415U0pgAyXBeuHmL5YKkQJfL9Jw\u002Fh35ofpn87JKyfl\u002FJDeoXghtmkHanaXGMBznBpfO0Jo1LG\u002FUSjFrrTmZ6jlFkmpdtWmR2ozr5m7y2wPqdr+dOgVAaRBpVy5dbI0yzwZCccek8gAsZQxCEZFEqRrwzq+hKpjObdDah5VWhqgpG6MAWJrJoQYKJeRCwe9PgcFpbQurRfBImxaWO7We08i0es83Pbb+64RN1l9wFyfoZo7fKGISg5bl+qVSSdu4iUrDx4XyJm86Mrwhxwap6FiyeT7U35DPwOVzgbw4xDLlrJJOtbPbh1Ci3nid6eNrT7drItDGiDhthdI2HArkJp\u002FeZ1crCQ04UQxkegTcZFpoZp+WXoe8B4DG+SVOD1YEMClyV8DSS4VCCg6IUV3SLf2eBlrN49Tqj9l\u002Fg79mEsjSnlLLKzi2lNAPNcv1obcqP7kMUlSnL++W0MatQntsCpBTWaHqA5h5KxcHKRoEu91aSDzB8RIengfUMZarLKT7y2pDmfoBzbGypPqfZlDYgNwce98I4zADSmqkbNmMLjInEbbpzG2g+d0sLImndpMKanL0NNI9eG27NJx8A7NNgX1VnEjU5b+g2MvpNMPGgnKArI1wdY3jlY0rlPGK7jqslC5kGUnFUO43gWUtNoT9I03CMrijmuUIqqE6W6P05MAYNg2lJPqqSALmrpRGlxJIr2BFDWcSUKjVEeXA\u002FNlMSBbQZzIC6sLGNs9UEMIEojQ6tgS30ZmnOTuzFzf4J5gphyz\u002F6zFqj4hV\u002FLB5lWhS4NcUbcTEaEgLg5BGCYlyMxvAMk78bWoKj83PeEtw9vKV2piErMTco9z2R+wbo4L8wH4Y71A8iSctozD6AQ5p8QJ89NkJJKqzhadRsXW7E76bA4Tl5Bzz0G4M09RCktUZbZrYg7QzK93LADdDbiqdpdwRNH6D2whoce\u002FMi4eF6XtoeQtdmdSBjdOOhE6kuZiGduW9HYzia50U92ecvXiLKGg9ZA7g0VlBO0MWn4AIs3aGbu+AdGHgsFR8SMdghdgLiCHql5Nm+alIz4ujmICqslb0pZSVTebaKa6rsbit307gIbenKyqZUKNLF8WrfqtGLQHdT1fyucm+p2nPnICxMhIsLZeeRFsfAKYAe4DsPIGmEjantuTgK2VWqRcZCsfrM8tAIHV4FH\u002FNgGrhxDYNhHBkyc3b3dMcaO71GNlMDee5g1p0uODrmFhnmhqX0ZTmnQ8OukJDnn2HAuLLwEDkjmIMH7UTCgtlGExoycpjSAIrnLwn5XyM8h2amKE9taLTJV2OEwUwRQ0YofsFAGNtG8WsUvFqWeopFl9FXjMyCwWW\u002FnDfKo9kJZBRc\u002FOGO1aRheidFC+nK\u002FVVU8Yw+aW6IoaBcy4F7+3EJjsz0npxZ0Doyq3hQmR7XFCJcBXqH0NDDJ5yODOvVzCdQLQ3sDlmWydnBg\u002FcDWwi2g3SL+lVlb4xOz3EiZAlFKhGag8nAHXs2EDtgEx6QMmr8DQfkS1fnAoUwu3zEuUCoGUX4BfGKEI+GQ9avLrAPA1QRi7OpCycdfiGGgkJJbZm+JgCtQOVFGhsEYfFLQr63VBNkYKPikMAi1AAYuFCToM3LkxsgAZGsDQhBt4A8N4OvmBU3BbJoYAsxowV4EA2YmVqDjBdmMahaotfKAiymQIK1szuKFq4\u002F7hcC\u002Fh+ZoNH1Ry+5wIIyxuACwT7kC1MfPbNBDbMZSRvd+Hq6i+rnh80Wx0xuVdVLD94Sg4GijYbKIR3MqLIaRuYaznCvMtODdifqloZIHnII1Sl9M+4Lhw6wvWGpMCKPJzFqKBVHpb316tImicPpbgyFGi+EO7sDInNztseAvq54oGu7jf8geAy5R1z7SmZ2omb1MMHDEXT3gIFSI+Mw5gHeogRa88rqiFLqN7A+9LDTzF1lxMSSMVGeDqzWTCgPZeB0LTzUwGWELaTtMP8bT7rCADGPsQyjhYTJndCFhRNIE6GZPmPun5bFRLMPkAaI49I59lkNBuAN7uPNx1yXMFIJy5tXVw+rPNIKqz5IWzxqqjWzvOrRPoB1Z93iwX91GmXZWvvlPMtp\u002FMd\u002FXnARl5OcP8eskj+SJodTBbTRib9fzrfa7eR8RySmvg6FyYWLpEMMeX1BIXp1v5xntEI+++yi2qTmvYEzacTJwMYyfy7tl1MxcpY47E63liOgtsBNYKjqzWYtNh+S+V0P5MhkwDNu8SDz+If\u002FRz06r2fH5l1OUzKculktdXS\u002FnG9ustvJgRZ0ercJkq9Bu8kkTYmlR6woIoqd2pLgx4E1EQy5EKfdnKWHorkwqme1wilJzkvbo9pnQDQ3R3O5ajKrLlL3MLAMWH42WNqEFr9zfYbDaf4bLY1FKufJ119fQIZo8Caos7Dvh9rIpb8GwpGYjXn7I6oRIM\u002F1VKdyRiOAf7ihn0B0DzFtABJeDMkj5kx8gt9t4JoZ2OXJXUidZe4tiHJPZbS0ER6qZ9s1zsWyMo3ZisbMED09gEfR1uSxbTre\u002F2YJvOUEBB6Drz\u002Fgm2shyjJxhQ5\u002FwB+\u002FafH8Q3\u002FBUnD5C53r4JeWM0Py3ZQxtYuHSG+njbEziE2ocUI6PE+H5yF7RzXdapwxgDTurONWKxsev6XDuJg5L4zLTiRv7VMC5pWqbA3gxPLQr\u002FTxXYxuNXUYuBGGyniQkmVk4gmpkaYQUwBJQNx6Ai1wftVLzGOHzsZWEhHjQhdQ8n457XaTS10qVQP9G9K3IIvn4aThoxJIrkVRrZ2Ro\u002FKJgmEvmNJQuMAaEKJXRM4b9K9uPzO2IV9BG64CXar98hbA4iVaMFLNEE47mtWzhBFWHo+ujbHiZ2wMBg4jt4HvLloMB3u\u002FnFbZxIF8CnliXn5y1zgUcR5kmHVDGsGI4I2D7yF2MxYXg9YuUYh3R8WYNRxSGSZmmX\u002FBWoLv4R+sJfkrb0nCIQ4aY745g0uzQCAEujHGv3jKpfWsKUaZ+BA+r1xMIgR5rp4a8z74kQJ3tagM9k1xhtZDvyDAnWD0Ld9iP6cdD+05+YQ7nfOOSwaFApjk5iAkiutO0OpCPx1Igdo39JucyhHneVSd9uaU0hqdnUA6orsTXEVjiXfO8+c0yYE6JEZQwH0B9v7jylOeySOvLYJhvHEXYUl8\u002Fs5OMcRYvEGoNnfwsEdterjypMTM9CJxNENsFuw1FJXN7qtEQ9HBYU9MehExChaYEPK+iPQu6i1NpOcl2ocwB07e7bQWqQMMoP8rU+DfQhsU1I1t0nqOaPYK4TFpzAnAQP5+Oa2NlZ2oTuUgFRnTEpCfazEuTL84f\u002FEb9RNp0N31VTVhmiUmZxqx0anvBJ2bqIzCh7+5ZbqxjT2MC2JfvQ3jJ2\u002FGLlobeLuZNU4i6pNo8R5+OhmDh91ztIL9bPxqjIMMExGVaeB1BgLgK8FXxnnf1MltO0ym1zEolNKdynrR4sF\u002FtVAO\u002F2xvcgPS3IrLXHb27dDhaVYUgPGlgSfVsXU6Pa88he8HdWID20bjBJp0R57HFMY0d6szAYfnWg2uqm1w\u002Fmyumrj3ZoHnsNfne\u002F8fMtSuUdDyAAA=",1793389736744]