[{"data":1,"prerenderedAt":391},["ShallowReactive",2],{"posts":3},[4,15,22,31,40,47,54,61,68,75,82,89,95,102,111,118,125,132,139,148,155,162,168,177,185,193,200,207,216,224,232,240,248,257,265,274,283,291,298,306,313,321,328,335,343,351,359,367,374,383],{"abbrlink":5,"title":6,"date":7,"categories":8,"tags":10,"excerpt":14},"deepseek-v4-parallelism","千卡、万卡集群的 LLM 训练并行调优：以 Megatron-LM 训练 DeepSeek-V4 为例","2026-10-26",[9],"懵逼的深度学习",[11,12,13],"分布式训练框架","Megatron-LM","MoE","把训练从 1024 张卡扩到 8192 张卡，如果模型和全局 batch 都不变，每个数据并行副本分到的工作就会减少。原本能够连续送进流水线的 32 个 microbatch，可能只剩下 4 个。卡多了，流水线却更难填满。 这里整理一下这类问题的分析方法。以 DeepSeek-V4-Pro 为例，从显存和并行分组开始，逐步讨论 EP、PP、CP 的选择，以及怎样判断一次优化是否有效。文中的算例可以复算，性能数字则保留论文中的模型、硬件与",{"abbrlink":16,"title":17,"date":18,"categories":19,"tags":20,"excerpt":21},"820979d0","Debug 救星！用 ObjWatch 助力复杂 Python 项目的高效代码理解与调试","2024-12-21",[],[],"摘要 使用 ObjWatch 助力复杂 Python 项目的高效代码理解与调试。",{"abbrlink":23,"title":24,"date":25,"categories":26,"tags":28,"excerpt":30},"80c749f","Jax编程: 踩坑记录","2024-03-30",[27],"Jax 编程",[29],"Jax","摘要 记录 jax 使用过程中的坑。",{"abbrlink":32,"title":33,"date":34,"categories":35,"tags":37,"excerpt":39},"40cda68f","CUDA编程: CUDA流,并发与上下文","2024-02-06",[36],"CUDA 编程",[38],"CUDA","摘要 深入介绍流的使用，包括流的并发，多线程调度流，并发限制，创建流间依赖关系，重叠主机与设备的执行等，同时，结合实例讲解了 CUDA 上下文以及 MPS 的概念，简单介绍了 CUDA Driver API 对 Context 的管理。",{"abbrlink":41,"title":42,"date":43,"categories":44,"tags":45,"excerpt":46},"5a01a3bf","CUDA编程: CUDA流,事件与同步","2024-01-30",[36],[38],"摘要 详细讲解 CUDA 流，事件的概念以及声明，以此为基础，深入了解 CUDA 流操作之间的依赖关系，流的同步与异步，以及如何优化事件的创建与管理等等。",{"abbrlink":48,"title":49,"date":50,"categories":51,"tags":52,"excerpt":53},"d7bb3d7c","CUDA编程: CUDA内存管理（三）","2024-01-23",[36],[38],"摘要 CUDA C 编程中的简单的内存管理，线程操作，如何编写核函数，使用 Thrust 库，并行计算，性能分析工具。",{"abbrlink":55,"title":56,"date":57,"categories":58,"tags":59,"excerpt":60},"9aa59e2e","CUDA编程: CUDA内存管理（二）","2024-01-16",[36],[38],"摘要 详细讲解共享内存存储体、存储体冲突、访问模式以及内存填充等知识点。",{"abbrlink":62,"title":63,"date":64,"categories":65,"tags":66,"excerpt":67},"aab372f6","CUDA编程: CUDA内存管理（一）","2024-01-09",[36],[38],"摘要 各种内存的对齐访问以及实验，避免带宽浪费。还对比了 AoS 与 SoA 结构体在 GPU 上的性能表现。",{"abbrlink":69,"title":70,"date":71,"categories":72,"tags":73,"excerpt":74},"62a8be74","CUDA编程: CUDA内存模型概述","2024-01-02",[36],[38],"摘要 讲解 CUDA 内存层次结构，如寄存器，共享内存，纹理内存，全局内存等。",{"abbrlink":76,"title":77,"date":78,"categories":79,"tags":80,"excerpt":81},"ec855d6e","CUDA编程: PyCUDA编程简介","2023-12-26",[36],[38],"摘要 PyCUDA GPU 编程。主要内容是掌握几种PyCUDA 的基本内核函数，在 Python 中生成和调用并行化的 CUDA C 内核函数，以及了解 PyCUDA 的并行前缀算法。",{"abbrlink":83,"title":84,"date":85,"categories":86,"tags":87,"excerpt":88},"55d1319f","CUDA编程: CUDA C编程简介","2023-12-19",[36],[38],"摘要 讲解 CUDA C 编程中的简单的内存管理，线程操作，如何编写核函数，使用 Thrust 库，并行计算，性能分析工具。",{"abbrlink":90,"title":91,"date":85,"categories":92,"tags":93,"excerpt":94},"e07c9171","CUDA编程: CUDA模型概述",[36],[38],"摘要 讨论 GPU 的并行计算是如何在硬件上实现的，CUDA 中的模块理解以及 CPU和 GPU 之间的交互，指令的同步。",{"abbrlink":96,"title":97,"date":98,"categories":99,"tags":100,"excerpt":101},"b3409785","CUDA编程: GPU编程概述和CUDA环境搭建","2023-12-12",[36],[38],"摘要 模型变得越来越深，参数愈加庞大，虽然准确率不断增长，由于硬件受限，对实际场景部署的要求也越来越高，CUDA 编程成为了一门必备的武林绝学。",{"abbrlink":103,"title":104,"date":105,"categories":106,"tags":108,"excerpt":110},"f3209fd4","记录第n次创建启用并清理临时swap","2023-12-05",[107],"linux",[109],"swap","摘要 时常要用 chatgpt 辅助生成 swap，记录下来方便",{"abbrlink":112,"title":113,"date":114,"categories":115,"tags":116,"excerpt":117},"82ab1321","论文阅读: ZeRO++: Extremely Eficient Collective Communication for Giant Model Training","2023-11-25",[9],[11],"摘要 原文链接：https:\u002F\u002Farxiv.org\u002Fpdf\u002F2306.10209.pdf 开源代码：https:\u002F\u002Fgithub.com\u002Fmicrosoft\u002Fdeepspeed",{"abbrlink":119,"title":120,"date":121,"categories":122,"tags":123,"excerpt":124},"d343b623","开个新坑: 精读pytorch源码","2023-11-09",[9],[],"摘要 毕设打算做个简单的深度学习框架，调研了几种语言目前的实现，各有优劣，先从最经典的 torch 读起。欢迎朋友们提出建议",{"abbrlink":126,"title":127,"date":128,"categories":129,"tags":130,"excerpt":131},"fc5b020","论文阅读: ZeRO-Infinity: Breaking the GPU Memory Wall for Extreme Scale Deep Learning","2023-10-26",[9],[11],"摘要 ZeRO-Infinity 是基于 ZeRO 的扩展，Infinity 离线引擎可以同时利用 GPU、CPU 和 NVMe 内存，还提出了其他的优化技术。",{"abbrlink":133,"title":134,"date":135,"categories":136,"tags":137,"excerpt":138},"17582ba3","论文阅读: ZeRO-Offload: Democratizing Billion-Scale Model Training","2023-10-17",[9],[11],"摘要 ZeRO-Offload 主要优化在于尽量减少数据在 GPU 与 CPU 之间的移动，并减少 CPU 计算时间，同时最大限度地节省 GPU 上的内存。",{"abbrlink":140,"title":141,"date":142,"categories":143,"tags":145,"excerpt":147},"bddb5e0d","记录第n次修复USB设备无法识别挂载","2023-10-11",[144],"archlinux",[146],"USB","摘要 archlinux 滚着滚着就不能识别 USB 设备了，记录一下修复过程",{"abbrlink":149,"title":150,"date":151,"categories":152,"tags":153,"excerpt":154},"706a94b5","论文阅读: ZeRO: Memory Optimizations Toward Training Trillion Parameter Models","2023-09-30",[9],[11],"摘要 ZeRO 是一种用于大规模深度学习模型的内存优化解决方案，通过消除数据和模型并行训练中的内存冗余，同时保持了低通信量和高计算粒度。",{"abbrlink":156,"title":157,"date":158,"categories":159,"tags":160,"excerpt":161},"b353fa5f","论文阅读: Horovod: fast and easy distributed deep learning in TensorFlow","2023-09-17",[9],[11],"摘要 最近忙实习去了，断更大半年。整理了一下论文，有十几篇没读，后面有时间慢慢更。 这篇论文主要工作是对 TensorFlow 框架 API 的重写，使用 ring-allreduce 和 broadcast 方法，进行数据并行。",{"abbrlink":163,"title":164,"date":158,"categories":165,"tags":166,"excerpt":167},"a8cf276f","论文阅读: PyTorch Distributed: Experiences on Accelerating Data Parallel Training",[9],[11],"摘要 本文提出的 DistributedDataParallel 在优化器运行之前进行梯度平均，用相同的梯度集更新所有模型副本，这样在数学上和本地训练完全等价，而且可以实现异步，比参数平均更加高效。",{"abbrlink":169,"title":170,"date":171,"categories":172,"tags":174,"excerpt":176},"c1bfab13","2023MCM-ICM美赛C题第二问思路","2023-02-23",[173],"数学建模",[175],"数据分析","摘要 在睡了十一个小时后，蹦出的思路，目前网上没有看到相同的方法（不知道比赛刚结束就发思路会不会出事），第一题和第三题大家大同小异，就不讲解了。",{"abbrlink":178,"title":179,"date":180,"categories":181,"tags":183,"excerpt":184},"f100aba","HGame_2023_Week3_3ctu4_card_problem","2023-01-31",[182],"misc",[],"摘要 干扰特征非常多的二分类问题，解题步骤如下",{"abbrlink":186,"title":187,"date":188,"categories":189,"tags":190,"excerpt":192},"19affb6","linux下中文路径命名规范化","2023-01-29",[],[191],"工具","摘要 shell 中无法正常访问带空格或某些字符的目录，写了个脚本规范一下中文路径命名",{"abbrlink":194,"title":195,"date":196,"categories":197,"tags":198,"excerpt":199},"5513d824","caj文档批量转换为pdf-shell脚本多线程","2023-01-10",[],[],"摘要 最近看的论文有些多，但知网上的 caj 文档又卡又难做笔记，遂写了一个 caj 文档批量转换为 pdf 的 shell 脚本。",{"abbrlink":201,"title":202,"date":203,"categories":204,"tags":205,"excerpt":206},"68deaade","CUDA编程性能分析工具 nvprof\u002Fncu --metrics参数含义","2023-01-07",[36],[191],"摘要 在网上没有比较全的中文 ncu --metrics 参数含义，于是自己整理了一下官方和外国友人的笔记。",{"abbrlink":208,"title":209,"date":210,"categories":211,"tags":213,"excerpt":215},"b1e8187e","CT-Windowing医学CT图像增强","2022-10-06",[212],"图像处理",[214],"知识点","摘要 CT winding 是CT扫描的一项图像处理任务，它有助于突出关键的组织结构，通过修改 HU (Hounsfield Units) 参数，使图像更易于分析。",{"abbrlink":217,"title":218,"date":219,"categories":220,"tags":221,"excerpt":223},"28e012f","GroupFormer简要学习笔记","2022-08-31",[9],[222],"群体行为识别","摘要 快速记录下 GroupFormer 网络的核心思想。",{"abbrlink":225,"title":226,"date":227,"categories":228,"tags":229,"excerpt":231},"908291b2","高斯函数与高斯核函数","2022-08-07",[212],[230],"空间滤波","摘要 图像处理领域中，高斯核函数常用于图像低通滤波，记录下原理，同时用代码实现下。",{"abbrlink":233,"title":234,"date":235,"categories":236,"tags":238,"excerpt":239},"5f24596c","我的2022暑假独居生活","2022-06-12",[237],"生活",[],"摘要 一张小方桌，有一荤一素",{"abbrlink":241,"title":242,"date":243,"categories":244,"tags":245,"excerpt":247},"68ed93d0","第二届网刃杯misc所见非所见wp","2022-04-25",[182],[246],"tensorflow","摘要 记录一下这道深度学习入门的wp",{"abbrlink":249,"title":250,"date":251,"categories":252,"tags":253,"excerpt":256},"d43ce4e7","多目标优化笔记","2022-04-19",[173],[254,255],"多目标优化","最优化方法","摘要 多目标优化 ( MOO ) - Multi-Objective Optimization 从问题定义，单目标、多目标，无约束、有约束方面了解多目标优化",{"abbrlink":258,"title":259,"date":260,"categories":261,"tags":262,"excerpt":264},"66d4364b","使用平行线段的相机标定[计划更新]","2022-04-16",[212],[263],"论文复现","摘要 一种使用平行线段来进行相机标定以及形状重建的方法",{"abbrlink":266,"title":267,"date":268,"categories":269,"tags":271,"excerpt":273},"7f21964e","设计实现内存的分配和回收算法","2022-04-13",[270],"操作系统",[272],"作业","摘要 操作系统作业",{"abbrlink":275,"title":276,"date":277,"categories":278,"tags":279,"excerpt":282},"ddce6477","制作目标检测数据集常用python脚本整理","2022-04-09",[9],[280,281],"python","数据集","摘要 制作个人目标检测数据集时，有查找爬取的错误图片，更改标注路径等等需要，写了以下python脚本",{"abbrlink":284,"title":285,"date":286,"categories":287,"tags":288,"excerpt":290},"6e84daf2","Faster-RCNN原理笔记","2022-03-30",[9],[289],"目标检测","摘要 详细记录了 Faster RCNN 网络原理和个人的理解",{"abbrlink":292,"title":293,"date":294,"categories":295,"tags":296,"excerpt":297},"ec86e25e","卷积运算的输入输出shape","2022-03-16",[9],[214],"摘要 记录一下二维卷积神经网络中卷积运算的输入输出shape大小",{"abbrlink":299,"title":300,"date":301,"categories":302,"tags":303,"excerpt":305},"7aac45fd","最小化交叉熵损失与最大似然估计的推导","2022-03-05",[9],[304],"概率与信息论","摘要 机器学习所使用的交叉熵损失函数与信息论里的交叉熵的推导与思考",{"abbrlink":307,"title":308,"date":309,"categories":310,"tags":311,"excerpt":312},"e8ff6ba7","使用Opencv+Python的AR小demo","2022-02-23",[212],[272],"摘要 浅浅了解一下 Python OpenCV，试着给自己的 iphone 8 做一下相机标定",{"abbrlink":314,"title":315,"date":316,"categories":317,"tags":318,"excerpt":320},"a524281d","labelImg闪退错误修复","2021-12-05",[9],[319,281],"labelImg","摘要 使用labelImg时，在未选中数据标签时按Ctrl+D时会闪退，在终端中有以下报错",{"abbrlink":322,"title":323,"date":324,"categories":325,"tags":326,"excerpt":327},"ae4fbfca","2021亚太数学建模竞赛A题简要思路","2021-11-29",[173],[212],"摘要 小记一下第一次参加数模赛",{"abbrlink":329,"title":330,"date":331,"categories":332,"tags":333,"excerpt":334},"3b4e1f24","和周小姐高中的对话（一）","2021-11-08",[237],[],"摘要 和周小姐的高中日常之一",{"abbrlink":336,"title":337,"date":338,"categories":339,"tags":340,"excerpt":342},"93d3d7c3","记录第n次修复引导区","2021-11-07",[144],[341],"grub","摘要 给U盘装了archlinux后，没注意把固态上archlinux的\u002Fetc\u002Ffstab分区挂载配置文件修改了，以为是grub又坏了，干脆把挂载在\u002Fboot下的EFI 系统格式化了，孙子云：“置之死地而后生。” 从头开始",{"abbrlink":344,"title":345,"date":346,"categories":347,"tags":349,"excerpt":350},"86ad67e4","求解微分方程","2021-11-02",[348],"高等数学",[214],"摘要 解微分方程是我的弱项，就靠这个了",{"abbrlink":352,"title":353,"date":354,"categories":355,"tags":357,"excerpt":358},"62c5a99c","拟合和统计中的检验参数","2021-10-20",[356],"统计学",[214],"摘要 数据分析中无处不在的知识点",{"abbrlink":360,"title":361,"date":362,"categories":363,"tags":365,"excerpt":366},"7d2c710d","伽马函数","2021-10-17",[364],"数学",[214],"摘要 遇到了这个伽马函数， 许多博客说对考研很重要，就浅浅记录一下",{"abbrlink":368,"title":369,"date":362,"categories":370,"tags":372,"excerpt":373},"6b91a7a9","正交多项式",[371],"数值分析",[214],"摘要 总结一下可能会遇到的正交多项式",{"abbrlink":375,"title":376,"date":377,"categories":378,"tags":380,"excerpt":382},"1d9b66c4","matlab画图功能再探","2021-10-08",[379],"使用教程",[381],"matlab","摘要 复习了一下 matlab 里面的基本画图函数的属性",{"abbrlink":384,"title":385,"date":386,"categories":387,"tags":388,"excerpt":390},"b8ab2279","拉格朗日插值多项式","2021-10-06",[371],[389],"多项式插值","摘要 在数值分析中，多项式插值是最常提到的内容",1792995406066]