[{"data":1,"prerenderedAt":482},["ShallowReactive",2],{"content-doc-\u002Fzh\u002Fblogs\u002F20251121-HiFloat8":3},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":7,"date":9,"author":10,"tags":11,"body":13,"_type":476,"_id":477,"_source":478,"_file":479,"_stem":480,"_extension":481},"\u002Fzh\u002Fblogs\u002F20251121-HiFloat8","blogs",false,"","HiFloat8 浮点数据格式：既要又要之路","2025\u002F11\u002F21","全球计算联盟GCC",[12],"HiFloat",{"type":14,"children":15,"toc":467},"root",[16,27,33,40,45,80,85,93,126,138,144,163,168,173,181,192,197,202,208,213,218,244,252,260,272,317,325,333,338,346,354,359,364,369,377,385,390,396,401,415],{"type":17,"tag":18,"props":19,"children":20},"element","p",{},[21],{"type":17,"tag":22,"props":23,"children":26},"img",{"alt":24,"src":25},"1","\u002Fblogs\u002F20251121-HiFloat8\u002F1.jpeg",[],{"type":17,"tag":18,"props":28,"children":29},{},[30],{"type":31,"value":32},"text","介绍 HiFloat8（HiF8）之前，我们先回顾一下 AI 浮点数据格式的发展及相应的基本概念，并作简单的对比分析。以便读者更容易理解这篇技术分析的内容。",{"type":17,"tag":34,"props":35,"children":37},"h2",{"id":36},"传统-16-位浮点数",[38],{"type":31,"value":39},"传统 16 位浮点数",{"type":17,"tag":18,"props":41,"children":42},{},[43],{"type":31,"value":44},"计算机用 IEEE 754 标准定义的数据格式存储数据，其核心类似于十进制的科学计数法，例如 −1.25×10^(-3)只不过是二进制版本。以半精度 FP16 浮点数据格式为例，它用 16 比特（2 字节）存储一个数。这 16 位被划分为 3 块：",{"type":17,"tag":46,"props":47,"children":48},"ul",{},[49,55,75],{"type":17,"tag":50,"props":51,"children":52},"li",{},[53],{"type":31,"value":54},"符号位（Sign bit）：1bit，1 表示负，0 表示正",{"type":17,"tag":50,"props":56,"children":57},{},[58,60,66,68,73],{"type":31,"value":59},"阶码位（Exponent Field）：5 bits，编码",{"type":17,"tag":61,"props":62,"children":63},"span",{},[64],{"type":31,"value":65},"0, 31",{"type":31,"value":67},"–15=",{"type":17,"tag":61,"props":69,"children":70},{},[71],{"type":31,"value":72},"-15, 16",{"type":31,"value":74},"，其中 15 是阶码偏置。E=-15，表示非常规数值（Subnormal\u002FDenormal），E=16 时编码无穷和 NaN，E 等于其他值时表示常规数值。",{"type":17,"tag":50,"props":76,"children":77},{},[78],{"type":31,"value":79},"尾数位（Mantissa Field）：10bits，都表示小数位，小数位之前有 1-bit 值固定的隐藏位（Normal 模式是 1，Subnormal 模式是 0），共 11 比特有效位",{"type":17,"tag":18,"props":81,"children":82},{},[83],{"type":31,"value":84},"因此 FP16 的二进制科学计数法表达式如下（阶码 E 已经过偏置还原）：",{"type":17,"tag":18,"props":86,"children":87},{},[88],{"type":17,"tag":22,"props":89,"children":92},{"alt":90,"src":91},"2","\u002Fblogs\u002F20251121-HiFloat8\u002F2.png",[],{"type":17,"tag":18,"props":94,"children":95},{},[96,98,103,105,110,112,117,119,124],{"type":31,"value":97},"浮点数相比定点数，通过二进制的阶码和尾数分离的科学计数法结构，在有限的硬件位宽下，实现了巨大的动态范围。比如 INT16，最多只能表示 ±",{"type":17,"tag":61,"props":99,"children":100},{},[101],{"type":31,"value":102},"2^0, 2^15−1",{"type":31,"value":104},"范围的非 0 数值，等效支持的二进制指数只有",{"type":17,"tag":61,"props":106,"children":107},{},[108],{"type":31,"value":109},"0, 15",{"type":31,"value":111},"，共 16 个；而 FP16 却可以表达 ±",{"type":17,"tag":61,"props":113,"children":114},{},[115],{"type":31,"value":116},"2^(−24), 2^15×(2−2^(−10))",{"type":31,"value":118},"范围的非 0 数值，等效支持的二进制指数高达",{"type":17,"tag":61,"props":120,"children":121},{},[122],{"type":31,"value":123},"-24, 15",{"type":31,"value":125},"，共 40 个。",{"type":17,"tag":18,"props":127,"children":128},{},[129,131,136],{"type":31,"value":130},"FP16 在 CNN 流行的时代，是主流的训练数据格式。但是随着 LLM 的爆发和流行，人们逐渐发现语言模型训练过程中的数值范围非常弥散，仅有 40 个指数支持的 FP16 经常有大量数值下溢变成 0，影响模型训练精度甚至直接跑飞（即便已经支持了反向全局 Loss Scaling）。因此谷歌早期提出的 BF16 数据格式逐渐替代 FP16，成为了 LLM 时代的主流训练格式。BF16 数据格式具有 1-bit 符号位，8-bit 阶码和 7-bit 尾数，等价于直接把 FP32 数据格式的低 16-bit 尾数直接砍掉。相比于 FP16，BF16 虽然只有 8 位有效位，但却能支持",{"type":17,"tag":61,"props":132,"children":133},{},[134],{"type":31,"value":135},"-133,127",{"type":31,"value":137},"的超大二进制指数范围（包括 Subnormal），对 LLM 训练稳定性极其友好。",{"type":17,"tag":34,"props":139,"children":141},{"id":140},"fp8-浮点数",[142],{"type":31,"value":143},"FP8 浮点数",{"type":17,"tag":18,"props":145,"children":146},{},[147,149,154,156,161],{"type":31,"value":148},"FP8 通常包括 E4M3 和 E5M2 两种（默认最高位存在 1bit 符号位），直接继承了 IEEE 754 定义的浮点数特征，使用固定位宽的阶码域和尾数域表达数值大小。其中 E4M3 设定 4-bit 阶码和 3-bit 尾数，精度相比 E5M2 较高，有 4 位有效位，但是动态范围很窄，只能支持",{"type":17,"tag":61,"props":150,"children":151},{},[152],{"type":31,"value":153},"-9,8",{"type":31,"value":155},"共 18 个二进制指数表达（包括 Subnormal）；而 E5M2 设定了 5-bit 阶码和 2-bit 尾数，精度相对 E4M3 较低，只有 3 位有效位，但是动态范围相对较大，能支持",{"type":17,"tag":61,"props":157,"children":158},{},[159],{"type":31,"value":160},"-16, 15",{"type":31,"value":162},"共 32 个二进制指数表达（包括 Subnormal）。通常 E4M3 用于神经网络前向传播的激活值 A 和权重 W，E5M2 用于神经网络后向传播的激活值梯度 dA。但近期业界认为 E5M2 精度过低，像 DeepSeek_V3 在训练的时候，dA 也用了 E4M3。",{"type":17,"tag":34,"props":164,"children":166},{"id":165},"现有浮点格式分析",[167],{"type":31,"value":165},{"type":17,"tag":18,"props":169,"children":170},{},[171],{"type":31,"value":172},"图 1 展示了 FP16、BF16 和 FP8 的域位宽结构。他们都包括三个域，符合 IEEE 754 标准的基本结构。其中阶码域决定基础动态范围，尾数域决定精度（有效位）。同时较宽的尾数域在 Subnormal 模式下能补充支持一定的二进制指数值，扩大一些动态范围。",{"type":17,"tag":18,"props":174,"children":175},{},[176],{"type":17,"tag":22,"props":177,"children":180},{"alt":178,"src":179},"3","\u002Fblogs\u002F20251121-HiFloat8\u002F3.png",[],{"type":17,"tag":182,"props":183,"children":185},"div",{"style":184},"font-size: 14px;",[186],{"type":17,"tag":187,"props":188,"children":189},"center",{},[190],{"type":31,"value":191},"图1. Float16和Float8域位宽结构示意图",{"type":17,"tag":18,"props":193,"children":194},{},[195],{"type":31,"value":196},"如前所述，具有 40 个二进制指数支持的 FP16 在 LLM 训练竞争中，已经逐渐败给了具备超大指数范围的 BF16 格式。毕竟现在大模型的训练成本太高了，训练不稳定的成本，很多企业和机构都难以承受。而现有的 FP8-E4M3，最多只能支持 18 个二进制指数，远远落后于 FP16。这使得 FP8 训练的稳定性面临极大的挑战。为了解决这一问题，细粒度 scaling 和 current scaling（FP8 delayed scaling 训练因为极其不稳定，尚未看见成功案例）成为了稳定训练的常见手段。但这些技术方案会明显降低 8 位浮点的训练加速。比如 Ling-1T 是目前已知最大的 FP8 训练的基础模型，但其 FP8 混合精度训练仅仅只带来了 15%多的端到端加速，远远低于预期，毕竟 8-bit 浮点的矩阵乘算力，相对 16-bit 浮点是翻倍了的。",{"type":17,"tag":18,"props":198,"children":199},{},[200],{"type":31,"value":201},"综上，LLM 训练对动态范围的诉求极大，导致 FP16 逐渐落败于 BF16。而 FP8-E4M3 的动态范围比 FP16 的一半还要小，这使得大家只能牺牲大量的训练性能提升，来换取训练稳定性和精度收敛。",{"type":17,"tag":34,"props":203,"children":205},{"id":204},"hif8-浮点数",[206],{"type":31,"value":207},"HiF8 浮点数",{"type":17,"tag":18,"props":209,"children":210},{},[211],{"type":31,"value":212},"FP8 困境的根源在于，对于 LLM 来说，精度很重要，动态范围也很重要，在 8-bit 位宽限制下，简单沿用传统 IEEE 754 的技术方案定义 8 比特数据格式，无法做到“既要又要”。为此，设计者提出了锥形精度的浮点数据格式 HiF8。",{"type":17,"tag":18,"props":214,"children":215},{},[216],{"type":31,"value":217},"如图 2 所示，HiF8 在传统的符号域、阶码域和尾数域的基础上，引入了即时可译的变长前缀码编码的点位域 Dot，直接显示阶码存储的位宽以及作为 Denormal 标志。Dot 分别编码了 0~4 五个值，因此阶码位宽最少 0bit，最多 4bits。在已知符号位、点位域和阶码域位宽的基础上，并且格式总共只有 8-bit，尾数位宽可对应求出。",{"type":17,"tag":18,"props":219,"children":220},{},[221,223,228,230,235,237,242],{"type":31,"value":222},"为确保不同位宽的阶码域所表达的指数值不重复，实现无冗余编码。HiF8 采用 Sign-Magnitude 的原码编码阶码域，并且固定幅值的最高位、不存储。比如说，当 Dot=0 时，说明阶码不占据位宽，并且 E 直接等于 0；而当 Dot=1~4 时，固定阶码幅值的最高位为 1（示意图中的红色数字），不占据存储位宽。基于这种规则，Dot=1，阶码 E 只能等于 ±1，不包含已经编码的 0；Dot=2 时，阶码 E 只能等于 ±",{"type":17,"tag":61,"props":224,"children":225},{},[226],{"type":31,"value":227},"2,3",{"type":31,"value":229},"，不包含已经编码的",{"type":17,"tag":61,"props":231,"children":232},{},[233],{"type":31,"value":234},"-1,1",{"type":31,"value":236},"。以此类推，Dot 域指示的 0~4 五种位宽的阶码，共编码了",{"type":17,"tag":61,"props":238,"children":239},{},[240],{"type":31,"value":241},"-15, 15",{"type":31,"value":243},"的指数，且彼此之间不重复，实现了无冗余编码。",{"type":17,"tag":18,"props":245,"children":246},{},[247],{"type":17,"tag":22,"props":248,"children":251},{"alt":249,"src":250},"4","\u002Fblogs\u002F20251121-HiFloat8\u002F4.png",[],{"type":17,"tag":182,"props":253,"children":254},{"style":184},[255],{"type":17,"tag":187,"props":256,"children":257},{},[258],{"type":31,"value":259},"图2. HiF8编码示意图",{"type":17,"tag":18,"props":261,"children":262},{},[263,265,270],{"type":31,"value":264},"同时，设计者特意用大位宽的 Dot 域指示小位宽和小值的阶码域，实现了尾数精度的渐变，而不是跳变。可以看到，在 E=",{"type":17,"tag":61,"props":266,"children":267},{},[268],{"type":31,"value":269},"-3, 3",{"type":31,"value":271},"时，有 3-bit 尾数。然后随着阶码幅值变大，Normal 模式下的尾数位宽逐渐减少到 1-bit。这是 HiF8 的老师、Posit 数据格式所不具备的。",{"type":17,"tag":18,"props":273,"children":274},{},[275,277,281,283,288,290,295,297,302,304,309,311,315],{"type":31,"value":276},"Normal 模式下，HiF8 的数值解析式和传统 IEEE 754 格式一样，是标准的二进制科学计数法表示。为了支持更大的动态范围，HiF8 还采用了不同于 IEEE 754 风格的 Subnormal\u002FDenormal 方案。如图 2 的 Denormal 表达式，当 Dot 域指示该数值为 Denromal 模式时，HiF8 没有指数域，多的 3-bit“尾数”（编码 0~7 八个数值），直接用于扩展 HiF8 的小值指数范围。HiF8 Denormal 模式在 Normal 模式支持的",{"type":17,"tag":61,"props":278,"children":279},{},[280],{"type":31,"value":241},{"type":31,"value":282},"共 31 个指数的基础上，额外扩展了",{"type":17,"tag":61,"props":284,"children":285},{},[286],{"type":31,"value":287},"-22,-16",{"type":31,"value":289},"共 7 个指数（-23 用于表示特殊值 Zero 和 NaN），综合形成了",{"type":17,"tag":61,"props":291,"children":292},{},[293],{"type":31,"value":294},"-22,15",{"type":31,"value":296},"共 38 个指数的大动态范围，非常接近 FP16 的",{"type":17,"tag":61,"props":298,"children":299},{},[300],{"type":31,"value":301},"-24,15",{"type":31,"value":303},"的 40 个指数范围。并且由于",{"type":17,"tag":61,"props":305,"children":306},{},[307],{"type":31,"value":308},"-15,-8",{"type":31,"value":310},"本身是 1-bit 尾数，扩展的",{"type":17,"tag":61,"props":312,"children":313},{},[314],{"type":31,"value":287},{"type":31,"value":316},"的 0-bit 尾数也属于精度渐变，并非跳变。",{"type":17,"tag":18,"props":318,"children":319},{},[320],{"type":17,"tag":22,"props":321,"children":324},{"alt":322,"src":323},"5","\u002Fblogs\u002F20251121-HiFloat8\u002F5.png",[],{"type":17,"tag":182,"props":326,"children":327},{"style":184},[328],{"type":17,"tag":187,"props":329,"children":330},{},[331],{"type":31,"value":332},"图3. Float8有效位-指数示意图",{"type":17,"tag":18,"props":334,"children":335},{},[336],{"type":31,"value":337},"根据前述 HiF8 编码描述，和 FP8 的回顾，可以画出图 3 所示的 Float8 有效位-指数示意图。其中 HiF8 精度明显不同于域位宽固定的 E4M3 和 E5M2，具备一种锥形的精度特征。锥形精度的概念受启发于 Posit 数据格式——数据工程中的绝大多数数值分布，都具备类高斯分布的聚合特征。神经网络的训练和推理也不例外。只要众数数值能用高精度充分表达即可，两边占比较小的数据（幅值较大或者较小），精度可以逐渐降低一些，不会显著影响端到端的功能和效果。但 Posit 的编码方式，尾数精度在指数变化时存在严重跳变，不够均衡。在 8-bit 限制下，Posit 无法很好匹配当前神经网络的诉求。HiF8 作为 Posit 的学生，希望能将锥形精度的思想发扬光大。",{"type":17,"tag":18,"props":339,"children":340},{},[341],{"type":17,"tag":22,"props":342,"children":345},{"alt":343,"src":344},"6","\u002Fblogs\u002F20251121-HiFloat8\u002F6.png",[],{"type":17,"tag":182,"props":347,"children":348},{"style":184},[349],{"type":17,"tag":187,"props":350,"children":351},{},[352],{"type":31,"value":353},"图4. HiF8特殊值支持",{"type":17,"tag":18,"props":355,"children":356},{},[357],{"type":31,"value":358},"除了常规数值编码，HiF8 也编码了 4 个特殊值，具体如图 4 所示。可以看到，HiF8 编码了 Zero、NaN 和正负无穷。其中 HiF8 不区分正 0 和负 0，或者可以理解为 HiF8 用单一 pattern 同时表示了（神经网络区分正负 0 意义不大，8-bit 表达空间十分有限，浪费不划算）。因此，HiF8 是一个数值表达完备的 8-bit 单数据格式。",{"type":17,"tag":34,"props":360,"children":362},{"id":361},"总结与展望",[363],{"type":31,"value":361},{"type":17,"tag":18,"props":365,"children":366},{},[367],{"type":31,"value":368},"表格 1 详细总结对比了 HiF8、FP8 和 FP16 的典型值和关键特征。可以看到，HiF8 利用匹配数据分布的锥形精度特征，成功地在 8-bit 限制下，在保证神经网络需求精度的前提下，显著扩大了格式的动态范围。进而为神经网络训推提供了能力更全面的 8-bit 单数据格式表达。",{"type":17,"tag":18,"props":370,"children":371},{},[372],{"type":17,"tag":22,"props":373,"children":376},{"alt":374,"src":375},"7","\u002Fblogs\u002F20251121-HiFloat8\u002F7.png",[],{"type":17,"tag":182,"props":378,"children":379},{"style":184},[380],{"type":17,"tag":187,"props":381,"children":382},{},[383],{"type":31,"value":384},"表1. HiF8、FP8和FP16的典型值和关键特征",{"type":17,"tag":18,"props":386,"children":387},{},[388],{"type":31,"value":389},"FP16 在和 BF16 的 LLM 训练竞争中，40 个指数的动态范围都显不足（LLM 训练真的太吃动态范围了！），HiF8 的 38 个指数表达显然也是不够的。但是相比于 FP8，尤其是 E4M3 的 18 个指数表达，其动态范围已经是大幅提升。有理由相信这种优势最终会转换到神经网络 E2E 性能或者精度上。在 8-bit 的极低开销限制下，HiF8 已经算是成功找到了一条“既要又要”的道路。",{"type":17,"tag":34,"props":391,"children":393},{"id":392},"敬请锁定-gcc-ip-栏目-compuwave",[394],{"type":31,"value":395},"敬请锁定 GCC IP 栏目 CompuWave！",{"type":17,"tag":18,"props":397,"children":398},{},[399],{"type":31,"value":400},"GCC 智算产业发展委员会将持续带来「HiF8」系列专题分享，后续将邀请技术专家与联创伙伴持续发掘并向大家呈现 HiF8 在训练和推理中的优势。",{"type":17,"tag":18,"props":402,"children":403},{},[404,406,413],{"type":31,"value":405},"更多 HiF8 联创招募、进展等信息可联系 GCC 智算产发委执行秘书长熊华（",{"type":17,"tag":407,"props":408,"children":410},"a",{"href":409},"mailto:xionghua@gccorg.com",[411],{"type":31,"value":412},"xionghua@gccorg.com",{"type":31,"value":414},"）获取。",{"type":17,"tag":182,"props":416,"children":417},{"style":184},[418],{"type":17,"tag":187,"props":419,"children":420},{},[421,423,427,429,432,434,437,439,442,444,447,449,452,454,457,460,462,465],{"type":31,"value":422},"欢迎参与！\n  与我们共同破浪前行，解锁计算世界双重精彩",{"type":17,"tag":424,"props":425,"children":426},"br",{},[],{"type":31,"value":428},"汇聚全球专家洞见，碰撞多元思维火花",{"type":17,"tag":424,"props":430,"children":431},{},[],{"type":31,"value":433},"聚焦计算前沿突破，捕捉技术创新脉搏",{"type":17,"tag":424,"props":435,"children":436},{},[],{"type":31,"value":438},"这里既有深度产业观察、趋势研判，",{"type":17,"tag":424,"props":440,"children":441},{},[],{"type":31,"value":443},"也有前沿技术探索、创新路径解析",{"type":17,"tag":424,"props":445,"children":446},{},[],{"type":31,"value":448},"让每一份专业表达与先锋探索，",{"type":17,"tag":424,"props":450,"children":451},{},[],{"type":31,"value":453},"都能共振行业、影响未来现在，",{"type":17,"tag":424,"props":455,"children":456},{},[],{"type":17,"tag":424,"props":458,"children":459},{},[],{"type":31,"value":461},"加入 CompuWave",{"type":17,"tag":424,"props":463,"children":464},{},[],{"type":31,"value":466},"与全球计算先锋共探技术边界，共逐蓝海征程",{"title":7,"searchDepth":468,"depth":468,"links":469},2,[470,471,472,473,474,475],{"id":36,"depth":468,"text":39},{"id":140,"depth":468,"text":143},{"id":165,"depth":468,"text":165},{"id":204,"depth":468,"text":207},{"id":361,"depth":468,"text":361},{"id":392,"depth":468,"text":395},"markdown","app:zh:blogs:20251121-HiFloat8.md","app","zh\u002Fblogs\u002F20251121-HiFloat8.md","zh\u002Fblogs\u002F20251121-HiFloat8","md",1783069436457]