def rmsnorm

bend-ml-llama@0.2.0.0 · main.bend:385

import bend-ml-llama@0.2.0.0/main.bend as BendMlLlama

import Base

385# RMSNorm: g * (x * (1 / sqrt(mean(x²) + eps))), in this order (LlamaRMSNorm.forward)
386def rmsnorm(+c: Cfg, +x: List<&2, F32>, +g: List<&2, F32>) -> List<&2, F32>:
387  list.mul(g, list.scale((1.0 / F32.sqrt(((tree_sum(Cfg.dim(c), squares(x)) / F32.from_nat(Cfg.dim(c)) : F32) + Cfg.eps(c) : F32)) : F32), x))

Uses (13)

Used by (3)