~/jendhub

def Cfg.layer_size

bend-ml-llama@0.2.1.0 · main.bend:188

import bend-ml-llama@0.2.1.0/main.bend as BendMlLlama
import Base

188# the numbers of one layer, from the shapes of LW<c>: two norms, q and o (dim × dim), k and v (kvw × dim),
189# gate and up (ffn × dim) and down (dim × ffn)
190def Cfg.layer_size(+c: Cfg) -> U32:
191  ((2 : U32) * n32(Cfg.dim(c)) + (2 : U32) * n32(Cfg.dim(c)) * n32(Cfg.dim(c)) + (2 : U32) * n32(Cfg.kvw(c)) * n32(Cfg.dim(c)) + (3 : U32) * n32(Cfg.ffn(c)) * n32(Cfg.dim(c)) : U32)

Uses (8)

Used by (1)