def forward_gen

bend-ml-llama@0.2.0.0 · main.bend:1039

import Base
import bend-ml-tensor-array@0.1.14.0/main.bend as TA

1039# a position for generation: the same logits, caches and model. With bands (d > 0, more than two threads) the
1040# activations are left empty, about a fifth faster per token on 16 threads; with one band (d = 0) they are kept, since
1041# leaving them out measured 4-11% slower per token on one thread (llama.bend NOTES, 2026-10-16), for no reason found yet
1042def forward_gen(+c: Cfg, -cap: Nat, +t: Nat, m: Model<c>, cs: List<&1, KV<cap, t>>, -room: {True{} == TA.leb(1n+t, cap) : Bool}, +tok: Nat) -> Out<c, cap, t>:
1043  match m:
1044    case Model{+d, emb, ls, +lnf}:
1045      forward.go(Nat.is_eq(d, 0n), c, cap, t, room, d, ls, lnf, cs, TA.Bands.read_row(Cfg.vocab(c), Cfg.dim(c), emb, tok))