~/jendhub

def grow

bendtorch@1.3.0.0 · main.bend:2642

import bendtorch@1.3.0.0/main.bend as Bendtorch
import Base

2642# row i of G (from i·t, t wide; softmax's backward pass, then the scale): dP's row against values 0..i only (drow, as
2643# qrow reads keys 0..i), written from the row's start, and s and G over as many positions as were written: i + 1 (the
2644# law causal_grad). G starts as zeros and nothing else writes the row, so its positions after i stay 0: no gradient
2645# reaches a key or value after the query's position. (P is 0 there too, so the full row gave the same G.)
2646def grow(+i: Nat, +t: Nat, +hd: Nat, +sc: F32, +ds: U32, +dc: U32, +vs: U32, +vc: U32, d: Array<F32>, v: Array<F32>, p: Array<F32>, g: Array<F32>) -> GR:
2647  gr2(sc, (U32.from_nat(i) * U32.from_nat(t) : U32), p, g, drow(i, hd, ((U32.from_nat(i) * ds : U32) + dc : U32), vc, vs, Nil{}, dot0(hd, ((U32.from_nat(i) * ds : U32) + dc : U32), vc, 1, 1, d, v)))

Uses (12)

Used by (3)