def frow

bendtorch@1.3.0.0 · main.bend:2445

import Base

2445# ---- the same P with every key visible: full attention. `qrow` stops at the
2446# query's own position (i + 1 keys, the causal law); `frow` reads all t keys and
2447# `fprow` softmaxes over all of them, so the row's zeros after position i do not
2448# exist here. Everything else (the scaling, the write order, the softmax) is the
2449# causal path's own functions, so the two agree bit for bit on the triangle the
2450# causal one covers.
2451def frow(+i: Nat, +m: Nat, +hd: Nat, +qs: U32, +qc: U32, +ks: U32, +kc: U32, q: Array<F32>, k: Array<F32>) -> DR:
2452  drow(m, hd, ((U32.from_nat(i) * qs : U32) + qc : U32), kc, ks, Nil{}, dot0(hd, ((U32.from_nat(i) * qs : U32) + qc : U32), kc, 1, 1, q, k))