bendtorch-fast Releases (6): 1.3.0.0 0x262e539f65210ab30c4bcb8167815e98 indexed https://jend.leeeet.dev/files.txt?path=bendtorch-fast@1.3.0.0 1.2.0.0 0x89b3380e86f5a59734944cc2f327badb unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-fast@1.2.0.0 1.1.1.0 0x7185f5844710249c5ac77956f6514eb7 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-fast@1.1.1.0 1.1.0.0 0x154ce8ea744f0491f3a42ae0881fc224 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-fast@1.1.0.0 1.0.1.0 0xb91744a472860b4a9d4321aff37d6994 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-fast@1.0.1.0 1.0.0.0 0x6475a9340a3eb94acc1748bc05999432 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-fast@1.0.0.0 Definitions (66): BendtorchFast.ew_d2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ew_d2) # log2 of the blocks for cnt numbers: one block below 131072 (MNIST's and the tiny Llama's matrices; at 100352 the # blocks cost MNIST more than they gave on 16 threads), else one per 16384, at most 64 def ew_d2(small: Bool, big: Bool, +cnt: Nat) -> Nat: BendtorchFast.ew_depth (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ew_depth) def ew_depth(+cnt: Nat) -> Nat: BendtorchFast.p2_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=p2_b2) def p2_b2(q: TA.P2) -> TF.B2: BendtorchFast.sgd_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=sgd_leaf) def sgd_leaf(+lr: F32, +i0: U32, +cnt: Nat, r: TF.B2) -> TF.B2: BendtorchFast.mom_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mom_b) def mom_b(q: T.Q3) -> TF.B3: BendtorchFast.mom_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mom_leaf) def mom_leaf(+op: T.Opt, +i0: U32, +cnt: Nat, r: TF.B3) -> TF.B3: BendtorchFast.q4_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=q4_b) def q4_b(q: T.Q4) -> TF.B4: BendtorchFast.opt_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=opt_leaf) def opt_leaf(+o: T.Opt, +i0: U32, +cnt: Nat, r: TF.B4) -> TF.B4: BendtorchFast.sgd_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=sgd_fin) def sgd_fin(-r: Nat, -c: Nat, q: TF.B2) -> T.MatStep: BendtorchFast.mom_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mom_fin) def mom_fin(-r: Nat, -c: Nat, q: TF.B3) -> T.MatStep: BendtorchFast.opt_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=opt_fin) def opt_fin(-r: Nat, -c: Nat, q: TF.B4) -> T.MatStep: BendtorchFast.PE (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=PE) type PE is Data: PE{o: T.Opt, n: U32} BendtorchFast.ph_cnt (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_cnt) # the elements of a block of cnt words from element e0, the last block stopping at n def ph_cnt(+n: U32, +e0: U32, +cnt: Nat) -> Nat: BendtorchFast.pq_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=pq_b) def pq_b(q: T.PQ) -> TF.B4: BendtorchFast.pqv_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=pqv_b) def pqv_b(q: T.PQV) -> TF.B4: BendtorchFast.ph_go (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_go) def ph_go(+o: T.Opt, +n: U32, +i0: U32, +cnt: Nat, r: TF.B4) -> TF.B4: BendtorchFast.ph_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_leaf) def ph_leaf(+e: PE, +i0: U32, +cnt: Nat, r: TF.B4) -> TF.B4: BendtorchFast.phv_go (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=phv_go) def phv_go(+o: T.Opt, +n: U32, +i0: U32, +cnt: Nat, r: TF.B4) -> TF.B4: BendtorchFast.phv_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=phv_leaf) def phv_leaf(+e: PE, +i0: U32, +cnt: Nat, r: TF.B4) -> TF.B4: BendtorchFast.ph_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_fin) def ph_fin(-r: Nat, -c: Nat, +n: U32, q: TF.B4) -> T.MatStep: BendtorchFast.phv_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=phv_fin) def phv_fin(-r: Nat, -c: Nat, +n: U32, q: TF.B4) -> T.MatStep: BendtorchFast.ph_nw (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_nw) # the words of r·c elements, ceil(r·c / 2) def ph_nw(+r: Nat, +c: Nat) -> Nat: BendtorchFast.ph_leafs (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_leafs) def ph_leafs(+r: Nat, +c: Nat, +o: T.Opt, wa: Array, ga: Array, t: T.PTree) -> T.MatStep: BendtorchFast.ph_mat (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ph_mat) def ph_mat(+r: Nat, +c: Nat, +o: T.Opt, wa: Array, ga: Array, t: T.PTree) -> T.MatStep: BendtorchFast.opt_mat (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=opt_mat) # SGD splits two arrays, momentum three, Adam four. No unused parameter-sized buffers. def opt_mat(+r: Nat, +c: Nat, +o: T.Opt, w: TA.Mat, g: TA.Mat, st: T.OptState>) -> T.MatStep: BendtorchFast.mat_step (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mat_step) def mat_step(+r: Nat, +c: Nat, +op: T.Opt, p: TA.Mat, g: TA.Mat, st: T.OptState>) -> T.Step>: BendtorchFast.lin_step2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=lin_step2) def lin_step2(+i: Nat, +o: Nat, +op: T.Opt, p: T.Lin, g: T.Lin, st: T.Two>, T.OptState>>) -> T.Step>: BendtorchFast.lin_step (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=lin_step) def lin_step(+i: Nat, +o: Nat, +op: T.Opt, p: T.Lin, g: T.Lin, st: T.OptState>) -> T.Step>: BendtorchFast.ew1_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ew1_leaf) def ew1_leaf(~f: @+a: F32 -> F32, +e: Unit, +i0: U32, +cnt: Nat, a: Array) -> Array: BendtorchFast.map (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=map) def map(~f: @+a: F32 -> F32, +r: Nat, +c: Nat, m: TA.Mat) -> TA.Mat: BendtorchFast.ew2_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ew2_leaf) def ew2_leaf(~f: @+a: F32 -> @+b: F32 -> F32, +e: Unit, +i0: U32, +cnt: Nat, r: TF.B2) -> TF.B2: BendtorchFast.zip_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=zip_fin) def zip_fin(-r: Nat, -c: Nat, q: TF.B2) -> TA.MPair: BendtorchFast.zip (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=zip) # a[i] = f(a[i], b[i]); b comes back unchanged def zip(~f: @+a: F32 -> @+b: F32 -> F32, +r: Nat, +c: Nat, a: TA.Mat, b: TA.Mat) -> TA.MPair: BendtorchFast.a2_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=a2_b2) def a2_b2(q: T.A2) -> TF.B2: BendtorchFast.copy_leaf (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=copy_leaf) # numbers i0 .. i0+cnt-1 of a into b def copy_leaf(+e: Unit, +i0: U32, +cnt: Nat, r: TF.B2) -> TF.B2: BendtorchFast.dup_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=dup_fin) def dup_fin(-r: Nat, -c: Nat, q: TF.B2) -> T.Two, TA.Mat>: BendtorchFast.Mat.dup (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Mat.dup) # two copies of a matrix (T.Mat.dup) def Mat.dup(+r: Nat, +c: Nat, m: TA.Mat) -> T.Two, TA.Mat>: BendtorchFast.mpair_a (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mpair_a) def mpair_a(-r: Nat, -c: Nat, q: TA.MPair) -> TA.Mat: BendtorchFast.Mat.sum2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Mat.sum2) # a + b, dropping b (T.Mat.sum2) def Mat.sum2(+r: Nat, +c: Nat, a: TA.Mat, b: TA.Mat) -> TA.Mat: BendtorchFast.lin_fwd (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=lin_fwd) def lin_fwd(+n: Nat, +i: Nat, +o: Nat, par: Nat, p: T.Lin, x: TA.Mat) -> T.Fwd, TA.Mat, TA.Mat>: BendtorchFast.lin_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=lin_b2) def lin_b2(+n: Nat, +i: Nat, +o: Nat, par: Nat, b: TA.Mat<1n, o>, x: TA.Mat, g: T.Lin, r: TA.MMul) -> T.Bwd, TA.Mat>: BendtorchFast.lin_bwd (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=lin_bwd) def lin_bwd(+n: Nat, +i: Nat, +o: Nat, +par: Nat, p: T.Lin, x: TA.Mat, dy: TA.Mat, g: T.Lin) -> T.Bwd, TA.Mat>: BendtorchFast.LinearF (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=LinearF) def LinearF(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchFast.linf_bwd (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=linf_bwd) def linf_bwd(+n: Nat, +i: Nat, +o: Nat, +par: Nat, p: T.Lin, x: TA.Mat, dy: TA.Mat, g: T.Lin) -> T.Bwd, TA.Mat>: BendtorchFast.LinearFirstF (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=LinearFirstF) def LinearFirstF(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchFast.lnb_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=lnb_b2) def lnb_b2(+n: Nat, +i: Nat, +o: Nat, +par: Nat, x: TA.Mat, g: TA.Mat, r: TA.MMul) -> T.Bwd, TA.Mat>: BendtorchFast.LinearNBF (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=LinearNBF) def LinearNBF(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchFast.acti_f3 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=acti_f3) def acti_f3(~f: @+a: F32 -> F32, +n: Nat, +c: Nat, m: T.Two, TA.Mat>) -> T.Fwd, TA.Mat>: BendtorchFast.ActIn (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=ActIn) # T.ActIn: f on a copy, the input saved def ActIn(~f: @+a: F32 -> F32, ~d: @+a: F32 -> @+b: F32 -> F32, +n: Nat, +c: Nat) -> T.Layer, TA.Mat, TA.Mat>: BendtorchFast.SiLU (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=SiLU) def SiLU(+n: Nat, +c: Nat) -> T.Layer, TA.Mat, TA.Mat>: BendtorchFast.add2_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=add2_f) def add2_f(+n: Nat, +c: Nat, x: T.Two, TA.Mat>) -> T.Fwd, Unit>: BendtorchFast.Add2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Add2) def Add2(+n: Nat, +c: Nat) -> T.Layer, TA.Mat>, TA.Mat, Unit>: BendtorchFast.mul2_f2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mul2_f2) def mul2_f2(+n: Nat, +c: Nat, da: T.Two, TA.Mat>, db: T.Two, TA.Mat>) -> T.Fwd, T.Two, TA.Mat>>: BendtorchFast.mul2_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mul2_f) def mul2_f(+n: Nat, +c: Nat, x: T.Two, TA.Mat>) -> T.Fwd, T.Two, TA.Mat>>: BendtorchFast.mul2_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=mul2_b2) def mul2_b2(+n: Nat, +c: Nat, s: T.Two, TA.Mat>, d: T.Two, TA.Mat>) -> T.Bwd, TA.Mat>>: BendtorchFast.Mul2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Mul2) def Mul2(+n: Nat, +c: Nat) -> T.Layer, TA.Mat>, TA.Mat, T.Two, TA.Mat>>: BendtorchFast.Residual (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Residual) # T.Residual with these copies and sums (T.ResidualOf, the composition the law residual is about) def Residual(~n: Nat, ~c: Nat, ~P: Type, ~S: Type, ~f: T.Layer, TA.Mat, S>) -> T.Layer, Unit>, TA.Mat, TA.Mat, T.Two, Unit>>: BendtorchFast.RMSNorm (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=RMSNorm) # main.bend's RMSNorm, RoPE, Attention, MSE and CrossEntropy under their own names (Embedding below, with its step in # blocks), so that a program that takes its layers from here can take device versions of the same names from another # file by its import line alone (they run as in main.bend) def RMSNorm(+n: Nat, +d: Nat, +eps: F32) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchFast.RoPE (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=RoPE) def RoPE(+n: Nat, +w: Nat, +t: Nat, +h: Nat, +half: Nat, +base: F32, -e: {w == Nat.mul(h, Nat.add(half, half)) : Nat}) -> T.Layer, TA.Mat, Unit>: BendtorchFast.Attention (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Attention) def Attention(+n: Nat, +w: Nat, +kvw: Nat, +b: Nat, +t: Nat, +kv: Nat, +g: Nat, +hd: Nat, +sc: F32, -e1: {n == Nat.mul(b, t) : Nat}, -e2: {w == Nat.mul(Nat.mul(kv, g), hd) : Nat}, -e3: {kvw == Nat.mul(kv, hd) : Nat}) -> T.Layer, T.Two, TA.Mat>>, TA.Mat, T.AttnS>: BendtorchFast.MSE (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=MSE) def MSE(+n: Nat, +c: Nat, y: TA.Mat, t: TA.Mat) -> T.LossOut>: BendtorchFast.Embedding (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Embedding) # main.bend's Embedding with its optimizer step in blocks (mat_step): the table is SmolLM2's largest matrix (49152 x 576, # tied to the output head), and main.bend's step updates it on one thread def Embedding(+n: Nat, +v: Nat, +d: Nat) -> T.Layer, List<&2, Nat>, TA.Mat, List<&2, Nat>>: BendtorchFast.CrossEntropy (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=CrossEntropy) def CrossEntropy(+n: Nat, +c: Nat, z: TA.Mat, +labels: List<&2, Nat>) -> T.LossOut>: BendtorchFast.Linear (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=Linear) def Linear(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchFast.LinearFirst (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=LinearFirst) def LinearFirst(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchFast.LinearNB (https://jend.leeeet.dev/definition.txt?path=bendtorch-fast@1.3.0.0/main.bend&name=LinearNB) def LinearNB(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: