bendtorch-gpu Releases (6): 1.3.0.0 0x4c5cda5550959e621d91895fcad928df indexed https://jend.leeeet.dev/files.txt?path=bendtorch-gpu@1.3.0.0 1.2.0.0 0x2687dd9704f52182e1ea4e66472c6fcd unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-gpu@1.2.0.0 1.1.1.0 0x3c22148d676c2988827dae103c18f499 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-gpu@1.1.1.0 1.1.0.0 0xd450835bf87b8ebe49fc7baf18bdbc46 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-gpu@1.1.0.0 1.0.1.0 0x357d99c3e9fc95428e11c1d5f106e0a4 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-gpu@1.0.1.0 1.0.0.0 0x88faa733c17ccc2bd79ba26bdf7a4c89 unindexed https://jend.leeeet.dev/files.txt?path=bendtorch-gpu@1.0.0.0 Definitions (324, 1 to 100 shown): BendtorchGpu.lin_fwd (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lin_fwd) def lin_fwd(+n: Nat, +i: Nat, +o: Nat, p: T.Lin, x: TA.Mat) -> T.Fwd, TA.Mat, TA.Mat>: BendtorchGpu.lin_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lin_b2) def lin_b2(+n: Nat, +i: Nat, +o: Nat, b: TA.Mat<1n, o>, x: TA.Mat, g: T.Lin, r: TA.MMul) -> T.Bwd, TA.Mat>: BendtorchGpu.lin_bwd (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lin_bwd) def lin_bwd(+n: Nat, +i: Nat, +o: Nat, p: T.Lin, x: TA.Mat, dy: TA.Mat, g: T.Lin) -> T.Bwd, TA.Mat>: BendtorchGpu.EN (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=EN) type EN is Data: EN{n: U32} BendtorchGpu.nn (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=nn) def nn(u: Unit) -> Array: BendtorchGpu.ew1_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ew1_loop) # out[i] = f(x[i]) for the leaf's cells def ew1_loop(~f: @+a: F32 -> F32, k: Nat, +i: U32, +l: U32, o: Array, r: Array & F32) -> Array: BendtorchGpu.ew1 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ew1) def ew1(~f: @+a: F32 -> F32, +t: U32, +l: U32, +e: EN, x: Array, o: Array) -> Array: BendtorchGpu.ew2_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ew2_loop) # out[i] = f(x[i], y[i]) for the leaf's cells def ew2_loop(~f: @+a: F32 -> @+b: F32 -> F32, k: Nat, +i: U32, +l: U32, o: Array, ra: Array & F32, rb: Array & F32) -> Array: BendtorchGpu.ew2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ew2) def ew2(~f: @+a: F32 -> @+b: F32 -> F32, +t: U32, +l: U32, +e: EN, x: Array, y: Array, o: Array) -> Array: BendtorchGpu.ewi_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ewi_loop) # o[i] = f(o[i], y[i]) for the leaf's cells, in place (o's old value is the first argument) def ewi_loop(~f: @+a: F32 -> @+b: F32 -> F32, k: Nat, +i: U32, +l: U32, ro: Array & F32, ry: Array & F32) -> Array: BendtorchGpu.ewi (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ewi) def ewi(~f: @+a: F32 -> @+b: F32 -> F32, +t: U32, +l: U32, +e: EN, o: Array, y: Array) -> Array: BendtorchGpu.f32_id (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=f32_id) def f32_id(+x: F32) -> F32: BendtorchGpu.lf_copy (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_copy) def lf_copy(+t: U32, +l: U32, +e: EN, x1: Array, x2: Array, x3: Array, x4: Array, o: Array) -> Array: BendtorchGpu.lf_silu (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_silu) def lf_silu(+t: U32, +l: U32, +e: EN, x1: Array, x2: Array, x3: Array, x4: Array, o: Array) -> Array: BendtorchGpu.lf_silud (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_silud) # o = dy, written in place, x1 = x: main.bend's silu_d(dy, x) def lf_silud(+t: U32, +l: U32, +e: EN, x1: Array, x2: Array, x3: Array, x4: Array, o: Array) -> Array: BendtorchGpu.lf_add (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_add) # o = a, written in place, x1 = b: a + b def lf_add(+t: U32, +l: U32, +e: EN, x1: Array, x2: Array, x3: Array, x4: Array, o: Array) -> Array: BendtorchGpu.lf_mul (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_mul) def lf_mul(+t: U32, +l: U32, +e: EN, x1: Array, x2: Array, x3: Array, x4: Array, o: Array) -> Array: BendtorchGpu.mulb_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mulb_loop) # x1 = dy; the saved a and b, written in place: a becomes db = dy * a and b becomes da = dy * b, in one bang def mulb_loop(k: Nat, +i: U32, +l: U32, rd: Array & F32, ra: Array & F32, rb: Array & F32) -> TG.O2: BendtorchGpu.lf_mulb (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_mulb) def lf_mulb(+t: U32, +l: U32, +e: EN, x1: Array, x2: Array, x3: Array, x4: Array, o: TG.O2) -> TG.O2: BendtorchGpu.en (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=en) def en(+cnt: Nat) -> EN: BendtorchGpu.r1_a (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=r1_a) def r1_a(r: TG.R1) -> Array: BendtorchGpu.dup_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=dup_fin) def dup_fin(-r: Nat, -c: Nat, q: TG.R1) -> T.Two, TA.Mat>: BendtorchGpu.Mat.dup (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=Mat.dup) # two copies of a matrix, the second made on the device def Mat.dup(+r: Nat, +c: Nat, m: TA.Mat) -> T.Two, TA.Mat>: BendtorchGpu.Mat.sum2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=Mat.sum2) # a + b, as main.bend's Mat.sum2 adds each cell, written over a (both are consumed) def Mat.sum2(+r: Nat, +c: Nat, a: TA.Mat, b: TA.Mat) -> TA.Mat: BendtorchGpu.silu_f2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=silu_f2) def silu_f2(-n: Nat, -c: Nat, q: TG.R1) -> T.Fwd, TA.Mat>: BendtorchGpu.silu_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=silu_f) def silu_f(+n: Nat, +c: Nat, x: TA.Mat) -> T.Fwd, TA.Mat>: BendtorchGpu.silu_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=silu_b) def silu_b(+n: Nat, +c: Nat, s: TA.Mat, dy: TA.Mat) -> T.Bwd>: BendtorchGpu.SiLU (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=SiLU) # SiLU: y = silu(x), the input saved; dx = silu'(x) * dy def SiLU(+n: Nat, +c: Nat) -> T.Layer, TA.Mat, TA.Mat>: BendtorchGpu.mul2_f2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mul2_f2) def mul2_f2(-n: Nat, -c: Nat, q: TG.R1) -> T.Fwd, T.Two, TA.Mat>>: BendtorchGpu.mul2_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mul2_f) def mul2_f(+n: Nat, +c: Nat, x: T.Two, TA.Mat>) -> T.Fwd, T.Two, TA.Mat>>: BendtorchGpu.mul2_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mul2_b2) def mul2_b2(-n: Nat, -c: Nat, q: TG.R2) -> T.Bwd, TA.Mat>>: BendtorchGpu.mul2_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mul2_b) def mul2_b(+n: Nat, +c: Nat, s: T.Two, TA.Mat>, dy: TA.Mat) -> T.Bwd, TA.Mat>>: BendtorchGpu.Mul2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=Mul2) # Mul2: y = a * b, the inputs saved; da = dy * b, db = dy * a def Mul2(+n: Nat, +c: Nat) -> T.Layer, TA.Mat>, TA.Mat, T.Two, TA.Mat>>: BendtorchGpu.add2_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=add2_f) def add2_f(+n: Nat, +c: Nat, x: T.Two, TA.Mat>) -> T.Fwd, Unit>: BendtorchGpu.Add2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=Add2) def Add2(+n: Nat, +c: Nat) -> T.Layer, TA.Mat>, TA.Mat, Unit>: BendtorchGpu.Residual (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=Residual) # T.Residual with these copies and sums def Residual(~n: Nat, ~c: Nat, ~P: Type, ~S: Type, ~f: T.Layer, TA.Mat, S>) -> T.Layer, Unit>, TA.Mat, TA.Mat, T.Two, Unit>>: BendtorchGpu.ESGD (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ESGD) type ESGD is Data: ESGD{n: U32, lr: F32} BendtorchGpu.sgd_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=sgd_loop) def sgd_loop(k: Nat, +i: U32, +l: U32, +lr: F32, rw: Array & F32, rg: Array & F32) -> TG.O2: BendtorchGpu.lf_sgd (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_sgd) def lf_sgd(+t: U32, +l: U32, +e: ESGD, x1: Array, x2: Array, x3: Array, x4: Array, o: TG.O2) -> TG.O2: BendtorchGpu.EMOM (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=EMOM) type EMOM is Data: EMOM{n: U32, lr: F32, mu: F32} BendtorchGpu.mom_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mom_loop) def mom_loop(k: Nat, +i: U32, +l: U32, +lr: F32, +mu: F32, rw: Array & F32, rg: Array & F32, rm: Array & F32) -> TG.O3: BendtorchGpu.lf_mom (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_mom) def lf_mom(+t: U32, +l: U32, +e: EMOM, x1: Array, x2: Array, x3: Array, x4: Array, o: TG.O3) -> TG.O3: BendtorchGpu.EADAM (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=EADAM) type EADAM is Data: EADAM{n: U32, lr: F32, c1: F32, b2: F32, c2: F32, eps: F32, wd: F32, dec: Bool, step: F32, bc2: F32} BendtorchGpu.adam_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=adam_loop) def adam_loop(k: Nat, +i: U32, +l: U32, +lr: F32, +c1: F32, +b2: F32, +c2: F32, +eps: F32, +wd: F32, +dec: Bool, +step: F32, +bc2: F32, rw: Array & F32, rg: Array & F32, rm: Array & F32, rv: Array & F32) -> TG.O4: BendtorchGpu.lf_adam (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_adam) def lf_adam(+t: U32, +l: U32, +e: EADAM, x1: Array, x2: Array, x3: Array, x4: Array, o: TG.O4) -> TG.O4: BendtorchGpu.sgd_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=sgd_fin) def sgd_fin(-r: Nat, -c: Nat, q: TG.R2) -> T.MatStep: BendtorchGpu.mom_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mom_fin) def mom_fin(-r: Nat, -c: Nat, q: TG.R3) -> T.MatStep: BendtorchGpu.opt_fin (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=opt_fin) def opt_fin(-r: Nat, -c: Nat, q: TG.R4) -> T.MatStep: BendtorchGpu.opt_mat (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=opt_mat) # one matrix's step, in place: SGD writes two arrays, momentum three, Adam four def opt_mat(+r: Nat, +c: Nat, +o: T.Opt, w: TA.Mat, g: TA.Mat, st: T.OptState>) -> T.MatStep: BendtorchGpu.mat_step (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=mat_step) def mat_step(+r: Nat, +c: Nat, +op: T.Opt, p: TA.Mat, g: TA.Mat, st: T.OptState>) -> T.Step>: BendtorchGpu.lin_step2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lin_step2) def lin_step2(+i: Nat, +o: Nat, +op: T.Opt, p: T.Lin, g: T.Lin, st: T.Two>, T.OptState>>) -> T.Step>: BendtorchGpu.lin_step (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lin_step) def lin_step(+i: Nat, +o: Nat, +op: T.Opt, p: T.Lin, g: T.Lin, st: T.OptState>) -> T.Step>: BendtorchGpu.LinearF (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=LinearF) def LinearF(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchGpu.linf_bwd (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=linf_bwd) # LinearFirst: no dX (the first layer's input needs no gradient) def linf_bwd(+n: Nat, +i: Nat, +o: Nat, p: T.Lin, x: TA.Mat, dy: TA.Mat, g: T.Lin) -> T.Bwd, TA.Mat>: BendtorchGpu.LinearFirstF (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=LinearFirstF) def LinearFirstF(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchGpu.lnb_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lnb_b2) # LinearNB: y = x·Wᵀ, no bias (Llama's projections) def lnb_b2(+n: Nat, +i: Nat, +o: Nat, x: TA.Mat, g: TA.Mat, r: TA.MMul) -> T.Bwd, TA.Mat>: BendtorchGpu.LinearNBF (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=LinearNBF) def LinearNBF(+n: Nat, +i: Nat, +o: Nat, +par: Nat) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchGpu.ERF (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=ERF) type ERF is Data: ERF{n: U32, d: Nat, d32: U32, df: F32, eps: F32} BendtorchGpu.r3_y (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=r3_y) def r3_y(s: T.R3) -> Array: BendtorchGpu.rmsf_if (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=rmsf_if) def rmsf_if(ok: Bool, +t: U32, +d: Nat, +d32: U32, +df: F32, +eps: F32, x: Array, g: Array, y: Array) -> Array: BendtorchGpu.lf_rmsf (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_rmsf) # x1 = x, x2 = the gain g; y a row a leaf def lf_rmsf(+t: U32, +l: U32, +e: ERF, x1: Array, x2: Array, x3: Array, x4: Array, y: Array) -> Array: BendtorchGpu.erf (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=erf) def erf(+n: Nat, +d: Nat, +eps: F32) -> ERF: BendtorchGpu.rms_f2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=rms_f2) def rms_f2(-n: Nat, -d: Nat, q: TG.R1) -> T.Fwd, TA.Mat, TA.Mat>: BendtorchGpu.rms_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=rms_f) def rms_f(+n: Nat, +d: Nat, +eps: F32, p: TA.Mat<1n, d>, x: TA.Mat) -> T.Fwd, TA.Mat, TA.Mat>: BendtorchGpu.gdx_w (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdx_w) def gdx_w(k: Nat, +j: U32, +off: U32, +r: F32, +m: F32, dx: Array, rx: Array & F32, rd: Array & F32, rg: Array & F32) -> Array: BendtorchGpu.gdx_3 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdx_3) def gdx_3(+d: Nat, +off: U32, +df: F32, +r: F32, +t: U32, dx: Array, rr: Array, q: T.R3S) -> TG.O2: BendtorchGpu.gdx_2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdx_2) def gdx_2(+d: Nat, +off: U32, +df: F32, +eps: F32, +t: U32, dy: Array, g: Array, dx: Array, rr: Array, q: T.AS) -> TG.O2: BendtorchGpu.gdx_if (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdx_if) def gdx_if(ok: Bool, +t: U32, +d: Nat, +d32: U32, +df: F32, +eps: F32, x: Array, dy: Array, g: Array, dx: Array, rr: Array) -> TG.O2: BendtorchGpu.lf_gdx (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_gdx) # x1 = x, x2 = dy, x3 = g; outputs dx and 1/rms of each row def lf_gdx(+t: U32, +l: U32, +e: ERF, x1: Array, x2: Array, x3: Array, x4: Array, o: TG.O2) -> TG.O2: BendtorchGpu.gdg_loop (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdg_loop) # acc + dy[i]·(x[i]·r[row]) down a column, i from j by d def gdg_loop(k: Nat, +i: U32, +d: U32, +row: U32, acc: F32, rx: Array & F32, rd: Array & F32, rr: Array & F32) -> F32: BendtorchGpu.gdg_put (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdg_put) def gdg_put(+j: U32, +s: F32, r: Array & F32) -> Array: BendtorchGpu.EDG (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=EDG) type EDG is Data: EDG{d: U32, n: Nat} BendtorchGpu.gdg_if (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdg_if) def gdg_if(ok: Bool, +j: U32, +n: Nat, +d: U32, x: Array, dy: Array, rr: Array, ga: Array) -> Array: BendtorchGpu.lf_gdg (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_gdg) # x1 = x, x2 = dy, x3 = 1/rms; the gradient of the gain, in place: a leaf a column def lf_gdg(+t: U32, +l: U32, +e: EDG, x1: Array, x2: Array, x3: Array, x4: Array, ga: Array) -> Array: BendtorchGpu.rms_b3 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=rms_b3) def rms_b3(-n: Nat, -d: Nat, g: Array, dx: Array, q: TG.R1) -> T.Bwd, TA.Mat>: BendtorchGpu.rms_b2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=rms_b2) def rms_b2(+n: Nat, +d: Nat, ga: Array, q: TG.R2) -> T.Bwd, TA.Mat>: BendtorchGpu.rms_b (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=rms_b) def rms_b(+n: Nat, +d: Nat, +eps: F32, p: TA.Mat<1n, d>, x: TA.Mat, dy: TA.Mat, gacc: TA.Mat<1n, d>) -> T.Bwd, TA.Mat>: BendtorchGpu.RMSNorm (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=RMSNorm) # RMSNorm over the last dimension, its gain trained (main.bend's RMSNorm) def RMSNorm(+n: Nat, +d: Nat, +eps: F32) -> T.Layer, TA.Mat, TA.Mat, TA.Mat>: BendtorchGpu.EGN (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=EGN) type EGN is Data: EGN{nb: U32, ng: U32, hw: Nat, hwc: U32, c: U32, cg: Nat, cg32: U32, nf: F32, eps: F32} BendtorchGpu.egn (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=egn) def egn(+n: Nat, +hw: Nat, +ng: Nat, +cg: Nat, +eps: F32) -> EGN: BendtorchGpu.r4_y (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=r4_y) def r4_y(s: T.R4) -> Array: BendtorchGpu.gnf_if (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnf_if) def gnf_if(ok: Bool, +t: U32, +ng: U32, +hw: Nat, +hwc: U32, +c: U32, +cg: Nat, +cg32: U32, +nf: F32, +eps: F32, x: Array, g: Array, b: Array, y: Array) -> Array: BendtorchGpu.lf_gnf (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_gnf) # x1 = x, x2 = the gain g, x3 = the bias b; y a block a leaf def lf_gnf(+t: U32, +l: U32, +e: EGN, x1: Array, x2: Array, x3: Array, x4: Array, y: Array) -> Array: BendtorchGpu.gn_f2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gn_f2) def gn_f2(-n: Nat, -d: Nat, q: TG.R1) -> T.Fwd, TA.Mat, TA.Mat>: BendtorchGpu.gn_f (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gn_f) def gn_f(+n: Nat, +hw: Nat, +ng: Nat, +cg: Nat, +eps: F32, p: T.LN, x: TA.Mat) -> T.Fwd, TA.Mat, TA.Mat>: BendtorchGpu.GB (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=GB) type GB is Type: GB{x: Array, dy: Array, g: Array, dx: Array} BendtorchGpu.gb_dx (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gb_dx) def gb_dx(s: GB) -> Array: BendtorchGpu.gnb_seg (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnb_seg) # dx[off + j] = (((dy·g - ma) - xh·mb)·r) along one row's segment def gnb_seg(k: Nat, +j: U32, +off: U32, +mu: F32, +r: F32, +ma: F32, +mb: F32, dx: Array, rx: Array & F32, rd: Array & F32, rg: Array & F32) -> GB: BendtorchGpu.gnb_segs (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnb_segs) def gnb_segs(left: Nat, +off: U32, +j: U32, +c: U32, +cg: Nat, +mu: F32, +r: F32, +ma: F32, +mb: F32, s: GB) -> GB: BendtorchGpu.gnb_blk3 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnb_blk3) def gnb_blk3(+hw: Nat, +off: U32, +j: U32, +c: U32, +cg: Nat, +nf: F32, +mu: F32, +r: F32, +t: U32, dx: Array, mr: Array, q: T.R3SS) -> TG.O2: BendtorchGpu.gnb_blk2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnb_blk2) def gnb_blk2(+hw: Nat, +off: U32, +j: U32, +c: U32, +cg: Nat, +nf: F32, +t: U32, dy: Array, g: Array, dx: Array, mr: Array, m: T.ASM) -> TG.O2: BendtorchGpu.gnb_blk (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnb_blk) def gnb_blk(+hw: Nat, +off: U32, +j: U32, +c: U32, +cg: Nat, +nf: F32, +eps: F32, +t: U32, x: Array, dy: Array, g: Array, dx: Array, mr: Array) -> TG.O2: BendtorchGpu.gnb_if (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gnb_if) def gnb_if(ok: Bool, +t: U32, +ng: U32, +hw: Nat, +hwc: U32, +c: U32, +cg: Nat, +cg32: U32, +nf: F32, +eps: F32, x: Array, dy: Array, g: Array, dx: Array, mr: Array) -> TG.O2: BendtorchGpu.lf_gnb (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=lf_gnb) # x1 = x, x2 = dy, x3 = g; outputs dx and each block's mean and 1/sqrt(var + eps) def lf_gnb(+t: U32, +l: U32, +e: EGN, x1: Array, x2: Array, x3: Array, x4: Array, o: TG.O2) -> TG.O2: BendtorchGpu.GS (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=GS) type GS is Type: GS{x: Array, dy: Array, mr: Array, sg: F32, sb: F32} BendtorchGpu.gdw_rows (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdw_rows) # sg + dy·xh and sb + dy down one image's rows of a channel, i its cell, by c def gdw_rows(k: Nat, +i: U32, +c: U32, +mu: F32, +r: F32, sg: F32, sb: F32, mr: Array, rx: Array & F32, rd: Array & F32) -> GS: BendtorchGpu.gdw_blk2 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdw_blk2) def gdw_blk2(+hw: Nat, +i: U32, +c: U32, +mu: F32, sg: F32, sb: F32, x: Array, dy: Array, rr: Array & F32) -> GS: BendtorchGpu.gdw_blk1 (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdw_blk1) def gdw_blk1(+hw: Nat, +i: U32, +bl: U32, +c: U32, sg: F32, sb: F32, x: Array, dy: Array, rm: Array & F32) -> GS: BendtorchGpu.gdw_blk (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdw_blk) # block bl's mean and 1/sqrt(var + eps), then its rows def gdw_blk(+hw: Nat, +i: U32, +bl: U32, +c: U32, s: GS) -> GS: BendtorchGpu.gdw_imgs (https://jend.leeeet.dev/definition.txt?path=bendtorch-gpu@1.3.0.0/main.bend&name=gdw_imgs) # a channel's images in order: i steps by hw·c to the next image's first row, bl by ng to its block of the same group def gdw_imgs(left: Nat, +i: U32, +bl: U32, +hw: Nat, +hwc: U32, +c: U32, +ng: U32, s: GS) -> GS: More: https://jend.leeeet.dev/packages/bendtorch-gpu.txt?definitions_offset=100