bend-ml-tensor-fork Releases (8): 0.1.6.0 0x00977eb08bd5a81deaf111443555211b indexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.6.0 0.1.5.0 0x236efa65086abc5955a9e74563e24668 unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.5.0 0.1.4.0 0xd13098b0f80022977b3ddb83a57e3c50 unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.4.0 0.1.3.0 0x75a8d2c438b6e6aeea7459234622dfd6 unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.3.0 0.1.2.0 0x5734508f2a6198d9a87a28d4c58bfbae unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.2.0 0.1.1.1 0x2a674c049e9c1285b25b99116b738ac5 unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.1.1 0.1.1.0 0x7426570b56a9f08f09fe41b970d661b9 unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.1.0 0.1.0.0 0xd8eb8357f30e6c7a242507aee97fab7e unindexed https://jend.leeeet.dev/files.txt?path=bend-ml-tensor-fork@0.1.0.0 Definitions (63): BendMlTensorFork.B2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=B2) type B2<-A: Type, -B: Type> is Type: B2{a: Array, b: Array} BendMlTensorFork.B3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=B3) type B3<-A: Type, -B: Type, -C: Type> is Type: B3{a: Array, b: Array, c: Array} BendMlTensorFork.B4 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=B4) type B4<-A: Type, -B: Type, -C: Type, -D: Type> is Type: B4{a: Array, b: Array, c: Array, d: Array} @unsafe BendMlTensorFork.blocks1 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=blocks1) def blocks1(~E: Data, ~A: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> Array -> Array, +e: E, d: Nat, +i0: U32, +cnt: Nat, pa: Array & Array) -> Array: BendMlTensorFork.join2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=join2) def join2(-A: Type, -B: Type, l: B2, r: B2) -> B2: BendMlTensorFork.blocks2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=blocks2) def blocks2(~E: Data, ~A: Type, ~B: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> B2 -> B2, +e: E, d: Nat, +i0: U32, +cnt: Nat, pa: Array & Array, pb: Array & Array) -> B2: BendMlTensorFork.join3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=join3) def join3(-A: Type, -B: Type, -C: Type, l: B3, r: B3) -> B3: BendMlTensorFork.blocks3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=blocks3) def blocks3(~E: Data, ~A: Type, ~B: Type, ~C: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> B3 -> B3, +e: E, d: Nat, +i0: U32, +cnt: Nat, pa: Array & Array, pb: Array & Array, pc: Array & Array) -> B3: BendMlTensorFork.join4 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=join4) def join4(-A: Type, -B: Type, -C: Type, -D: Type, l: B4, r: B4) -> B4: BendMlTensorFork.blocks4 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=blocks4) def blocks4(~E: Data, ~A: Type, ~B: Type, ~C: Type, ~D: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> B4 -> B4, +e: E, d: Nat, +i0: U32, +cnt: Nat, pa: Array & Array, pb: Array & Array, pc: Array & Array, pd: Array & Array) -> B4: BendMlTensorFork.par1 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=par1) def par1(~E: Data, ~A: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> Array -> Array, +e: E, d: Nat, +cnt: Nat, a: Array) -> Array: BendMlTensorFork.par2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=par2) def par2(~E: Data, ~A: Type, ~B: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> B2 -> B2, +e: E, d: Nat, +cnt: Nat, a: Array, b: Array) -> B2: BendMlTensorFork.par3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=par3) def par3(~E: Data, ~A: Type, ~B: Type, ~C: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> B3 -> B3, +e: E, d: Nat, +cnt: Nat, a: Array, b: Array, c: Array) -> B3: BendMlTensorFork.par4 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=par4) def par4(~E: Data, ~A: Type, ~B: Type, ~C: Type, ~D: Type, ~f: @+e: E -> @+i0: U32 -> @+cnt: Nat -> B4 -> B4, +e: E, d: Nat, +cnt: Nat, a: Array, b: Array, c: Array, x: Array) -> B4: BendMlTensorFork.depth_go (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=depth_go) # the smallest d with 2^d >= n (TA.cap_depth's answer) by doubling a U32: log2(n) steps, where cap_depth compares Nats # (a share of every product's time at 512 x 1536) def depth_go(f: Nat, ok: Bool, +n: U32, +p: U32, d: Nat) -> Nat: BendMlTensorFork.depth (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=depth) def depth(+n: Nat) -> Nat: BendMlTensorFork.PE (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=PE) type PE is Data: PE{tk: Nat, tku: U32, ku: U32, lstep: U32, st: U32} BendMlTensorFork.pk_b2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=pk_b2) def pk_b2(r: TA.PK) -> B2: BendMlTensorFork.pk_leaf (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=pk_leaf) # lines i0 .. i0+cnt-1 into their cells def pk_leaf(+e: PE, +i0: U32, +cnt: Nat, r: B2) -> B2: BendMlTensorFork.pk_of (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=pk_of) def pk_of(r: B2) -> TA.PK: BendMlTensorFork.pk_d (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=pk_d) # one block below 16384 cells (MNIST's 100 x 784 and 128 x 784 pack faster in one), else 2^d def pk_d(big: Bool, d: Nat) -> Nat: BendMlTensorFork.pk_par (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=pk_par) def pk_par(d: Nat, +cnt: Nat, +tk: Nat, +k: Nat, +lstep: U32, +st: U32, a: Array) -> TA.PK: BendMlTensorFork.tg_c (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=tg_c) def tg_c(g: TA.TG) -> Array: BendMlTensorFork.rblock (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=rblock) # rows i0 .. i0+cnt-1 of C, every column def rblock(+ac: Bool, cnt: Nat, +m: Nat, +tk: Nat, +tku: U32, +rot: Nat, +i0: U32, +mu: U32, a: Array, b: Array, c: Array) -> Array: BendMlTensorFork.cbrows (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=cbrows) # columns j0 .. j0+cnt-1 of C, rows i.. (n more) def cbrows(+ac: Bool, n: Nat, +cnt: Nat, +tk: Nat, +tku: U32, +rot: Nat, +i: U32, +j0: U32, +mu: U32, g: TA.TG) -> TA.TG: BendMlTensorFork.rpar (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=rpar) def rpar(+ac: Bool, d: Nat, +cnt: Nat, +m: Nat, +tk: Nat, +tku: U32, +rot: Nat, +i0: U32, +mu: U32, pa: Array & Array, pb: Array & Array, pc: Array & Array) -> Array: BendMlTensorFork.cpar (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=cpar) def cpar(+ac: Bool, d: Nat, +cnt: Nat, +n: Nat, +tk: Nat, +tku: U32, +rot: Nat, +j0: U32, +mu: U32, pa: Array & Array, pb: Array & Array, pc: Array & Array) -> Array: BendMlTensorFork.t8_tree (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=t8_tree) def t8_tree(byrows: Bool, +ac: Bool, d: Nat, +n: Nat, +m: Nat, +tk: Nat, +tku: U32, +rot: Nat, pa: Array & Array, pb: Array & Array, pc: Array & Array) -> Array: BendMlTensorFork.t8_go (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=t8_go) def t8_go(+ac: Bool, d: Nat, +n: Nat, +m: Nat, +tk: Nat, +tku: U32, +rot: Nat, c: Array, p: TA.PKK) -> TA.G: BendMlTensorFork.gemm_fork (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=gemm_fork) def gemm_fork(+ac: Bool, +d: Nat, +n: Nat, +m: Nat, +k: Nat, +arow: U32, +sa: U32, +sb: U32, +bcol: U32, a: Array, b: Array, c: Array) -> TA.G: BendMlTensorFork.gemm_rows (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=gemm_rows) # two or more rows: the shared tree when d > 0, tensor-array's sequential cells at d = 0 def gemm_rows(+ac: Bool, d: Nat, +n: Nat, +m: Nat, +k: Nat, +arow: U32, +sa: U32, +sb: U32, +bcol: U32, a: Array, b: Array, c: Array) -> TA.G: BendMlTensorFork.gemm_any2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=gemm_any2) def gemm_any2(+ac: Bool, one: Bool, d: Nat, +n: Nat, +m: Nat, +k: Nat, +arow: U32, +sa: U32, +sb: U32, +bcol: U32, a: Array, b: Array, c: Array) -> TA.G: BendMlTensorFork.gemm_any (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=gemm_any) # TA.gemm_any's arguments and result; one row runs the scalar loop at every depth, as in tensor-array 0.1.12.0 def gemm_any(+ac: Bool, d: Nat, +n: Nat, +m: Nat, +k: Nat, +arow: U32, +sa: U32, +sb: U32, +bcol: U32, a: Array, b: Array, c: Array) -> TA.G: BendMlTensorFork.matmul (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul) # C = A · B (n x k) · (k x m) def matmul(+n: Nat, +k: Nat, +m: Nat, par: Nat, a: TA.Mat, b: TA.Mat) -> TA.MMul: BendMlTensorFork.matmul_acc (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul_acc) # C + A · B, into C def matmul_acc(+n: Nat, +k: Nat, +m: Nat, par: Nat, a: TA.Mat, b: TA.Mat, c: TA.Mat) -> TA.MMul: BendMlTensorFork.matmul_nt (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul_nt) # C = A · Bᵀ (n x k) · (m x k)ᵀ: weights with one row per output def matmul_nt(+n: Nat, +k: Nat, +m: Nat, par: Nat, a: TA.Mat, b: TA.Mat) -> TA.MMulNT: BendMlTensorFork.matmul_nt_acc (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul_nt_acc) # C + A · Bᵀ, into C def matmul_nt_acc(+n: Nat, +k: Nat, +m: Nat, par: Nat, a: TA.Mat, b: TA.Mat, c: TA.Mat) -> TA.MMulNT: BendMlTensorFork.matmul_tn (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul_tn) # C = Aᵀ · B (k x n)ᵀ · (k x m): a weight gradient, Xᵀ · dY def matmul_tn(+n: Nat, +k: Nat, +m: Nat, par: Nat, a: TA.Mat, b: TA.Mat) -> TA.MMulTN: BendMlTensorFork.matmul_tn_acc (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul_tn_acc) # C + Aᵀ · B, into C: dW += Xᵀ · dY def matmul_tn_acc(+n: Nat, +k: Nat, +m: Nat, par: Nat, a: TA.Mat, b: TA.Mat, c: TA.Mat) -> TA.MMulTN: BendMlTensorFork.mp_s3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=mp_s3) def mp_s3(+ac: Bool, +par: Nat, +n: Nat, +m: Nat, +w: Nat, +w32: U32, +m32: U32, a: Array, as: Array, c: Array, sb: TA.SL) -> TA.G: BendMlTensorFork.mp_s2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=mp_s2) def mp_s2(+ac: Bool, +par: Nat, +n: Nat, +m: Nat, +w: Nat, +w32: U32, +m32: U32, +o: U32, c: Array, b: Array, sa: TA.SL) -> TA.G: BendMlTensorFork.mp_step (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=mp_step) def mp_step(+ac: Bool, +par: Nat, +n: Nat, +m: Nat, +k32: U32, +bi: U32, a: Array, b: Array, c: Array) -> TA.G: BendMlTensorFork.mp_go (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=mp_go) def mp_go(nb: Nat, +bi: U32, +par: Nat, +n: Nat, +m: Nat, +k32: U32, g: TA.G) -> TA.G: BendMlTensorFork.mm_precise2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=mm_precise2) def mm_precise2(long: Bool, +n: Nat, +k: Nat, +m: Nat, +par: Nat, a: TA.Mat, b: TA.Mat) -> TA.MMul: BendMlTensorFork.matmul_precise (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=matmul_precise) def matmul_precise(+n: Nat, +k: Nat, +m: Nat, +par: Nat, a: TA.Mat, b: TA.Mat) -> TA.MMul: BendMlTensorFork.SE (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=SE) type SE is Data: SE{mm1: Nat, mu: U32, nf: F32} BendMlTensorFork.sr3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=sr3) # a row's loss term log(max(p, 1e-12)) and its gradient: cea_row3..cea_row0 without the running loss def sr3(+mm1: Nat, +o: U32, +label: Nat, +s: F32, +nf: F32, q: TA.AF) -> TA.L2: BendMlTensorFork.sr2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=sr2) def sr2(+mm1: Nat, +o: U32, +label: Nat, +nf: F32, q: TA.AF) -> TA.L2: BendMlTensorFork.sr1 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=sr1) def sr1(+mm1: Nat, +o: U32, +label: Nat, +nf: F32, q: TA.AF) -> TA.L2: BendMlTensorFork.sr0 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=sr0) def sr0(+mm1: Nat, +o: U32, +label: Nat, +nf: F32, r: Array & F32) -> TA.L2: BendMlTensorFork.ce_put (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_put) def ce_put(+i: U32, lab: Array, t: Array, q: TA.L2) -> B3: BendMlTensorFork.ce_row (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_row) def ce_row(+mm1: Nat, +mu: U32, +nf: F32, +i: U32, z: Array, t: Array, r: Array & U32) -> B3: BendMlTensorFork.ce_step (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_step) def ce_step(+mm1: Nat, +mu: U32, +nf: F32, +i: U32, g: B3) -> B3: BendMlTensorFork.ce_loop (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_loop) def ce_loop(k: Nat, +mm1: Nat, +mu: U32, +nf: F32, +i: U32, g: B3) -> B3: BendMlTensorFork.ce_leaf (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_leaf) # rows i0 .. i0+cnt-1: their gradients in the logits, their loss terms in t def ce_leaf(+e: SE, +i0: U32, +cnt: Nat, g: B3) -> B3: BendMlTensorFork.ce_sum (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_sum) # the loss over k terms, in row order from +0.0, as cea_rows adds them def ce_sum(k: Nat, +i: U32, +loss: F32, r: Array & F32) -> F32: BendMlTensorFork.CL (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=CL) type CL is Type: CL{a: Array, k: Nat, rest: List<&2, Nat>} BendMlTensorFork.lab_fit (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=lab_fit) # a label past the row gives p = 0 and no one-hot cell, as c does: kept as c so that it fits a U32 def lab_fit(inside: Bool, +y: Nat, +c: Nat) -> Nat: BendMlTensorFork.lab_take (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=lab_take) # up to n labels into a, counted in k; the rest stays a list def lab_take(n: Nat, labels: List<&2, Nat>, +mm1: Nat, +c: Nat, +i: U32, +k: Nat, a: Array) -> CL: BendMlTensorFork.ce_fin2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_fin2) # after the blocks: the loss from t, then any labels past the n rows, in order, then tensor-array's mean def ce_fin2(-n: Nat, -c: Nat, +nf: F32, +mm1: Nat, +mu: U32, +k: Nat, rest: List<&2, Nat>, g: B3) -> TA.MCE: BendMlTensorFork.ce_par3 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=ce_par3) def ce_par3(-n: Nat, -c: Nat, +nf: F32, +mm1: Nat, +mu: U32, +d: Nat, z: Array, lt: CL) -> TA.MCE: BendMlTensorFork.softmax_ce2 (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=softmax_ce2) def softmax_ce2(seq: Bool, +n: Nat, +c: Nat, +d: Nat, z: TA.Mat, labels: List<&2, Nat>) -> TA.MCE: BendMlTensorFork.softmax_ce (https://jend.leeeet.dev/definition.txt?path=bend-ml-tensor-fork@0.1.6.0/main.bend&name=softmax_ce) # mean cross-entropy and its gradient, TA.Mat.softmax_ce's arguments and bits, with the rows in up to 2^par blocks # (par = 0: tensor-array's own loop) def softmax_ce(+n: Nat, +c: Nat, +par: Nat, z: TA.Mat, labels: List<&2, Nat>) -> TA.MCE: