mlbend-generative Releases (2): 0.1.1.0 0xd48e7ce018b6fa7dc613b629e4260a23 indexed https://jend.leeeet.dev/files.txt?path=mlbend-generative@0.1.1.0 0.1.0.0 0xa51d4d9fa3661462984f336054bbc146 unindexed https://jend.leeeet.dev/files.txt?path=mlbend-generative@0.1.0.0 Definitions (36): MlbendGenerative.mix32 (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=mix32) # murmur3's fmix32 finaliser, over U32 arithmetic that wraps def mix32(+z: U32) -> U32: MlbendGenerative.unxor13 (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=unxor13) # the 13-shift undone: xor the shifted value twice more def unxor13(+y: U32) -> U32: MlbendGenerative.mix32_inv (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=mix32_inv) # the inverse of mix32, in closed form: the 16-shift is an involution on # 32 bits; the 13-shift inverts by the triple xor; the multiplies invert # by their modular inverses mod 2^32 def mix32_inv(+z: U32) -> U32: MlbendGenerative.hash_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=hash_of) # the hash of one counter: the step salt times the golden ratio, the # index added beside it def hash_of(+s: Nat, +i: Nat) -> U32: MlbendGenerative.uniform (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=uniform) # a uniform in (0, 1]: the low 24 bits of the hash plus one, exactly # representable, so log(u) is never log(0) def uniform(+s: Nat, +i: Nat) -> F32: MlbendGenerative.gauss_body (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=gauss_body) # one Gaussian of a counter, by parity: the even index of the pair brings # the radius, the odd one the angle def gauss_body(is_even: Bool, +r: F32, +ang: F32) -> F32: MlbendGenerative.gauss_at (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=gauss_at) def gauss_at(+s: Nat, +j: Nat) -> F32: MlbendGenerative.noise_from (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=noise_from) # The noise of counter s, left numbers from index `from` on. The count # left is the recursion; the index rides after it. def noise_from(+s: Nat, +left: Nat, +from: Nat) -> List<&2, F32>: MlbendGenerative.noise_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=noise_of) # the noise of counter s: m numbers, the index 0 first def noise_of(+s: Nat, +m: Nat) -> List<&2, F32>: MlbendGenerative.noise_length (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=noise_length) # LAW 1 (noise_length): m numbers asked for are m numbers built. law noise_length: for +s: Nat for +m: Nat for +from: Nat {List.length(&2, F32, noise_from(s, m, from)) == m : Nat} MlbendGenerative.cos_f (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=cos_f) def cos_f(+t: Nat, +steps: Nat) -> F32: MlbendGenerative.alpha_at (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=alpha_at) # ᾱ_t of the cosine schedule over `steps` steps (t = 1 .. steps) def alpha_at(+t: Nat, +steps: Nat) -> F32: MlbendGenerative.beta_at (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=beta_at) # β_t, the ratio the reverse step removes: 1 - ᾱ_t/ᾱ_{t-1}, clamped at # 0.99 — the cosine schedule's last ratio is 1 in F32 (ᾱ_T ~ 1e-15), and # the reverse step's 1/sqrt(1-β) would amplify any prediction error def beta_at(+t: Nat, +steps: Nat) -> F32: MlbendGenerative.alphas_from (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=alphas_from) # The schedule from step `from` on, `left` entries. The count left is the # recursion; the step rides after it. def alphas_from(+steps: Nat, +left: Nat, +from: Nat) -> List<&2, F32>: MlbendGenerative.alphas (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=alphas) # ᾱ_1 .. ᾱ_steps, in order def alphas(+steps: Nat) -> List<&2, F32>: MlbendGenerative.alphas_length (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=alphas_length) # LAW 2 (alphas_length): a schedule of t entries is t entries. law alphas_length: for +t: Nat for +left: Nat for +from: Nat {List.length(&2, F32, alphas_from(t, left, from)) == left : Nat} MlbendGenerative.lin_beta_at (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=lin_beta_at) # β_t of the linear schedule over `steps` steps (t = 1 .. steps) def lin_beta_at(+t: Nat, +steps: Nat) -> F32: MlbendGenerative.lin_alphas_from (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=lin_alphas_from) # The schedule ᾱ_1 .. from `from` on, `left` entries, the running product # riding beside. The count left is the recursion; the step and the # product ride after it. def lin_alphas_from(+steps: Nat, +left: Nat, +from: Nat, +acc: F32) -> List<&2, F32>: MlbendGenerative.lin_alphas (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=lin_alphas) # ᾱ_1 .. ᾱ_steps of the linear schedule, in order def lin_alphas(+steps: Nat) -> List<&2, F32>: MlbendGenerative.lin_alphas_length (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=lin_alphas_length) # LAW 4 (lin_alphas_length): a linear schedule of t entries is t entries. law lin_alphas_length: for +t: Nat for +left: Nat for +from: Nat for +acc: F32 {List.length(&2, F32, lin_alphas_from(t, left, from, acc)) == left : Nat} MlbendGenerative.freq_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=freq_of) # the frequency of rung p: a decade ladder 0.001 .. 1.0 def freq_of(+p: Nat) -> F32: MlbendGenerative.temb_from (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=temb_from) # The embedding of step t over `steps`, from rung `p` on, `left` rungs — # each rung a sine and a cosine. The rungs left are the recursion; the # rung index rides after them. def temb_from(+t: Nat, +steps: Nat, +left: Nat, +p: Nat) -> List<&2, F32>: MlbendGenerative.temb_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=temb_of) # the embedding of step t: k rungs, the lowest frequency first def temb_of(+t: Nat, +steps: Nat, +k: Nat) -> List<&2, F32>: MlbendGenerative.temb_length (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=temb_length) # LAW 3 (temb_length): an embedding of k rungs is 2k numbers. law temb_length: for +t: Nat for +steps: Nat for +k: Nat for +p: Nat {List.length(&2, F32, temb_from(t, steps, k, p)) == Nat.mul(k, 2n) : Nat} MlbendGenerative.groups_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=groups_of) # How many entries the walk emits over n entries. The count recurses on # the count left, exactly as the walk recurses on the list left, and the # two matches (on the count, on the skip) are in the same order — that # is what lets the law below follow both by induction. def groups_of(+n: Nat, +k: Nat, +skip: Nat) -> Nat: MlbendGenerative.strided_from (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=strided_from) # Every k-th entry of xs, the (skip+1)-th first: drop `skip`, emit one, # drop k-1, emit one, and so on to the end of the list. The list is # matched first (binders in order), the skip beside it. def strided_from(-A: Data, xs: List<&2, A>, +k: Nat, +skip: Nat) -> List<&2, A>: MlbendGenerative.strided (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=strided) # every k-th entry of xs, the k-th first def strided(-A: Data, xs: List<&2, A>, +k: Nat) -> List<&2, A>: MlbendGenerative.strided_length (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=strided_length) # LAW 5 (strided_length): every k-th entry of n entries is exactly the # number the count says. law strided_length: for -A: Data for +xs: List<&2, A> for +k: Nat for +skip: Nat {List.length(&2, A, strided_from(A, xs, k, skip)) == groups_of(List.length(&2, A, xs), k, skip) : Nat} MlbendGenerative.sigma_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=sigma_of) def sigma_of(+abar: F32, +abar_prev: F32, +eta: F32) -> F32: MlbendGenerative.Step (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=Step) # The five coefficients of one step: the four roots and the noise scale. # They are computed once per step, not once per element — the CIFAR # probe measures 786k elements in a step, and five sqrts each would be # paid 786k times. Every radicand is clamped at 0, so float rounding # cannot hand sqrt a negative and turn a whole grid into NaN. type Step is Data: Step{c0: F32, c1: F32, c2: F32, c3: F32, sig: F32} MlbendGenerative.step_of (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=step_of) def step_of(+abar: F32, +abar_prev: F32, +eta: F32) -> Step: MlbendGenerative.ddim_elem (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=ddim_elem) # one element of the step: x0 folded into x_prev def ddim_elem(+co: Step, +x: F32, +eps: F32, +z: F32) -> F32: MlbendGenerative.head_or_zero (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=head_or_zero) # the head of a list, or 0.0 when it is empty: the missing eps or z of a # step whose z is not drawn at all (eta = 0 passes Nil, and sigma·0.0 is # 0.0) def head_or_zero(xs: List<&2, F32>) -> F32: MlbendGenerative.tail_or_nil (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=tail_or_nil) # the rest of a list, or empty: the tail beside head_or_zero above def tail_or_nil(xs: List<&2, F32>) -> List<&2, F32>: MlbendGenerative.ddim (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=ddim) # One step over the whole sample: elementwise, and the shape follows x — # n elements in, n elements out (LAW 6). eps and z ride beside it: a # shorter list pads with 0.0, which is exactly right for the z of an # eta = 0 step and is caught by the twin's numbers otherwise. def ddim(+co: Step, x: List<&2, F32>, +eps: List<&2, F32>, +z: List<&2, F32>) -> List<&2, F32>: MlbendGenerative.ddim_length (https://jend.leeeet.dev/definition.txt?path=mlbend-generative@0.1.1.0/main.bend&name=ddim_length) # LAW 6 (ddim_length): a step over n elements is n elements. law ddim_length: for +co: Step for +x: List<&2, F32> for +eps: List<&2, F32> for +z: List<&2, F32> {List.length(&2, F32, ddim(co, x, eps, z)) == List.length(&2, F32, x) : Nat}