~/jendhub

def adam_grad

bendtorch@1.3.0.0 · main.bend:256

import bendtorch@1.3.0.0/main.bend as Bendtorch
import Base

256# the Adam family, in PyTorch's order of operations (torch.optim.adam._single_tensor_adam):
257#   g' = g + wd*w (Adam) or w' = w*(1 - lr*wd) (AdamW); m = m + (1-b1)*(g' - m) (lerp); v = v*b2 + (1-b2)*g'*g';
258#   w = w' + (-step*m) / (sqrt(v)/bc2 + eps)
259def adam_grad(dec: Bool, +wd: F32, +w: F32, +g: F32) -> F32:
260  match dec:
261    case True{}:
262      g
263    case False{}:
264      (g + (w * wd : F32) : F32)

Uses (4)

Used by (4)