~/jendhub

def gelu_erf_cdf

bendtorch@1.3.0.0 · main.bend:1208

import bendtorch@1.3.0.0/main.bend as Bendtorch
import Base

1208# GELU in its exact form, PyTorch's default (approximate="none"): gelu(x) = x/2 (1 + erf(x / sqrt 2)), in PyTorch's CPU
1209# order, (x * 0.5) * (1 + erf(x * 0.70710677)). Its gradient is cdf + x pdf, with cdf = 0.5 (1 + erf(x / sqrt 2)) and
1210# pdf = exp(-x^2 / 2) / sqrt(2 pi), computed as PyTorch does: 0.5 (1 + erf(x * 0.70710677)) and
1211# 0.3989423 * exp((x * x) * -0.5). The exponential is F32.exp, as in SiLU (so on a GPU the gradient would take CUDA's
1212# float exp).
1213def gelu_erf_cdf(+x: F32) -> F32:
1214  (0.5 * (1.0 + erf((x * 0.7071067811865476 : F32)) : F32) : F32)

Uses (4)

Used by (1)