def operations

stelliferous@0.0.2.0 · math_activation.bend:107

import Base

107# Operations per element, excluding the pass that applies them.
108def operations(activation: Activation) -> F32:
109  match activation:
110    case Identity{}: 0.0
111    # compare and select
112    case ReLU{}: 2.0
113    case ReLU6{}: 4.0
114    # compare, multiply and select
115    case LeakyReLU{slope}: 3.0
116    # 31 arithmetic, comparison, select and word instructions per eight lanes
117    case Sigmoid{}: 3.875
118    # 34 per eight lanes: scaled numerator and positive-tail selection
119    case SiLU{}: 4.25
120    # 54 per eight lanes
121    case Tanh{}: 6.75
122    # add, relu6 and divide; hard_swish also multiplies
123    case HardSigmoid{}: 6.0
124    case HardSwish{}: 7.0
125    # 62 per eight lanes: the PyTorch expression around the tanh
126    case GELUTanh{}: 7.75
127    case Custom{operations}: operations