def smollm2_360m

bend-ml-llama@0.2.1.0 · main.bend:154

import Base

154# SmolLM2-360M: 5 key/value heads of 3 query heads each, heads of 64
155def smollm2_360m() -> Cfg:
156  Cfg{5n, 3n, 32n, 2560n, 32n, 49152n, 0.00001, 100000.0}