def smollm2_1_7b

bend-ml-llama@0.2.1.0 · main.bend:158

import Base

158# SmolLM2-1.7B: 32 key/value heads of one query head each (plain multi-head attention), heads of 64
159def smollm2_1_7b() -> Cfg:
160  Cfg{32n, 1n, 32n, 8192n, 24n, 49152n, 0.00001, 130000.0}