distillGPTNeoConfig.json

{
   "activation_function":"gelu_new",
   "attention_dropout":0.0,
   "attention_layers":[
      "global",
      "global",
      "global",
      "global",
      "global",
      "global",
      "global",
      "global",
      "global",
      "global",
      "global",
      "global"
   ],
   "attention_types":[
      [
         [
            "global"
         ],
         12
      ]
   ],
   "bos_token_id":50256,
   "embed_dropout":0.0,
   "eos_token_id":50256,
   "gradient_checkpointing":false,
   "hidden_size":2048,
   "initializer_range":0.02,
   "intermediate_size":null,
   "layer_norm_epsilon":1e-05,
   "max_position_embeddings":1024,
   "model_type":"gpt_neo",
   "num_heads":16,
   "num_layers":12,
   "resid_dropout":0.0,
   "summary_activation":null,
   "summary_first_dropout":0.1,
   "summary_proj_to_labels":true,
   "summary_type":"cls_index",
   "summary_use_proj":true,
   "transformers_version":"4.5.0.dev0",
   "use_cache":true,
   "vocab_size":50257,
   "window_size":256
}