Convert_to_hf_LED.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a876cf66",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import LEDForConditionalGeneration\n",
    "from transformers import LEDConfig\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e6f96aa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_name='wcep'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "676464c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "states = torch.load('../github/PRIMER/PRIMER_%s_state.pt'%(dataset_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b540e021",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "odict_keys(['final_logits_bias', 'model.shared.weight', 'model.encoder.embed_tokens.weight', 'model.encoder.embed_positions.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.0.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.0.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.0.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.0.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.0.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.0.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.0.self_attn.output.weight', 'model.encoder.layers.0.self_attn.output.bias', 'model.encoder.layers.0.self_attn_layer_norm.weight', 'model.encoder.layers.0.self_attn_layer_norm.bias', 'model.encoder.layers.0.fc1.weight', 'model.encoder.layers.0.fc1.bias', 'model.encoder.layers.0.fc2.weight', 'model.encoder.layers.0.fc2.bias', 'model.encoder.layers.0.final_layer_norm.weight', 'model.encoder.layers.0.final_layer_norm.bias', 'model.encoder.layers.1.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.1.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.1.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.1.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.1.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.1.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.1.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.1.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.1.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.1.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.1.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.1.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.1.self_attn.output.weight', 'model.encoder.layers.1.self_attn.output.bias', 'model.encoder.layers.1.self_attn_layer_norm.weight', 'model.encoder.layers.1.self_attn_layer_norm.bias', 'model.encoder.layers.1.fc1.weight', 'model.encoder.layers.1.fc1.bias', 'model.encoder.layers.1.fc2.weight', 'model.encoder.layers.1.fc2.bias', 'model.encoder.layers.1.final_layer_norm.weight', 'model.encoder.layers.1.final_layer_norm.bias', 'model.encoder.layers.2.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.2.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.2.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.2.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.2.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.2.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.2.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.2.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.2.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.2.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.2.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.2.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.2.self_attn.output.weight', 'model.encoder.layers.2.self_attn.output.bias', 'model.encoder.layers.2.self_attn_layer_norm.weight', 'model.encoder.layers.2.self_attn_layer_norm.bias', 'model.encoder.layers.2.fc1.weight', 'model.encoder.layers.2.fc1.bias', 'model.encoder.layers.2.fc2.weight', 'model.encoder.layers.2.fc2.bias', 'model.encoder.layers.2.final_layer_norm.weight', 'model.encoder.layers.2.final_layer_norm.bias', 'model.encoder.layers.3.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.3.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.3.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.3.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.3.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.3.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.3.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.3.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.3.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.3.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.3.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.3.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.3.self_attn.output.weight', 'model.encoder.layers.3.self_attn.output.bias', 'model.encoder.layers.3.self_attn_layer_norm.weight', 'model.encoder.layers.3.self_attn_layer_norm.bias', 'model.encoder.layers.3.fc1.weight', 'model.encoder.layers.3.fc1.bias', 'model.encoder.layers.3.fc2.weight', 'model.encoder.layers.3.fc2.bias', 'model.encoder.layers.3.final_layer_norm.weight', 'model.encoder.layers.3.final_layer_norm.bias', 'model.encoder.layers.4.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.4.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.4.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.4.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.4.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.4.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.4.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.4.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.4.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.4.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.4.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.4.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.4.self_attn.output.weight', 'model.encoder.layers.4.self_attn.output.bias', 'model.encoder.layers.4.self_attn_layer_norm.weight', 'model.encoder.layers.4.self_attn_layer_norm.bias', 'model.encoder.layers.4.fc1.weight', 'model.encoder.layers.4.fc1.bias', 'model.encoder.layers.4.fc2.weight', 'model.encoder.layers.4.fc2.bias', 'model.encoder.layers.4.final_layer_norm.weight', 'model.encoder.layers.4.final_layer_norm.bias', 'model.encoder.layers.5.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.5.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.5.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.5.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.5.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.5.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.5.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.5.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.5.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.5.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.5.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.5.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.5.self_attn.output.weight', 'model.encoder.layers.5.self_attn.output.bias', 'model.encoder.layers.5.self_attn_layer_norm.weight', 'model.encoder.layers.5.self_attn_layer_norm.bias', 'model.encoder.layers.5.fc1.weight', 'model.encoder.layers.5.fc1.bias', 'model.encoder.layers.5.fc2.weight', 'model.encoder.layers.5.fc2.bias', 'model.encoder.layers.5.final_layer_norm.weight', 'model.encoder.layers.5.final_layer_norm.bias', 'model.encoder.layers.6.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.6.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.6.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.6.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.6.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.6.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.6.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.6.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.6.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.6.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.6.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.6.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.6.self_attn.output.weight', 'model.encoder.layers.6.self_attn.output.bias', 'model.encoder.layers.6.self_attn_layer_norm.weight', 'model.encoder.layers.6.self_attn_layer_norm.bias', 'model.encoder.layers.6.fc1.weight', 'model.encoder.layers.6.fc1.bias', 'model.encoder.layers.6.fc2.weight', 'model.encoder.layers.6.fc2.bias', 'model.encoder.layers.6.final_layer_norm.weight', 'model.encoder.layers.6.final_layer_norm.bias', 'model.encoder.layers.7.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.7.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.7.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.7.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.7.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.7.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.7.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.7.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.7.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.7.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.7.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.7.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.7.self_attn.output.weight', 'model.encoder.layers.7.self_attn.output.bias', 'model.encoder.layers.7.self_attn_layer_norm.weight', 'model.encoder.layers.7.self_attn_layer_norm.bias', 'model.encoder.layers.7.fc1.weight', 'model.encoder.layers.7.fc1.bias', 'model.encoder.layers.7.fc2.weight', 'model.encoder.layers.7.fc2.bias', 'model.encoder.layers.7.final_layer_norm.weight', 'model.encoder.layers.7.final_layer_norm.bias', 'model.encoder.layers.8.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.8.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.8.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.8.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.8.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.8.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.8.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.8.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.8.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.8.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.8.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.8.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.8.self_attn.output.weight', 'model.encoder.layers.8.self_attn.output.bias', 'model.encoder.layers.8.self_attn_layer_norm.weight', 'model.encoder.layers.8.self_attn_layer_norm.bias', 'model.encoder.layers.8.fc1.weight', 'model.encoder.layers.8.fc1.bias', 'model.encoder.layers.8.fc2.weight', 'model.encoder.layers.8.fc2.bias', 'model.encoder.layers.8.final_layer_norm.weight', 'model.encoder.layers.8.final_layer_norm.bias', 'model.encoder.layers.9.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.9.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.9.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.9.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.9.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.9.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.9.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.9.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.9.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.9.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.9.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.9.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.9.self_attn.output.weight', 'model.encoder.layers.9.self_attn.output.bias', 'model.encoder.layers.9.self_attn_layer_norm.weight', 'model.encoder.layers.9.self_attn_layer_norm.bias', 'model.encoder.layers.9.fc1.weight', 'model.encoder.layers.9.fc1.bias', 'model.encoder.layers.9.fc2.weight', 'model.encoder.layers.9.fc2.bias', 'model.encoder.layers.9.final_layer_norm.weight', 'model.encoder.layers.9.final_layer_norm.bias', 'model.encoder.layers.10.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.10.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.10.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.10.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.10.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.10.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.10.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.10.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.10.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.10.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.10.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.10.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.10.self_attn.output.weight', 'model.encoder.layers.10.self_attn.output.bias', 'model.encoder.layers.10.self_attn_layer_norm.weight', 'model.encoder.layers.10.self_attn_layer_norm.bias', 'model.encoder.layers.10.fc1.weight', 'model.encoder.layers.10.fc1.bias', 'model.encoder.layers.10.fc2.weight', 'model.encoder.layers.10.fc2.bias', 'model.encoder.layers.10.final_layer_norm.weight', 'model.encoder.layers.10.final_layer_norm.bias', 'model.encoder.layers.11.self_attn.longformer_self_attn.query.weight', 'model.encoder.layers.11.self_attn.longformer_self_attn.query.bias', 'model.encoder.layers.11.self_attn.longformer_self_attn.key.weight', 'model.encoder.layers.11.self_attn.longformer_self_attn.key.bias', 'model.encoder.layers.11.self_attn.longformer_self_attn.value.weight', 'model.encoder.layers.11.self_attn.longformer_self_attn.value.bias', 'model.encoder.layers.11.self_attn.longformer_self_attn.query_global.weight', 'model.encoder.layers.11.self_attn.longformer_self_attn.query_global.bias', 'model.encoder.layers.11.self_attn.longformer_self_attn.key_global.weight', 'model.encoder.layers.11.self_attn.longformer_self_attn.key_global.bias', 'model.encoder.layers.11.self_attn.longformer_self_attn.value_global.weight', 'model.encoder.layers.11.self_attn.longformer_self_attn.value_global.bias', 'model.encoder.layers.11.self_attn.output.weight', 'model.encoder.layers.11.self_attn.output.bias', 'model.encoder.layers.11.self_attn_layer_norm.weight', 'model.encoder.layers.11.self_attn_layer_norm.bias', 'model.encoder.layers.11.fc1.weight', 'model.encoder.layers.11.fc1.bias', 'model.encoder.layers.11.fc2.weight', 'model.encoder.layers.11.fc2.bias', 'model.encoder.layers.11.final_layer_norm.weight', 'model.encoder.layers.11.final_layer_norm.bias', 'model.encoder.layernorm_embedding.weight', 'model.encoder.layernorm_embedding.bias', 'model.decoder.embed_tokens.weight', 'model.decoder.embed_positions.weight', 'model.decoder.layers.0.self_attn.k_proj.weight', 'model.decoder.layers.0.self_attn.k_proj.bias', 'model.decoder.layers.0.self_attn.v_proj.weight', 'model.decoder.layers.0.self_attn.v_proj.bias', 'model.decoder.layers.0.self_attn.q_proj.weight', 'model.decoder.layers.0.self_attn.q_proj.bias', 'model.decoder.layers.0.self_attn.out_proj.weight', 'model.decoder.layers.0.self_attn.out_proj.bias', 'model.decoder.layers.0.self_attn_layer_norm.weight', 'model.decoder.layers.0.self_attn_layer_norm.bias', 'model.decoder.layers.0.encoder_attn.k_proj.weight', 'model.decoder.layers.0.encoder_attn.k_proj.bias', 'model.decoder.layers.0.encoder_attn.v_proj.weight', 'model.decoder.layers.0.encoder_attn.v_proj.bias', 'model.decoder.layers.0.encoder_attn.q_proj.weight', 'model.decoder.layers.0.encoder_attn.q_proj.bias', 'model.decoder.layers.0.encoder_attn.out_proj.weight', 'model.decoder.layers.0.encoder_attn.out_proj.bias', 'model.decoder.layers.0.encoder_attn_layer_norm.weight', 'model.decoder.layers.0.encoder_attn_layer_norm.bias', 'model.decoder.layers.0.fc1.weight', 'model.decoder.layers.0.fc1.bias', 'model.decoder.layers.0.fc2.weight', 'model.decoder.layers.0.fc2.bias', 'model.decoder.layers.0.final_layer_norm.weight', 'model.decoder.layers.0.final_layer_norm.bias', 'model.decoder.layers.1.self_attn.k_proj.weight', 'model.decoder.layers.1.self_attn.k_proj.bias', 'model.decoder.layers.1.self_attn.v_proj.weight', 'model.decoder.layers.1.self_attn.v_proj.bias', 'model.decoder.layers.1.self_attn.q_proj.weight', 'model.decoder.layers.1.self_attn.q_proj.bias', 'model.decoder.layers.1.self_attn.out_proj.weight', 'model.decoder.layers.1.self_attn.out_proj.bias', 'model.decoder.layers.1.self_attn_layer_norm.weight', 'model.decoder.layers.1.self_attn_layer_norm.bias', 'model.decoder.layers.1.encoder_attn.k_proj.weight', 'model.decoder.layers.1.encoder_attn.k_proj.bias', 'model.decoder.layers.1.encoder_attn.v_proj.weight', 'model.decoder.layers.1.encoder_attn.v_proj.bias', 'model.decoder.layers.1.encoder_attn.q_proj.weight', 'model.decoder.layers.1.encoder_attn.q_proj.bias', 'model.decoder.layers.1.encoder_attn.out_proj.weight', 'model.decoder.layers.1.encoder_attn.out_proj.bias', 'model.decoder.layers.1.encoder_attn_layer_norm.weight', 'model.decoder.layers.1.encoder_attn_layer_norm.bias', 'model.decoder.layers.1.fc1.weight', 'model.decoder.layers.1.fc1.bias', 'model.decoder.layers.1.fc2.weight', 'model.decoder.layers.1.fc2.bias', 'model.decoder.layers.1.final_layer_norm.weight', 'model.decoder.layers.1.final_layer_norm.bias', 'model.decoder.layers.2.self_attn.k_proj.weight', 'model.decoder.layers.2.self_attn.k_proj.bias', 'model.decoder.layers.2.self_attn.v_proj.weight', 'model.decoder.layers.2.self_attn.v_proj.bias', 'model.decoder.layers.2.self_attn.q_proj.weight', 'model.decoder.layers.2.self_attn.q_proj.bias', 'model.decoder.layers.2.self_attn.out_proj.weight', 'model.decoder.layers.2.self_attn.out_proj.bias', 'model.decoder.layers.2.self_attn_layer_norm.weight', 'model.decoder.layers.2.self_attn_layer_norm.bias', 'model.decoder.layers.2.encoder_attn.k_proj.weight', 'model.decoder.layers.2.encoder_attn.k_proj.bias', 'model.decoder.layers.2.encoder_attn.v_proj.weight', 'model.decoder.layers.2.encoder_attn.v_proj.bias', 'model.decoder.layers.2.encoder_attn.q_proj.weight', 'model.decoder.layers.2.encoder_attn.q_proj.bias', 'model.decoder.layers.2.encoder_attn.out_proj.weight', 'model.decoder.layers.2.encoder_attn.out_proj.bias', 'model.decoder.layers.2.encoder_attn_layer_norm.weight', 'model.decoder.layers.2.encoder_attn_layer_norm.bias', 'model.decoder.layers.2.fc1.weight', 'model.decoder.layers.2.fc1.bias', 'model.decoder.layers.2.fc2.weight', 'model.decoder.layers.2.fc2.bias', 'model.decoder.layers.2.final_layer_norm.weight', 'model.decoder.layers.2.final_layer_norm.bias', 'model.decoder.layers.3.self_attn.k_proj.weight', 'model.decoder.layers.3.self_attn.k_proj.bias', 'model.decoder.layers.3.self_attn.v_proj.weight', 'model.decoder.layers.3.self_attn.v_proj.bias', 'model.decoder.layers.3.self_attn.q_proj.weight', 'model.decoder.layers.3.self_attn.q_proj.bias', 'model.decoder.layers.3.self_attn.out_proj.weight', 'model.decoder.layers.3.self_attn.out_proj.bias', 'model.decoder.layers.3.self_attn_layer_norm.weight', 'model.decoder.layers.3.self_attn_layer_norm.bias', 'model.decoder.layers.3.encoder_attn.k_proj.weight', 'model.decoder.layers.3.encoder_attn.k_proj.bias', 'model.decoder.layers.3.encoder_attn.v_proj.weight', 'model.decoder.layers.3.encoder_attn.v_proj.bias', 'model.decoder.layers.3.encoder_attn.q_proj.weight', 'model.decoder.layers.3.encoder_attn.q_proj.bias', 'model.decoder.layers.3.encoder_attn.out_proj.weight', 'model.decoder.layers.3.encoder_attn.out_proj.bias', 'model.decoder.layers.3.encoder_attn_layer_norm.weight', 'model.decoder.layers.3.encoder_attn_layer_norm.bias', 'model.decoder.layers.3.fc1.weight', 'model.decoder.layers.3.fc1.bias', 'model.decoder.layers.3.fc2.weight', 'model.decoder.layers.3.fc2.bias', 'model.decoder.layers.3.final_layer_norm.weight', 'model.decoder.layers.3.final_layer_norm.bias', 'model.decoder.layers.4.self_attn.k_proj.weight', 'model.decoder.layers.4.self_attn.k_proj.bias', 'model.decoder.layers.4.self_attn.v_proj.weight', 'model.decoder.layers.4.self_attn.v_proj.bias', 'model.decoder.layers.4.self_attn.q_proj.weight', 'model.decoder.layers.4.self_attn.q_proj.bias', 'model.decoder.layers.4.self_attn.out_proj.weight', 'model.decoder.layers.4.self_attn.out_proj.bias', 'model.decoder.layers.4.self_attn_layer_norm.weight', 'model.decoder.layers.4.self_attn_layer_norm.bias', 'model.decoder.layers.4.encoder_attn.k_proj.weight', 'model.decoder.layers.4.encoder_attn.k_proj.bias', 'model.decoder.layers.4.encoder_attn.v_proj.weight', 'model.decoder.layers.4.encoder_attn.v_proj.bias', 'model.decoder.layers.4.encoder_attn.q_proj.weight', 'model.decoder.layers.4.encoder_attn.q_proj.bias', 'model.decoder.layers.4.encoder_attn.out_proj.weight', 'model.decoder.layers.4.encoder_attn.out_proj.bias', 'model.decoder.layers.4.encoder_attn_layer_norm.weight', 'model.decoder.layers.4.encoder_attn_layer_norm.bias', 'model.decoder.layers.4.fc1.weight', 'model.decoder.layers.4.fc1.bias', 'model.decoder.layers.4.fc2.weight', 'model.decoder.layers.4.fc2.bias', 'model.decoder.layers.4.final_layer_norm.weight', 'model.decoder.layers.4.final_layer_norm.bias', 'model.decoder.layers.5.self_attn.k_proj.weight', 'model.decoder.layers.5.self_attn.k_proj.bias', 'model.decoder.layers.5.self_attn.v_proj.weight', 'model.decoder.layers.5.self_attn.v_proj.bias', 'model.decoder.layers.5.self_attn.q_proj.weight', 'model.decoder.layers.5.self_attn.q_proj.bias', 'model.decoder.layers.5.self_attn.out_proj.weight', 'model.decoder.layers.5.self_attn.out_proj.bias', 'model.decoder.layers.5.self_attn_layer_norm.weight', 'model.decoder.layers.5.self_attn_layer_norm.bias', 'model.decoder.layers.5.encoder_attn.k_proj.weight', 'model.decoder.layers.5.encoder_attn.k_proj.bias', 'model.decoder.layers.5.encoder_attn.v_proj.weight', 'model.decoder.layers.5.encoder_attn.v_proj.bias', 'model.decoder.layers.5.encoder_attn.q_proj.weight', 'model.decoder.layers.5.encoder_attn.q_proj.bias', 'model.decoder.layers.5.encoder_attn.out_proj.weight', 'model.decoder.layers.5.encoder_attn.out_proj.bias', 'model.decoder.layers.5.encoder_attn_layer_norm.weight', 'model.decoder.layers.5.encoder_attn_layer_norm.bias', 'model.decoder.layers.5.fc1.weight', 'model.decoder.layers.5.fc1.bias', 'model.decoder.layers.5.fc2.weight', 'model.decoder.layers.5.fc2.bias', 'model.decoder.layers.5.final_layer_norm.weight', 'model.decoder.layers.5.final_layer_norm.bias', 'model.decoder.layers.6.self_attn.k_proj.weight', 'model.decoder.layers.6.self_attn.k_proj.bias', 'model.decoder.layers.6.self_attn.v_proj.weight', 'model.decoder.layers.6.self_attn.v_proj.bias', 'model.decoder.layers.6.self_attn.q_proj.weight', 'model.decoder.layers.6.self_attn.q_proj.bias', 'model.decoder.layers.6.self_attn.out_proj.weight', 'model.decoder.layers.6.self_attn.out_proj.bias', 'model.decoder.layers.6.self_attn_layer_norm.weight', 'model.decoder.layers.6.self_attn_layer_norm.bias', 'model.decoder.layers.6.encoder_attn.k_proj.weight', 'model.decoder.layers.6.encoder_attn.k_proj.bias', 'model.decoder.layers.6.encoder_attn.v_proj.weight', 'model.decoder.layers.6.encoder_attn.v_proj.bias', 'model.decoder.layers.6.encoder_attn.q_proj.weight', 'model.decoder.layers.6.encoder_attn.q_proj.bias', 'model.decoder.layers.6.encoder_attn.out_proj.weight', 'model.decoder.layers.6.encoder_attn.out_proj.bias', 'model.decoder.layers.6.encoder_attn_layer_norm.weight', 'model.decoder.layers.6.encoder_attn_layer_norm.bias', 'model.decoder.layers.6.fc1.weight', 'model.decoder.layers.6.fc1.bias', 'model.decoder.layers.6.fc2.weight', 'model.decoder.layers.6.fc2.bias', 'model.decoder.layers.6.final_layer_norm.weight', 'model.decoder.layers.6.final_layer_norm.bias', 'model.decoder.layers.7.self_attn.k_proj.weight', 'model.decoder.layers.7.self_attn.k_proj.bias', 'model.decoder.layers.7.self_attn.v_proj.weight', 'model.decoder.layers.7.self_attn.v_proj.bias', 'model.decoder.layers.7.self_attn.q_proj.weight', 'model.decoder.layers.7.self_attn.q_proj.bias', 'model.decoder.layers.7.self_attn.out_proj.weight', 'model.decoder.layers.7.self_attn.out_proj.bias', 'model.decoder.layers.7.self_attn_layer_norm.weight', 'model.decoder.layers.7.self_attn_layer_norm.bias', 'model.decoder.layers.7.encoder_attn.k_proj.weight', 'model.decoder.layers.7.encoder_attn.k_proj.bias', 'model.decoder.layers.7.encoder_attn.v_proj.weight', 'model.decoder.layers.7.encoder_attn.v_proj.bias', 'model.decoder.layers.7.encoder_attn.q_proj.weight', 'model.decoder.layers.7.encoder_attn.q_proj.bias', 'model.decoder.layers.7.encoder_attn.out_proj.weight', 'model.decoder.layers.7.encoder_attn.out_proj.bias', 'model.decoder.layers.7.encoder_attn_layer_norm.weight', 'model.decoder.layers.7.encoder_attn_layer_norm.bias', 'model.decoder.layers.7.fc1.weight', 'model.decoder.layers.7.fc1.bias', 'model.decoder.layers.7.fc2.weight', 'model.decoder.layers.7.fc2.bias', 'model.decoder.layers.7.final_layer_norm.weight', 'model.decoder.layers.7.final_layer_norm.bias', 'model.decoder.layers.8.self_attn.k_proj.weight', 'model.decoder.layers.8.self_attn.k_proj.bias', 'model.decoder.layers.8.self_attn.v_proj.weight', 'model.decoder.layers.8.self_attn.v_proj.bias', 'model.decoder.layers.8.self_attn.q_proj.weight', 'model.decoder.layers.8.self_attn.q_proj.bias', 'model.decoder.layers.8.self_attn.out_proj.weight', 'model.decoder.layers.8.self_attn.out_proj.bias', 'model.decoder.layers.8.self_attn_layer_norm.weight', 'model.decoder.layers.8.self_attn_layer_norm.bias', 'model.decoder.layers.8.encoder_attn.k_proj.weight', 'model.decoder.layers.8.encoder_attn.k_proj.bias', 'model.decoder.layers.8.encoder_attn.v_proj.weight', 'model.decoder.layers.8.encoder_attn.v_proj.bias', 'model.decoder.layers.8.encoder_attn.q_proj.weight', 'model.decoder.layers.8.encoder_attn.q_proj.bias', 'model.decoder.layers.8.encoder_attn.out_proj.weight', 'model.decoder.layers.8.encoder_attn.out_proj.bias', 'model.decoder.layers.8.encoder_attn_layer_norm.weight', 'model.decoder.layers.8.encoder_attn_layer_norm.bias', 'model.decoder.layers.8.fc1.weight', 'model.decoder.layers.8.fc1.bias', 'model.decoder.layers.8.fc2.weight', 'model.decoder.layers.8.fc2.bias', 'model.decoder.layers.8.final_layer_norm.weight', 'model.decoder.layers.8.final_layer_norm.bias', 'model.decoder.layers.9.self_attn.k_proj.weight', 'model.decoder.layers.9.self_attn.k_proj.bias', 'model.decoder.layers.9.self_attn.v_proj.weight', 'model.decoder.layers.9.self_attn.v_proj.bias', 'model.decoder.layers.9.self_attn.q_proj.weight', 'model.decoder.layers.9.self_attn.q_proj.bias', 'model.decoder.layers.9.self_attn.out_proj.weight', 'model.decoder.layers.9.self_attn.out_proj.bias', 'model.decoder.layers.9.self_attn_layer_norm.weight', 'model.decoder.layers.9.self_attn_layer_norm.bias', 'model.decoder.layers.9.encoder_attn.k_proj.weight', 'model.decoder.layers.9.encoder_attn.k_proj.bias', 'model.decoder.layers.9.encoder_attn.v_proj.weight', 'model.decoder.layers.9.encoder_attn.v_proj.bias', 'model.decoder.layers.9.encoder_attn.q_proj.weight', 'model.decoder.layers.9.encoder_attn.q_proj.bias', 'model.decoder.layers.9.encoder_attn.out_proj.weight', 'model.decoder.layers.9.encoder_attn.out_proj.bias', 'model.decoder.layers.9.encoder_attn_layer_norm.weight', 'model.decoder.layers.9.encoder_attn_layer_norm.bias', 'model.decoder.layers.9.fc1.weight', 'model.decoder.layers.9.fc1.bias', 'model.decoder.layers.9.fc2.weight', 'model.decoder.layers.9.fc2.bias', 'model.decoder.layers.9.final_layer_norm.weight', 'model.decoder.layers.9.final_layer_norm.bias', 'model.decoder.layers.10.self_attn.k_proj.weight', 'model.decoder.layers.10.self_attn.k_proj.bias', 'model.decoder.layers.10.self_attn.v_proj.weight', 'model.decoder.layers.10.self_attn.v_proj.bias', 'model.decoder.layers.10.self_attn.q_proj.weight', 'model.decoder.layers.10.self_attn.q_proj.bias', 'model.decoder.layers.10.self_attn.out_proj.weight', 'model.decoder.layers.10.self_attn.out_proj.bias', 'model.decoder.layers.10.self_attn_layer_norm.weight', 'model.decoder.layers.10.self_attn_layer_norm.bias', 'model.decoder.layers.10.encoder_attn.k_proj.weight', 'model.decoder.layers.10.encoder_attn.k_proj.bias', 'model.decoder.layers.10.encoder_attn.v_proj.weight', 'model.decoder.layers.10.encoder_attn.v_proj.bias', 'model.decoder.layers.10.encoder_attn.q_proj.weight', 'model.decoder.layers.10.encoder_attn.q_proj.bias', 'model.decoder.layers.10.encoder_attn.out_proj.weight', 'model.decoder.layers.10.encoder_attn.out_proj.bias', 'model.decoder.layers.10.encoder_attn_layer_norm.weight', 'model.decoder.layers.10.encoder_attn_layer_norm.bias', 'model.decoder.layers.10.fc1.weight', 'model.decoder.layers.10.fc1.bias', 'model.decoder.layers.10.fc2.weight', 'model.decoder.layers.10.fc2.bias', 'model.decoder.layers.10.final_layer_norm.weight', 'model.decoder.layers.10.final_layer_norm.bias', 'model.decoder.layers.11.self_attn.k_proj.weight', 'model.decoder.layers.11.self_attn.k_proj.bias', 'model.decoder.layers.11.self_attn.v_proj.weight', 'model.decoder.layers.11.self_attn.v_proj.bias', 'model.decoder.layers.11.self_attn.q_proj.weight', 'model.decoder.layers.11.self_attn.q_proj.bias', 'model.decoder.layers.11.self_attn.out_proj.weight', 'model.decoder.layers.11.self_attn.out_proj.bias', 'model.decoder.layers.11.self_attn_layer_norm.weight', 'model.decoder.layers.11.self_attn_layer_norm.bias', 'model.decoder.layers.11.encoder_attn.k_proj.weight', 'model.decoder.layers.11.encoder_attn.k_proj.bias', 'model.decoder.layers.11.encoder_attn.v_proj.weight', 'model.decoder.layers.11.encoder_attn.v_proj.bias', 'model.decoder.layers.11.encoder_attn.q_proj.weight', 'model.decoder.layers.11.encoder_attn.q_proj.bias', 'model.decoder.layers.11.encoder_attn.out_proj.weight', 'model.decoder.layers.11.encoder_attn.out_proj.bias', 'model.decoder.layers.11.encoder_attn_layer_norm.weight', 'model.decoder.layers.11.encoder_attn_layer_norm.bias', 'model.decoder.layers.11.fc1.weight', 'model.decoder.layers.11.fc1.bias', 'model.decoder.layers.11.fc2.weight', 'model.decoder.layers.11.fc2.bias', 'model.decoder.layers.11.final_layer_norm.weight', 'model.decoder.layers.11.final_layer_norm.bias', 'model.decoder.layernorm_embedding.weight', 'model.decoder.layernorm_embedding.bias'])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "states.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "81d239e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_states = OrderedDict()\n",
    "for k in states:\n",
    "    new_k=k.replace('model','led')\n",
    "    new_states[new_k]=states[k]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5cd68d58",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_states['led.encoder.embed_positions.weight'] = states['model.encoder.embed_positions.weight'][2:]\n",
    "new_states['led.decoder.embed_positions.weight'] = states['model.decoder.embed_positions.weight'][2:]\n",
    "new_states['lm_head.weight'] = states['model.shared.weight']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1e9afdf5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1024, 1024])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_states['led.decoder.embed_positions.weight'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "56507b34",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a model of type bart to instantiate a model of type led. This is not supported for all configurations of models and can yield errors.\n"
     ]
    }
   ],
   "source": [
    "config=LEDConfig.from_pretrained('../github/PRIMER/PRIMER_%s/'%(dataset_name))\n",
    "model=LEDForConditionalGeneration(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b9ac2c9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_state_dict(new_states,strict=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "65c99c57",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.config.save_pretrained('../github/PRIMER/PRIMER_%s_hf/'%(dataset_name))\n",
    "model.save_pretrained('../github/PRIMER/PRIMER_%s_hf/'%(dataset_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "f203f73a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ce49642",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}