testing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1fe2e738-1c7c-4625-b82c-10e3c027500a",
   "metadata": {},
   "source": [
    "### OPEN CLIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e980a5fa-34f1-4999-b311-ad7260dcd41d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import open_clip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22f8566e-3657-49bc-8533-a4948c242c8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "model, _, _ = open_clip.create_model_and_transforms('ViT-L-14', pretrained='openai')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "127c3300-793b-4bc7-8a0e-01c2949db3f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "visual_encoder = model.visual\n",
    "text_encoder = model.transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6bb6f122-a675-45ed-bd5b-6d5db0f413b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VisionTransformer(\n",
      "  (conv1): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
      "  (patch_dropout): Identity()\n",
      "  (ln_pre): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (transformer): Transformer(\n",
      "    (resblocks): ModuleList(\n",
      "      (0-23): 24 x ResidualAttentionBlock(\n",
      "        (ln_1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (attn): MultiheadAttention(\n",
      "          (out_proj): NonDynamicallyQuantizableLinear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (ls_1): Identity()\n",
      "        (ln_2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): Sequential(\n",
      "          (c_fc): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (gelu): QuickGELU()\n",
      "          (c_proj): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        )\n",
      "        (ls_2): Identity()\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (ln_post): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(visual_encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "12e18df5-8b14-4744-8e58-69fddb63e2b8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transformer(\n",
      "  (resblocks): ModuleList(\n",
      "    (0-11): 12 x ResidualAttentionBlock(\n",
      "      (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "      (attn): MultiheadAttention(\n",
      "        (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)\n",
      "      )\n",
      "      (ls_1): Identity()\n",
      "      (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "      (mlp): Sequential(\n",
      "        (c_fc): Linear(in_features=768, out_features=3072, bias=True)\n",
      "        (gelu): QuickGELU()\n",
      "        (c_proj): Linear(in_features=3072, out_features=768, bias=True)\n",
      "      )\n",
      "      (ls_2): Identity()\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(text_encoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2617c605-c352-4738-a768-5596cfb172c3",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "encode_text() missing 1 required positional argument: 'text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m#model.encode_image()\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencode_text\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mTypeError\u001b[0m: encode_text() missing 1 required positional argument: 'text'"
     ]
    }
   ],
   "source": [
    "model.encode_image()\n",
    "model.encode_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ec3a0eaa-f5d4-4af5-8343-fd3ddc83844f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gridsan/manderson/.conda/envs/train-clip/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import open_clip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "45a64e4c-129e-423a-8823-e305dcba4d03",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1+cu121\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a0fb4465-1e3d-4184-94ea-d4a188c2b93e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34f488ae-7cd2-460b-9313-2b017790cda4",
   "metadata": {},
   "source": [
    "### CLIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3f2da5d0-c556-43ca-b9e8-3924ffba5e18",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_lightning import Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bfb07190-b81a-4e64-96e7-084f3fc91ccf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "HPU available: False, using: 0 HPUs\n",
      "/home/gridsan/manderson/.conda/envs/train-clip/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/logger_connector/logger_connector.py:75: Starting from v1.9.0, `tensorboardX` has been removed as a dependency of the `pytorch_lightning` package, due to potential conflicts with other packages in the ML ecosystem. For this reason, `logger=True` will use `CSVLogger` as the default logger, unless the `tensorboard` or `tensorboardX` packages are found. Please `pip install lightning[extra]` or one of them to enable TensorBoard support by default\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a68b009e-4a37-4312-b92d-c0eaefe01d6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gridsan/manderson/.conda/envs/train-clip/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import CLIPModel, CLIPTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f975176a-af75-41ec-921d-2304aaae343a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CLIPModel.from_pretrained(\"/home/gridsan/manderson/ovdsat/weights/clip-vit-large-patch14\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fc225280-75fe-46c3-9ad2-46cf5df82be9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gridsan/manderson/.conda/envs/train-clip/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "tokenizer = CLIPTokenizer.from_pretrained(\"/home/gridsan/manderson/ovdsat/weights/clip-vit-large-patch14\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4758d94b-d569-412c-bc80-621a80f7f8fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLIPTextTransformer(\n",
      "  (embeddings): CLIPTextEmbeddings(\n",
      "    (token_embedding): Embedding(49408, 768)\n",
      "    (position_embedding): Embedding(77, 768)\n",
      "  )\n",
      "  (encoder): CLIPEncoder(\n",
      "    (layers): ModuleList(\n",
      "      (0-11): 12 x CLIPEncoderLayer(\n",
      "        (self_attn): CLIPSdpaAttention(\n",
      "          (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "        )\n",
      "        (layer_norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): CLIPMLP(\n",
      "          (activation_fn): QuickGELUActivation()\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "        )\n",
      "        (layer_norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      ")\n",
      "\n",
      "CLIPVisionTransformer(\n",
      "  (embeddings): CLIPVisionEmbeddings(\n",
      "    (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
      "    (position_embedding): Embedding(257, 1024)\n",
      "  )\n",
      "  (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (encoder): CLIPEncoder(\n",
      "    (layers): ModuleList(\n",
      "      (0-23): 24 x CLIPEncoderLayer(\n",
      "        (self_attn): CLIPSdpaAttention(\n",
      "          (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): CLIPMLP(\n",
      "          (activation_fn): QuickGELUActivation()\n",
      "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model.text_model)\n",
    "print()\n",
    "print(model.vision_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "9a1c0857-f48f-48eb-b5ad-db63aff3d5ac",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer logit_scale is trainable.\n",
      "Layer text_model.embeddings.token_embedding.weight is trainable.\n",
      "Layer text_model.embeddings.position_embedding.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer text_model.final_layer_norm.weight is trainable.\n",
      "Layer text_model.final_layer_norm.bias is trainable.\n",
      "Layer vision_model.embeddings.class_embedding is trainable.\n",
      "Layer vision_model.embeddings.patch_embedding.weight is trainable.\n",
      "Layer vision_model.embeddings.position_embedding.weight is trainable.\n",
      "Layer vision_model.pre_layrnorm.weight is trainable.\n",
      "Layer vision_model.pre_layrnorm.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer vision_model.post_layernorm.weight is trainable.\n",
      "Layer vision_model.post_layernorm.bias is trainable.\n",
      "Layer visual_projection.weight is trainable.\n",
      "Layer text_projection.weight is trainable.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for name, param in model.named_parameters():\n",
    "    if not param.requires_grad:\n",
    "        print(f\"Layer {name} is frozen.\")\n",
    "    else:\n",
    "        print(f\"Layer {name} is trainable.\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "de1a6cf1-aa03-4518-a67c-a21735224302",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer sink_temp is trainable.\n",
      "Layer model.positional_embedding is trainable.\n",
      "Layer model.text_projection is trainable.\n",
      "Layer model.logit_scale is trainable.\n",
      "Layer model.token_embedding.weight is trainable.\n",
      "Layer model.ln_final.weight is trainable.\n",
      "Layer model.ln_final.bias is trainable.\n",
      "Layer model.visual_projection.weight is trainable.\n",
      "Layer model.visual_projection.bias is trainable.\n",
      "Layer model.visual.embeddings.class_embedding is trainable.\n",
      "Layer model.visual.embeddings.patch_embedding.weight is trainable.\n",
      "Layer model.visual.embeddings.position_embedding.weight is trainable.\n",
      "Layer model.visual.pre_layrnorm.weight is trainable.\n",
      "Layer model.visual.pre_layrnorm.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer model.visual.post_layernorm.weight is trainable.\n",
      "Layer model.visual.post_layernorm.bias is trainable.\n",
      "Layer model.transformer.embeddings.token_embedding.weight is trainable.\n",
      "Layer model.transformer.embeddings.position_embedding.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.final_layer_norm.weight is trainable.\n",
      "Layer model.transformer.final_layer_norm.bias is trainable.\n",
      "Layer teacher.positional_embedding is trainable.\n",
      "Layer teacher.text_projection is trainable.\n",
      "Layer teacher.logit_scale is trainable.\n",
      "Layer teacher.token_embedding.weight is trainable.\n",
      "Layer teacher.ln_final.weight is trainable.\n",
      "Layer teacher.ln_final.bias is trainable.\n",
      "Layer teacher.visual_projection.weight is trainable.\n",
      "Layer teacher.visual_projection.bias is trainable.\n",
      "Layer teacher.visual.embeddings.class_embedding is trainable.\n",
      "Layer teacher.visual.embeddings.patch_embedding.weight is trainable.\n",
      "Layer teacher.visual.embeddings.position_embedding.weight is trainable.\n",
      "Layer teacher.visual.pre_layrnorm.weight is trainable.\n",
      "Layer teacher.visual.pre_layrnorm.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.post_layernorm.weight is trainable.\n",
      "Layer teacher.visual.post_layernorm.bias is trainable.\n",
      "Layer teacher.transformer.embeddings.token_embedding.weight is trainable.\n",
      "Layer teacher.transformer.embeddings.position_embedding.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.final_layer_norm.weight is trainable.\n",
      "Layer teacher.transformer.final_layer_norm.bias is trainable.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for name, param in custom_model.named_parameters():\n",
    "    if not param.requires_grad:\n",
    "        print(f\"Layer {name} is frozen.\")\n",
    "    else:\n",
    "        print(f\"Layer {name} is trainable.\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c7abdfa9-775d-4410-916c-c1dcc68242b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([768, 1024])\n"
     ]
    }
   ],
   "source": [
    "print(model.visual_projection.weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8de1a20c-b1e0-4fb0-9402-6e758d98a196",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0224, grad_fn=<SelectBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(model.visual_projection.weight[0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f8bd2945-db20-4fea-8f9a-8167a5393fc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([768, 768])\n"
     ]
    }
   ],
   "source": [
    "print(model.text_projection.weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5dcf4efc-0615-4af8-ac9d-21a4ab518ab8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'transformers.models.clip.modeling_clip.CLIPModel'>\n"
     ]
    }
   ],
   "source": [
    "print(type(model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ad0c8968-aca4-484a-a389-8ad3e8082d6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'transformers.models.clip.modeling_clip.CLIPVisionTransformer'>\n"
     ]
    }
   ],
   "source": [
    "print(type(model.vision_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b917d031-8240-4561-b857-53e69bef1127",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLIPVisionTransformer(\n",
      "  (embeddings): CLIPVisionEmbeddings(\n",
      "    (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
      "    (position_embedding): Embedding(257, 1024)\n",
      "  )\n",
      "  (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (encoder): CLIPEncoder(\n",
      "    (layers): ModuleList(\n",
      "      (0-23): 24 x CLIPEncoderLayer(\n",
      "        (self_attn): CLIPSdpaAttention(\n",
      "          (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): CLIPMLP(\n",
      "          (activation_fn): QuickGELUActivation()\n",
      "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model.vision_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3f23117c-44ed-48b1-8969-c4676b4620fe",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer embeddings.class_embedding is trainable.\n",
      "Layer embeddings.patch_embedding.weight is trainable.\n",
      "Layer embeddings.position_embedding.weight is trainable.\n",
      "Layer pre_layrnorm.weight is trainable.\n",
      "Layer pre_layrnorm.bias is trainable.\n",
      "Layer encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer post_layernorm.weight is trainable.\n",
      "Layer post_layernorm.bias is trainable.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for name, param in model.vision_model.named_parameters():\n",
    "    if not param.requires_grad:\n",
    "        print(f\"Layer {name} is frozen.\")\n",
    "    else:\n",
    "        print(f\"Layer {name} is trainable.\")\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "448807de-6542-48dd-abe0-8e8fe0d49021",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.vision_model.visual_projection = model.visual_projection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e6f5421e-e7d2-484a-b708-a7e31759e7b1",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "embeddings.class_embedding\n",
      "embeddings.patch_embedding.weight\n",
      "embeddings.position_embedding.weight\n",
      "pre_layrnorm.weight\n",
      "pre_layrnorm.bias\n",
      "encoder.layers.0.self_attn.k_proj.weight\n",
      "encoder.layers.0.self_attn.k_proj.bias\n",
      "encoder.layers.0.self_attn.v_proj.weight\n",
      "encoder.layers.0.self_attn.v_proj.bias\n",
      "encoder.layers.0.self_attn.q_proj.weight\n",
      "encoder.layers.0.self_attn.q_proj.bias\n",
      "encoder.layers.0.self_attn.out_proj.weight\n",
      "encoder.layers.0.self_attn.out_proj.bias\n",
      "encoder.layers.0.layer_norm1.weight\n",
      "encoder.layers.0.layer_norm1.bias\n",
      "encoder.layers.0.mlp.fc1.weight\n",
      "encoder.layers.0.mlp.fc1.bias\n",
      "encoder.layers.0.mlp.fc2.weight\n",
      "encoder.layers.0.mlp.fc2.bias\n",
      "encoder.layers.0.layer_norm2.weight\n",
      "encoder.layers.0.layer_norm2.bias\n",
      "encoder.layers.1.self_attn.k_proj.weight\n",
      "encoder.layers.1.self_attn.k_proj.bias\n",
      "encoder.layers.1.self_attn.v_proj.weight\n",
      "encoder.layers.1.self_attn.v_proj.bias\n",
      "encoder.layers.1.self_attn.q_proj.weight\n",
      "encoder.layers.1.self_attn.q_proj.bias\n",
      "encoder.layers.1.self_attn.out_proj.weight\n",
      "encoder.layers.1.self_attn.out_proj.bias\n",
      "encoder.layers.1.layer_norm1.weight\n",
      "encoder.layers.1.layer_norm1.bias\n",
      "encoder.layers.1.mlp.fc1.weight\n",
      "encoder.layers.1.mlp.fc1.bias\n",
      "encoder.layers.1.mlp.fc2.weight\n",
      "encoder.layers.1.mlp.fc2.bias\n",
      "encoder.layers.1.layer_norm2.weight\n",
      "encoder.layers.1.layer_norm2.bias\n",
      "encoder.layers.2.self_attn.k_proj.weight\n",
      "encoder.layers.2.self_attn.k_proj.bias\n",
      "encoder.layers.2.self_attn.v_proj.weight\n",
      "encoder.layers.2.self_attn.v_proj.bias\n",
      "encoder.layers.2.self_attn.q_proj.weight\n",
      "encoder.layers.2.self_attn.q_proj.bias\n",
      "encoder.layers.2.self_attn.out_proj.weight\n",
      "encoder.layers.2.self_attn.out_proj.bias\n",
      "encoder.layers.2.layer_norm1.weight\n",
      "encoder.layers.2.layer_norm1.bias\n",
      "encoder.layers.2.mlp.fc1.weight\n",
      "encoder.layers.2.mlp.fc1.bias\n",
      "encoder.layers.2.mlp.fc2.weight\n",
      "encoder.layers.2.mlp.fc2.bias\n",
      "encoder.layers.2.layer_norm2.weight\n",
      "encoder.layers.2.layer_norm2.bias\n",
      "encoder.layers.3.self_attn.k_proj.weight\n",
      "encoder.layers.3.self_attn.k_proj.bias\n",
      "encoder.layers.3.self_attn.v_proj.weight\n",
      "encoder.layers.3.self_attn.v_proj.bias\n",
      "encoder.layers.3.self_attn.q_proj.weight\n",
      "encoder.layers.3.self_attn.q_proj.bias\n",
      "encoder.layers.3.self_attn.out_proj.weight\n",
      "encoder.layers.3.self_attn.out_proj.bias\n",
      "encoder.layers.3.layer_norm1.weight\n",
      "encoder.layers.3.layer_norm1.bias\n",
      "encoder.layers.3.mlp.fc1.weight\n",
      "encoder.layers.3.mlp.fc1.bias\n",
      "encoder.layers.3.mlp.fc2.weight\n",
      "encoder.layers.3.mlp.fc2.bias\n",
      "encoder.layers.3.layer_norm2.weight\n",
      "encoder.layers.3.layer_norm2.bias\n",
      "encoder.layers.4.self_attn.k_proj.weight\n",
      "encoder.layers.4.self_attn.k_proj.bias\n",
      "encoder.layers.4.self_attn.v_proj.weight\n",
      "encoder.layers.4.self_attn.v_proj.bias\n",
      "encoder.layers.4.self_attn.q_proj.weight\n",
      "encoder.layers.4.self_attn.q_proj.bias\n",
      "encoder.layers.4.self_attn.out_proj.weight\n",
      "encoder.layers.4.self_attn.out_proj.bias\n",
      "encoder.layers.4.layer_norm1.weight\n",
      "encoder.layers.4.layer_norm1.bias\n",
      "encoder.layers.4.mlp.fc1.weight\n",
      "encoder.layers.4.mlp.fc1.bias\n",
      "encoder.layers.4.mlp.fc2.weight\n",
      "encoder.layers.4.mlp.fc2.bias\n",
      "encoder.layers.4.layer_norm2.weight\n",
      "encoder.layers.4.layer_norm2.bias\n",
      "encoder.layers.5.self_attn.k_proj.weight\n",
      "encoder.layers.5.self_attn.k_proj.bias\n",
      "encoder.layers.5.self_attn.v_proj.weight\n",
      "encoder.layers.5.self_attn.v_proj.bias\n",
      "encoder.layers.5.self_attn.q_proj.weight\n",
      "encoder.layers.5.self_attn.q_proj.bias\n",
      "encoder.layers.5.self_attn.out_proj.weight\n",
      "encoder.layers.5.self_attn.out_proj.bias\n",
      "encoder.layers.5.layer_norm1.weight\n",
      "encoder.layers.5.layer_norm1.bias\n",
      "encoder.layers.5.mlp.fc1.weight\n",
      "encoder.layers.5.mlp.fc1.bias\n",
      "encoder.layers.5.mlp.fc2.weight\n",
      "encoder.layers.5.mlp.fc2.bias\n",
      "encoder.layers.5.layer_norm2.weight\n",
      "encoder.layers.5.layer_norm2.bias\n",
      "encoder.layers.6.self_attn.k_proj.weight\n",
      "encoder.layers.6.self_attn.k_proj.bias\n",
      "encoder.layers.6.self_attn.v_proj.weight\n",
      "encoder.layers.6.self_attn.v_proj.bias\n",
      "encoder.layers.6.self_attn.q_proj.weight\n",
      "encoder.layers.6.self_attn.q_proj.bias\n",
      "encoder.layers.6.self_attn.out_proj.weight\n",
      "encoder.layers.6.self_attn.out_proj.bias\n",
      "encoder.layers.6.layer_norm1.weight\n",
      "encoder.layers.6.layer_norm1.bias\n",
      "encoder.layers.6.mlp.fc1.weight\n",
      "encoder.layers.6.mlp.fc1.bias\n",
      "encoder.layers.6.mlp.fc2.weight\n",
      "encoder.layers.6.mlp.fc2.bias\n",
      "encoder.layers.6.layer_norm2.weight\n",
      "encoder.layers.6.layer_norm2.bias\n",
      "encoder.layers.7.self_attn.k_proj.weight\n",
      "encoder.layers.7.self_attn.k_proj.bias\n",
      "encoder.layers.7.self_attn.v_proj.weight\n",
      "encoder.layers.7.self_attn.v_proj.bias\n",
      "encoder.layers.7.self_attn.q_proj.weight\n",
      "encoder.layers.7.self_attn.q_proj.bias\n",
      "encoder.layers.7.self_attn.out_proj.weight\n",
      "encoder.layers.7.self_attn.out_proj.bias\n",
      "encoder.layers.7.layer_norm1.weight\n",
      "encoder.layers.7.layer_norm1.bias\n",
      "encoder.layers.7.mlp.fc1.weight\n",
      "encoder.layers.7.mlp.fc1.bias\n",
      "encoder.layers.7.mlp.fc2.weight\n",
      "encoder.layers.7.mlp.fc2.bias\n",
      "encoder.layers.7.layer_norm2.weight\n",
      "encoder.layers.7.layer_norm2.bias\n",
      "encoder.layers.8.self_attn.k_proj.weight\n",
      "encoder.layers.8.self_attn.k_proj.bias\n",
      "encoder.layers.8.self_attn.v_proj.weight\n",
      "encoder.layers.8.self_attn.v_proj.bias\n",
      "encoder.layers.8.self_attn.q_proj.weight\n",
      "encoder.layers.8.self_attn.q_proj.bias\n",
      "encoder.layers.8.self_attn.out_proj.weight\n",
      "encoder.layers.8.self_attn.out_proj.bias\n",
      "encoder.layers.8.layer_norm1.weight\n",
      "encoder.layers.8.layer_norm1.bias\n",
      "encoder.layers.8.mlp.fc1.weight\n",
      "encoder.layers.8.mlp.fc1.bias\n",
      "encoder.layers.8.mlp.fc2.weight\n",
      "encoder.layers.8.mlp.fc2.bias\n",
      "encoder.layers.8.layer_norm2.weight\n",
      "encoder.layers.8.layer_norm2.bias\n",
      "encoder.layers.9.self_attn.k_proj.weight\n",
      "encoder.layers.9.self_attn.k_proj.bias\n",
      "encoder.layers.9.self_attn.v_proj.weight\n",
      "encoder.layers.9.self_attn.v_proj.bias\n",
      "encoder.layers.9.self_attn.q_proj.weight\n",
      "encoder.layers.9.self_attn.q_proj.bias\n",
      "encoder.layers.9.self_attn.out_proj.weight\n",
      "encoder.layers.9.self_attn.out_proj.bias\n",
      "encoder.layers.9.layer_norm1.weight\n",
      "encoder.layers.9.layer_norm1.bias\n",
      "encoder.layers.9.mlp.fc1.weight\n",
      "encoder.layers.9.mlp.fc1.bias\n",
      "encoder.layers.9.mlp.fc2.weight\n",
      "encoder.layers.9.mlp.fc2.bias\n",
      "encoder.layers.9.layer_norm2.weight\n",
      "encoder.layers.9.layer_norm2.bias\n",
      "encoder.layers.10.self_attn.k_proj.weight\n",
      "encoder.layers.10.self_attn.k_proj.bias\n",
      "encoder.layers.10.self_attn.v_proj.weight\n",
      "encoder.layers.10.self_attn.v_proj.bias\n",
      "encoder.layers.10.self_attn.q_proj.weight\n",
      "encoder.layers.10.self_attn.q_proj.bias\n",
      "encoder.layers.10.self_attn.out_proj.weight\n",
      "encoder.layers.10.self_attn.out_proj.bias\n",
      "encoder.layers.10.layer_norm1.weight\n",
      "encoder.layers.10.layer_norm1.bias\n",
      "encoder.layers.10.mlp.fc1.weight\n",
      "encoder.layers.10.mlp.fc1.bias\n",
      "encoder.layers.10.mlp.fc2.weight\n",
      "encoder.layers.10.mlp.fc2.bias\n",
      "encoder.layers.10.layer_norm2.weight\n",
      "encoder.layers.10.layer_norm2.bias\n",
      "encoder.layers.11.self_attn.k_proj.weight\n",
      "encoder.layers.11.self_attn.k_proj.bias\n",
      "encoder.layers.11.self_attn.v_proj.weight\n",
      "encoder.layers.11.self_attn.v_proj.bias\n",
      "encoder.layers.11.self_attn.q_proj.weight\n",
      "encoder.layers.11.self_attn.q_proj.bias\n",
      "encoder.layers.11.self_attn.out_proj.weight\n",
      "encoder.layers.11.self_attn.out_proj.bias\n",
      "encoder.layers.11.layer_norm1.weight\n",
      "encoder.layers.11.layer_norm1.bias\n",
      "encoder.layers.11.mlp.fc1.weight\n",
      "encoder.layers.11.mlp.fc1.bias\n",
      "encoder.layers.11.mlp.fc2.weight\n",
      "encoder.layers.11.mlp.fc2.bias\n",
      "encoder.layers.11.layer_norm2.weight\n",
      "encoder.layers.11.layer_norm2.bias\n",
      "encoder.layers.12.self_attn.k_proj.weight\n",
      "encoder.layers.12.self_attn.k_proj.bias\n",
      "encoder.layers.12.self_attn.v_proj.weight\n",
      "encoder.layers.12.self_attn.v_proj.bias\n",
      "encoder.layers.12.self_attn.q_proj.weight\n",
      "encoder.layers.12.self_attn.q_proj.bias\n",
      "encoder.layers.12.self_attn.out_proj.weight\n",
      "encoder.layers.12.self_attn.out_proj.bias\n",
      "encoder.layers.12.layer_norm1.weight\n",
      "encoder.layers.12.layer_norm1.bias\n",
      "encoder.layers.12.mlp.fc1.weight\n",
      "encoder.layers.12.mlp.fc1.bias\n",
      "encoder.layers.12.mlp.fc2.weight\n",
      "encoder.layers.12.mlp.fc2.bias\n",
      "encoder.layers.12.layer_norm2.weight\n",
      "encoder.layers.12.layer_norm2.bias\n",
      "encoder.layers.13.self_attn.k_proj.weight\n",
      "encoder.layers.13.self_attn.k_proj.bias\n",
      "encoder.layers.13.self_attn.v_proj.weight\n",
      "encoder.layers.13.self_attn.v_proj.bias\n",
      "encoder.layers.13.self_attn.q_proj.weight\n",
      "encoder.layers.13.self_attn.q_proj.bias\n",
      "encoder.layers.13.self_attn.out_proj.weight\n",
      "encoder.layers.13.self_attn.out_proj.bias\n",
      "encoder.layers.13.layer_norm1.weight\n",
      "encoder.layers.13.layer_norm1.bias\n",
      "encoder.layers.13.mlp.fc1.weight\n",
      "encoder.layers.13.mlp.fc1.bias\n",
      "encoder.layers.13.mlp.fc2.weight\n",
      "encoder.layers.13.mlp.fc2.bias\n",
      "encoder.layers.13.layer_norm2.weight\n",
      "encoder.layers.13.layer_norm2.bias\n",
      "encoder.layers.14.self_attn.k_proj.weight\n",
      "encoder.layers.14.self_attn.k_proj.bias\n",
      "encoder.layers.14.self_attn.v_proj.weight\n",
      "encoder.layers.14.self_attn.v_proj.bias\n",
      "encoder.layers.14.self_attn.q_proj.weight\n",
      "encoder.layers.14.self_attn.q_proj.bias\n",
      "encoder.layers.14.self_attn.out_proj.weight\n",
      "encoder.layers.14.self_attn.out_proj.bias\n",
      "encoder.layers.14.layer_norm1.weight\n",
      "encoder.layers.14.layer_norm1.bias\n",
      "encoder.layers.14.mlp.fc1.weight\n",
      "encoder.layers.14.mlp.fc1.bias\n",
      "encoder.layers.14.mlp.fc2.weight\n",
      "encoder.layers.14.mlp.fc2.bias\n",
      "encoder.layers.14.layer_norm2.weight\n",
      "encoder.layers.14.layer_norm2.bias\n",
      "encoder.layers.15.self_attn.k_proj.weight\n",
      "encoder.layers.15.self_attn.k_proj.bias\n",
      "encoder.layers.15.self_attn.v_proj.weight\n",
      "encoder.layers.15.self_attn.v_proj.bias\n",
      "encoder.layers.15.self_attn.q_proj.weight\n",
      "encoder.layers.15.self_attn.q_proj.bias\n",
      "encoder.layers.15.self_attn.out_proj.weight\n",
      "encoder.layers.15.self_attn.out_proj.bias\n",
      "encoder.layers.15.layer_norm1.weight\n",
      "encoder.layers.15.layer_norm1.bias\n",
      "encoder.layers.15.mlp.fc1.weight\n",
      "encoder.layers.15.mlp.fc1.bias\n",
      "encoder.layers.15.mlp.fc2.weight\n",
      "encoder.layers.15.mlp.fc2.bias\n",
      "encoder.layers.15.layer_norm2.weight\n",
      "encoder.layers.15.layer_norm2.bias\n",
      "encoder.layers.16.self_attn.k_proj.weight\n",
      "encoder.layers.16.self_attn.k_proj.bias\n",
      "encoder.layers.16.self_attn.v_proj.weight\n",
      "encoder.layers.16.self_attn.v_proj.bias\n",
      "encoder.layers.16.self_attn.q_proj.weight\n",
      "encoder.layers.16.self_attn.q_proj.bias\n",
      "encoder.layers.16.self_attn.out_proj.weight\n",
      "encoder.layers.16.self_attn.out_proj.bias\n",
      "encoder.layers.16.layer_norm1.weight\n",
      "encoder.layers.16.layer_norm1.bias\n",
      "encoder.layers.16.mlp.fc1.weight\n",
      "encoder.layers.16.mlp.fc1.bias\n",
      "encoder.layers.16.mlp.fc2.weight\n",
      "encoder.layers.16.mlp.fc2.bias\n",
      "encoder.layers.16.layer_norm2.weight\n",
      "encoder.layers.16.layer_norm2.bias\n",
      "encoder.layers.17.self_attn.k_proj.weight\n",
      "encoder.layers.17.self_attn.k_proj.bias\n",
      "encoder.layers.17.self_attn.v_proj.weight\n",
      "encoder.layers.17.self_attn.v_proj.bias\n",
      "encoder.layers.17.self_attn.q_proj.weight\n",
      "encoder.layers.17.self_attn.q_proj.bias\n",
      "encoder.layers.17.self_attn.out_proj.weight\n",
      "encoder.layers.17.self_attn.out_proj.bias\n",
      "encoder.layers.17.layer_norm1.weight\n",
      "encoder.layers.17.layer_norm1.bias\n",
      "encoder.layers.17.mlp.fc1.weight\n",
      "encoder.layers.17.mlp.fc1.bias\n",
      "encoder.layers.17.mlp.fc2.weight\n",
      "encoder.layers.17.mlp.fc2.bias\n",
      "encoder.layers.17.layer_norm2.weight\n",
      "encoder.layers.17.layer_norm2.bias\n",
      "encoder.layers.18.self_attn.k_proj.weight\n",
      "encoder.layers.18.self_attn.k_proj.bias\n",
      "encoder.layers.18.self_attn.v_proj.weight\n",
      "encoder.layers.18.self_attn.v_proj.bias\n",
      "encoder.layers.18.self_attn.q_proj.weight\n",
      "encoder.layers.18.self_attn.q_proj.bias\n",
      "encoder.layers.18.self_attn.out_proj.weight\n",
      "encoder.layers.18.self_attn.out_proj.bias\n",
      "encoder.layers.18.layer_norm1.weight\n",
      "encoder.layers.18.layer_norm1.bias\n",
      "encoder.layers.18.mlp.fc1.weight\n",
      "encoder.layers.18.mlp.fc1.bias\n",
      "encoder.layers.18.mlp.fc2.weight\n",
      "encoder.layers.18.mlp.fc2.bias\n",
      "encoder.layers.18.layer_norm2.weight\n",
      "encoder.layers.18.layer_norm2.bias\n",
      "encoder.layers.19.self_attn.k_proj.weight\n",
      "encoder.layers.19.self_attn.k_proj.bias\n",
      "encoder.layers.19.self_attn.v_proj.weight\n",
      "encoder.layers.19.self_attn.v_proj.bias\n",
      "encoder.layers.19.self_attn.q_proj.weight\n",
      "encoder.layers.19.self_attn.q_proj.bias\n",
      "encoder.layers.19.self_attn.out_proj.weight\n",
      "encoder.layers.19.self_attn.out_proj.bias\n",
      "encoder.layers.19.layer_norm1.weight\n",
      "encoder.layers.19.layer_norm1.bias\n",
      "encoder.layers.19.mlp.fc1.weight\n",
      "encoder.layers.19.mlp.fc1.bias\n",
      "encoder.layers.19.mlp.fc2.weight\n",
      "encoder.layers.19.mlp.fc2.bias\n",
      "encoder.layers.19.layer_norm2.weight\n",
      "encoder.layers.19.layer_norm2.bias\n",
      "encoder.layers.20.self_attn.k_proj.weight\n",
      "encoder.layers.20.self_attn.k_proj.bias\n",
      "encoder.layers.20.self_attn.v_proj.weight\n",
      "encoder.layers.20.self_attn.v_proj.bias\n",
      "encoder.layers.20.self_attn.q_proj.weight\n",
      "encoder.layers.20.self_attn.q_proj.bias\n",
      "encoder.layers.20.self_attn.out_proj.weight\n",
      "encoder.layers.20.self_attn.out_proj.bias\n",
      "encoder.layers.20.layer_norm1.weight\n",
      "encoder.layers.20.layer_norm1.bias\n",
      "encoder.layers.20.mlp.fc1.weight\n",
      "encoder.layers.20.mlp.fc1.bias\n",
      "encoder.layers.20.mlp.fc2.weight\n",
      "encoder.layers.20.mlp.fc2.bias\n",
      "encoder.layers.20.layer_norm2.weight\n",
      "encoder.layers.20.layer_norm2.bias\n",
      "encoder.layers.21.self_attn.k_proj.weight\n",
      "encoder.layers.21.self_attn.k_proj.bias\n",
      "encoder.layers.21.self_attn.v_proj.weight\n",
      "encoder.layers.21.self_attn.v_proj.bias\n",
      "encoder.layers.21.self_attn.q_proj.weight\n",
      "encoder.layers.21.self_attn.q_proj.bias\n",
      "encoder.layers.21.self_attn.out_proj.weight\n",
      "encoder.layers.21.self_attn.out_proj.bias\n",
      "encoder.layers.21.layer_norm1.weight\n",
      "encoder.layers.21.layer_norm1.bias\n",
      "encoder.layers.21.mlp.fc1.weight\n",
      "encoder.layers.21.mlp.fc1.bias\n",
      "encoder.layers.21.mlp.fc2.weight\n",
      "encoder.layers.21.mlp.fc2.bias\n",
      "encoder.layers.21.layer_norm2.weight\n",
      "encoder.layers.21.layer_norm2.bias\n",
      "encoder.layers.22.self_attn.k_proj.weight\n",
      "encoder.layers.22.self_attn.k_proj.bias\n",
      "encoder.layers.22.self_attn.v_proj.weight\n",
      "encoder.layers.22.self_attn.v_proj.bias\n",
      "encoder.layers.22.self_attn.q_proj.weight\n",
      "encoder.layers.22.self_attn.q_proj.bias\n",
      "encoder.layers.22.self_attn.out_proj.weight\n",
      "encoder.layers.22.self_attn.out_proj.bias\n",
      "encoder.layers.22.layer_norm1.weight\n",
      "encoder.layers.22.layer_norm1.bias\n",
      "encoder.layers.22.mlp.fc1.weight\n",
      "encoder.layers.22.mlp.fc1.bias\n",
      "encoder.layers.22.mlp.fc2.weight\n",
      "encoder.layers.22.mlp.fc2.bias\n",
      "encoder.layers.22.layer_norm2.weight\n",
      "encoder.layers.22.layer_norm2.bias\n",
      "encoder.layers.23.self_attn.k_proj.weight\n",
      "encoder.layers.23.self_attn.k_proj.bias\n",
      "encoder.layers.23.self_attn.v_proj.weight\n",
      "encoder.layers.23.self_attn.v_proj.bias\n",
      "encoder.layers.23.self_attn.q_proj.weight\n",
      "encoder.layers.23.self_attn.q_proj.bias\n",
      "encoder.layers.23.self_attn.out_proj.weight\n",
      "encoder.layers.23.self_attn.out_proj.bias\n",
      "encoder.layers.23.layer_norm1.weight\n",
      "encoder.layers.23.layer_norm1.bias\n",
      "encoder.layers.23.mlp.fc1.weight\n",
      "encoder.layers.23.mlp.fc1.bias\n",
      "encoder.layers.23.mlp.fc2.weight\n",
      "encoder.layers.23.mlp.fc2.bias\n",
      "encoder.layers.23.layer_norm2.weight\n",
      "encoder.layers.23.layer_norm2.bias\n",
      "post_layernorm.weight\n",
      "post_layernorm.bias\n",
      "visual_projection.weight\n"
     ]
    }
   ],
   "source": [
    "for name, param in model.vision_model.named_parameters():\n",
    "    print(name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90a5ec5d-e642-48f8-998a-5be4db6308f9",
   "metadata": {},
   "source": [
    "### Load checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "93f19150-c53a-4279-a3c0-e4b227255927",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gridsan/manderson/.conda/envs/train-clip/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import CLIPModel\n",
    "from models import CustomCLIPWrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "bb509351-7880-415d-9b10-3164f3b73606",
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH_CKPT_clip14_test = '/home/gridsan/manderson/train-CLIP/run/test/last.ckpt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7615b4ca-4b07-48cf-af08-3bf14da0f24b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/state/partition1/slurm_tmp/27105912.0.0/ipykernel_2330604/3157462980.py:1: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  ckpt = torch.load(PATH_CKPT_clip14_test, map_location='cpu')\n"
     ]
    }
   ],
   "source": [
    "ckpt = torch.load(PATH_CKPT_clip14_test, map_location='cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7dab83b6-fac7-44ba-99b6-60d1f4bfaa46",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['epoch', 'global_step', 'pytorch-lightning_version', 'state_dict', 'callbacks', 'optimizer_states', 'lr_schedulers', 'native_amp_scaling_state'])\n"
     ]
    }
   ],
   "source": [
    "print(ckpt.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2819ee6e-8f9d-4e8e-a340-486719ac6d1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict = ckpt['state_dict']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "97cd2e42-c009-4882-a042-9c574b0282c9",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "odict_keys(['sink_temp', 'model.positional_embedding', 'model.text_projection', 'model.logit_scale', 'model.token_embedding.weight', 'model.ln_final.weight', 'model.ln_final.bias', 'model.visual_projection.weight', 'model.visual.embeddings.class_embedding', 'model.visual.embeddings.patch_embedding.weight', 'model.visual.embeddings.position_embedding.weight', 'model.visual.pre_layrnorm.weight', 'model.visual.pre_layrnorm.bias', 'model.visual.encoder.layers.0.self_attn.k_proj.weight', 'model.visual.encoder.layers.0.self_attn.k_proj.bias', 'model.visual.encoder.layers.0.self_attn.v_proj.weight', 'model.visual.encoder.layers.0.self_attn.v_proj.bias', 'model.visual.encoder.layers.0.self_attn.q_proj.weight', 'model.visual.encoder.layers.0.self_attn.q_proj.bias', 'model.visual.encoder.layers.0.self_attn.out_proj.weight', 'model.visual.encoder.layers.0.self_attn.out_proj.bias', 'model.visual.encoder.layers.0.layer_norm1.weight', 'model.visual.encoder.layers.0.layer_norm1.bias', 'model.visual.encoder.layers.0.mlp.fc1.weight', 'model.visual.encoder.layers.0.mlp.fc1.bias', 'model.visual.encoder.layers.0.mlp.fc2.weight', 'model.visual.encoder.layers.0.mlp.fc2.bias', 'model.visual.encoder.layers.0.layer_norm2.weight', 'model.visual.encoder.layers.0.layer_norm2.bias', 'model.visual.encoder.layers.1.self_attn.k_proj.weight', 'model.visual.encoder.layers.1.self_attn.k_proj.bias', 'model.visual.encoder.layers.1.self_attn.v_proj.weight', 'model.visual.encoder.layers.1.self_attn.v_proj.bias', 'model.visual.encoder.layers.1.self_attn.q_proj.weight', 'model.visual.encoder.layers.1.self_attn.q_proj.bias', 'model.visual.encoder.layers.1.self_attn.out_proj.weight', 'model.visual.encoder.layers.1.self_attn.out_proj.bias', 'model.visual.encoder.layers.1.layer_norm1.weight', 'model.visual.encoder.layers.1.layer_norm1.bias', 'model.visual.encoder.layers.1.mlp.fc1.weight', 'model.visual.encoder.layers.1.mlp.fc1.bias', 'model.visual.encoder.layers.1.mlp.fc2.weight', 'model.visual.encoder.layers.1.mlp.fc2.bias', 'model.visual.encoder.layers.1.layer_norm2.weight', 'model.visual.encoder.layers.1.layer_norm2.bias', 'model.visual.encoder.layers.2.self_attn.k_proj.weight', 'model.visual.encoder.layers.2.self_attn.k_proj.bias', 'model.visual.encoder.layers.2.self_attn.v_proj.weight', 'model.visual.encoder.layers.2.self_attn.v_proj.bias', 'model.visual.encoder.layers.2.self_attn.q_proj.weight', 'model.visual.encoder.layers.2.self_attn.q_proj.bias', 'model.visual.encoder.layers.2.self_attn.out_proj.weight', 'model.visual.encoder.layers.2.self_attn.out_proj.bias', 'model.visual.encoder.layers.2.layer_norm1.weight', 'model.visual.encoder.layers.2.layer_norm1.bias', 'model.visual.encoder.layers.2.mlp.fc1.weight', 'model.visual.encoder.layers.2.mlp.fc1.bias', 'model.visual.encoder.layers.2.mlp.fc2.weight', 'model.visual.encoder.layers.2.mlp.fc2.bias', 'model.visual.encoder.layers.2.layer_norm2.weight', 'model.visual.encoder.layers.2.layer_norm2.bias', 'model.visual.encoder.layers.3.self_attn.k_proj.weight', 'model.visual.encoder.layers.3.self_attn.k_proj.bias', 'model.visual.encoder.layers.3.self_attn.v_proj.weight', 'model.visual.encoder.layers.3.self_attn.v_proj.bias', 'model.visual.encoder.layers.3.self_attn.q_proj.weight', 'model.visual.encoder.layers.3.self_attn.q_proj.bias', 'model.visual.encoder.layers.3.self_attn.out_proj.weight', 'model.visual.encoder.layers.3.self_attn.out_proj.bias', 'model.visual.encoder.layers.3.layer_norm1.weight', 'model.visual.encoder.layers.3.layer_norm1.bias', 'model.visual.encoder.layers.3.mlp.fc1.weight', 'model.visual.encoder.layers.3.mlp.fc1.bias', 'model.visual.encoder.layers.3.mlp.fc2.weight', 'model.visual.encoder.layers.3.mlp.fc2.bias', 'model.visual.encoder.layers.3.layer_norm2.weight', 'model.visual.encoder.layers.3.layer_norm2.bias', 'model.visual.encoder.layers.4.self_attn.k_proj.weight', 'model.visual.encoder.layers.4.self_attn.k_proj.bias', 'model.visual.encoder.layers.4.self_attn.v_proj.weight', 'model.visual.encoder.layers.4.self_attn.v_proj.bias', 'model.visual.encoder.layers.4.self_attn.q_proj.weight', 'model.visual.encoder.layers.4.self_attn.q_proj.bias', 'model.visual.encoder.layers.4.self_attn.out_proj.weight', 'model.visual.encoder.layers.4.self_attn.out_proj.bias', 'model.visual.encoder.layers.4.layer_norm1.weight', 'model.visual.encoder.layers.4.layer_norm1.bias', 'model.visual.encoder.layers.4.mlp.fc1.weight', 'model.visual.encoder.layers.4.mlp.fc1.bias', 'model.visual.encoder.layers.4.mlp.fc2.weight', 'model.visual.encoder.layers.4.mlp.fc2.bias', 'model.visual.encoder.layers.4.layer_norm2.weight', 'model.visual.encoder.layers.4.layer_norm2.bias', 'model.visual.encoder.layers.5.self_attn.k_proj.weight', 'model.visual.encoder.layers.5.self_attn.k_proj.bias', 'model.visual.encoder.layers.5.self_attn.v_proj.weight', 'model.visual.encoder.layers.5.self_attn.v_proj.bias', 'model.visual.encoder.layers.5.self_attn.q_proj.weight', 'model.visual.encoder.layers.5.self_attn.q_proj.bias', 'model.visual.encoder.layers.5.self_attn.out_proj.weight', 'model.visual.encoder.layers.5.self_attn.out_proj.bias', 'model.visual.encoder.layers.5.layer_norm1.weight', 'model.visual.encoder.layers.5.layer_norm1.bias', 'model.visual.encoder.layers.5.mlp.fc1.weight', 'model.visual.encoder.layers.5.mlp.fc1.bias', 'model.visual.encoder.layers.5.mlp.fc2.weight', 'model.visual.encoder.layers.5.mlp.fc2.bias', 'model.visual.encoder.layers.5.layer_norm2.weight', 'model.visual.encoder.layers.5.layer_norm2.bias', 'model.visual.encoder.layers.6.self_attn.k_proj.weight', 'model.visual.encoder.layers.6.self_attn.k_proj.bias', 'model.visual.encoder.layers.6.self_attn.v_proj.weight', 'model.visual.encoder.layers.6.self_attn.v_proj.bias', 'model.visual.encoder.layers.6.self_attn.q_proj.weight', 'model.visual.encoder.layers.6.self_attn.q_proj.bias', 'model.visual.encoder.layers.6.self_attn.out_proj.weight', 'model.visual.encoder.layers.6.self_attn.out_proj.bias', 'model.visual.encoder.layers.6.layer_norm1.weight', 'model.visual.encoder.layers.6.layer_norm1.bias', 'model.visual.encoder.layers.6.mlp.fc1.weight', 'model.visual.encoder.layers.6.mlp.fc1.bias', 'model.visual.encoder.layers.6.mlp.fc2.weight', 'model.visual.encoder.layers.6.mlp.fc2.bias', 'model.visual.encoder.layers.6.layer_norm2.weight', 'model.visual.encoder.layers.6.layer_norm2.bias', 'model.visual.encoder.layers.7.self_attn.k_proj.weight', 'model.visual.encoder.layers.7.self_attn.k_proj.bias', 'model.visual.encoder.layers.7.self_attn.v_proj.weight', 'model.visual.encoder.layers.7.self_attn.v_proj.bias', 'model.visual.encoder.layers.7.self_attn.q_proj.weight', 'model.visual.encoder.layers.7.self_attn.q_proj.bias', 'model.visual.encoder.layers.7.self_attn.out_proj.weight', 'model.visual.encoder.layers.7.self_attn.out_proj.bias', 'model.visual.encoder.layers.7.layer_norm1.weight', 'model.visual.encoder.layers.7.layer_norm1.bias', 'model.visual.encoder.layers.7.mlp.fc1.weight', 'model.visual.encoder.layers.7.mlp.fc1.bias', 'model.visual.encoder.layers.7.mlp.fc2.weight', 'model.visual.encoder.layers.7.mlp.fc2.bias', 'model.visual.encoder.layers.7.layer_norm2.weight', 'model.visual.encoder.layers.7.layer_norm2.bias', 'model.visual.encoder.layers.8.self_attn.k_proj.weight', 'model.visual.encoder.layers.8.self_attn.k_proj.bias', 'model.visual.encoder.layers.8.self_attn.v_proj.weight', 'model.visual.encoder.layers.8.self_attn.v_proj.bias', 'model.visual.encoder.layers.8.self_attn.q_proj.weight', 'model.visual.encoder.layers.8.self_attn.q_proj.bias', 'model.visual.encoder.layers.8.self_attn.out_proj.weight', 'model.visual.encoder.layers.8.self_attn.out_proj.bias', 'model.visual.encoder.layers.8.layer_norm1.weight', 'model.visual.encoder.layers.8.layer_norm1.bias', 'model.visual.encoder.layers.8.mlp.fc1.weight', 'model.visual.encoder.layers.8.mlp.fc1.bias', 'model.visual.encoder.layers.8.mlp.fc2.weight', 'model.visual.encoder.layers.8.mlp.fc2.bias', 'model.visual.encoder.layers.8.layer_norm2.weight', 'model.visual.encoder.layers.8.layer_norm2.bias', 'model.visual.encoder.layers.9.self_attn.k_proj.weight', 'model.visual.encoder.layers.9.self_attn.k_proj.bias', 'model.visual.encoder.layers.9.self_attn.v_proj.weight', 'model.visual.encoder.layers.9.self_attn.v_proj.bias', 'model.visual.encoder.layers.9.self_attn.q_proj.weight', 'model.visual.encoder.layers.9.self_attn.q_proj.bias', 'model.visual.encoder.layers.9.self_attn.out_proj.weight', 'model.visual.encoder.layers.9.self_attn.out_proj.bias', 'model.visual.encoder.layers.9.layer_norm1.weight', 'model.visual.encoder.layers.9.layer_norm1.bias', 'model.visual.encoder.layers.9.mlp.fc1.weight', 'model.visual.encoder.layers.9.mlp.fc1.bias', 'model.visual.encoder.layers.9.mlp.fc2.weight', 'model.visual.encoder.layers.9.mlp.fc2.bias', 'model.visual.encoder.layers.9.layer_norm2.weight', 'model.visual.encoder.layers.9.layer_norm2.bias', 'model.visual.encoder.layers.10.self_attn.k_proj.weight', 'model.visual.encoder.layers.10.self_attn.k_proj.bias', 'model.visual.encoder.layers.10.self_attn.v_proj.weight', 'model.visual.encoder.layers.10.self_attn.v_proj.bias', 'model.visual.encoder.layers.10.self_attn.q_proj.weight', 'model.visual.encoder.layers.10.self_attn.q_proj.bias', 'model.visual.encoder.layers.10.self_attn.out_proj.weight', 'model.visual.encoder.layers.10.self_attn.out_proj.bias', 'model.visual.encoder.layers.10.layer_norm1.weight', 'model.visual.encoder.layers.10.layer_norm1.bias', 'model.visual.encoder.layers.10.mlp.fc1.weight', 'model.visual.encoder.layers.10.mlp.fc1.bias', 'model.visual.encoder.layers.10.mlp.fc2.weight', 'model.visual.encoder.layers.10.mlp.fc2.bias', 'model.visual.encoder.layers.10.layer_norm2.weight', 'model.visual.encoder.layers.10.layer_norm2.bias', 'model.visual.encoder.layers.11.self_attn.k_proj.weight', 'model.visual.encoder.layers.11.self_attn.k_proj.bias', 'model.visual.encoder.layers.11.self_attn.v_proj.weight', 'model.visual.encoder.layers.11.self_attn.v_proj.bias', 'model.visual.encoder.layers.11.self_attn.q_proj.weight', 'model.visual.encoder.layers.11.self_attn.q_proj.bias', 'model.visual.encoder.layers.11.self_attn.out_proj.weight', 'model.visual.encoder.layers.11.self_attn.out_proj.bias', 'model.visual.encoder.layers.11.layer_norm1.weight', 'model.visual.encoder.layers.11.layer_norm1.bias', 'model.visual.encoder.layers.11.mlp.fc1.weight', 'model.visual.encoder.layers.11.mlp.fc1.bias', 'model.visual.encoder.layers.11.mlp.fc2.weight', 'model.visual.encoder.layers.11.mlp.fc2.bias', 'model.visual.encoder.layers.11.layer_norm2.weight', 'model.visual.encoder.layers.11.layer_norm2.bias', 'model.visual.encoder.layers.12.self_attn.k_proj.weight', 'model.visual.encoder.layers.12.self_attn.k_proj.bias', 'model.visual.encoder.layers.12.self_attn.v_proj.weight', 'model.visual.encoder.layers.12.self_attn.v_proj.bias', 'model.visual.encoder.layers.12.self_attn.q_proj.weight', 'model.visual.encoder.layers.12.self_attn.q_proj.bias', 'model.visual.encoder.layers.12.self_attn.out_proj.weight', 'model.visual.encoder.layers.12.self_attn.out_proj.bias', 'model.visual.encoder.layers.12.layer_norm1.weight', 'model.visual.encoder.layers.12.layer_norm1.bias', 'model.visual.encoder.layers.12.mlp.fc1.weight', 'model.visual.encoder.layers.12.mlp.fc1.bias', 'model.visual.encoder.layers.12.mlp.fc2.weight', 'model.visual.encoder.layers.12.mlp.fc2.bias', 'model.visual.encoder.layers.12.layer_norm2.weight', 'model.visual.encoder.layers.12.layer_norm2.bias', 'model.visual.encoder.layers.13.self_attn.k_proj.weight', 'model.visual.encoder.layers.13.self_attn.k_proj.bias', 'model.visual.encoder.layers.13.self_attn.v_proj.weight', 'model.visual.encoder.layers.13.self_attn.v_proj.bias', 'model.visual.encoder.layers.13.self_attn.q_proj.weight', 'model.visual.encoder.layers.13.self_attn.q_proj.bias', 'model.visual.encoder.layers.13.self_attn.out_proj.weight', 'model.visual.encoder.layers.13.self_attn.out_proj.bias', 'model.visual.encoder.layers.13.layer_norm1.weight', 'model.visual.encoder.layers.13.layer_norm1.bias', 'model.visual.encoder.layers.13.mlp.fc1.weight', 'model.visual.encoder.layers.13.mlp.fc1.bias', 'model.visual.encoder.layers.13.mlp.fc2.weight', 'model.visual.encoder.layers.13.mlp.fc2.bias', 'model.visual.encoder.layers.13.layer_norm2.weight', 'model.visual.encoder.layers.13.layer_norm2.bias', 'model.visual.encoder.layers.14.self_attn.k_proj.weight', 'model.visual.encoder.layers.14.self_attn.k_proj.bias', 'model.visual.encoder.layers.14.self_attn.v_proj.weight', 'model.visual.encoder.layers.14.self_attn.v_proj.bias', 'model.visual.encoder.layers.14.self_attn.q_proj.weight', 'model.visual.encoder.layers.14.self_attn.q_proj.bias', 'model.visual.encoder.layers.14.self_attn.out_proj.weight', 'model.visual.encoder.layers.14.self_attn.out_proj.bias', 'model.visual.encoder.layers.14.layer_norm1.weight', 'model.visual.encoder.layers.14.layer_norm1.bias', 'model.visual.encoder.layers.14.mlp.fc1.weight', 'model.visual.encoder.layers.14.mlp.fc1.bias', 'model.visual.encoder.layers.14.mlp.fc2.weight', 'model.visual.encoder.layers.14.mlp.fc2.bias', 'model.visual.encoder.layers.14.layer_norm2.weight', 'model.visual.encoder.layers.14.layer_norm2.bias', 'model.visual.encoder.layers.15.self_attn.k_proj.weight', 'model.visual.encoder.layers.15.self_attn.k_proj.bias', 'model.visual.encoder.layers.15.self_attn.v_proj.weight', 'model.visual.encoder.layers.15.self_attn.v_proj.bias', 'model.visual.encoder.layers.15.self_attn.q_proj.weight', 'model.visual.encoder.layers.15.self_attn.q_proj.bias', 'model.visual.encoder.layers.15.self_attn.out_proj.weight', 'model.visual.encoder.layers.15.self_attn.out_proj.bias', 'model.visual.encoder.layers.15.layer_norm1.weight', 'model.visual.encoder.layers.15.layer_norm1.bias', 'model.visual.encoder.layers.15.mlp.fc1.weight', 'model.visual.encoder.layers.15.mlp.fc1.bias', 'model.visual.encoder.layers.15.mlp.fc2.weight', 'model.visual.encoder.layers.15.mlp.fc2.bias', 'model.visual.encoder.layers.15.layer_norm2.weight', 'model.visual.encoder.layers.15.layer_norm2.bias', 'model.visual.encoder.layers.16.self_attn.k_proj.weight', 'model.visual.encoder.layers.16.self_attn.k_proj.bias', 'model.visual.encoder.layers.16.self_attn.v_proj.weight', 'model.visual.encoder.layers.16.self_attn.v_proj.bias', 'model.visual.encoder.layers.16.self_attn.q_proj.weight', 'model.visual.encoder.layers.16.self_attn.q_proj.bias', 'model.visual.encoder.layers.16.self_attn.out_proj.weight', 'model.visual.encoder.layers.16.self_attn.out_proj.bias', 'model.visual.encoder.layers.16.layer_norm1.weight', 'model.visual.encoder.layers.16.layer_norm1.bias', 'model.visual.encoder.layers.16.mlp.fc1.weight', 'model.visual.encoder.layers.16.mlp.fc1.bias', 'model.visual.encoder.layers.16.mlp.fc2.weight', 'model.visual.encoder.layers.16.mlp.fc2.bias', 'model.visual.encoder.layers.16.layer_norm2.weight', 'model.visual.encoder.layers.16.layer_norm2.bias', 'model.visual.encoder.layers.17.self_attn.k_proj.weight', 'model.visual.encoder.layers.17.self_attn.k_proj.bias', 'model.visual.encoder.layers.17.self_attn.v_proj.weight', 'model.visual.encoder.layers.17.self_attn.v_proj.bias', 'model.visual.encoder.layers.17.self_attn.q_proj.weight', 'model.visual.encoder.layers.17.self_attn.q_proj.bias', 'model.visual.encoder.layers.17.self_attn.out_proj.weight', 'model.visual.encoder.layers.17.self_attn.out_proj.bias', 'model.visual.encoder.layers.17.layer_norm1.weight', 'model.visual.encoder.layers.17.layer_norm1.bias', 'model.visual.encoder.layers.17.mlp.fc1.weight', 'model.visual.encoder.layers.17.mlp.fc1.bias', 'model.visual.encoder.layers.17.mlp.fc2.weight', 'model.visual.encoder.layers.17.mlp.fc2.bias', 'model.visual.encoder.layers.17.layer_norm2.weight', 'model.visual.encoder.layers.17.layer_norm2.bias', 'model.visual.encoder.layers.18.self_attn.k_proj.weight', 'model.visual.encoder.layers.18.self_attn.k_proj.bias', 'model.visual.encoder.layers.18.self_attn.v_proj.weight', 'model.visual.encoder.layers.18.self_attn.v_proj.bias', 'model.visual.encoder.layers.18.self_attn.q_proj.weight', 'model.visual.encoder.layers.18.self_attn.q_proj.bias', 'model.visual.encoder.layers.18.self_attn.out_proj.weight', 'model.visual.encoder.layers.18.self_attn.out_proj.bias', 'model.visual.encoder.layers.18.layer_norm1.weight', 'model.visual.encoder.layers.18.layer_norm1.bias', 'model.visual.encoder.layers.18.mlp.fc1.weight', 'model.visual.encoder.layers.18.mlp.fc1.bias', 'model.visual.encoder.layers.18.mlp.fc2.weight', 'model.visual.encoder.layers.18.mlp.fc2.bias', 'model.visual.encoder.layers.18.layer_norm2.weight', 'model.visual.encoder.layers.18.layer_norm2.bias', 'model.visual.encoder.layers.19.self_attn.k_proj.weight', 'model.visual.encoder.layers.19.self_attn.k_proj.bias', 'model.visual.encoder.layers.19.self_attn.v_proj.weight', 'model.visual.encoder.layers.19.self_attn.v_proj.bias', 'model.visual.encoder.layers.19.self_attn.q_proj.weight', 'model.visual.encoder.layers.19.self_attn.q_proj.bias', 'model.visual.encoder.layers.19.self_attn.out_proj.weight', 'model.visual.encoder.layers.19.self_attn.out_proj.bias', 'model.visual.encoder.layers.19.layer_norm1.weight', 'model.visual.encoder.layers.19.layer_norm1.bias', 'model.visual.encoder.layers.19.mlp.fc1.weight', 'model.visual.encoder.layers.19.mlp.fc1.bias', 'model.visual.encoder.layers.19.mlp.fc2.weight', 'model.visual.encoder.layers.19.mlp.fc2.bias', 'model.visual.encoder.layers.19.layer_norm2.weight', 'model.visual.encoder.layers.19.layer_norm2.bias', 'model.visual.encoder.layers.20.self_attn.k_proj.weight', 'model.visual.encoder.layers.20.self_attn.k_proj.bias', 'model.visual.encoder.layers.20.self_attn.v_proj.weight', 'model.visual.encoder.layers.20.self_attn.v_proj.bias', 'model.visual.encoder.layers.20.self_attn.q_proj.weight', 'model.visual.encoder.layers.20.self_attn.q_proj.bias', 'model.visual.encoder.layers.20.self_attn.out_proj.weight', 'model.visual.encoder.layers.20.self_attn.out_proj.bias', 'model.visual.encoder.layers.20.layer_norm1.weight', 'model.visual.encoder.layers.20.layer_norm1.bias', 'model.visual.encoder.layers.20.mlp.fc1.weight', 'model.visual.encoder.layers.20.mlp.fc1.bias', 'model.visual.encoder.layers.20.mlp.fc2.weight', 'model.visual.encoder.layers.20.mlp.fc2.bias', 'model.visual.encoder.layers.20.layer_norm2.weight', 'model.visual.encoder.layers.20.layer_norm2.bias', 'model.visual.encoder.layers.21.self_attn.k_proj.weight', 'model.visual.encoder.layers.21.self_attn.k_proj.bias', 'model.visual.encoder.layers.21.self_attn.v_proj.weight', 'model.visual.encoder.layers.21.self_attn.v_proj.bias', 'model.visual.encoder.layers.21.self_attn.q_proj.weight', 'model.visual.encoder.layers.21.self_attn.q_proj.bias', 'model.visual.encoder.layers.21.self_attn.out_proj.weight', 'model.visual.encoder.layers.21.self_attn.out_proj.bias', 'model.visual.encoder.layers.21.layer_norm1.weight', 'model.visual.encoder.layers.21.layer_norm1.bias', 'model.visual.encoder.layers.21.mlp.fc1.weight', 'model.visual.encoder.layers.21.mlp.fc1.bias', 'model.visual.encoder.layers.21.mlp.fc2.weight', 'model.visual.encoder.layers.21.mlp.fc2.bias', 'model.visual.encoder.layers.21.layer_norm2.weight', 'model.visual.encoder.layers.21.layer_norm2.bias', 'model.visual.encoder.layers.22.self_attn.k_proj.weight', 'model.visual.encoder.layers.22.self_attn.k_proj.bias', 'model.visual.encoder.layers.22.self_attn.v_proj.weight', 'model.visual.encoder.layers.22.self_attn.v_proj.bias', 'model.visual.encoder.layers.22.self_attn.q_proj.weight', 'model.visual.encoder.layers.22.self_attn.q_proj.bias', 'model.visual.encoder.layers.22.self_attn.out_proj.weight', 'model.visual.encoder.layers.22.self_attn.out_proj.bias', 'model.visual.encoder.layers.22.layer_norm1.weight', 'model.visual.encoder.layers.22.layer_norm1.bias', 'model.visual.encoder.layers.22.mlp.fc1.weight', 'model.visual.encoder.layers.22.mlp.fc1.bias', 'model.visual.encoder.layers.22.mlp.fc2.weight', 'model.visual.encoder.layers.22.mlp.fc2.bias', 'model.visual.encoder.layers.22.layer_norm2.weight', 'model.visual.encoder.layers.22.layer_norm2.bias', 'model.visual.encoder.layers.23.self_attn.k_proj.weight', 'model.visual.encoder.layers.23.self_attn.k_proj.bias', 'model.visual.encoder.layers.23.self_attn.v_proj.weight', 'model.visual.encoder.layers.23.self_attn.v_proj.bias', 'model.visual.encoder.layers.23.self_attn.q_proj.weight', 'model.visual.encoder.layers.23.self_attn.q_proj.bias', 'model.visual.encoder.layers.23.self_attn.out_proj.weight', 'model.visual.encoder.layers.23.self_attn.out_proj.bias', 'model.visual.encoder.layers.23.layer_norm1.weight', 'model.visual.encoder.layers.23.layer_norm1.bias', 'model.visual.encoder.layers.23.mlp.fc1.weight', 'model.visual.encoder.layers.23.mlp.fc1.bias', 'model.visual.encoder.layers.23.mlp.fc2.weight', 'model.visual.encoder.layers.23.mlp.fc2.bias', 'model.visual.encoder.layers.23.layer_norm2.weight', 'model.visual.encoder.layers.23.layer_norm2.bias', 'model.visual.post_layernorm.weight', 'model.visual.post_layernorm.bias', 'model.transformer.embeddings.token_embedding.weight', 'model.transformer.embeddings.position_embedding.weight', 'model.transformer.encoder.layers.0.self_attn.k_proj.weight', 'model.transformer.encoder.layers.0.self_attn.k_proj.bias', 'model.transformer.encoder.layers.0.self_attn.v_proj.weight', 'model.transformer.encoder.layers.0.self_attn.v_proj.bias', 'model.transformer.encoder.layers.0.self_attn.q_proj.weight', 'model.transformer.encoder.layers.0.self_attn.q_proj.bias', 'model.transformer.encoder.layers.0.self_attn.out_proj.weight', 'model.transformer.encoder.layers.0.self_attn.out_proj.bias', 'model.transformer.encoder.layers.0.layer_norm1.weight', 'model.transformer.encoder.layers.0.layer_norm1.bias', 'model.transformer.encoder.layers.0.mlp.fc1.weight', 'model.transformer.encoder.layers.0.mlp.fc1.bias', 'model.transformer.encoder.layers.0.mlp.fc2.weight', 'model.transformer.encoder.layers.0.mlp.fc2.bias', 'model.transformer.encoder.layers.0.layer_norm2.weight', 'model.transformer.encoder.layers.0.layer_norm2.bias', 'model.transformer.encoder.layers.1.self_attn.k_proj.weight', 'model.transformer.encoder.layers.1.self_attn.k_proj.bias', 'model.transformer.encoder.layers.1.self_attn.v_proj.weight', 'model.transformer.encoder.layers.1.self_attn.v_proj.bias', 'model.transformer.encoder.layers.1.self_attn.q_proj.weight', 'model.transformer.encoder.layers.1.self_attn.q_proj.bias', 'model.transformer.encoder.layers.1.self_attn.out_proj.weight', 'model.transformer.encoder.layers.1.self_attn.out_proj.bias', 'model.transformer.encoder.layers.1.layer_norm1.weight', 'model.transformer.encoder.layers.1.layer_norm1.bias', 'model.transformer.encoder.layers.1.mlp.fc1.weight', 'model.transformer.encoder.layers.1.mlp.fc1.bias', 'model.transformer.encoder.layers.1.mlp.fc2.weight', 'model.transformer.encoder.layers.1.mlp.fc2.bias', 'model.transformer.encoder.layers.1.layer_norm2.weight', 'model.transformer.encoder.layers.1.layer_norm2.bias', 'model.transformer.encoder.layers.2.self_attn.k_proj.weight', 'model.transformer.encoder.layers.2.self_attn.k_proj.bias', 'model.transformer.encoder.layers.2.self_attn.v_proj.weight', 'model.transformer.encoder.layers.2.self_attn.v_proj.bias', 'model.transformer.encoder.layers.2.self_attn.q_proj.weight', 'model.transformer.encoder.layers.2.self_attn.q_proj.bias', 'model.transformer.encoder.layers.2.self_attn.out_proj.weight', 'model.transformer.encoder.layers.2.self_attn.out_proj.bias', 'model.transformer.encoder.layers.2.layer_norm1.weight', 'model.transformer.encoder.layers.2.layer_norm1.bias', 'model.transformer.encoder.layers.2.mlp.fc1.weight', 'model.transformer.encoder.layers.2.mlp.fc1.bias', 'model.transformer.encoder.layers.2.mlp.fc2.weight', 'model.transformer.encoder.layers.2.mlp.fc2.bias', 'model.transformer.encoder.layers.2.layer_norm2.weight', 'model.transformer.encoder.layers.2.layer_norm2.bias', 'model.transformer.encoder.layers.3.self_attn.k_proj.weight', 'model.transformer.encoder.layers.3.self_attn.k_proj.bias', 'model.transformer.encoder.layers.3.self_attn.v_proj.weight', 'model.transformer.encoder.layers.3.self_attn.v_proj.bias', 'model.transformer.encoder.layers.3.self_attn.q_proj.weight', 'model.transformer.encoder.layers.3.self_attn.q_proj.bias', 'model.transformer.encoder.layers.3.self_attn.out_proj.weight', 'model.transformer.encoder.layers.3.self_attn.out_proj.bias', 'model.transformer.encoder.layers.3.layer_norm1.weight', 'model.transformer.encoder.layers.3.layer_norm1.bias', 'model.transformer.encoder.layers.3.mlp.fc1.weight', 'model.transformer.encoder.layers.3.mlp.fc1.bias', 'model.transformer.encoder.layers.3.mlp.fc2.weight', 'model.transformer.encoder.layers.3.mlp.fc2.bias', 'model.transformer.encoder.layers.3.layer_norm2.weight', 'model.transformer.encoder.layers.3.layer_norm2.bias', 'model.transformer.encoder.layers.4.self_attn.k_proj.weight', 'model.transformer.encoder.layers.4.self_attn.k_proj.bias', 'model.transformer.encoder.layers.4.self_attn.v_proj.weight', 'model.transformer.encoder.layers.4.self_attn.v_proj.bias', 'model.transformer.encoder.layers.4.self_attn.q_proj.weight', 'model.transformer.encoder.layers.4.self_attn.q_proj.bias', 'model.transformer.encoder.layers.4.self_attn.out_proj.weight', 'model.transformer.encoder.layers.4.self_attn.out_proj.bias', 'model.transformer.encoder.layers.4.layer_norm1.weight', 'model.transformer.encoder.layers.4.layer_norm1.bias', 'model.transformer.encoder.layers.4.mlp.fc1.weight', 'model.transformer.encoder.layers.4.mlp.fc1.bias', 'model.transformer.encoder.layers.4.mlp.fc2.weight', 'model.transformer.encoder.layers.4.mlp.fc2.bias', 'model.transformer.encoder.layers.4.layer_norm2.weight', 'model.transformer.encoder.layers.4.layer_norm2.bias', 'model.transformer.encoder.layers.5.self_attn.k_proj.weight', 'model.transformer.encoder.layers.5.self_attn.k_proj.bias', 'model.transformer.encoder.layers.5.self_attn.v_proj.weight', 'model.transformer.encoder.layers.5.self_attn.v_proj.bias', 'model.transformer.encoder.layers.5.self_attn.q_proj.weight', 'model.transformer.encoder.layers.5.self_attn.q_proj.bias', 'model.transformer.encoder.layers.5.self_attn.out_proj.weight', 'model.transformer.encoder.layers.5.self_attn.out_proj.bias', 'model.transformer.encoder.layers.5.layer_norm1.weight', 'model.transformer.encoder.layers.5.layer_norm1.bias', 'model.transformer.encoder.layers.5.mlp.fc1.weight', 'model.transformer.encoder.layers.5.mlp.fc1.bias', 'model.transformer.encoder.layers.5.mlp.fc2.weight', 'model.transformer.encoder.layers.5.mlp.fc2.bias', 'model.transformer.encoder.layers.5.layer_norm2.weight', 'model.transformer.encoder.layers.5.layer_norm2.bias', 'model.transformer.encoder.layers.6.self_attn.k_proj.weight', 'model.transformer.encoder.layers.6.self_attn.k_proj.bias', 'model.transformer.encoder.layers.6.self_attn.v_proj.weight', 'model.transformer.encoder.layers.6.self_attn.v_proj.bias', 'model.transformer.encoder.layers.6.self_attn.q_proj.weight', 'model.transformer.encoder.layers.6.self_attn.q_proj.bias', 'model.transformer.encoder.layers.6.self_attn.out_proj.weight', 'model.transformer.encoder.layers.6.self_attn.out_proj.bias', 'model.transformer.encoder.layers.6.layer_norm1.weight', 'model.transformer.encoder.layers.6.layer_norm1.bias', 'model.transformer.encoder.layers.6.mlp.fc1.weight', 'model.transformer.encoder.layers.6.mlp.fc1.bias', 'model.transformer.encoder.layers.6.mlp.fc2.weight', 'model.transformer.encoder.layers.6.mlp.fc2.bias', 'model.transformer.encoder.layers.6.layer_norm2.weight', 'model.transformer.encoder.layers.6.layer_norm2.bias', 'model.transformer.encoder.layers.7.self_attn.k_proj.weight', 'model.transformer.encoder.layers.7.self_attn.k_proj.bias', 'model.transformer.encoder.layers.7.self_attn.v_proj.weight', 'model.transformer.encoder.layers.7.self_attn.v_proj.bias', 'model.transformer.encoder.layers.7.self_attn.q_proj.weight', 'model.transformer.encoder.layers.7.self_attn.q_proj.bias', 'model.transformer.encoder.layers.7.self_attn.out_proj.weight', 'model.transformer.encoder.layers.7.self_attn.out_proj.bias', 'model.transformer.encoder.layers.7.layer_norm1.weight', 'model.transformer.encoder.layers.7.layer_norm1.bias', 'model.transformer.encoder.layers.7.mlp.fc1.weight', 'model.transformer.encoder.layers.7.mlp.fc1.bias', 'model.transformer.encoder.layers.7.mlp.fc2.weight', 'model.transformer.encoder.layers.7.mlp.fc2.bias', 'model.transformer.encoder.layers.7.layer_norm2.weight', 'model.transformer.encoder.layers.7.layer_norm2.bias', 'model.transformer.encoder.layers.8.self_attn.k_proj.weight', 'model.transformer.encoder.layers.8.self_attn.k_proj.bias', 'model.transformer.encoder.layers.8.self_attn.v_proj.weight', 'model.transformer.encoder.layers.8.self_attn.v_proj.bias', 'model.transformer.encoder.layers.8.self_attn.q_proj.weight', 'model.transformer.encoder.layers.8.self_attn.q_proj.bias', 'model.transformer.encoder.layers.8.self_attn.out_proj.weight', 'model.transformer.encoder.layers.8.self_attn.out_proj.bias', 'model.transformer.encoder.layers.8.layer_norm1.weight', 'model.transformer.encoder.layers.8.layer_norm1.bias', 'model.transformer.encoder.layers.8.mlp.fc1.weight', 'model.transformer.encoder.layers.8.mlp.fc1.bias', 'model.transformer.encoder.layers.8.mlp.fc2.weight', 'model.transformer.encoder.layers.8.mlp.fc2.bias', 'model.transformer.encoder.layers.8.layer_norm2.weight', 'model.transformer.encoder.layers.8.layer_norm2.bias', 'model.transformer.encoder.layers.9.self_attn.k_proj.weight', 'model.transformer.encoder.layers.9.self_attn.k_proj.bias', 'model.transformer.encoder.layers.9.self_attn.v_proj.weight', 'model.transformer.encoder.layers.9.self_attn.v_proj.bias', 'model.transformer.encoder.layers.9.self_attn.q_proj.weight', 'model.transformer.encoder.layers.9.self_attn.q_proj.bias', 'model.transformer.encoder.layers.9.self_attn.out_proj.weight', 'model.transformer.encoder.layers.9.self_attn.out_proj.bias', 'model.transformer.encoder.layers.9.layer_norm1.weight', 'model.transformer.encoder.layers.9.layer_norm1.bias', 'model.transformer.encoder.layers.9.mlp.fc1.weight', 'model.transformer.encoder.layers.9.mlp.fc1.bias', 'model.transformer.encoder.layers.9.mlp.fc2.weight', 'model.transformer.encoder.layers.9.mlp.fc2.bias', 'model.transformer.encoder.layers.9.layer_norm2.weight', 'model.transformer.encoder.layers.9.layer_norm2.bias', 'model.transformer.encoder.layers.10.self_attn.k_proj.weight', 'model.transformer.encoder.layers.10.self_attn.k_proj.bias', 'model.transformer.encoder.layers.10.self_attn.v_proj.weight', 'model.transformer.encoder.layers.10.self_attn.v_proj.bias', 'model.transformer.encoder.layers.10.self_attn.q_proj.weight', 'model.transformer.encoder.layers.10.self_attn.q_proj.bias', 'model.transformer.encoder.layers.10.self_attn.out_proj.weight', 'model.transformer.encoder.layers.10.self_attn.out_proj.bias', 'model.transformer.encoder.layers.10.layer_norm1.weight', 'model.transformer.encoder.layers.10.layer_norm1.bias', 'model.transformer.encoder.layers.10.mlp.fc1.weight', 'model.transformer.encoder.layers.10.mlp.fc1.bias', 'model.transformer.encoder.layers.10.mlp.fc2.weight', 'model.transformer.encoder.layers.10.mlp.fc2.bias', 'model.transformer.encoder.layers.10.layer_norm2.weight', 'model.transformer.encoder.layers.10.layer_norm2.bias', 'model.transformer.encoder.layers.11.self_attn.k_proj.weight', 'model.transformer.encoder.layers.11.self_attn.k_proj.bias', 'model.transformer.encoder.layers.11.self_attn.v_proj.weight', 'model.transformer.encoder.layers.11.self_attn.v_proj.bias', 'model.transformer.encoder.layers.11.self_attn.q_proj.weight', 'model.transformer.encoder.layers.11.self_attn.q_proj.bias', 'model.transformer.encoder.layers.11.self_attn.out_proj.weight', 'model.transformer.encoder.layers.11.self_attn.out_proj.bias', 'model.transformer.encoder.layers.11.layer_norm1.weight', 'model.transformer.encoder.layers.11.layer_norm1.bias', 'model.transformer.encoder.layers.11.mlp.fc1.weight', 'model.transformer.encoder.layers.11.mlp.fc1.bias', 'model.transformer.encoder.layers.11.mlp.fc2.weight', 'model.transformer.encoder.layers.11.mlp.fc2.bias', 'model.transformer.encoder.layers.11.layer_norm2.weight', 'model.transformer.encoder.layers.11.layer_norm2.bias', 'model.transformer.final_layer_norm.weight', 'model.transformer.final_layer_norm.bias', 'teacher.positional_embedding', 'teacher.text_projection', 'teacher.logit_scale', 'teacher.token_embedding.weight', 'teacher.ln_final.weight', 'teacher.ln_final.bias', 'teacher.visual_projection.weight', 'teacher.visual.embeddings.class_embedding', 'teacher.visual.embeddings.patch_embedding.weight', 'teacher.visual.embeddings.position_embedding.weight', 'teacher.visual.pre_layrnorm.weight', 'teacher.visual.pre_layrnorm.bias', 'teacher.visual.encoder.layers.0.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.0.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.0.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.0.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.0.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.0.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.0.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.0.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.0.layer_norm1.weight', 'teacher.visual.encoder.layers.0.layer_norm1.bias', 'teacher.visual.encoder.layers.0.mlp.fc1.weight', 'teacher.visual.encoder.layers.0.mlp.fc1.bias', 'teacher.visual.encoder.layers.0.mlp.fc2.weight', 'teacher.visual.encoder.layers.0.mlp.fc2.bias', 'teacher.visual.encoder.layers.0.layer_norm2.weight', 'teacher.visual.encoder.layers.0.layer_norm2.bias', 'teacher.visual.encoder.layers.1.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.1.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.1.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.1.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.1.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.1.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.1.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.1.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.1.layer_norm1.weight', 'teacher.visual.encoder.layers.1.layer_norm1.bias', 'teacher.visual.encoder.layers.1.mlp.fc1.weight', 'teacher.visual.encoder.layers.1.mlp.fc1.bias', 'teacher.visual.encoder.layers.1.mlp.fc2.weight', 'teacher.visual.encoder.layers.1.mlp.fc2.bias', 'teacher.visual.encoder.layers.1.layer_norm2.weight', 'teacher.visual.encoder.layers.1.layer_norm2.bias', 'teacher.visual.encoder.layers.2.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.2.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.2.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.2.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.2.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.2.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.2.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.2.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.2.layer_norm1.weight', 'teacher.visual.encoder.layers.2.layer_norm1.bias', 'teacher.visual.encoder.layers.2.mlp.fc1.weight', 'teacher.visual.encoder.layers.2.mlp.fc1.bias', 'teacher.visual.encoder.layers.2.mlp.fc2.weight', 'teacher.visual.encoder.layers.2.mlp.fc2.bias', 'teacher.visual.encoder.layers.2.layer_norm2.weight', 'teacher.visual.encoder.layers.2.layer_norm2.bias', 'teacher.visual.encoder.layers.3.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.3.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.3.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.3.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.3.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.3.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.3.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.3.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.3.layer_norm1.weight', 'teacher.visual.encoder.layers.3.layer_norm1.bias', 'teacher.visual.encoder.layers.3.mlp.fc1.weight', 'teacher.visual.encoder.layers.3.mlp.fc1.bias', 'teacher.visual.encoder.layers.3.mlp.fc2.weight', 'teacher.visual.encoder.layers.3.mlp.fc2.bias', 'teacher.visual.encoder.layers.3.layer_norm2.weight', 'teacher.visual.encoder.layers.3.layer_norm2.bias', 'teacher.visual.encoder.layers.4.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.4.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.4.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.4.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.4.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.4.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.4.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.4.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.4.layer_norm1.weight', 'teacher.visual.encoder.layers.4.layer_norm1.bias', 'teacher.visual.encoder.layers.4.mlp.fc1.weight', 'teacher.visual.encoder.layers.4.mlp.fc1.bias', 'teacher.visual.encoder.layers.4.mlp.fc2.weight', 'teacher.visual.encoder.layers.4.mlp.fc2.bias', 'teacher.visual.encoder.layers.4.layer_norm2.weight', 'teacher.visual.encoder.layers.4.layer_norm2.bias', 'teacher.visual.encoder.layers.5.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.5.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.5.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.5.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.5.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.5.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.5.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.5.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.5.layer_norm1.weight', 'teacher.visual.encoder.layers.5.layer_norm1.bias', 'teacher.visual.encoder.layers.5.mlp.fc1.weight', 'teacher.visual.encoder.layers.5.mlp.fc1.bias', 'teacher.visual.encoder.layers.5.mlp.fc2.weight', 'teacher.visual.encoder.layers.5.mlp.fc2.bias', 'teacher.visual.encoder.layers.5.layer_norm2.weight', 'teacher.visual.encoder.layers.5.layer_norm2.bias', 'teacher.visual.encoder.layers.6.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.6.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.6.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.6.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.6.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.6.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.6.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.6.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.6.layer_norm1.weight', 'teacher.visual.encoder.layers.6.layer_norm1.bias', 'teacher.visual.encoder.layers.6.mlp.fc1.weight', 'teacher.visual.encoder.layers.6.mlp.fc1.bias', 'teacher.visual.encoder.layers.6.mlp.fc2.weight', 'teacher.visual.encoder.layers.6.mlp.fc2.bias', 'teacher.visual.encoder.layers.6.layer_norm2.weight', 'teacher.visual.encoder.layers.6.layer_norm2.bias', 'teacher.visual.encoder.layers.7.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.7.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.7.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.7.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.7.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.7.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.7.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.7.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.7.layer_norm1.weight', 'teacher.visual.encoder.layers.7.layer_norm1.bias', 'teacher.visual.encoder.layers.7.mlp.fc1.weight', 'teacher.visual.encoder.layers.7.mlp.fc1.bias', 'teacher.visual.encoder.layers.7.mlp.fc2.weight', 'teacher.visual.encoder.layers.7.mlp.fc2.bias', 'teacher.visual.encoder.layers.7.layer_norm2.weight', 'teacher.visual.encoder.layers.7.layer_norm2.bias', 'teacher.visual.encoder.layers.8.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.8.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.8.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.8.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.8.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.8.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.8.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.8.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.8.layer_norm1.weight', 'teacher.visual.encoder.layers.8.layer_norm1.bias', 'teacher.visual.encoder.layers.8.mlp.fc1.weight', 'teacher.visual.encoder.layers.8.mlp.fc1.bias', 'teacher.visual.encoder.layers.8.mlp.fc2.weight', 'teacher.visual.encoder.layers.8.mlp.fc2.bias', 'teacher.visual.encoder.layers.8.layer_norm2.weight', 'teacher.visual.encoder.layers.8.layer_norm2.bias', 'teacher.visual.encoder.layers.9.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.9.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.9.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.9.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.9.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.9.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.9.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.9.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.9.layer_norm1.weight', 'teacher.visual.encoder.layers.9.layer_norm1.bias', 'teacher.visual.encoder.layers.9.mlp.fc1.weight', 'teacher.visual.encoder.layers.9.mlp.fc1.bias', 'teacher.visual.encoder.layers.9.mlp.fc2.weight', 'teacher.visual.encoder.layers.9.mlp.fc2.bias', 'teacher.visual.encoder.layers.9.layer_norm2.weight', 'teacher.visual.encoder.layers.9.layer_norm2.bias', 'teacher.visual.encoder.layers.10.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.10.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.10.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.10.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.10.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.10.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.10.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.10.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.10.layer_norm1.weight', 'teacher.visual.encoder.layers.10.layer_norm1.bias', 'teacher.visual.encoder.layers.10.mlp.fc1.weight', 'teacher.visual.encoder.layers.10.mlp.fc1.bias', 'teacher.visual.encoder.layers.10.mlp.fc2.weight', 'teacher.visual.encoder.layers.10.mlp.fc2.bias', 'teacher.visual.encoder.layers.10.layer_norm2.weight', 'teacher.visual.encoder.layers.10.layer_norm2.bias', 'teacher.visual.encoder.layers.11.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.11.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.11.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.11.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.11.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.11.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.11.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.11.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.11.layer_norm1.weight', 'teacher.visual.encoder.layers.11.layer_norm1.bias', 'teacher.visual.encoder.layers.11.mlp.fc1.weight', 'teacher.visual.encoder.layers.11.mlp.fc1.bias', 'teacher.visual.encoder.layers.11.mlp.fc2.weight', 'teacher.visual.encoder.layers.11.mlp.fc2.bias', 'teacher.visual.encoder.layers.11.layer_norm2.weight', 'teacher.visual.encoder.layers.11.layer_norm2.bias', 'teacher.visual.encoder.layers.12.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.12.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.12.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.12.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.12.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.12.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.12.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.12.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.12.layer_norm1.weight', 'teacher.visual.encoder.layers.12.layer_norm1.bias', 'teacher.visual.encoder.layers.12.mlp.fc1.weight', 'teacher.visual.encoder.layers.12.mlp.fc1.bias', 'teacher.visual.encoder.layers.12.mlp.fc2.weight', 'teacher.visual.encoder.layers.12.mlp.fc2.bias', 'teacher.visual.encoder.layers.12.layer_norm2.weight', 'teacher.visual.encoder.layers.12.layer_norm2.bias', 'teacher.visual.encoder.layers.13.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.13.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.13.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.13.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.13.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.13.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.13.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.13.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.13.layer_norm1.weight', 'teacher.visual.encoder.layers.13.layer_norm1.bias', 'teacher.visual.encoder.layers.13.mlp.fc1.weight', 'teacher.visual.encoder.layers.13.mlp.fc1.bias', 'teacher.visual.encoder.layers.13.mlp.fc2.weight', 'teacher.visual.encoder.layers.13.mlp.fc2.bias', 'teacher.visual.encoder.layers.13.layer_norm2.weight', 'teacher.visual.encoder.layers.13.layer_norm2.bias', 'teacher.visual.encoder.layers.14.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.14.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.14.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.14.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.14.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.14.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.14.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.14.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.14.layer_norm1.weight', 'teacher.visual.encoder.layers.14.layer_norm1.bias', 'teacher.visual.encoder.layers.14.mlp.fc1.weight', 'teacher.visual.encoder.layers.14.mlp.fc1.bias', 'teacher.visual.encoder.layers.14.mlp.fc2.weight', 'teacher.visual.encoder.layers.14.mlp.fc2.bias', 'teacher.visual.encoder.layers.14.layer_norm2.weight', 'teacher.visual.encoder.layers.14.layer_norm2.bias', 'teacher.visual.encoder.layers.15.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.15.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.15.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.15.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.15.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.15.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.15.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.15.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.15.layer_norm1.weight', 'teacher.visual.encoder.layers.15.layer_norm1.bias', 'teacher.visual.encoder.layers.15.mlp.fc1.weight', 'teacher.visual.encoder.layers.15.mlp.fc1.bias', 'teacher.visual.encoder.layers.15.mlp.fc2.weight', 'teacher.visual.encoder.layers.15.mlp.fc2.bias', 'teacher.visual.encoder.layers.15.layer_norm2.weight', 'teacher.visual.encoder.layers.15.layer_norm2.bias', 'teacher.visual.encoder.layers.16.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.16.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.16.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.16.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.16.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.16.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.16.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.16.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.16.layer_norm1.weight', 'teacher.visual.encoder.layers.16.layer_norm1.bias', 'teacher.visual.encoder.layers.16.mlp.fc1.weight', 'teacher.visual.encoder.layers.16.mlp.fc1.bias', 'teacher.visual.encoder.layers.16.mlp.fc2.weight', 'teacher.visual.encoder.layers.16.mlp.fc2.bias', 'teacher.visual.encoder.layers.16.layer_norm2.weight', 'teacher.visual.encoder.layers.16.layer_norm2.bias', 'teacher.visual.encoder.layers.17.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.17.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.17.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.17.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.17.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.17.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.17.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.17.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.17.layer_norm1.weight', 'teacher.visual.encoder.layers.17.layer_norm1.bias', 'teacher.visual.encoder.layers.17.mlp.fc1.weight', 'teacher.visual.encoder.layers.17.mlp.fc1.bias', 'teacher.visual.encoder.layers.17.mlp.fc2.weight', 'teacher.visual.encoder.layers.17.mlp.fc2.bias', 'teacher.visual.encoder.layers.17.layer_norm2.weight', 'teacher.visual.encoder.layers.17.layer_norm2.bias', 'teacher.visual.encoder.layers.18.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.18.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.18.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.18.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.18.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.18.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.18.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.18.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.18.layer_norm1.weight', 'teacher.visual.encoder.layers.18.layer_norm1.bias', 'teacher.visual.encoder.layers.18.mlp.fc1.weight', 'teacher.visual.encoder.layers.18.mlp.fc1.bias', 'teacher.visual.encoder.layers.18.mlp.fc2.weight', 'teacher.visual.encoder.layers.18.mlp.fc2.bias', 'teacher.visual.encoder.layers.18.layer_norm2.weight', 'teacher.visual.encoder.layers.18.layer_norm2.bias', 'teacher.visual.encoder.layers.19.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.19.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.19.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.19.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.19.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.19.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.19.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.19.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.19.layer_norm1.weight', 'teacher.visual.encoder.layers.19.layer_norm1.bias', 'teacher.visual.encoder.layers.19.mlp.fc1.weight', 'teacher.visual.encoder.layers.19.mlp.fc1.bias', 'teacher.visual.encoder.layers.19.mlp.fc2.weight', 'teacher.visual.encoder.layers.19.mlp.fc2.bias', 'teacher.visual.encoder.layers.19.layer_norm2.weight', 'teacher.visual.encoder.layers.19.layer_norm2.bias', 'teacher.visual.encoder.layers.20.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.20.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.20.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.20.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.20.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.20.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.20.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.20.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.20.layer_norm1.weight', 'teacher.visual.encoder.layers.20.layer_norm1.bias', 'teacher.visual.encoder.layers.20.mlp.fc1.weight', 'teacher.visual.encoder.layers.20.mlp.fc1.bias', 'teacher.visual.encoder.layers.20.mlp.fc2.weight', 'teacher.visual.encoder.layers.20.mlp.fc2.bias', 'teacher.visual.encoder.layers.20.layer_norm2.weight', 'teacher.visual.encoder.layers.20.layer_norm2.bias', 'teacher.visual.encoder.layers.21.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.21.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.21.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.21.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.21.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.21.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.21.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.21.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.21.layer_norm1.weight', 'teacher.visual.encoder.layers.21.layer_norm1.bias', 'teacher.visual.encoder.layers.21.mlp.fc1.weight', 'teacher.visual.encoder.layers.21.mlp.fc1.bias', 'teacher.visual.encoder.layers.21.mlp.fc2.weight', 'teacher.visual.encoder.layers.21.mlp.fc2.bias', 'teacher.visual.encoder.layers.21.layer_norm2.weight', 'teacher.visual.encoder.layers.21.layer_norm2.bias', 'teacher.visual.encoder.layers.22.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.22.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.22.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.22.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.22.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.22.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.22.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.22.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.22.layer_norm1.weight', 'teacher.visual.encoder.layers.22.layer_norm1.bias', 'teacher.visual.encoder.layers.22.mlp.fc1.weight', 'teacher.visual.encoder.layers.22.mlp.fc1.bias', 'teacher.visual.encoder.layers.22.mlp.fc2.weight', 'teacher.visual.encoder.layers.22.mlp.fc2.bias', 'teacher.visual.encoder.layers.22.layer_norm2.weight', 'teacher.visual.encoder.layers.22.layer_norm2.bias', 'teacher.visual.encoder.layers.23.self_attn.k_proj.weight', 'teacher.visual.encoder.layers.23.self_attn.k_proj.bias', 'teacher.visual.encoder.layers.23.self_attn.v_proj.weight', 'teacher.visual.encoder.layers.23.self_attn.v_proj.bias', 'teacher.visual.encoder.layers.23.self_attn.q_proj.weight', 'teacher.visual.encoder.layers.23.self_attn.q_proj.bias', 'teacher.visual.encoder.layers.23.self_attn.out_proj.weight', 'teacher.visual.encoder.layers.23.self_attn.out_proj.bias', 'teacher.visual.encoder.layers.23.layer_norm1.weight', 'teacher.visual.encoder.layers.23.layer_norm1.bias', 'teacher.visual.encoder.layers.23.mlp.fc1.weight', 'teacher.visual.encoder.layers.23.mlp.fc1.bias', 'teacher.visual.encoder.layers.23.mlp.fc2.weight', 'teacher.visual.encoder.layers.23.mlp.fc2.bias', 'teacher.visual.encoder.layers.23.layer_norm2.weight', 'teacher.visual.encoder.layers.23.layer_norm2.bias', 'teacher.visual.post_layernorm.weight', 'teacher.visual.post_layernorm.bias', 'teacher.transformer.embeddings.token_embedding.weight', 'teacher.transformer.embeddings.position_embedding.weight', 'teacher.transformer.encoder.layers.0.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.0.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.0.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.0.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.0.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.0.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.0.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.0.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.0.layer_norm1.weight', 'teacher.transformer.encoder.layers.0.layer_norm1.bias', 'teacher.transformer.encoder.layers.0.mlp.fc1.weight', 'teacher.transformer.encoder.layers.0.mlp.fc1.bias', 'teacher.transformer.encoder.layers.0.mlp.fc2.weight', 'teacher.transformer.encoder.layers.0.mlp.fc2.bias', 'teacher.transformer.encoder.layers.0.layer_norm2.weight', 'teacher.transformer.encoder.layers.0.layer_norm2.bias', 'teacher.transformer.encoder.layers.1.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.1.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.1.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.1.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.1.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.1.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.1.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.1.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.1.layer_norm1.weight', 'teacher.transformer.encoder.layers.1.layer_norm1.bias', 'teacher.transformer.encoder.layers.1.mlp.fc1.weight', 'teacher.transformer.encoder.layers.1.mlp.fc1.bias', 'teacher.transformer.encoder.layers.1.mlp.fc2.weight', 'teacher.transformer.encoder.layers.1.mlp.fc2.bias', 'teacher.transformer.encoder.layers.1.layer_norm2.weight', 'teacher.transformer.encoder.layers.1.layer_norm2.bias', 'teacher.transformer.encoder.layers.2.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.2.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.2.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.2.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.2.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.2.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.2.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.2.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.2.layer_norm1.weight', 'teacher.transformer.encoder.layers.2.layer_norm1.bias', 'teacher.transformer.encoder.layers.2.mlp.fc1.weight', 'teacher.transformer.encoder.layers.2.mlp.fc1.bias', 'teacher.transformer.encoder.layers.2.mlp.fc2.weight', 'teacher.transformer.encoder.layers.2.mlp.fc2.bias', 'teacher.transformer.encoder.layers.2.layer_norm2.weight', 'teacher.transformer.encoder.layers.2.layer_norm2.bias', 'teacher.transformer.encoder.layers.3.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.3.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.3.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.3.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.3.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.3.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.3.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.3.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.3.layer_norm1.weight', 'teacher.transformer.encoder.layers.3.layer_norm1.bias', 'teacher.transformer.encoder.layers.3.mlp.fc1.weight', 'teacher.transformer.encoder.layers.3.mlp.fc1.bias', 'teacher.transformer.encoder.layers.3.mlp.fc2.weight', 'teacher.transformer.encoder.layers.3.mlp.fc2.bias', 'teacher.transformer.encoder.layers.3.layer_norm2.weight', 'teacher.transformer.encoder.layers.3.layer_norm2.bias', 'teacher.transformer.encoder.layers.4.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.4.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.4.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.4.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.4.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.4.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.4.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.4.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.4.layer_norm1.weight', 'teacher.transformer.encoder.layers.4.layer_norm1.bias', 'teacher.transformer.encoder.layers.4.mlp.fc1.weight', 'teacher.transformer.encoder.layers.4.mlp.fc1.bias', 'teacher.transformer.encoder.layers.4.mlp.fc2.weight', 'teacher.transformer.encoder.layers.4.mlp.fc2.bias', 'teacher.transformer.encoder.layers.4.layer_norm2.weight', 'teacher.transformer.encoder.layers.4.layer_norm2.bias', 'teacher.transformer.encoder.layers.5.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.5.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.5.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.5.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.5.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.5.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.5.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.5.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.5.layer_norm1.weight', 'teacher.transformer.encoder.layers.5.layer_norm1.bias', 'teacher.transformer.encoder.layers.5.mlp.fc1.weight', 'teacher.transformer.encoder.layers.5.mlp.fc1.bias', 'teacher.transformer.encoder.layers.5.mlp.fc2.weight', 'teacher.transformer.encoder.layers.5.mlp.fc2.bias', 'teacher.transformer.encoder.layers.5.layer_norm2.weight', 'teacher.transformer.encoder.layers.5.layer_norm2.bias', 'teacher.transformer.encoder.layers.6.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.6.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.6.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.6.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.6.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.6.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.6.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.6.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.6.layer_norm1.weight', 'teacher.transformer.encoder.layers.6.layer_norm1.bias', 'teacher.transformer.encoder.layers.6.mlp.fc1.weight', 'teacher.transformer.encoder.layers.6.mlp.fc1.bias', 'teacher.transformer.encoder.layers.6.mlp.fc2.weight', 'teacher.transformer.encoder.layers.6.mlp.fc2.bias', 'teacher.transformer.encoder.layers.6.layer_norm2.weight', 'teacher.transformer.encoder.layers.6.layer_norm2.bias', 'teacher.transformer.encoder.layers.7.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.7.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.7.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.7.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.7.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.7.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.7.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.7.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.7.layer_norm1.weight', 'teacher.transformer.encoder.layers.7.layer_norm1.bias', 'teacher.transformer.encoder.layers.7.mlp.fc1.weight', 'teacher.transformer.encoder.layers.7.mlp.fc1.bias', 'teacher.transformer.encoder.layers.7.mlp.fc2.weight', 'teacher.transformer.encoder.layers.7.mlp.fc2.bias', 'teacher.transformer.encoder.layers.7.layer_norm2.weight', 'teacher.transformer.encoder.layers.7.layer_norm2.bias', 'teacher.transformer.encoder.layers.8.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.8.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.8.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.8.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.8.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.8.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.8.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.8.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.8.layer_norm1.weight', 'teacher.transformer.encoder.layers.8.layer_norm1.bias', 'teacher.transformer.encoder.layers.8.mlp.fc1.weight', 'teacher.transformer.encoder.layers.8.mlp.fc1.bias', 'teacher.transformer.encoder.layers.8.mlp.fc2.weight', 'teacher.transformer.encoder.layers.8.mlp.fc2.bias', 'teacher.transformer.encoder.layers.8.layer_norm2.weight', 'teacher.transformer.encoder.layers.8.layer_norm2.bias', 'teacher.transformer.encoder.layers.9.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.9.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.9.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.9.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.9.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.9.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.9.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.9.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.9.layer_norm1.weight', 'teacher.transformer.encoder.layers.9.layer_norm1.bias', 'teacher.transformer.encoder.layers.9.mlp.fc1.weight', 'teacher.transformer.encoder.layers.9.mlp.fc1.bias', 'teacher.transformer.encoder.layers.9.mlp.fc2.weight', 'teacher.transformer.encoder.layers.9.mlp.fc2.bias', 'teacher.transformer.encoder.layers.9.layer_norm2.weight', 'teacher.transformer.encoder.layers.9.layer_norm2.bias', 'teacher.transformer.encoder.layers.10.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.10.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.10.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.10.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.10.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.10.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.10.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.10.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.10.layer_norm1.weight', 'teacher.transformer.encoder.layers.10.layer_norm1.bias', 'teacher.transformer.encoder.layers.10.mlp.fc1.weight', 'teacher.transformer.encoder.layers.10.mlp.fc1.bias', 'teacher.transformer.encoder.layers.10.mlp.fc2.weight', 'teacher.transformer.encoder.layers.10.mlp.fc2.bias', 'teacher.transformer.encoder.layers.10.layer_norm2.weight', 'teacher.transformer.encoder.layers.10.layer_norm2.bias', 'teacher.transformer.encoder.layers.11.self_attn.k_proj.weight', 'teacher.transformer.encoder.layers.11.self_attn.k_proj.bias', 'teacher.transformer.encoder.layers.11.self_attn.v_proj.weight', 'teacher.transformer.encoder.layers.11.self_attn.v_proj.bias', 'teacher.transformer.encoder.layers.11.self_attn.q_proj.weight', 'teacher.transformer.encoder.layers.11.self_attn.q_proj.bias', 'teacher.transformer.encoder.layers.11.self_attn.out_proj.weight', 'teacher.transformer.encoder.layers.11.self_attn.out_proj.bias', 'teacher.transformer.encoder.layers.11.layer_norm1.weight', 'teacher.transformer.encoder.layers.11.layer_norm1.bias', 'teacher.transformer.encoder.layers.11.mlp.fc1.weight', 'teacher.transformer.encoder.layers.11.mlp.fc1.bias', 'teacher.transformer.encoder.layers.11.mlp.fc2.weight', 'teacher.transformer.encoder.layers.11.mlp.fc2.bias', 'teacher.transformer.encoder.layers.11.layer_norm2.weight', 'teacher.transformer.encoder.layers.11.layer_norm2.bias', 'teacher.transformer.final_layer_norm.weight', 'teacher.transformer.final_layer_norm.bias'])\n"
     ]
    }
   ],
   "source": [
    "print(state_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1e0de760-53cd-4d91-83cc-1c97f05405b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CLIPModel.from_pretrained('/home/gridsan/manderson/ovdsat/weights/clip-vit-large-patch14')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "05e18499-f3d3-4d9d-897d-f58b9635037a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CLIPVisionTransformer(\n",
       "  (embeddings): CLIPVisionEmbeddings(\n",
       "    (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
       "    (position_embedding): Embedding(257, 1024)\n",
       "  )\n",
       "  (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "  (encoder): CLIPEncoder(\n",
       "    (layers): ModuleList(\n",
       "      (0-23): 24 x CLIPEncoderLayer(\n",
       "        (self_attn): CLIPSdpaAttention(\n",
       "          (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "          (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
       "        )\n",
       "        (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): CLIPMLP(\n",
       "          (activation_fn): QuickGELUActivation()\n",
       "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
       "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
       "        )\n",
       "        (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.vision_model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f6f231ba-9d8f-40f5-b86a-0ff674905596",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CLIPTextTransformer(\n",
       "  (embeddings): CLIPTextEmbeddings(\n",
       "    (token_embedding): Embedding(49408, 768)\n",
       "    (position_embedding): Embedding(77, 768)\n",
       "  )\n",
       "  (encoder): CLIPEncoder(\n",
       "    (layers): ModuleList(\n",
       "      (0-11): 12 x CLIPEncoderLayer(\n",
       "        (self_attn): CLIPSdpaAttention(\n",
       "          (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "          (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "          (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "          (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        )\n",
       "        (layer_norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "        (mlp): CLIPMLP(\n",
       "          (activation_fn): QuickGELUActivation()\n",
       "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        )\n",
       "        (layer_norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (final_layer_norm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
       ")"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.text_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "114dadfd-86f5-4423-ab61-e7cf97158617",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init custom clip model\n",
    "custom_model = CustomCLIPWrapper(\n",
    "    model.vision_model,\n",
    "    model.text_model,\n",
    "    minibatch_size=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "24d8e78e-8a35-4dfc-bd4b-13988a83883e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "custom_model.load_state_dict(state_dict, strict=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "69efedcf-cefc-4ce0-bc9e-eea4f175a9df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([768, 768])\n",
      "torch.Size([768, 1024])\n",
      "tensor(-0.0109, grad_fn=<SelectBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(model.text_projection.weight.shape)\n",
    "print(model.visual_projection.weight.shape)\n",
    "print(model.text_projection.weight[0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a4bf79e-d967-4e53-9301-dbab4a8e8767",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(custom_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "482f55fd-bb48-40d0-a5e8-5b722a021494",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLIPVisionTransformer(\n",
      "  (embeddings): CLIPVisionEmbeddings(\n",
      "    (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
      "    (position_embedding): Embedding(257, 1024)\n",
      "  )\n",
      "  (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (encoder): CLIPEncoder(\n",
      "    (layers): ModuleList(\n",
      "      (0-23): 24 x CLIPEncoderLayer(\n",
      "        (self_attn): CLIPSdpaAttention(\n",
      "          (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): CLIPMLP(\n",
      "          (activation_fn): QuickGELUActivation()\n",
      "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(custom_model.model.visual)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "0cdfdf7e-c5eb-4aa8-a424-dbc893c211fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.vision_model = custom_model.model.visual\n",
    "model.text_model = custom_model.model.transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c36d7c25-33b8-4fc4-a799-b091c45de330",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLIPVisionTransformer(\n",
      "  (embeddings): CLIPVisionEmbeddings(\n",
      "    (patch_embedding): Conv2d(3, 1024, kernel_size=(14, 14), stride=(14, 14), bias=False)\n",
      "    (position_embedding): Embedding(257, 1024)\n",
      "  )\n",
      "  (pre_layrnorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "  (encoder): CLIPEncoder(\n",
      "    (layers): ModuleList(\n",
      "      (0-23): 24 x CLIPEncoderLayer(\n",
      "        (self_attn): CLIPSdpaAttention(\n",
      "          (k_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (v_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (q_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "          (out_proj): Linear(in_features=1024, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm1): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "        (mlp): CLIPMLP(\n",
      "          (activation_fn): QuickGELUActivation()\n",
      "          (fc1): Linear(in_features=1024, out_features=4096, bias=True)\n",
      "          (fc2): Linear(in_features=4096, out_features=1024, bias=True)\n",
      "        )\n",
      "        (layer_norm2): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (post_layernorm): LayerNorm((1024,), eps=1e-05, elementwise_affine=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(model.vision_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e55f7b06-31b0-43e3-80d6-b5839a799b73",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer logit_scale is trainable.\n",
      "Layer text_model.embeddings.token_embedding.weight is trainable.\n",
      "Layer text_model.embeddings.position_embedding.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer text_model.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer text_model.final_layer_norm.weight is trainable.\n",
      "Layer text_model.final_layer_norm.bias is trainable.\n",
      "Layer vision_model.embeddings.class_embedding is trainable.\n",
      "Layer vision_model.embeddings.patch_embedding.weight is trainable.\n",
      "Layer vision_model.embeddings.position_embedding.weight is trainable.\n",
      "Layer vision_model.pre_layrnorm.weight is trainable.\n",
      "Layer vision_model.pre_layrnorm.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer vision_model.encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer vision_model.post_layernorm.weight is trainable.\n",
      "Layer vision_model.post_layernorm.bias is trainable.\n",
      "Layer visual_projection.weight is trainable.\n",
      "Layer text_projection.weight is trainable.\n",
      "590 total params\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "for name, param in model.named_parameters():\n",
    "    if not param.requires_grad:\n",
    "        print(f\"Layer {name} is frozen.\")\n",
    "    else:\n",
    "        print(f\"Layer {name} is trainable.\")\n",
    "    i+=1\n",
    "print(f'{i} total params')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7586a3df-f099-4b66-b0ee-2bb1f51627b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([768, 768])\n",
      "torch.Size([768, 1024])\n"
     ]
    }
   ],
   "source": [
    "print(model.text_projection.weight.shape)\n",
    "print(model.visual_projection.weight.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "11956579-57a1-4d9e-bbd8-80c9b6e6d60f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.0522, grad_fn=<SelectBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(custom_model.model.text_projection[0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "73e668b2-f6ff-4171-a11d-dacc14eb7684",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Layer sink_temp is trainable.\n",
      "Layer model.positional_embedding is trainable.\n",
      "Layer model.text_projection is trainable.\n",
      "Layer model.logit_scale is trainable.\n",
      "Layer model.token_embedding.weight is trainable.\n",
      "Layer model.ln_final.weight is trainable.\n",
      "Layer model.ln_final.bias is trainable.\n",
      "Layer model.visual_projection.weight is trainable.\n",
      "Layer model.visual.embeddings.class_embedding is trainable.\n",
      "Layer model.visual.embeddings.patch_embedding.weight is trainable.\n",
      "Layer model.visual.embeddings.position_embedding.weight is trainable.\n",
      "Layer model.visual.pre_layrnorm.weight is trainable.\n",
      "Layer model.visual.pre_layrnorm.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer model.visual.encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer model.visual.post_layernorm.weight is trainable.\n",
      "Layer model.visual.post_layernorm.bias is trainable.\n",
      "Layer model.transformer.embeddings.token_embedding.weight is trainable.\n",
      "Layer model.transformer.embeddings.position_embedding.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer model.transformer.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer model.transformer.final_layer_norm.weight is trainable.\n",
      "Layer model.transformer.final_layer_norm.bias is trainable.\n",
      "Layer teacher.positional_embedding is trainable.\n",
      "Layer teacher.text_projection is trainable.\n",
      "Layer teacher.logit_scale is trainable.\n",
      "Layer teacher.token_embedding.weight is trainable.\n",
      "Layer teacher.ln_final.weight is trainable.\n",
      "Layer teacher.ln_final.bias is trainable.\n",
      "Layer teacher.visual_projection.weight is trainable.\n",
      "Layer teacher.visual.embeddings.class_embedding is trainable.\n",
      "Layer teacher.visual.embeddings.patch_embedding.weight is trainable.\n",
      "Layer teacher.visual.embeddings.position_embedding.weight is trainable.\n",
      "Layer teacher.visual.pre_layrnorm.weight is trainable.\n",
      "Layer teacher.visual.pre_layrnorm.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.12.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.13.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.14.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.15.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.16.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.17.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.18.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.19.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.20.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.21.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.22.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc1.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc1.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.mlp.fc2.bias is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm2.weight is trainable.\n",
      "Layer teacher.visual.encoder.layers.23.layer_norm2.bias is trainable.\n",
      "Layer teacher.visual.post_layernorm.weight is trainable.\n",
      "Layer teacher.visual.post_layernorm.bias is trainable.\n",
      "Layer teacher.transformer.embeddings.token_embedding.weight is trainable.\n",
      "Layer teacher.transformer.embeddings.position_embedding.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.0.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.1.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.2.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.3.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.4.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.5.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.6.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.7.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.8.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.9.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.10.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.k_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.k_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.v_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.v_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.q_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.q_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.out_proj.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.self_attn.out_proj.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc1.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc1.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.mlp.fc2.bias is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm2.weight is trainable.\n",
      "Layer teacher.transformer.encoder.layers.11.layer_norm2.bias is trainable.\n",
      "Layer teacher.transformer.final_layer_norm.weight is trainable.\n",
      "Layer teacher.transformer.final_layer_norm.bias is trainable.\n",
      "1189 total params\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "for name, param in custom_model.named_parameters():\n",
    "    if not param.requires_grad:\n",
    "        print(f\"Layer {name} is frozen.\")\n",
    "    else:\n",
    "        print(f\"Layer {name} is trainable.\")\n",
    "    i+=1\n",
    "print(f'{i} total params')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9eea9c3b-1935-41d3-aa4b-10c3aecec687",
   "metadata": {},
   "source": [
    "### Convert to CLIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "312b398f-ef80-4677-ad63-57da61401fc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import CLIPModel\n",
    "from models import CustomCLIPWrapper\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8ba94a6d-453d-4659-bf68-50f118733e92",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_dir = '/home/gridsan/manderson/train-CLIP/run/train'\n",
    "clip_pretrain = '/home/gridsan/manderson/ovdsat/weights/clip-vit-large-patch14'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4f4204a-66b1-4a52-9e90-76ccc4862cc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "ckpt_files = glob.glob(f'{model_dir}/*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dd41a829-784b-4c51-adae-d0f935d3f533",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/home/gridsan/manderson/train-CLIP/run/train/last.ckpt', '/home/gridsan/manderson/train-CLIP/run/train/model-epoch=24-val_loss=1.00.ckpt', '/home/gridsan/manderson/train-CLIP/run/train/model-epoch=20-val_loss=1.00.ckpt', '/home/gridsan/manderson/train-CLIP/run/train/model-epoch=33-val_loss=0.99.ckpt']\n"
     ]
    }
   ],
   "source": [
    "print(ckpt_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32196bc5-d7c3-4c90-9b64-3540e2f46479",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load clip model\n",
    "clip_model = CLIPModel.from_pretrained(clip_pretrain)\n",
    "clip_keys = list(clip_model.state_dict().keys())\n",
    "print(f'{len(clip_keys)} parameters in model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "224e244a-d544-4fca-ae02-b6355e147f4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converting /home/gridsan/manderson/train-CLIP/run/train/last.ckpt...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/state/partition1/slurm_tmp/27123069.0.0/ipykernel_1068323/333349102.py:6: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  ckpt = torch.load(custom_ckpt, map_location='cpu')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "590 parameters in model\n",
      "590 parameters converted\n",
      "Saved converted model to /home/gridsan/manderson/train-CLIP/run/train/last_cvt.pth\n",
      "Converting /home/gridsan/manderson/train-CLIP/run/train/model-epoch=24-val_loss=1.00.ckpt...\n",
      "590 parameters in model\n",
      "590 parameters converted\n",
      "Saved converted model to /home/gridsan/manderson/train-CLIP/run/train/model-epoch=24-val_loss=1_cvt.pth\n",
      "Converting /home/gridsan/manderson/train-CLIP/run/train/model-epoch=20-val_loss=1.00.ckpt...\n",
      "590 parameters in model\n",
      "590 parameters converted\n",
      "Saved converted model to /home/gridsan/manderson/train-CLIP/run/train/model-epoch=20-val_loss=1_cvt.pth\n",
      "Converting /home/gridsan/manderson/train-CLIP/run/train/model-epoch=33-val_loss=0.99.ckpt...\n",
      "590 parameters in model\n",
      "590 parameters converted\n",
      "Saved converted model to /home/gridsan/manderson/train-CLIP/run/train/model-epoch=33-val_loss=0_cvt.pth\n"
     ]
    }
   ],
   "source": [
    "for custom_ckpt in ckpt_files:\n",
    "    print(f'Converting {custom_ckpt}...')\n",
    "    convert_name = custom_ckpt.split('.')[0]+'_clip.pth'\n",
    "    \n",
    "    # Load custom model parameters\n",
    "    ckpt = torch.load(custom_ckpt, map_location='cpu')\n",
    "    custom_state_dict = ckpt['state_dict']\n",
    "    custom_keys = list(custom_state_dict.keys())\n",
    "    \n",
    "    # Convert custom to clip\n",
    "    convert_state_dict = {}\n",
    "\n",
    "    i = 0\n",
    "    for k in clip_keys:\n",
    "        parts = k.split('.')\n",
    "        if len(parts) <= 2:\n",
    "            i+=1\n",
    "            custom_name = [name for name in custom_keys if parts[0] in name][0]\n",
    "        elif parts[0] == 'text_model':\n",
    "            i+=1\n",
    "            custom_name = 'model.transformer.'+'.'.join(parts[1:])\n",
    "        elif parts[0] == 'vision_model':\n",
    "            i+=1\n",
    "            custom_name = 'model.visual.'+'.'.join(parts[1:])\n",
    "        v = custom_state_dict[custom_name]\n",
    "        convert_state_dict[k] = v\n",
    "    print(f'{i} parameters converted')\n",
    "    \n",
    "    # Check compatibility\n",
    "    ret = clip_model.load_state_dict(convert_state_dict)\n",
    "    all_keys_matched = len(ret.missing_keys) == 0 and len(ret.unexpected_keys) == 0\n",
    "    \n",
    "    # If successful, save new state dict\n",
    "    if ret:\n",
    "        torch.save(convert_state_dict, convert_name)\n",
    "        print(f'Saved converted model to {convert_name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c852d972-7cb9-4073-8488-26e58e5671a6",
   "metadata": {},
   "source": [
    "#### one at a time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9b45e78d-3357-455a-8083-d5fec75a4620",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/gridsan/manderson/train-CLIP/run/train/last_cvt.pth\n"
     ]
    }
   ],
   "source": [
    "custom_ckpt = '/home/gridsan/manderson/train-CLIP/run/train/last.ckpt'\n",
    "clip_pretrain = '/home/gridsan/manderson/ovdsat/weights/clip-vit-large-patch14'\n",
    "convert_name = custom_ckpt.split('.')[0]+'_cvt.pth'\n",
    "print(convert_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "857c8ba0-2b45-4518-914b-174b64e4a98e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/state/partition1/slurm_tmp/27123069.0.0/ipykernel_1068323/2380601795.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  ckpt = torch.load(custom_ckpt, map_location='cpu')\n"
     ]
    }
   ],
   "source": [
    "# Load custom model parameters\n",
    "ckpt = torch.load(custom_ckpt, map_location='cpu')\n",
    "custom_state_dict = ckpt['state_dict']\n",
    "custom_keys = list(custom_state_dict.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "47b16c80-854d-4f72-8f45-9ef619003302",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "590 parameters in model\n"
     ]
    }
   ],
   "source": [
    "# Load clip model\n",
    "clip_model = CLIPModel.from_pretrained(clip_pretrain)\n",
    "clip_keys = list(clip_model.state_dict().keys())\n",
    "print(f'{len(clip_keys)} parameters in model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "16d0aa28-a1b4-4530-9e71-968a4ccf845f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "590 parameters converted\n"
     ]
    }
   ],
   "source": [
    "# Convert custom to clip\n",
    "convert_state_dict = {}\n",
    "\n",
    "i = 0\n",
    "for k in clip_keys:\n",
    "    parts = k.split('.')\n",
    "    if len(parts) <= 2:\n",
    "        i+=1\n",
    "        custom_name = [name for name in custom_keys if parts[0] in name][0]\n",
    "    elif parts[0] == 'text_model':\n",
    "        i+=1\n",
    "        custom_name = 'model.transformer.'+'.'.join(parts[1:])\n",
    "    elif parts[0] == 'vision_model':\n",
    "        i+=1\n",
    "        custom_name = 'model.visual.'+'.'.join(parts[1:])\n",
    "    v = custom_state_dict[custom_name]\n",
    "    convert_state_dict[k] = v\n",
    "print(f'{i} parameters converted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d365cfb0-3f34-4482-8691-95e70935eacd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<All keys matched successfully>\n"
     ]
    }
   ],
   "source": [
    "# Check compatibility\n",
    "print(clip_model.load_state_dict(convert_state_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0cc0f666-5b58-43aa-abc2-4a14f7f8b6e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "ret = clip_model.load_state_dict(convert_state_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fe94a510-bd53-4593-8635-fb4319ad819e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'torch.nn.modules.module._IncompatibleKeys'>\n"
     ]
    }
   ],
   "source": [
    "print(type(ret))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c059030c-7f36-4b01-83ec-ce2a0e87ad44",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "all_keys_matched = len(ret.missing_keys) == 0 and len(ret.unexpected_keys) == 0\n",
    "print(all_keys_matched)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "00bfe301-aceb-4268-ad2a-f978268710e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If successful, save new state dict\n",
    "torch.save(convert_state_dict, convert_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "bc55fb81-7390-44f3-8a7e-030a26f92ec3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/gridsan/manderson/train-CLIP/run/test/last_cvt.pth'"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "convert_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "ec7dbf98-7c64-450d-b5fb-6b74692d76de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/state/partition1/slurm_tmp/27105912.0.0/ipykernel_2330604/2724428146.py:2: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  ckpt = torch.load(convert_name, map_location=\"cpu\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using checkpoint /home/gridsan/manderson/train-CLIP/run/test/last_cvt.pth\n"
     ]
    }
   ],
   "source": [
    "model = CLIPModel.from_pretrained(\"/home/gridsan/manderson/ovdsat/weights/clip-vit-large-patch14\")\n",
    "ckpt = torch.load(convert_name, map_location=\"cpu\")\n",
    "model.load_state_dict(ckpt)\n",
    "model = model.vision_model\n",
    "model.output_tokens = True\n",
    "print(f'Using checkpoint {convert_name}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4cb3e624-950f-4ae7-b391-779e64f545f2",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-train-clip]",
   "language": "python",
   "name": "conda-env-.conda-train-clip-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}