fine_tune_normal_pytorch_trainer_api_also.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyMgaXcBBL/C18/tqPHkZ8nG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shakilkhan24/Playground_DL/blob/main/fine_tune_normal_pytorch_trainer_api_also.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GYAVuJwol_4_"
      },
      "outputs": [],
      "source": [
        "# hugging face fine tuning a model\n",
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        "# Fine tuning vit image classifier using pytorch on mnist data\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "D-DhxcwfpFpw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from transformers import ViTForImageClassification, ViTFeatureExtractor\n",
        "\n",
        "# Step 1: Load the MNIST dataset\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),  # Resize MNIST images to fit the vision transformer input size\n",
        "    transforms.Grayscale(num_output_channels=3),  # Convert grayscale to RGB\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5,), (0.5,))\n",
        "])\n",
        "\n",
        "train_dataset = torchvision.datasets.MNIST(root='./data', train=True, transform=transform, download=True)\n",
        "train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
        "\n",
        "# Step 2: Prepare the Vision Transformer model\n",
        "model_name = 'google/vit-base-patch16-224-in21k'\n",
        "feature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\n",
        "model = ViTForImageClassification.from_pretrained(model_name, num_labels=10)  # MNIST has 10 classes\n",
        "\n",
        "# Step 3: Define Fine-tuning Parameters\n",
        "optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\n",
        "criterion = torch.nn.CrossEntropyLoss()\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "model.to(device)\n",
        "\n",
        "# Step 4: Fine-tuning Loop\n",
        "num_epochs = 1\n",
        "for epoch in range(num_epochs):\n",
        "    model.train()\n",
        "    running_loss = 0.0\n",
        "    correct = 0\n",
        "    total = 0\n",
        "\n",
        "    for images, labels in train_loader:\n",
        "        images, labels = images.to(device), labels.to(device)\n",
        "        outputs = model(images)\n",
        "        loss = criterion(outputs.logits, labels)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        running_loss += loss.item()\n",
        "        _, predicted = torch.max(outputs.logits, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "\n",
        "    epoch_loss = running_loss / len(train_loader)\n",
        "    epoch_accuracy = 100 * correct / total\n",
        "    print(f\"Epoch {epoch+1}/{num_epochs}, Loss: {epoch_loss:.4f}, Accuracy: {epoch_accuracy:.2f}%\")\n",
        "\n",
        "# Step 5: Evaluation (optional)\n",
        "# You can evaluate the model on a validation set if available.\n",
        "\n",
        "# Step 6: Save the Fine-tuned Model\n",
        "torch.save(model.state_dict(), 'fine_tuned_vision_transformer_mnist.pth')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "E5GNCGpgmFlT",
        "outputId": "85a3b624-c520-4f3a-9aa4-b9df0dc673f7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n",
            "The secret `HF_TOKEN` does not exist in your Colab secrets.\n",
            "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n",
            "You will be able to reuse this secret in all of your notebooks.\n",
            "Please note that authentication is recommended but still optional to access public models or datasets.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/models/vit/feature_extraction_vit.py:28: FutureWarning: The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use ViTImageProcessor instead.\n",
            "  warnings.warn(\n",
            "Some weights of ViTForImageClassification were not initialized from the model checkpoint at google/vit-base-patch16-224-in21k and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        "# we can use TQDM (tqdm) with the training loop\n",
        "# that's also easy\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "R4Xb12TWrm1V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from tqdm import tqdm\n",
        "import time"
      ],
      "metadata": {
        "id": "eU9MYYC3rXPT"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in tqdm(range(10), desc=\"Processing\"):\n",
        "  print(f'{i: } shakil')\n",
        "  time.sleep(0.1)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fZWsqjvrmdGi",
        "outputId": "eca615c4-c1ec-42d5-9891-6faaf56a6a4f"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing:  10%|█         | 1/10 [00:00<00:00,  9.98it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 0 shakil\n",
            " 1 shakil\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing:  30%|███       | 3/10 [00:00<00:00,  9.58it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 2 shakil\n",
            " 3 shakil\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing:  60%|██████    | 6/10 [00:00<00:00,  9.60it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 4 shakil\n",
            " 5 shakil\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing:  80%|████████  | 8/10 [00:00<00:00,  9.62it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 6 shakil\n",
            " 7 shakil\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Processing: 100%|██████████| 10/10 [00:01<00:00,  9.65it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " 8 shakil\n",
            " 9 shakil\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\rProcessing: 100%|██████████| 10/10 [00:01<00:00,  9.52it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        "# Same intention, but highly featured training from hugging_face\n",
        "# TrainingArguments and Trainer api\n",
        "# callbacks, checkpoints, different logs and many more\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "CxteTzbCqjiF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "from transformers import Trainer, TrainingArguments, ViTForImageClassification, ViTFeatureExtractor\n",
        "\n",
        "# Step 1: Load the MNIST dataset\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),  # Resize MNIST images to fit the vision transformer input size\n",
        "    transforms.Grayscale(num_output_channels=3),  # Convert grayscale to RGB\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5,), (0.5,))\n",
        "])\n",
        "\n",
        "train_dataset = torchvision.datasets.MNIST(root='./data', train=True, transform=transform, download=True)\n",
        "\n",
        "# Step 2: Prepare the Vision Transformer model\n",
        "model_name = 'google/vit-base-patch16-224-in21k'\n",
        "feature_extractor = ViTFeatureExtractor.from_pretrained(model_name)\n",
        "model = ViTForImageClassification.from_pretrained(model_name, num_labels=10)  # MNIST has 10 classes\n",
        "\n",
        "# Step 3: Define TrainingArguments\n",
        "training_args = TrainingArguments(\n",
        "    per_device_train_batch_size=32,\n",
        "    num_train_epochs=5,\n",
        "    logging_dir='./logs',\n",
        "    logging_steps=100,\n",
        "    save_steps=500,\n",
        "    output_dir='./results',\n",
        "    overwrite_output_dir=True,\n",
        "    evaluation_strategy=\"epoch\",\n",
        ")\n",
        "\n",
        "# Step 4: Define Trainer\n",
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        ")\n",
        "\n",
        "# Step 5: Fine-tuning\n",
        "trainer.train()\n",
        "\n",
        "# Step 6: Save the Fine-tuned Model\n",
        "trainer.save_model('fine_tuned_vision_transformer_mnist')\n"
      ],
      "metadata": {
        "id": "9q_-M45tq2Jl"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}