project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "f6efab8d-fab1-41db-8487-bd649a48237d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-08T17:27:41.161529Z",
     "start_time": "2024-11-08T13:09:53.257536Z"
    }
   },
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.metrics import roc_auc_score, confusion_matrix, roc_curve, auc\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "output_dir = 'ex1'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "class SVHNDataModule:\n",
    "    def __init__(self):\n",
    "        self.transform = transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=(0.4377, 0.4438, 0.4728), std=(0.1980, 0.2010, 0.1970))\n",
    "        ])\n",
    "\n",
    "    def load_data(self, batch_size):\n",
    "        train_dataset = datasets.SVHN(root='./data', split='train', download=True, transform=self.transform)\n",
    "        test_dataset = datasets.SVHN(root='./data', split='test', download=True, transform=self.transform)\n",
    "\n",
    "        train_size = int(0.8 * len(train_dataset))\n",
    "        val_size = len(train_dataset) - train_size\n",
    "        train_subset, val_subset = torch.utils.data.random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "        train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True)\n",
    "        val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "        return train_loader, val_loader, test_loader\n",
    "\n",
    "\n",
    "class SmallVGG(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SmallVGG, self).__init__()\n",
    "        self.conv_layers = nn.Sequential(\n",
    "            nn.Conv2d(3, 8, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(8, 16, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.fc_layers = nn.Sequential(\n",
    "            nn.Linear(32 * 4 * 4, 256),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv_layers(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc_layers(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "def train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs=10, device='cuda'):\n",
    "    model.train()\n",
    "    device = torch.device(device)\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0\n",
    "        for images, labels in train_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "\n",
    "        train_loss = running_loss / len(train_loader)\n",
    "        train_losses.append(train_loss)\n",
    "        val_loss = evaluate_model(model, val_loader, device, return_loss=True)\n",
    "        val_losses.append(val_loss)\n",
    "\n",
    "        print(f\"Epoch [{epoch + 1}/{num_epochs}], Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}\")\n",
    "\n",
    "    return train_losses, val_losses\n",
    "\n",
    "\n",
    "def evaluate_model(model, data_loader, device='cuda', return_loss=False):\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    all_labels = []\n",
    "    all_preds = []\n",
    "    all_probs = []\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    with torch.no_grad():\n",
    "        for images, labels in data_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            running_loss += loss.item()\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            probs = torch.softmax(outputs, dim=1)\n",
    "            all_labels.extend(labels.cpu().numpy())\n",
    "            all_preds.extend(predicted.cpu().numpy())\n",
    "            all_probs.extend(probs.cpu().numpy())\n",
    "    avg_loss = running_loss / len(data_loader)\n",
    "    accuracy = np.mean(np.array(all_labels) == np.array(all_preds))\n",
    "    num_classes = 10\n",
    "    all_labels_onehot = np.eye(num_classes)[all_labels]\n",
    "    all_probs = np.array(all_probs)\n",
    "    micro_roc_auc = roc_auc_score(all_labels_onehot, all_probs, average='micro')\n",
    "    macro_roc_auc = roc_auc_score(all_labels_onehot, all_probs, average='macro')\n",
    "    print(f'Accuracy: {accuracy * 100:.2f}%, Micro ROC AUC: {micro_roc_auc:.4f}, Macro ROC AUC: {macro_roc_auc:.4f}')\n",
    "    if return_loss:\n",
    "        return avg_loss\n",
    "    else:\n",
    "        return avg_loss, accuracy, micro_roc_auc, macro_roc_auc, all_labels, all_preds, all_probs\n",
    "\n",
    "def grid_search_hyperparameters(device='cuda'):\n",
    "    learning_rates = [0.001, 0.0001]\n",
    "    batch_sizes = [64, 128]\n",
    "    num_epochs_list = [10, 50, 200]\n",
    "\n",
    "    results = []\n",
    "\n",
    "    # 遍历所有的超参数组合\n",
    "    for lr, batch_size, num_epochs in itertools.product(learning_rates, batch_sizes, num_epochs_list):\n",
    "        experiment_details = f\"lr={lr}, batch_size={batch_size}, epochs={num_epochs}\"\n",
    "        print(f\"Training with {experiment_details}\")\n",
    "\n",
    "        # 数据加载\n",
    "        data_module = SVHNDataModule()\n",
    "        train_loader, val_loader, test_loader = data_module.load_data(batch_size)\n",
    "\n",
    "        # 初始化模型、损失函数和优化器\n",
    "        model = SmallVGG().to(device)\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        optimizer = optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "        # 训练模型\n",
    "        train_losses, val_losses = train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs=num_epochs, device=device)\n",
    "\n",
    "        # 评估模型在测试集上的表现\n",
    "        test_loss, accuracy, micro_roc_auc, macro_roc_auc, all_labels, all_preds, all_probs = evaluate_model(model, test_loader, device)\n",
    "\n",
    "        # 绘制 Train Loss 和 Validation Loss 并保存\n",
    "        plt.figure(figsize=(10, 5))\n",
    "        plt.plot(range(1, num_epochs + 1), train_losses, label='Train Loss')\n",
    "        plt.plot(range(1, num_epochs + 1), val_losses, label='Validation Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.title(f'Train and Validation Loss for {experiment_details}')\n",
    "        plt.legend()\n",
    "        loss_plot_path = os.path.join(output_dir, f'loss_{lr}_{batch_size}_{num_epochs}.png')\n",
    "        plt.savefig(loss_plot_path)\n",
    "        plt.close()\n",
    "\n",
    "        # 绘制 ROC 曲线并保存\n",
    "        fpr, tpr, _ = roc_curve(np.eye(10)[all_labels].ravel(), np.array(all_probs).ravel())\n",
    "        plt.figure()\n",
    "        plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (area = {micro_roc_auc:.2f})')\n",
    "        plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "        plt.xlabel('False Positive Rate')\n",
    "        plt.ylabel('True Positive Rate')\n",
    "        plt.title(f'ROC Curve for {experiment_details}')\n",
    "        plt.legend(loc=\"lower right\")\n",
    "        roc_plot_path = os.path.join(output_dir, f'roc_{lr}_{batch_size}_{num_epochs}.png')\n",
    "        plt.savefig(roc_plot_path)\n",
    "        plt.close()\n",
    "\n",
    "        # 绘制混淆矩阵并保存\n",
    "        cm = confusion_matrix(all_labels, all_preds)\n",
    "        plt.figure(figsize=(10, 8))\n",
    "        sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "        plt.xlabel('Predicted')\n",
    "        plt.ylabel('True')\n",
    "        plt.title(f'Confusion Matrix for {experiment_details}')\n",
    "        cm_plot_path = os.path.join(output_dir, f'confusion_matrix_{lr}_{batch_size}_{num_epochs}.png')\n",
    "        plt.savefig(cm_plot_path)\n",
    "        plt.close()\n",
    "\n",
    "        # 计算精确率和召回率\n",
    "        precision = np.diag(cm) / np.sum(cm, axis=0)\n",
    "        recall = np.diag(cm) / np.sum(cm, axis=1)\n",
    "\n",
    "        # 记录实验结果\n",
    "        results.append({\n",
    "            'Learning Rate': lr,\n",
    "            'Batch Size': batch_size,\n",
    "            'Epochs': num_epochs,\n",
    "            'Test Loss': test_loss,\n",
    "            'Accuracy': accuracy,\n",
    "            'Micro ROC AUC': micro_roc_auc,\n",
    "            'Macro ROC AUC': macro_roc_auc,\n",
    "            'Precision': np.nanmean(precision),\n",
    "            'Recall': np.nanmean(recall)\n",
    "        })\n",
    "\n",
    "    # 绘制精确率-召回率柱状图\n",
    "    metrics_df = pd.DataFrame(results)\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    metrics_df[['Precision', 'Recall']].plot(kind='bar', figsize=(14, 7))\n",
    "    plt.xticks(\n",
    "        ticks=range(len(metrics_df)),\n",
    "        labels=metrics_df[['Learning Rate', 'Batch Size', 'Epochs']].apply(\n",
    "            lambda row: f\"lr={row['Learning Rate']}, bs={row['Batch Size']}, ep={row['Epochs']}\", axis=1\n",
    "        ),\n",
    "        rotation=45,\n",
    "        ha='right'\n",
    "    )\n",
    "    plt.title('Precision and Recall by Hyperparameter Settings')\n",
    "    plt.ylabel('Score')\n",
    "    precision_recall_plot_path = os.path.join(output_dir, 'precision_recall_comparison.png')\n",
    "    plt.savefig(precision_recall_plot_path)\n",
    "    plt.close()\n",
    "\n",
    "    # 将结果保存到 ex1 目录中的 CSV 文件\n",
    "    results_csv_path = os.path.join(output_dir, 'hyperparameter_tuning_results.csv')\n",
    "    metrics_df.to_csv(results_csv_path, index=False)\n",
    "    print(f\"结果已保存到 {results_csv_path}\")\n",
    "# 主函数\n",
    "if __name__ == \"__main__\":\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    grid_search_hyperparameters(device=device)\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with lr=0.001, batch_size=64, epochs=10\n",
      "Accuracy: 84.83%, Micro ROC AUC: 0.9851, Macro ROC AUC: 0.9843\n",
      "Epoch [1/10], Train Loss: 1.303923, Val Loss: 0.495875\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9912\n",
      "Epoch [2/10], Train Loss: 0.417262, Val Loss: 0.349478\n",
      "Accuracy: 90.85%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9929\n",
      "Epoch [3/10], Train Loss: 0.334015, Val Loss: 0.309632\n",
      "Accuracy: 90.46%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9929\n",
      "Epoch [4/10], Train Loss: 0.292320, Val Loss: 0.318851\n",
      "Accuracy: 91.50%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9936\n",
      "Epoch [5/10], Train Loss: 0.260541, Val Loss: 0.295484\n",
      "Accuracy: 91.01%, Micro ROC AUC: 0.9940, Macro ROC AUC: 0.9937\n",
      "Epoch [6/10], Train Loss: 0.238997, Val Loss: 0.296160\n",
      "Accuracy: 91.78%, Micro ROC AUC: 0.9942, Macro ROC AUC: 0.9939\n",
      "Epoch [7/10], Train Loss: 0.216976, Val Loss: 0.290043\n",
      "Accuracy: 91.12%, Micro ROC AUC: 0.9938, Macro ROC AUC: 0.9938\n",
      "Epoch [8/10], Train Loss: 0.197991, Val Loss: 0.322038\n",
      "Accuracy: 91.46%, Micro ROC AUC: 0.9940, Macro ROC AUC: 0.9937\n",
      "Epoch [9/10], Train Loss: 0.180027, Val Loss: 0.324376\n",
      "Accuracy: 91.46%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9934\n",
      "Epoch [10/10], Train Loss: 0.163738, Val Loss: 0.322768\n",
      "Accuracy: 91.12%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9923\n",
      "Training with lr=0.001, batch_size=64, epochs=50\n",
      "Accuracy: 78.06%, Micro ROC AUC: 0.9714, Macro ROC AUC: 0.9713\n",
      "Epoch [1/50], Train Loss: 1.582051, Val Loss: 0.694675\n",
      "Accuracy: 85.78%, Micro ROC AUC: 0.9872, Macro ROC AUC: 0.9874\n",
      "Epoch [2/50], Train Loss: 0.519070, Val Loss: 0.454691\n",
      "Accuracy: 88.77%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9912\n",
      "Epoch [3/50], Train Loss: 0.371737, Val Loss: 0.365124\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9924\n",
      "Epoch [4/50], Train Loss: 0.314781, Val Loss: 0.331557\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9923\n",
      "Epoch [5/50], Train Loss: 0.276418, Val Loss: 0.338008\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9925\n",
      "Epoch [6/50], Train Loss: 0.250789, Val Loss: 0.340325\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9933\n",
      "Epoch [7/50], Train Loss: 0.227183, Val Loss: 0.327412\n",
      "Accuracy: 91.02%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9934\n",
      "Epoch [8/50], Train Loss: 0.209584, Val Loss: 0.312657\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9934\n",
      "Epoch [9/50], Train Loss: 0.191052, Val Loss: 0.328159\n",
      "Accuracy: 90.77%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9932\n",
      "Epoch [10/50], Train Loss: 0.175624, Val Loss: 0.335459\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9932\n",
      "Epoch [11/50], Train Loss: 0.160982, Val Loss: 0.336820\n",
      "Accuracy: 90.94%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9931\n",
      "Epoch [12/50], Train Loss: 0.147387, Val Loss: 0.348834\n",
      "Accuracy: 90.72%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9929\n",
      "Epoch [13/50], Train Loss: 0.136469, Val Loss: 0.360054\n",
      "Accuracy: 90.94%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9929\n",
      "Epoch [14/50], Train Loss: 0.122434, Val Loss: 0.393097\n",
      "Accuracy: 90.75%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9925\n",
      "Epoch [15/50], Train Loss: 0.115340, Val Loss: 0.398483\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9921\n",
      "Epoch [16/50], Train Loss: 0.108365, Val Loss: 0.436591\n",
      "Accuracy: 90.98%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927\n",
      "Epoch [17/50], Train Loss: 0.101623, Val Loss: 0.412600\n",
      "Accuracy: 90.81%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9923\n",
      "Epoch [18/50], Train Loss: 0.094880, Val Loss: 0.475134\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [19/50], Train Loss: 0.091819, Val Loss: 0.473107\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9922\n",
      "Epoch [20/50], Train Loss: 0.084551, Val Loss: 0.477841\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9916\n",
      "Epoch [21/50], Train Loss: 0.083086, Val Loss: 0.479757\n",
      "Accuracy: 89.98%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9913\n",
      "Epoch [22/50], Train Loss: 0.069575, Val Loss: 0.571793\n",
      "Accuracy: 90.12%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9916\n",
      "Epoch [23/50], Train Loss: 0.074083, Val Loss: 0.542883\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9917\n",
      "Epoch [24/50], Train Loss: 0.069746, Val Loss: 0.530301\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9916\n",
      "Epoch [25/50], Train Loss: 0.067982, Val Loss: 0.595853\n",
      "Accuracy: 90.20%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914\n",
      "Epoch [26/50], Train Loss: 0.068633, Val Loss: 0.571388\n",
      "Accuracy: 90.45%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9918\n",
      "Epoch [27/50], Train Loss: 0.064031, Val Loss: 0.618813\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9921\n",
      "Epoch [28/50], Train Loss: 0.064660, Val Loss: 0.605053\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9919\n",
      "Epoch [29/50], Train Loss: 0.066088, Val Loss: 0.610131\n",
      "Accuracy: 90.47%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [30/50], Train Loss: 0.058184, Val Loss: 0.627619\n",
      "Accuracy: 90.67%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [31/50], Train Loss: 0.059204, Val Loss: 0.643314\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911\n",
      "Epoch [32/50], Train Loss: 0.049712, Val Loss: 0.695088\n",
      "Accuracy: 90.15%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914\n",
      "Epoch [33/50], Train Loss: 0.062373, Val Loss: 0.668264\n",
      "Accuracy: 90.45%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9918\n",
      "Epoch [34/50], Train Loss: 0.057794, Val Loss: 0.651573\n",
      "Accuracy: 90.76%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921\n",
      "Epoch [35/50], Train Loss: 0.053962, Val Loss: 0.678015\n",
      "Accuracy: 90.31%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9914\n",
      "Epoch [36/50], Train Loss: 0.053340, Val Loss: 0.695617\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9911\n",
      "Epoch [37/50], Train Loss: 0.054514, Val Loss: 0.792325\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9909\n",
      "Epoch [38/50], Train Loss: 0.059768, Val Loss: 0.752578\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9909\n",
      "Epoch [39/50], Train Loss: 0.051322, Val Loss: 0.706126\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9912\n",
      "Epoch [40/50], Train Loss: 0.054331, Val Loss: 0.757231\n",
      "Accuracy: 90.11%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914\n",
      "Epoch [41/50], Train Loss: 0.056484, Val Loss: 0.726342\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9915\n",
      "Epoch [42/50], Train Loss: 0.051259, Val Loss: 0.785604\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [43/50], Train Loss: 0.051229, Val Loss: 0.733219\n",
      "Accuracy: 90.38%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9916\n",
      "Epoch [44/50], Train Loss: 0.044669, Val Loss: 0.800137\n",
      "Accuracy: 90.60%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9917\n",
      "Epoch [45/50], Train Loss: 0.053998, Val Loss: 0.747224\n",
      "Accuracy: 90.30%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914\n",
      "Epoch [46/50], Train Loss: 0.044250, Val Loss: 0.814426\n",
      "Accuracy: 90.26%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9915\n",
      "Epoch [47/50], Train Loss: 0.053137, Val Loss: 0.803740\n",
      "Accuracy: 90.27%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [48/50], Train Loss: 0.050381, Val Loss: 0.749756\n",
      "Accuracy: 90.52%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9917\n",
      "Epoch [49/50], Train Loss: 0.049267, Val Loss: 0.815035\n",
      "Accuracy: 90.63%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9916\n",
      "Epoch [50/50], Train Loss: 0.046477, Val Loss: 0.791613\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Training with lr=0.001, batch_size=64, epochs=200\n",
      "Accuracy: 82.53%, Micro ROC AUC: 0.9807, Macro ROC AUC: 0.9804\n",
      "Epoch [1/200], Train Loss: 1.488226, Val Loss: 0.562763\n",
      "Accuracy: 88.27%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [2/200], Train Loss: 0.429271, Val Loss: 0.384752\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [3/200], Train Loss: 0.337821, Val Loss: 0.342330\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9917\n",
      "Epoch [4/200], Train Loss: 0.289682, Val Loss: 0.355250\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9928\n",
      "Epoch [5/200], Train Loss: 0.260180, Val Loss: 0.337608\n",
      "Accuracy: 91.15%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9933\n",
      "Epoch [6/200], Train Loss: 0.238124, Val Loss: 0.310504\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9935, Macro ROC AUC: 0.9933\n",
      "Epoch [7/200], Train Loss: 0.218532, Val Loss: 0.309057\n",
      "Accuracy: 91.18%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9935\n",
      "Epoch [8/200], Train Loss: 0.200205, Val Loss: 0.309979\n",
      "Accuracy: 90.64%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9928\n",
      "Epoch [9/200], Train Loss: 0.185331, Val Loss: 0.335711\n",
      "Accuracy: 91.00%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9928\n",
      "Epoch [10/200], Train Loss: 0.171665, Val Loss: 0.325713\n",
      "Accuracy: 90.75%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9929\n",
      "Epoch [11/200], Train Loss: 0.158310, Val Loss: 0.341348\n",
      "Accuracy: 91.00%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9928\n",
      "Epoch [12/200], Train Loss: 0.146217, Val Loss: 0.350259\n",
      "Accuracy: 89.44%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9910\n",
      "Epoch [13/200], Train Loss: 0.136366, Val Loss: 0.378544\n",
      "Accuracy: 90.92%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9928\n",
      "Epoch [14/200], Train Loss: 0.125092, Val Loss: 0.360845\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9922\n",
      "Epoch [15/200], Train Loss: 0.119915, Val Loss: 0.388225\n",
      "Accuracy: 91.01%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9925\n",
      "Epoch [16/200], Train Loss: 0.109409, Val Loss: 0.402673\n",
      "Accuracy: 90.56%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9919\n",
      "Epoch [17/200], Train Loss: 0.101858, Val Loss: 0.412631\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9921\n",
      "Epoch [18/200], Train Loss: 0.095851, Val Loss: 0.474548\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9918\n",
      "Epoch [19/200], Train Loss: 0.093034, Val Loss: 0.439441\n",
      "Accuracy: 90.79%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9921\n",
      "Epoch [20/200], Train Loss: 0.088269, Val Loss: 0.509423\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [21/200], Train Loss: 0.085280, Val Loss: 0.517435\n",
      "Accuracy: 90.50%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9918\n",
      "Epoch [22/200], Train Loss: 0.082623, Val Loss: 0.529593\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9914\n",
      "Epoch [23/200], Train Loss: 0.074547, Val Loss: 0.541059\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9918\n",
      "Epoch [24/200], Train Loss: 0.080160, Val Loss: 0.521484\n",
      "Accuracy: 90.29%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9915\n",
      "Epoch [25/200], Train Loss: 0.071180, Val Loss: 0.616887\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9915\n",
      "Epoch [26/200], Train Loss: 0.070673, Val Loss: 0.657776\n",
      "Accuracy: 90.20%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9909\n",
      "Epoch [27/200], Train Loss: 0.069479, Val Loss: 0.604181\n",
      "Accuracy: 90.28%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9910\n",
      "Epoch [28/200], Train Loss: 0.070297, Val Loss: 0.642990\n",
      "Accuracy: 90.09%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9912\n",
      "Epoch [29/200], Train Loss: 0.063337, Val Loss: 0.654526\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9909\n",
      "Epoch [30/200], Train Loss: 0.067292, Val Loss: 0.654391\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9911\n",
      "Epoch [31/200], Train Loss: 0.062692, Val Loss: 0.673989\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9906\n",
      "Epoch [32/200], Train Loss: 0.058053, Val Loss: 0.645439\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9909\n",
      "Epoch [33/200], Train Loss: 0.063156, Val Loss: 0.671886\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911\n",
      "Epoch [34/200], Train Loss: 0.059427, Val Loss: 0.652225\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9910\n",
      "Epoch [35/200], Train Loss: 0.065976, Val Loss: 0.688018\n",
      "Accuracy: 89.92%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9904\n",
      "Epoch [36/200], Train Loss: 0.054859, Val Loss: 0.718726\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9911\n",
      "Epoch [37/200], Train Loss: 0.058177, Val Loss: 0.683944\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9903\n",
      "Epoch [38/200], Train Loss: 0.049710, Val Loss: 0.805135\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [39/200], Train Loss: 0.061493, Val Loss: 0.704874\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9910\n",
      "Epoch [40/200], Train Loss: 0.052241, Val Loss: 0.754167\n",
      "Accuracy: 90.06%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9905\n",
      "Epoch [41/200], Train Loss: 0.052987, Val Loss: 0.770465\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9904\n",
      "Epoch [42/200], Train Loss: 0.052992, Val Loss: 0.766189\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9908\n",
      "Epoch [43/200], Train Loss: 0.052418, Val Loss: 0.794199\n",
      "Accuracy: 89.54%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9900\n",
      "Epoch [44/200], Train Loss: 0.064473, Val Loss: 0.746763\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9906\n",
      "Epoch [45/200], Train Loss: 0.048975, Val Loss: 0.824344\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [46/200], Train Loss: 0.051039, Val Loss: 0.816952\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [47/200], Train Loss: 0.054389, Val Loss: 0.750407\n",
      "Accuracy: 90.09%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [48/200], Train Loss: 0.042117, Val Loss: 0.854677\n",
      "Accuracy: 90.16%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9904\n",
      "Epoch [49/200], Train Loss: 0.051024, Val Loss: 0.824547\n",
      "Accuracy: 90.02%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9904\n",
      "Epoch [50/200], Train Loss: 0.054773, Val Loss: 0.785020\n",
      "Accuracy: 89.96%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [51/200], Train Loss: 0.042652, Val Loss: 0.808058\n",
      "Accuracy: 89.82%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9906\n",
      "Epoch [52/200], Train Loss: 0.059010, Val Loss: 0.767469\n",
      "Accuracy: 90.27%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9904\n",
      "Epoch [53/200], Train Loss: 0.047436, Val Loss: 0.854845\n",
      "Accuracy: 90.16%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [54/200], Train Loss: 0.045867, Val Loss: 0.864972\n",
      "Accuracy: 89.88%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [55/200], Train Loss: 0.051395, Val Loss: 0.885495\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9910\n",
      "Epoch [56/200], Train Loss: 0.053882, Val Loss: 0.859743\n",
      "Accuracy: 90.43%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9905\n",
      "Epoch [57/200], Train Loss: 0.040967, Val Loss: 0.883029\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9908\n",
      "Epoch [58/200], Train Loss: 0.047964, Val Loss: 0.886777\n",
      "Accuracy: 90.12%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9908\n",
      "Epoch [59/200], Train Loss: 0.036793, Val Loss: 0.866411\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9906\n",
      "Epoch [60/200], Train Loss: 0.056035, Val Loss: 0.883473\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9906\n",
      "Epoch [61/200], Train Loss: 0.048376, Val Loss: 0.868454\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9908\n",
      "Epoch [62/200], Train Loss: 0.048815, Val Loss: 0.815821\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9908\n",
      "Epoch [63/200], Train Loss: 0.049544, Val Loss: 0.866226\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9909\n",
      "Epoch [64/200], Train Loss: 0.045367, Val Loss: 0.875948\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9907\n",
      "Epoch [65/200], Train Loss: 0.039684, Val Loss: 0.960409\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [66/200], Train Loss: 0.047761, Val Loss: 0.866469\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9902\n",
      "Epoch [67/200], Train Loss: 0.043539, Val Loss: 0.897132\n",
      "Accuracy: 89.92%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [68/200], Train Loss: 0.040559, Val Loss: 0.961650\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [69/200], Train Loss: 0.047283, Val Loss: 0.915634\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [70/200], Train Loss: 0.041018, Val Loss: 0.964744\n",
      "Accuracy: 90.21%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [71/200], Train Loss: 0.039957, Val Loss: 0.949998\n",
      "Accuracy: 90.31%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9906\n",
      "Epoch [72/200], Train Loss: 0.044254, Val Loss: 0.943414\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [73/200], Train Loss: 0.042889, Val Loss: 1.018793\n",
      "Accuracy: 90.59%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [74/200], Train Loss: 0.051013, Val Loss: 0.918733\n",
      "Accuracy: 89.94%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9902\n",
      "Epoch [75/200], Train Loss: 0.041062, Val Loss: 0.877550\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9895\n",
      "Epoch [76/200], Train Loss: 0.046935, Val Loss: 0.949936\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [77/200], Train Loss: 0.038441, Val Loss: 0.934662\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [78/200], Train Loss: 0.038549, Val Loss: 1.064913\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9907\n",
      "Epoch [79/200], Train Loss: 0.053451, Val Loss: 0.948195\n",
      "Accuracy: 90.46%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9909\n",
      "Epoch [80/200], Train Loss: 0.042137, Val Loss: 0.973458\n",
      "Accuracy: 90.52%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9907\n",
      "Epoch [81/200], Train Loss: 0.038873, Val Loss: 0.975318\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [82/200], Train Loss: 0.037677, Val Loss: 1.057795\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [83/200], Train Loss: 0.054780, Val Loss: 0.961729\n",
      "Accuracy: 90.67%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [84/200], Train Loss: 0.031659, Val Loss: 1.067980\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9902\n",
      "Epoch [85/200], Train Loss: 0.046024, Val Loss: 0.996462\n",
      "Accuracy: 90.31%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9907\n",
      "Epoch [86/200], Train Loss: 0.040869, Val Loss: 0.989765\n",
      "Accuracy: 90.09%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9904\n",
      "Epoch [87/200], Train Loss: 0.041300, Val Loss: 1.012451\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9899\n",
      "Epoch [88/200], Train Loss: 0.042314, Val Loss: 1.109279\n",
      "Accuracy: 90.25%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9904\n",
      "Epoch [89/200], Train Loss: 0.047121, Val Loss: 0.947963\n",
      "Accuracy: 89.84%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [90/200], Train Loss: 0.039142, Val Loss: 0.993854\n",
      "Accuracy: 90.41%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [91/200], Train Loss: 0.042783, Val Loss: 1.015871\n",
      "Accuracy: 90.65%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9910\n",
      "Epoch [92/200], Train Loss: 0.044860, Val Loss: 0.966894\n",
      "Accuracy: 90.09%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [93/200], Train Loss: 0.034759, Val Loss: 1.044305\n",
      "Accuracy: 90.11%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9898\n",
      "Epoch [94/200], Train Loss: 0.043499, Val Loss: 1.146762\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9906\n",
      "Epoch [95/200], Train Loss: 0.047401, Val Loss: 1.034262\n",
      "Accuracy: 90.32%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [96/200], Train Loss: 0.038383, Val Loss: 1.143066\n",
      "Accuracy: 90.01%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [97/200], Train Loss: 0.046771, Val Loss: 1.015130\n",
      "Accuracy: 90.49%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9902\n",
      "Epoch [98/200], Train Loss: 0.036313, Val Loss: 1.125719\n",
      "Accuracy: 89.63%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9894\n",
      "Epoch [99/200], Train Loss: 0.044685, Val Loss: 1.057237\n",
      "Accuracy: 90.30%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9903\n",
      "Epoch [100/200], Train Loss: 0.035765, Val Loss: 1.162936\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [101/200], Train Loss: 0.041605, Val Loss: 1.079634\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [102/200], Train Loss: 0.044844, Val Loss: 1.074426\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9898\n",
      "Epoch [103/200], Train Loss: 0.050604, Val Loss: 1.102229\n",
      "Accuracy: 90.14%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [104/200], Train Loss: 0.032536, Val Loss: 1.237110\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9898\n",
      "Epoch [105/200], Train Loss: 0.041503, Val Loss: 1.162914\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [106/200], Train Loss: 0.042181, Val Loss: 1.060956\n",
      "Accuracy: 90.21%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [107/200], Train Loss: 0.050481, Val Loss: 1.123173\n",
      "Accuracy: 89.76%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [108/200], Train Loss: 0.036973, Val Loss: 1.044444\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [109/200], Train Loss: 0.041065, Val Loss: 1.129198\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9907\n",
      "Epoch [110/200], Train Loss: 0.049770, Val Loss: 1.090465\n",
      "Accuracy: 90.12%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9903\n",
      "Epoch [111/200], Train Loss: 0.033056, Val Loss: 1.070985\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9894\n",
      "Epoch [112/200], Train Loss: 0.044789, Val Loss: 1.104029\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [113/200], Train Loss: 0.044703, Val Loss: 1.105049\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9904\n",
      "Epoch [114/200], Train Loss: 0.036138, Val Loss: 1.109916\n",
      "Accuracy: 90.24%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [115/200], Train Loss: 0.037567, Val Loss: 1.145311\n",
      "Accuracy: 90.00%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9898\n",
      "Epoch [116/200], Train Loss: 0.039041, Val Loss: 1.151436\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9907\n",
      "Epoch [117/200], Train Loss: 0.039293, Val Loss: 1.082683\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9895, Macro ROC AUC: 0.9892\n",
      "Epoch [118/200], Train Loss: 0.042096, Val Loss: 1.119705\n",
      "Accuracy: 90.47%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [119/200], Train Loss: 0.044955, Val Loss: 1.067203\n",
      "Accuracy: 89.98%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [120/200], Train Loss: 0.036390, Val Loss: 1.151155\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9899\n",
      "Epoch [121/200], Train Loss: 0.047100, Val Loss: 1.178898\n",
      "Accuracy: 90.28%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9905\n",
      "Epoch [122/200], Train Loss: 0.038042, Val Loss: 1.129544\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [123/200], Train Loss: 0.040529, Val Loss: 1.135977\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [124/200], Train Loss: 0.043763, Val Loss: 1.067925\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9898\n",
      "Epoch [125/200], Train Loss: 0.039841, Val Loss: 1.219817\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9895\n",
      "Epoch [126/200], Train Loss: 0.039958, Val Loss: 1.210478\n",
      "Accuracy: 90.11%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [127/200], Train Loss: 0.034216, Val Loss: 1.177494\n",
      "Accuracy: 89.64%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [128/200], Train Loss: 0.051479, Val Loss: 1.160939\n",
      "Accuracy: 89.63%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9896\n",
      "Epoch [129/200], Train Loss: 0.047115, Val Loss: 1.111702\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [130/200], Train Loss: 0.029230, Val Loss: 1.227450\n",
      "Accuracy: 89.90%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9896\n",
      "Epoch [131/200], Train Loss: 0.044649, Val Loss: 1.142550\n",
      "Accuracy: 89.51%, Micro ROC AUC: 0.9891, Macro ROC AUC: 0.9888\n",
      "Epoch [132/200], Train Loss: 0.033809, Val Loss: 1.347375\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9902\n",
      "Epoch [133/200], Train Loss: 0.040220, Val Loss: 1.219602\n",
      "Accuracy: 90.09%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [134/200], Train Loss: 0.040214, Val Loss: 1.220073\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [135/200], Train Loss: 0.040546, Val Loss: 1.200260\n",
      "Accuracy: 90.21%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9896\n",
      "Epoch [136/200], Train Loss: 0.031772, Val Loss: 1.323879\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9900\n",
      "Epoch [137/200], Train Loss: 0.048567, Val Loss: 1.156839\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9901\n",
      "Epoch [138/200], Train Loss: 0.042099, Val Loss: 1.236939\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [139/200], Train Loss: 0.045437, Val Loss: 1.226031\n",
      "Accuracy: 90.12%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9898\n",
      "Epoch [140/200], Train Loss: 0.032730, Val Loss: 1.303038\n",
      "Accuracy: 90.42%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [141/200], Train Loss: 0.043916, Val Loss: 1.173184\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [142/200], Train Loss: 0.039476, Val Loss: 1.237483\n",
      "Accuracy: 89.68%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9900\n",
      "Epoch [143/200], Train Loss: 0.047266, Val Loss: 1.220473\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9896\n",
      "Epoch [144/200], Train Loss: 0.028630, Val Loss: 1.337173\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [145/200], Train Loss: 0.047825, Val Loss: 1.176259\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9896\n",
      "Epoch [146/200], Train Loss: 0.030585, Val Loss: 1.233769\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [147/200], Train Loss: 0.048163, Val Loss: 1.204938\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [148/200], Train Loss: 0.039656, Val Loss: 1.217479\n",
      "Accuracy: 90.54%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9905\n",
      "Epoch [149/200], Train Loss: 0.033889, Val Loss: 1.221823\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [150/200], Train Loss: 0.044117, Val Loss: 1.254770\n",
      "Accuracy: 90.53%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9902\n",
      "Epoch [151/200], Train Loss: 0.040890, Val Loss: 1.202005\n",
      "Accuracy: 90.25%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [152/200], Train Loss: 0.037059, Val Loss: 1.242645\n",
      "Accuracy: 90.26%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9898\n",
      "Epoch [153/200], Train Loss: 0.050354, Val Loss: 1.155738\n",
      "Accuracy: 90.20%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9900\n",
      "Epoch [154/200], Train Loss: 0.029697, Val Loss: 1.282772\n",
      "Accuracy: 90.06%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9894\n",
      "Epoch [155/200], Train Loss: 0.043272, Val Loss: 1.314285\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9897\n",
      "Epoch [156/200], Train Loss: 0.041722, Val Loss: 1.224630\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [157/200], Train Loss: 0.040040, Val Loss: 1.244677\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9897\n",
      "Epoch [158/200], Train Loss: 0.035832, Val Loss: 1.338641\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [159/200], Train Loss: 0.041578, Val Loss: 1.323433\n",
      "Accuracy: 90.11%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9902\n",
      "Epoch [160/200], Train Loss: 0.046559, Val Loss: 1.245770\n",
      "Accuracy: 90.39%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9902\n",
      "Epoch [161/200], Train Loss: 0.034364, Val Loss: 1.314802\n",
      "Accuracy: 90.53%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [162/200], Train Loss: 0.046329, Val Loss: 1.271977\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9899\n",
      "Epoch [163/200], Train Loss: 0.035008, Val Loss: 1.247123\n",
      "Accuracy: 90.32%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9903\n",
      "Epoch [164/200], Train Loss: 0.034615, Val Loss: 1.345496\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9890\n",
      "Epoch [165/200], Train Loss: 0.046968, Val Loss: 1.254609\n",
      "Accuracy: 90.20%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9894\n",
      "Epoch [166/200], Train Loss: 0.037476, Val Loss: 1.302338\n",
      "Accuracy: 90.37%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [167/200], Train Loss: 0.037508, Val Loss: 1.350786\n",
      "Accuracy: 90.32%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [168/200], Train Loss: 0.046576, Val Loss: 1.273291\n",
      "Accuracy: 90.29%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9898\n",
      "Epoch [169/200], Train Loss: 0.031104, Val Loss: 1.365314\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9894, Macro ROC AUC: 0.9889\n",
      "Epoch [170/200], Train Loss: 0.050728, Val Loss: 1.422521\n",
      "Accuracy: 90.02%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9892\n",
      "Epoch [171/200], Train Loss: 0.040167, Val Loss: 1.418824\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9896\n",
      "Epoch [172/200], Train Loss: 0.044362, Val Loss: 1.273063\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9900\n",
      "Epoch [173/200], Train Loss: 0.030891, Val Loss: 1.350084\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9895\n",
      "Epoch [174/200], Train Loss: 0.041910, Val Loss: 1.343384\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [175/200], Train Loss: 0.041757, Val Loss: 1.312557\n",
      "Accuracy: 90.37%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [176/200], Train Loss: 0.043037, Val Loss: 1.300114\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [177/200], Train Loss: 0.031038, Val Loss: 1.276856\n",
      "Accuracy: 90.25%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [178/200], Train Loss: 0.044502, Val Loss: 1.338680\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9896\n",
      "Epoch [179/200], Train Loss: 0.039392, Val Loss: 1.382423\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [180/200], Train Loss: 0.037303, Val Loss: 1.352937\n",
      "Accuracy: 90.00%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [181/200], Train Loss: 0.039763, Val Loss: 1.444291\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [182/200], Train Loss: 0.037457, Val Loss: 1.436275\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [183/200], Train Loss: 0.038409, Val Loss: 1.385518\n",
      "Accuracy: 90.14%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [184/200], Train Loss: 0.042809, Val Loss: 1.350538\n",
      "Accuracy: 89.65%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9887\n",
      "Epoch [185/200], Train Loss: 0.039405, Val Loss: 1.410479\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9897\n",
      "Epoch [186/200], Train Loss: 0.034888, Val Loss: 1.422037\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9893, Macro ROC AUC: 0.9888\n",
      "Epoch [187/200], Train Loss: 0.045107, Val Loss: 1.405700\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [188/200], Train Loss: 0.036968, Val Loss: 1.416717\n",
      "Accuracy: 90.01%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [189/200], Train Loss: 0.043865, Val Loss: 1.391653\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [190/200], Train Loss: 0.044857, Val Loss: 1.414288\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9888\n",
      "Epoch [191/200], Train Loss: 0.037186, Val Loss: 1.492243\n",
      "Accuracy: 90.00%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [192/200], Train Loss: 0.050126, Val Loss: 1.351685\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9895, Macro ROC AUC: 0.9890\n",
      "Epoch [193/200], Train Loss: 0.037018, Val Loss: 1.347825\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [194/200], Train Loss: 0.034541, Val Loss: 1.351911\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9895, Macro ROC AUC: 0.9890\n",
      "Epoch [195/200], Train Loss: 0.039467, Val Loss: 1.460007\n",
      "Accuracy: 90.09%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [196/200], Train Loss: 0.047283, Val Loss: 1.392309\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [197/200], Train Loss: 0.041194, Val Loss: 1.438880\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9890\n",
      "Epoch [198/200], Train Loss: 0.036702, Val Loss: 1.432735\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9894\n",
      "Epoch [199/200], Train Loss: 0.046573, Val Loss: 1.431087\n",
      "Accuracy: 90.27%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [200/200], Train Loss: 0.034898, Val Loss: 1.461678\n",
      "Accuracy: 90.78%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Training with lr=0.001, batch_size=128, epochs=10\n",
      "Accuracy: 79.10%, Micro ROC AUC: 0.9734, Macro ROC AUC: 0.9716\n",
      "Epoch [1/10], Train Loss: 1.537043, Val Loss: 0.671046\n",
      "Accuracy: 86.43%, Micro ROC AUC: 0.9869, Macro ROC AUC: 0.9859\n",
      "Epoch [2/10], Train Loss: 0.521315, Val Loss: 0.456948\n",
      "Accuracy: 88.27%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [3/10], Train Loss: 0.386995, Val Loss: 0.396824\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [4/10], Train Loss: 0.324696, Val Loss: 0.338082\n",
      "Accuracy: 90.78%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9927\n",
      "Epoch [5/10], Train Loss: 0.284305, Val Loss: 0.317677\n",
      "Accuracy: 90.82%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9929\n",
      "Epoch [6/10], Train Loss: 0.255025, Val Loss: 0.317106\n",
      "Accuracy: 90.90%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9930\n",
      "Epoch [7/10], Train Loss: 0.231857, Val Loss: 0.318612\n",
      "Accuracy: 90.91%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9930\n",
      "Epoch [8/10], Train Loss: 0.209957, Val Loss: 0.323318\n",
      "Accuracy: 91.49%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9932\n",
      "Epoch [9/10], Train Loss: 0.192652, Val Loss: 0.300220\n",
      "Accuracy: 91.41%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9935\n",
      "Epoch [10/10], Train Loss: 0.175454, Val Loss: 0.309016\n",
      "Accuracy: 91.25%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9925\n",
      "Training with lr=0.001, batch_size=128, epochs=50\n",
      "Accuracy: 81.31%, Micro ROC AUC: 0.9783, Macro ROC AUC: 0.9765\n",
      "Epoch [1/50], Train Loss: 1.462154, Val Loss: 0.600188\n",
      "Accuracy: 87.88%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9890\n",
      "Epoch [2/50], Train Loss: 0.484416, Val Loss: 0.401127\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9916\n",
      "Epoch [3/50], Train Loss: 0.357081, Val Loss: 0.348597\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9927\n",
      "Epoch [4/50], Train Loss: 0.304704, Val Loss: 0.331283\n",
      "Accuracy: 91.25%, Micro ROC AUC: 0.9940, Macro ROC AUC: 0.9937\n",
      "Epoch [5/50], Train Loss: 0.265596, Val Loss: 0.293878\n",
      "Accuracy: 91.67%, Micro ROC AUC: 0.9940, Macro ROC AUC: 0.9939\n",
      "Epoch [6/50], Train Loss: 0.240513, Val Loss: 0.303093\n",
      "Accuracy: 91.46%, Micro ROC AUC: 0.9941, Macro ROC AUC: 0.9940\n",
      "Epoch [7/50], Train Loss: 0.216206, Val Loss: 0.297326\n",
      "Accuracy: 91.68%, Micro ROC AUC: 0.9942, Macro ROC AUC: 0.9940\n",
      "Epoch [8/50], Train Loss: 0.197093, Val Loss: 0.292744\n",
      "Accuracy: 92.01%, Micro ROC AUC: 0.9944, Macro ROC AUC: 0.9942\n",
      "Epoch [9/50], Train Loss: 0.180763, Val Loss: 0.291329\n",
      "Accuracy: 91.52%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9937\n",
      "Epoch [10/50], Train Loss: 0.161798, Val Loss: 0.294484\n",
      "Accuracy: 91.82%, Micro ROC AUC: 0.9941, Macro ROC AUC: 0.9938\n",
      "Epoch [11/50], Train Loss: 0.146366, Val Loss: 0.303266\n",
      "Accuracy: 91.74%, Micro ROC AUC: 0.9941, Macro ROC AUC: 0.9940\n",
      "Epoch [12/50], Train Loss: 0.132842, Val Loss: 0.337816\n",
      "Accuracy: 91.66%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9934\n",
      "Epoch [13/50], Train Loss: 0.116056, Val Loss: 0.355873\n",
      "Accuracy: 91.76%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9936\n",
      "Epoch [14/50], Train Loss: 0.106410, Val Loss: 0.349017\n",
      "Accuracy: 91.55%, Micro ROC AUC: 0.9935, Macro ROC AUC: 0.9933\n",
      "Epoch [15/50], Train Loss: 0.098419, Val Loss: 0.370340\n",
      "Accuracy: 91.78%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9933\n",
      "Epoch [16/50], Train Loss: 0.082437, Val Loss: 0.403945\n",
      "Accuracy: 91.73%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9936\n",
      "Epoch [17/50], Train Loss: 0.079553, Val Loss: 0.449070\n",
      "Accuracy: 91.67%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9934\n",
      "Epoch [18/50], Train Loss: 0.070977, Val Loss: 0.446924\n",
      "Accuracy: 91.78%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9934\n",
      "Epoch [19/50], Train Loss: 0.065152, Val Loss: 0.433253\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9929\n",
      "Epoch [20/50], Train Loss: 0.059974, Val Loss: 0.480302\n",
      "Accuracy: 91.50%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9929\n",
      "Epoch [21/50], Train Loss: 0.053317, Val Loss: 0.526665\n",
      "Accuracy: 91.24%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928\n",
      "Epoch [22/50], Train Loss: 0.053835, Val Loss: 0.517789\n",
      "Accuracy: 90.94%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9926\n",
      "Epoch [23/50], Train Loss: 0.044056, Val Loss: 0.586387\n",
      "Accuracy: 91.01%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9926\n",
      "Epoch [24/50], Train Loss: 0.045958, Val Loss: 0.549356\n",
      "Accuracy: 91.56%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9930\n",
      "Epoch [25/50], Train Loss: 0.047407, Val Loss: 0.548976\n",
      "Accuracy: 91.03%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9924\n",
      "Epoch [26/50], Train Loss: 0.041519, Val Loss: 0.651846\n",
      "Accuracy: 91.18%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9923\n",
      "Epoch [27/50], Train Loss: 0.045630, Val Loss: 0.590324\n",
      "Accuracy: 91.01%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9924\n",
      "Epoch [28/50], Train Loss: 0.042244, Val Loss: 0.674860\n",
      "Accuracy: 91.48%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9928\n",
      "Epoch [29/50], Train Loss: 0.042220, Val Loss: 0.636095\n",
      "Accuracy: 91.45%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9929\n",
      "Epoch [30/50], Train Loss: 0.036843, Val Loss: 0.667782\n",
      "Accuracy: 91.28%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928\n",
      "Epoch [31/50], Train Loss: 0.033121, Val Loss: 0.650399\n",
      "Accuracy: 91.41%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9927\n",
      "Epoch [32/50], Train Loss: 0.044949, Val Loss: 0.625805\n",
      "Accuracy: 91.22%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927\n",
      "Epoch [33/50], Train Loss: 0.031207, Val Loss: 0.674146\n",
      "Accuracy: 91.40%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9929\n",
      "Epoch [34/50], Train Loss: 0.039136, Val Loss: 0.629019\n",
      "Accuracy: 91.53%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9929\n",
      "Epoch [35/50], Train Loss: 0.035041, Val Loss: 0.673534\n",
      "Accuracy: 91.18%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9926\n",
      "Epoch [36/50], Train Loss: 0.033547, Val Loss: 0.684384\n",
      "Accuracy: 91.12%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927\n",
      "Epoch [37/50], Train Loss: 0.037011, Val Loss: 0.659364\n",
      "Accuracy: 91.11%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9924\n",
      "Epoch [38/50], Train Loss: 0.028857, Val Loss: 0.691232\n",
      "Accuracy: 91.32%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928\n",
      "Epoch [39/50], Train Loss: 0.029292, Val Loss: 0.741674\n",
      "Accuracy: 91.47%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928\n",
      "Epoch [40/50], Train Loss: 0.035668, Val Loss: 0.657858\n",
      "Accuracy: 91.36%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928\n",
      "Epoch [41/50], Train Loss: 0.036703, Val Loss: 0.699169\n",
      "Accuracy: 91.54%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9931\n",
      "Epoch [42/50], Train Loss: 0.031488, Val Loss: 0.760827\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9924\n",
      "Epoch [43/50], Train Loss: 0.035930, Val Loss: 0.732718\n",
      "Accuracy: 91.13%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9925\n",
      "Epoch [44/50], Train Loss: 0.029534, Val Loss: 0.853163\n",
      "Accuracy: 91.39%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927\n",
      "Epoch [45/50], Train Loss: 0.033836, Val Loss: 0.792999\n",
      "Accuracy: 91.30%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927\n",
      "Epoch [46/50], Train Loss: 0.030205, Val Loss: 0.766461\n",
      "Accuracy: 91.50%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9926\n",
      "Epoch [47/50], Train Loss: 0.026391, Val Loss: 0.831559\n",
      "Accuracy: 91.02%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9924\n",
      "Epoch [48/50], Train Loss: 0.026904, Val Loss: 0.878060\n",
      "Accuracy: 90.77%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [49/50], Train Loss: 0.027890, Val Loss: 0.789192\n",
      "Accuracy: 91.48%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9929\n",
      "Epoch [50/50], Train Loss: 0.038809, Val Loss: 0.735609\n",
      "Accuracy: 90.93%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Training with lr=0.001, batch_size=128, epochs=200\n",
      "Accuracy: 76.93%, Micro ROC AUC: 0.9684, Macro ROC AUC: 0.9671\n",
      "Epoch [1/200], Train Loss: 1.418944, Val Loss: 0.729031\n",
      "Accuracy: 85.63%, Micro ROC AUC: 0.9870, Macro ROC AUC: 0.9861\n",
      "Epoch [2/200], Train Loss: 0.554200, Val Loss: 0.458951\n",
      "Accuracy: 87.89%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9898\n",
      "Epoch [3/200], Train Loss: 0.389218, Val Loss: 0.391961\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9920\n",
      "Epoch [4/200], Train Loss: 0.320978, Val Loss: 0.344588\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9927\n",
      "Epoch [5/200], Train Loss: 0.282470, Val Loss: 0.331581\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9932\n",
      "Epoch [6/200], Train Loss: 0.249402, Val Loss: 0.325599\n",
      "Accuracy: 91.28%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9937\n",
      "Epoch [7/200], Train Loss: 0.227848, Val Loss: 0.299372\n",
      "Accuracy: 90.91%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9935\n",
      "Epoch [8/200], Train Loss: 0.208451, Val Loss: 0.323299\n",
      "Accuracy: 91.08%, Micro ROC AUC: 0.9938, Macro ROC AUC: 0.9936\n",
      "Epoch [9/200], Train Loss: 0.187018, Val Loss: 0.313356\n",
      "Accuracy: 91.02%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9935\n",
      "Epoch [10/200], Train Loss: 0.169123, Val Loss: 0.321614\n",
      "Accuracy: 91.19%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9932\n",
      "Epoch [11/200], Train Loss: 0.153823, Val Loss: 0.334315\n",
      "Accuracy: 91.07%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9931\n",
      "Epoch [12/200], Train Loss: 0.136099, Val Loss: 0.348232\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9926\n",
      "Epoch [13/200], Train Loss: 0.125540, Val Loss: 0.352656\n",
      "Accuracy: 90.84%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9928\n",
      "Epoch [14/200], Train Loss: 0.112290, Val Loss: 0.381915\n",
      "Accuracy: 91.03%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9926\n",
      "Epoch [15/200], Train Loss: 0.099737, Val Loss: 0.425172\n",
      "Accuracy: 90.85%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [16/200], Train Loss: 0.094226, Val Loss: 0.426148\n",
      "Accuracy: 90.90%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9923\n",
      "Epoch [17/200], Train Loss: 0.086367, Val Loss: 0.442068\n",
      "Accuracy: 91.11%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9923\n",
      "Epoch [18/200], Train Loss: 0.074393, Val Loss: 0.448148\n",
      "Accuracy: 90.81%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9923\n",
      "Epoch [19/200], Train Loss: 0.072987, Val Loss: 0.489549\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9921\n",
      "Epoch [20/200], Train Loss: 0.064790, Val Loss: 0.505822\n",
      "Accuracy: 90.89%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [21/200], Train Loss: 0.056689, Val Loss: 0.538871\n",
      "Accuracy: 90.49%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9919\n",
      "Epoch [22/200], Train Loss: 0.059427, Val Loss: 0.542009\n",
      "Accuracy: 90.52%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [23/200], Train Loss: 0.057667, Val Loss: 0.578615\n",
      "Accuracy: 90.89%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [24/200], Train Loss: 0.051021, Val Loss: 0.584380\n",
      "Accuracy: 90.79%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [25/200], Train Loss: 0.041600, Val Loss: 0.662208\n",
      "Accuracy: 90.74%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921\n",
      "Epoch [26/200], Train Loss: 0.053973, Val Loss: 0.672361\n",
      "Accuracy: 90.85%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [27/200], Train Loss: 0.045150, Val Loss: 0.674777\n",
      "Accuracy: 90.44%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921\n",
      "Epoch [28/200], Train Loss: 0.044002, Val Loss: 0.636944\n",
      "Accuracy: 90.75%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9925\n",
      "Epoch [29/200], Train Loss: 0.040468, Val Loss: 0.685687\n",
      "Accuracy: 90.92%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9924\n",
      "Epoch [30/200], Train Loss: 0.047233, Val Loss: 0.651097\n",
      "Accuracy: 90.64%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [31/200], Train Loss: 0.036348, Val Loss: 0.713348\n",
      "Accuracy: 90.73%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [32/200], Train Loss: 0.041067, Val Loss: 0.645574\n",
      "Accuracy: 90.81%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [33/200], Train Loss: 0.035219, Val Loss: 0.797771\n",
      "Accuracy: 90.68%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [34/200], Train Loss: 0.037213, Val Loss: 0.752918\n",
      "Accuracy: 90.94%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9921\n",
      "Epoch [35/200], Train Loss: 0.034884, Val Loss: 0.751121\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9912\n",
      "Epoch [36/200], Train Loss: 0.036385, Val Loss: 0.789214\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [37/200], Train Loss: 0.039093, Val Loss: 0.772310\n",
      "Accuracy: 90.41%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [38/200], Train Loss: 0.037533, Val Loss: 0.808100\n",
      "Accuracy: 90.50%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [39/200], Train Loss: 0.038695, Val Loss: 0.784224\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [40/200], Train Loss: 0.032449, Val Loss: 0.826296\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9914\n",
      "Epoch [41/200], Train Loss: 0.035643, Val Loss: 0.800942\n",
      "Accuracy: 90.88%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [42/200], Train Loss: 0.028831, Val Loss: 0.796735\n",
      "Accuracy: 90.76%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [43/200], Train Loss: 0.032467, Val Loss: 0.776659\n",
      "Accuracy: 90.53%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9918\n",
      "Epoch [44/200], Train Loss: 0.029828, Val Loss: 0.907109\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [45/200], Train Loss: 0.036373, Val Loss: 0.811021\n",
      "Accuracy: 90.52%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9918\n",
      "Epoch [46/200], Train Loss: 0.030447, Val Loss: 0.850819\n",
      "Accuracy: 90.44%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [47/200], Train Loss: 0.032904, Val Loss: 0.785112\n",
      "Accuracy: 90.64%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9918\n",
      "Epoch [48/200], Train Loss: 0.031279, Val Loss: 0.781727\n",
      "Accuracy: 90.38%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [49/200], Train Loss: 0.033841, Val Loss: 0.816397\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [50/200], Train Loss: 0.028137, Val Loss: 0.870277\n",
      "Accuracy: 90.75%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9919\n",
      "Epoch [51/200], Train Loss: 0.039084, Val Loss: 0.829832\n",
      "Accuracy: 90.65%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [52/200], Train Loss: 0.022180, Val Loss: 0.941373\n",
      "Accuracy: 90.97%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [53/200], Train Loss: 0.029112, Val Loss: 0.925643\n",
      "Accuracy: 90.90%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [54/200], Train Loss: 0.032727, Val Loss: 0.903112\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [55/200], Train Loss: 0.036348, Val Loss: 0.844140\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [56/200], Train Loss: 0.023836, Val Loss: 0.962393\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [57/200], Train Loss: 0.027340, Val Loss: 0.903422\n",
      "Accuracy: 90.63%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [58/200], Train Loss: 0.034553, Val Loss: 0.901439\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [59/200], Train Loss: 0.024602, Val Loss: 0.921644\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [60/200], Train Loss: 0.031731, Val Loss: 0.917546\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9918\n",
      "Epoch [61/200], Train Loss: 0.032303, Val Loss: 0.871436\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [62/200], Train Loss: 0.021029, Val Loss: 0.944065\n",
      "Accuracy: 90.49%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [63/200], Train Loss: 0.032050, Val Loss: 0.825155\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9920\n",
      "Epoch [64/200], Train Loss: 0.029752, Val Loss: 0.941156\n",
      "Accuracy: 90.69%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9918\n",
      "Epoch [65/200], Train Loss: 0.028172, Val Loss: 0.925725\n",
      "Accuracy: 90.69%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9920\n",
      "Epoch [66/200], Train Loss: 0.025211, Val Loss: 0.941695\n",
      "Accuracy: 90.31%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [67/200], Train Loss: 0.028702, Val Loss: 0.940337\n",
      "Accuracy: 90.54%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [68/200], Train Loss: 0.021293, Val Loss: 0.978606\n",
      "Accuracy: 90.81%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9921\n",
      "Epoch [69/200], Train Loss: 0.028909, Val Loss: 0.969192\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [70/200], Train Loss: 0.035018, Val Loss: 0.962233\n",
      "Accuracy: 90.72%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [71/200], Train Loss: 0.032713, Val Loss: 0.897205\n",
      "Accuracy: 90.85%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9921\n",
      "Epoch [72/200], Train Loss: 0.024950, Val Loss: 0.943365\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9919\n",
      "Epoch [73/200], Train Loss: 0.024718, Val Loss: 0.902691\n",
      "Accuracy: 89.94%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9911\n",
      "Epoch [74/200], Train Loss: 0.032898, Val Loss: 1.024245\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [75/200], Train Loss: 0.029094, Val Loss: 0.940144\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [76/200], Train Loss: 0.020866, Val Loss: 1.016766\n",
      "Accuracy: 90.37%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9914\n",
      "Epoch [77/200], Train Loss: 0.025951, Val Loss: 0.989201\n",
      "Accuracy: 90.53%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9916\n",
      "Epoch [78/200], Train Loss: 0.025290, Val Loss: 1.005850\n",
      "Accuracy: 90.47%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [79/200], Train Loss: 0.031198, Val Loss: 1.004178\n",
      "Accuracy: 90.68%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [80/200], Train Loss: 0.023855, Val Loss: 1.053289\n",
      "Accuracy: 90.49%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9914\n",
      "Epoch [81/200], Train Loss: 0.029662, Val Loss: 0.928165\n",
      "Accuracy: 90.72%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [82/200], Train Loss: 0.018906, Val Loss: 1.095582\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9921\n",
      "Epoch [83/200], Train Loss: 0.030077, Val Loss: 0.939632\n",
      "Accuracy: 90.44%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [84/200], Train Loss: 0.028192, Val Loss: 1.067969\n",
      "Accuracy: 90.90%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [85/200], Train Loss: 0.027373, Val Loss: 1.064982\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [86/200], Train Loss: 0.025920, Val Loss: 1.042241\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [87/200], Train Loss: 0.022731, Val Loss: 0.966601\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9918\n",
      "Epoch [88/200], Train Loss: 0.013683, Val Loss: 1.022766\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [89/200], Train Loss: 0.036316, Val Loss: 1.052138\n",
      "Accuracy: 90.79%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [90/200], Train Loss: 0.019643, Val Loss: 1.121806\n",
      "Accuracy: 90.23%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9912\n",
      "Epoch [91/200], Train Loss: 0.023502, Val Loss: 1.057088\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [92/200], Train Loss: 0.038051, Val Loss: 0.927514\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [93/200], Train Loss: 0.018124, Val Loss: 1.077697\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9914\n",
      "Epoch [94/200], Train Loss: 0.024353, Val Loss: 1.137599\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [95/200], Train Loss: 0.028429, Val Loss: 1.074986\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [96/200], Train Loss: 0.028716, Val Loss: 0.998014\n",
      "Accuracy: 89.73%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9907\n",
      "Epoch [97/200], Train Loss: 0.023858, Val Loss: 1.118166\n",
      "Accuracy: 90.64%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9917\n",
      "Epoch [98/200], Train Loss: 0.024718, Val Loss: 1.094420\n",
      "Accuracy: 90.38%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [99/200], Train Loss: 0.024939, Val Loss: 1.056934\n",
      "Accuracy: 90.53%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [100/200], Train Loss: 0.025465, Val Loss: 1.088871\n",
      "Accuracy: 90.45%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [101/200], Train Loss: 0.022205, Val Loss: 1.044768\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [102/200], Train Loss: 0.026936, Val Loss: 1.063635\n",
      "Accuracy: 90.41%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [103/200], Train Loss: 0.017344, Val Loss: 1.170038\n",
      "Accuracy: 90.49%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9918\n",
      "Epoch [104/200], Train Loss: 0.028867, Val Loss: 1.147144\n",
      "Accuracy: 90.99%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [105/200], Train Loss: 0.022563, Val Loss: 1.154750\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9917\n",
      "Epoch [106/200], Train Loss: 0.024081, Val Loss: 1.135996\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9909\n",
      "Epoch [107/200], Train Loss: 0.030686, Val Loss: 1.086804\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921\n",
      "Epoch [108/200], Train Loss: 0.021034, Val Loss: 1.132872\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [109/200], Train Loss: 0.014348, Val Loss: 1.104323\n",
      "Accuracy: 90.64%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [110/200], Train Loss: 0.027981, Val Loss: 1.006662\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [111/200], Train Loss: 0.020265, Val Loss: 1.109219\n",
      "Accuracy: 90.56%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [112/200], Train Loss: 0.026356, Val Loss: 1.092405\n",
      "Accuracy: 90.65%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [113/200], Train Loss: 0.023997, Val Loss: 1.080702\n",
      "Accuracy: 90.14%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [114/200], Train Loss: 0.023431, Val Loss: 1.119755\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Epoch [115/200], Train Loss: 0.027605, Val Loss: 1.104116\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [116/200], Train Loss: 0.024875, Val Loss: 1.239295\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [117/200], Train Loss: 0.024607, Val Loss: 1.123167\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [118/200], Train Loss: 0.019312, Val Loss: 1.228277\n",
      "Accuracy: 90.27%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [119/200], Train Loss: 0.033115, Val Loss: 1.094734\n",
      "Accuracy: 90.90%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921\n",
      "Epoch [120/200], Train Loss: 0.020792, Val Loss: 1.105079\n",
      "Accuracy: 90.46%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [121/200], Train Loss: 0.024416, Val Loss: 1.200690\n",
      "Accuracy: 90.43%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [122/200], Train Loss: 0.027292, Val Loss: 1.178347\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [123/200], Train Loss: 0.023703, Val Loss: 1.115831\n",
      "Accuracy: 90.81%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9916\n",
      "Epoch [124/200], Train Loss: 0.018958, Val Loss: 1.149935\n",
      "Accuracy: 90.47%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [125/200], Train Loss: 0.021993, Val Loss: 1.143723\n",
      "Accuracy: 90.14%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9909\n",
      "Epoch [126/200], Train Loss: 0.019742, Val Loss: 1.238347\n",
      "Accuracy: 90.75%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [127/200], Train Loss: 0.029486, Val Loss: 1.090692\n",
      "Accuracy: 90.68%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9916\n",
      "Epoch [128/200], Train Loss: 0.024166, Val Loss: 1.171144\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [129/200], Train Loss: 0.014432, Val Loss: 1.204073\n",
      "Accuracy: 90.12%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9908\n",
      "Epoch [130/200], Train Loss: 0.027248, Val Loss: 1.195259\n",
      "Accuracy: 90.72%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [131/200], Train Loss: 0.025773, Val Loss: 1.121702\n",
      "Accuracy: 90.29%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [132/200], Train Loss: 0.017343, Val Loss: 1.137876\n",
      "Accuracy: 90.28%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [133/200], Train Loss: 0.028660, Val Loss: 1.274167\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [134/200], Train Loss: 0.027069, Val Loss: 1.230501\n",
      "Accuracy: 90.54%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [135/200], Train Loss: 0.023592, Val Loss: 1.155056\n",
      "Accuracy: 90.44%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [136/200], Train Loss: 0.018582, Val Loss: 1.110676\n",
      "Accuracy: 90.40%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9911\n",
      "Epoch [137/200], Train Loss: 0.014945, Val Loss: 1.198473\n",
      "Accuracy: 90.29%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9911\n",
      "Epoch [138/200], Train Loss: 0.026572, Val Loss: 1.227339\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Epoch [139/200], Train Loss: 0.028950, Val Loss: 1.180837\n",
      "Accuracy: 90.54%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [140/200], Train Loss: 0.021412, Val Loss: 1.175151\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9909\n",
      "Epoch [141/200], Train Loss: 0.019879, Val Loss: 1.177318\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [142/200], Train Loss: 0.020514, Val Loss: 1.238557\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [143/200], Train Loss: 0.027204, Val Loss: 1.279076\n",
      "Accuracy: 90.59%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Epoch [144/200], Train Loss: 0.033457, Val Loss: 1.232666\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [145/200], Train Loss: 0.020449, Val Loss: 1.188824\n",
      "Accuracy: 90.69%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [146/200], Train Loss: 0.018028, Val Loss: 1.227115\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [147/200], Train Loss: 0.020910, Val Loss: 1.239673\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [148/200], Train Loss: 0.015634, Val Loss: 1.315686\n",
      "Accuracy: 90.30%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9907\n",
      "Epoch [149/200], Train Loss: 0.025063, Val Loss: 1.191406\n",
      "Accuracy: 90.41%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [150/200], Train Loss: 0.023510, Val Loss: 1.308958\n",
      "Accuracy: 90.48%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [151/200], Train Loss: 0.026272, Val Loss: 1.287307\n",
      "Accuracy: 90.80%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9911\n",
      "Epoch [152/200], Train Loss: 0.027980, Val Loss: 1.301246\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Epoch [153/200], Train Loss: 0.023859, Val Loss: 1.264162\n",
      "Accuracy: 90.63%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [154/200], Train Loss: 0.019393, Val Loss: 1.261237\n",
      "Accuracy: 90.24%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [155/200], Train Loss: 0.020777, Val Loss: 1.251226\n",
      "Accuracy: 90.26%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9908\n",
      "Epoch [156/200], Train Loss: 0.026330, Val Loss: 1.225539\n",
      "Accuracy: 90.79%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [157/200], Train Loss: 0.024572, Val Loss: 1.193549\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Epoch [158/200], Train Loss: 0.025758, Val Loss: 1.238061\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9909\n",
      "Epoch [159/200], Train Loss: 0.017805, Val Loss: 1.405318\n",
      "Accuracy: 90.76%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9913\n",
      "Epoch [160/200], Train Loss: 0.027690, Val Loss: 1.216302\n",
      "Accuracy: 90.46%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [161/200], Train Loss: 0.019734, Val Loss: 1.247635\n",
      "Accuracy: 90.39%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9909\n",
      "Epoch [162/200], Train Loss: 0.025329, Val Loss: 1.351024\n",
      "Accuracy: 90.89%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9917\n",
      "Epoch [163/200], Train Loss: 0.023042, Val Loss: 1.262239\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9909\n",
      "Epoch [164/200], Train Loss: 0.021614, Val Loss: 1.374706\n",
      "Accuracy: 90.45%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [165/200], Train Loss: 0.026053, Val Loss: 1.265247\n",
      "Accuracy: 90.97%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9919\n",
      "Epoch [166/200], Train Loss: 0.010357, Val Loss: 1.278592\n",
      "Accuracy: 89.65%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9906\n",
      "Epoch [167/200], Train Loss: 0.026234, Val Loss: 1.294569\n",
      "Accuracy: 90.54%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [168/200], Train Loss: 0.028287, Val Loss: 1.336645\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9914\n",
      "Epoch [169/200], Train Loss: 0.020201, Val Loss: 1.365572\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9910\n",
      "Epoch [170/200], Train Loss: 0.015354, Val Loss: 1.463141\n",
      "Accuracy: 90.28%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [171/200], Train Loss: 0.031427, Val Loss: 1.285129\n",
      "Accuracy: 90.80%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [172/200], Train Loss: 0.021792, Val Loss: 1.270048\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [173/200], Train Loss: 0.015913, Val Loss: 1.337059\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [174/200], Train Loss: 0.033370, Val Loss: 1.293677\n",
      "Accuracy: 90.14%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9907\n",
      "Epoch [175/200], Train Loss: 0.023808, Val Loss: 1.304145\n",
      "Accuracy: 90.56%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9912\n",
      "Epoch [176/200], Train Loss: 0.019402, Val Loss: 1.400190\n",
      "Accuracy: 90.29%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9910\n",
      "Epoch [177/200], Train Loss: 0.022694, Val Loss: 1.434967\n",
      "Accuracy: 90.54%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [178/200], Train Loss: 0.023811, Val Loss: 1.414203\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [179/200], Train Loss: 0.020898, Val Loss: 1.339727\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [180/200], Train Loss: 0.014030, Val Loss: 1.482737\n",
      "Accuracy: 90.50%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9908\n",
      "Epoch [181/200], Train Loss: 0.026786, Val Loss: 1.461781\n",
      "Accuracy: 90.60%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [182/200], Train Loss: 0.021546, Val Loss: 1.414563\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9908\n",
      "Epoch [183/200], Train Loss: 0.029960, Val Loss: 1.431654\n",
      "Accuracy: 90.88%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [184/200], Train Loss: 0.017975, Val Loss: 1.319668\n",
      "Accuracy: 90.67%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9915\n",
      "Epoch [185/200], Train Loss: 0.014936, Val Loss: 1.431177\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9908\n",
      "Epoch [186/200], Train Loss: 0.029404, Val Loss: 1.480298\n",
      "Accuracy: 90.60%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9914\n",
      "Epoch [187/200], Train Loss: 0.033375, Val Loss: 1.295683\n",
      "Accuracy: 90.25%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911\n",
      "Epoch [188/200], Train Loss: 0.023956, Val Loss: 1.362544\n",
      "Accuracy: 90.85%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [189/200], Train Loss: 0.025455, Val Loss: 1.351178\n",
      "Accuracy: 90.75%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9916\n",
      "Epoch [190/200], Train Loss: 0.018855, Val Loss: 1.376825\n",
      "Accuracy: 90.32%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9909\n",
      "Epoch [191/200], Train Loss: 0.021736, Val Loss: 1.497300\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914\n",
      "Epoch [192/200], Train Loss: 0.023110, Val Loss: 1.488139\n",
      "Accuracy: 90.77%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [193/200], Train Loss: 0.016093, Val Loss: 1.491535\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [194/200], Train Loss: 0.025099, Val Loss: 1.453735\n",
      "Accuracy: 90.72%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9911\n",
      "Epoch [195/200], Train Loss: 0.018435, Val Loss: 1.498165\n",
      "Accuracy: 90.65%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [196/200], Train Loss: 0.023041, Val Loss: 1.386848\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [197/200], Train Loss: 0.018905, Val Loss: 1.670676\n",
      "Accuracy: 89.92%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9903\n",
      "Epoch [198/200], Train Loss: 0.033641, Val Loss: 1.566772\n",
      "Accuracy: 90.32%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [199/200], Train Loss: 0.025135, Val Loss: 1.407584\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [200/200], Train Loss: 0.017415, Val Loss: 1.460736\n",
      "Accuracy: 90.78%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Training with lr=0.0001, batch_size=64, epochs=10\n",
      "Accuracy: 42.63%, Micro ROC AUC: 0.8267, Macro ROC AUC: 0.8024\n",
      "Epoch [1/10], Train Loss: 2.175978, Val Loss: 1.674318\n",
      "Accuracy: 69.72%, Micro ROC AUC: 0.9483, Macro ROC AUC: 0.9425\n",
      "Epoch [2/10], Train Loss: 1.213355, Val Loss: 0.955179\n",
      "Accuracy: 73.85%, Micro ROC AUC: 0.9600, Macro ROC AUC: 0.9573\n",
      "Epoch [3/10], Train Loss: 0.887158, Val Loss: 0.829879\n",
      "Accuracy: 77.54%, Micro ROC AUC: 0.9699, Macro ROC AUC: 0.9668\n",
      "Epoch [4/10], Train Loss: 0.768013, Val Loss: 0.715010\n",
      "Accuracy: 79.39%, Micro ROC AUC: 0.9739, Macro ROC AUC: 0.9718\n",
      "Epoch [5/10], Train Loss: 0.682836, Val Loss: 0.662710\n",
      "Accuracy: 81.53%, Micro ROC AUC: 0.9790, Macro ROC AUC: 0.9767\n",
      "Epoch [6/10], Train Loss: 0.617627, Val Loss: 0.590276\n",
      "Accuracy: 82.04%, Micro ROC AUC: 0.9801, Macro ROC AUC: 0.9794\n",
      "Epoch [7/10], Train Loss: 0.566461, Val Loss: 0.573821\n",
      "Accuracy: 84.32%, Micro ROC AUC: 0.9840, Macro ROC AUC: 0.9823\n",
      "Epoch [8/10], Train Loss: 0.521186, Val Loss: 0.511397\n",
      "Accuracy: 84.91%, Micro ROC AUC: 0.9851, Macro ROC AUC: 0.9840\n",
      "Epoch [9/10], Train Loss: 0.486589, Val Loss: 0.489007\n",
      "Accuracy: 85.80%, Micro ROC AUC: 0.9868, Macro ROC AUC: 0.9856\n",
      "Epoch [10/10], Train Loss: 0.457016, Val Loss: 0.460084\n",
      "Accuracy: 85.06%, Micro ROC AUC: 0.9850, Macro ROC AUC: 0.9833\n",
      "Training with lr=0.0001, batch_size=64, epochs=50\n",
      "Accuracy: 47.24%, Micro ROC AUC: 0.8456, Macro ROC AUC: 0.8421\n",
      "Epoch [1/50], Train Loss: 2.150071, Val Loss: 1.598505\n",
      "Accuracy: 70.78%, Micro ROC AUC: 0.9518, Macro ROC AUC: 0.9476\n",
      "Epoch [2/50], Train Loss: 1.112847, Val Loss: 0.916929\n",
      "Accuracy: 76.03%, Micro ROC AUC: 0.9658, Macro ROC AUC: 0.9629\n",
      "Epoch [3/50], Train Loss: 0.812157, Val Loss: 0.762694\n",
      "Accuracy: 78.61%, Micro ROC AUC: 0.9721, Macro ROC AUC: 0.9706\n",
      "Epoch [4/50], Train Loss: 0.697766, Val Loss: 0.684603\n",
      "Accuracy: 81.09%, Micro ROC AUC: 0.9774, Macro ROC AUC: 0.9756\n",
      "Epoch [5/50], Train Loss: 0.621260, Val Loss: 0.612484\n",
      "Accuracy: 82.54%, Micro ROC AUC: 0.9800, Macro ROC AUC: 0.9786\n",
      "Epoch [6/50], Train Loss: 0.563405, Val Loss: 0.574431\n",
      "Accuracy: 83.50%, Micro ROC AUC: 0.9820, Macro ROC AUC: 0.9811\n",
      "Epoch [7/50], Train Loss: 0.520906, Val Loss: 0.542484\n",
      "Accuracy: 84.75%, Micro ROC AUC: 0.9839, Macro ROC AUC: 0.9827\n",
      "Epoch [8/50], Train Loss: 0.482862, Val Loss: 0.511589\n",
      "Accuracy: 85.76%, Micro ROC AUC: 0.9857, Macro ROC AUC: 0.9846\n",
      "Epoch [9/50], Train Loss: 0.453285, Val Loss: 0.476206\n",
      "Accuracy: 86.31%, Micro ROC AUC: 0.9867, Macro ROC AUC: 0.9857\n",
      "Epoch [10/50], Train Loss: 0.426239, Val Loss: 0.460011\n",
      "Accuracy: 85.95%, Micro ROC AUC: 0.9864, Macro ROC AUC: 0.9859\n",
      "Epoch [11/50], Train Loss: 0.405254, Val Loss: 0.470085\n",
      "Accuracy: 86.89%, Micro ROC AUC: 0.9876, Macro ROC AUC: 0.9873\n",
      "Epoch [12/50], Train Loss: 0.381730, Val Loss: 0.441212\n",
      "Accuracy: 87.52%, Micro ROC AUC: 0.9884, Macro ROC AUC: 0.9878\n",
      "Epoch [13/50], Train Loss: 0.362218, Val Loss: 0.428088\n",
      "Accuracy: 88.02%, Micro ROC AUC: 0.9893, Macro ROC AUC: 0.9886\n",
      "Epoch [14/50], Train Loss: 0.345742, Val Loss: 0.406740\n",
      "Accuracy: 88.19%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9889\n",
      "Epoch [15/50], Train Loss: 0.329357, Val Loss: 0.406053\n",
      "Accuracy: 88.20%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9895\n",
      "Epoch [16/50], Train Loss: 0.315822, Val Loss: 0.394843\n",
      "Accuracy: 88.96%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [17/50], Train Loss: 0.301414, Val Loss: 0.381084\n",
      "Accuracy: 88.95%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [18/50], Train Loss: 0.286941, Val Loss: 0.377477\n",
      "Accuracy: 89.05%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9904\n",
      "Epoch [19/50], Train Loss: 0.275665, Val Loss: 0.376595\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9905\n",
      "Epoch [20/50], Train Loss: 0.263198, Val Loss: 0.378528\n",
      "Accuracy: 89.18%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9906\n",
      "Epoch [21/50], Train Loss: 0.251858, Val Loss: 0.385499\n",
      "Accuracy: 89.04%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9908\n",
      "Epoch [22/50], Train Loss: 0.241810, Val Loss: 0.373506\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9907\n",
      "Epoch [23/50], Train Loss: 0.231626, Val Loss: 0.372513\n",
      "Accuracy: 89.82%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9911\n",
      "Epoch [24/50], Train Loss: 0.222147, Val Loss: 0.363756\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [25/50], Train Loss: 0.210119, Val Loss: 0.364081\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9912\n",
      "Epoch [26/50], Train Loss: 0.200983, Val Loss: 0.372425\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9909\n",
      "Epoch [27/50], Train Loss: 0.191925, Val Loss: 0.382223\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [28/50], Train Loss: 0.182231, Val Loss: 0.379532\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9907\n",
      "Epoch [29/50], Train Loss: 0.172075, Val Loss: 0.388825\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9912\n",
      "Epoch [30/50], Train Loss: 0.165104, Val Loss: 0.408022\n",
      "Accuracy: 89.60%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9909\n",
      "Epoch [31/50], Train Loss: 0.155311, Val Loss: 0.399394\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9909\n",
      "Epoch [32/50], Train Loss: 0.146331, Val Loss: 0.410031\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9909\n",
      "Epoch [33/50], Train Loss: 0.139180, Val Loss: 0.419098\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9907\n",
      "Epoch [34/50], Train Loss: 0.130263, Val Loss: 0.443707\n",
      "Accuracy: 89.96%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9907\n",
      "Epoch [35/50], Train Loss: 0.123741, Val Loss: 0.435530\n",
      "Accuracy: 89.81%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9908\n",
      "Epoch [36/50], Train Loss: 0.115550, Val Loss: 0.451743\n",
      "Accuracy: 89.38%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9904\n",
      "Epoch [37/50], Train Loss: 0.107570, Val Loss: 0.489011\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9904\n",
      "Epoch [38/50], Train Loss: 0.100807, Val Loss: 0.479474\n",
      "Accuracy: 89.49%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [39/50], Train Loss: 0.094521, Val Loss: 0.495738\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9904\n",
      "Epoch [40/50], Train Loss: 0.086920, Val Loss: 0.529683\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9895\n",
      "Epoch [41/50], Train Loss: 0.079599, Val Loss: 0.538816\n",
      "Accuracy: 88.84%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9897\n",
      "Epoch [42/50], Train Loss: 0.076503, Val Loss: 0.546570\n",
      "Accuracy: 89.01%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [43/50], Train Loss: 0.068441, Val Loss: 0.579073\n",
      "Accuracy: 89.18%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [44/50], Train Loss: 0.062704, Val Loss: 0.593278\n",
      "Accuracy: 88.89%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [45/50], Train Loss: 0.058248, Val Loss: 0.610583\n",
      "Accuracy: 88.86%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9895\n",
      "Epoch [46/50], Train Loss: 0.053677, Val Loss: 0.654626\n",
      "Accuracy: 89.07%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9897\n",
      "Epoch [47/50], Train Loss: 0.049450, Val Loss: 0.680413\n",
      "Accuracy: 88.98%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9894\n",
      "Epoch [48/50], Train Loss: 0.046379, Val Loss: 0.702016\n",
      "Accuracy: 88.80%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9892\n",
      "Epoch [49/50], Train Loss: 0.043656, Val Loss: 0.706874\n",
      "Accuracy: 89.13%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [50/50], Train Loss: 0.040927, Val Loss: 0.733843\n",
      "Accuracy: 88.51%, Micro ROC AUC: 0.9887, Macro ROC AUC: 0.9879\n",
      "Training with lr=0.0001, batch_size=64, epochs=200\n",
      "Accuracy: 36.02%, Micro ROC AUC: 0.7855, Macro ROC AUC: 0.7656\n",
      "Epoch [1/200], Train Loss: 2.203058, Val Loss: 1.824740\n",
      "Accuracy: 71.62%, Micro ROC AUC: 0.9533, Macro ROC AUC: 0.9479\n",
      "Epoch [2/200], Train Loss: 1.203003, Val Loss: 0.900954\n",
      "Accuracy: 77.91%, Micro ROC AUC: 0.9696, Macro ROC AUC: 0.9661\n",
      "Epoch [3/200], Train Loss: 0.807313, Val Loss: 0.716874\n",
      "Accuracy: 80.94%, Micro ROC AUC: 0.9758, Macro ROC AUC: 0.9733\n",
      "Epoch [4/200], Train Loss: 0.674597, Val Loss: 0.633740\n",
      "Accuracy: 82.65%, Micro ROC AUC: 0.9795, Macro ROC AUC: 0.9775\n",
      "Epoch [5/200], Train Loss: 0.590923, Val Loss: 0.578788\n",
      "Accuracy: 83.74%, Micro ROC AUC: 0.9819, Macro ROC AUC: 0.9806\n",
      "Epoch [6/200], Train Loss: 0.530069, Val Loss: 0.540767\n",
      "Accuracy: 85.05%, Micro ROC AUC: 0.9843, Macro ROC AUC: 0.9830\n",
      "Epoch [7/200], Train Loss: 0.482907, Val Loss: 0.498624\n",
      "Accuracy: 85.97%, Micro ROC AUC: 0.9855, Macro ROC AUC: 0.9842\n",
      "Epoch [8/200], Train Loss: 0.447145, Val Loss: 0.479459\n",
      "Accuracy: 86.43%, Micro ROC AUC: 0.9865, Macro ROC AUC: 0.9856\n",
      "Epoch [9/200], Train Loss: 0.415759, Val Loss: 0.460316\n",
      "Accuracy: 86.96%, Micro ROC AUC: 0.9874, Macro ROC AUC: 0.9866\n",
      "Epoch [10/200], Train Loss: 0.390516, Val Loss: 0.446159\n",
      "Accuracy: 87.68%, Micro ROC AUC: 0.9887, Macro ROC AUC: 0.9877\n",
      "Epoch [11/200], Train Loss: 0.368686, Val Loss: 0.417899\n",
      "Accuracy: 88.04%, Micro ROC AUC: 0.9890, Macro ROC AUC: 0.9883\n",
      "Epoch [12/200], Train Loss: 0.348259, Val Loss: 0.412157\n",
      "Accuracy: 88.49%, Micro ROC AUC: 0.9895, Macro ROC AUC: 0.9890\n",
      "Epoch [13/200], Train Loss: 0.331688, Val Loss: 0.399004\n",
      "Accuracy: 87.91%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9888\n",
      "Epoch [14/200], Train Loss: 0.316546, Val Loss: 0.407533\n",
      "Accuracy: 89.03%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [15/200], Train Loss: 0.301589, Val Loss: 0.380912\n",
      "Accuracy: 89.17%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9900\n",
      "Epoch [16/200], Train Loss: 0.287702, Val Loss: 0.379664\n",
      "Accuracy: 89.43%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9904\n",
      "Epoch [17/200], Train Loss: 0.275719, Val Loss: 0.372287\n",
      "Accuracy: 89.65%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [18/200], Train Loss: 0.264623, Val Loss: 0.376354\n",
      "Accuracy: 89.76%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9908\n",
      "Epoch [19/200], Train Loss: 0.252911, Val Loss: 0.361017\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [20/200], Train Loss: 0.241968, Val Loss: 0.360747\n",
      "Accuracy: 89.96%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9910\n",
      "Epoch [21/200], Train Loss: 0.230940, Val Loss: 0.367730\n",
      "Accuracy: 89.90%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9911\n",
      "Epoch [22/200], Train Loss: 0.219968, Val Loss: 0.370016\n",
      "Accuracy: 89.94%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9911\n",
      "Epoch [23/200], Train Loss: 0.211922, Val Loss: 0.367523\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9911\n",
      "Epoch [24/200], Train Loss: 0.201834, Val Loss: 0.375045\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9913\n",
      "Epoch [25/200], Train Loss: 0.192999, Val Loss: 0.367991\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9910\n",
      "Epoch [26/200], Train Loss: 0.183541, Val Loss: 0.398852\n",
      "Accuracy: 90.03%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911\n",
      "Epoch [27/200], Train Loss: 0.173986, Val Loss: 0.384420\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9912\n",
      "Epoch [28/200], Train Loss: 0.165783, Val Loss: 0.394511\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9913\n",
      "Epoch [29/200], Train Loss: 0.157917, Val Loss: 0.382554\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911\n",
      "Epoch [30/200], Train Loss: 0.148453, Val Loss: 0.407927\n",
      "Accuracy: 89.81%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9909\n",
      "Epoch [31/200], Train Loss: 0.141358, Val Loss: 0.417047\n",
      "Accuracy: 90.10%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9910\n",
      "Epoch [32/200], Train Loss: 0.131970, Val Loss: 0.421968\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9905\n",
      "Epoch [33/200], Train Loss: 0.125089, Val Loss: 0.421554\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9908\n",
      "Epoch [34/200], Train Loss: 0.115962, Val Loss: 0.432708\n",
      "Accuracy: 89.52%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [35/200], Train Loss: 0.107965, Val Loss: 0.454855\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9906\n",
      "Epoch [36/200], Train Loss: 0.101760, Val Loss: 0.471270\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9898\n",
      "Epoch [37/200], Train Loss: 0.095131, Val Loss: 0.479503\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [38/200], Train Loss: 0.088518, Val Loss: 0.498446\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9902\n",
      "Epoch [39/200], Train Loss: 0.082313, Val Loss: 0.522171\n",
      "Accuracy: 89.39%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [40/200], Train Loss: 0.075255, Val Loss: 0.568692\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [41/200], Train Loss: 0.070282, Val Loss: 0.558421\n",
      "Accuracy: 89.30%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9896\n",
      "Epoch [42/200], Train Loss: 0.061745, Val Loss: 0.582164\n",
      "Accuracy: 89.45%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [43/200], Train Loss: 0.056890, Val Loss: 0.627372\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [44/200], Train Loss: 0.051505, Val Loss: 0.633066\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9895\n",
      "Epoch [45/200], Train Loss: 0.049199, Val Loss: 0.661222\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9899\n",
      "Epoch [46/200], Train Loss: 0.043969, Val Loss: 0.675635\n",
      "Accuracy: 89.60%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [47/200], Train Loss: 0.041360, Val Loss: 0.695037\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9893\n",
      "Epoch [48/200], Train Loss: 0.036037, Val Loss: 0.763594\n",
      "Accuracy: 89.21%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [49/200], Train Loss: 0.034161, Val Loss: 0.774255\n",
      "Accuracy: 89.27%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9894\n",
      "Epoch [50/200], Train Loss: 0.031977, Val Loss: 0.767954\n",
      "Accuracy: 89.02%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9892\n",
      "Epoch [51/200], Train Loss: 0.028099, Val Loss: 0.845569\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9890\n",
      "Epoch [52/200], Train Loss: 0.026859, Val Loss: 0.845442\n",
      "Accuracy: 89.18%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [53/200], Train Loss: 0.028550, Val Loss: 0.860541\n",
      "Accuracy: 89.38%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [54/200], Train Loss: 0.021340, Val Loss: 0.891146\n",
      "Accuracy: 89.34%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9892\n",
      "Epoch [55/200], Train Loss: 0.023255, Val Loss: 0.875885\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9895\n",
      "Epoch [56/200], Train Loss: 0.023674, Val Loss: 0.890477\n",
      "Accuracy: 89.27%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9894\n",
      "Epoch [57/200], Train Loss: 0.021579, Val Loss: 0.918124\n",
      "Accuracy: 89.20%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [58/200], Train Loss: 0.017135, Val Loss: 0.977462\n",
      "Accuracy: 89.17%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9891\n",
      "Epoch [59/200], Train Loss: 0.024770, Val Loss: 0.883368\n",
      "Accuracy: 89.20%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9891\n",
      "Epoch [60/200], Train Loss: 0.015908, Val Loss: 0.961058\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9893\n",
      "Epoch [61/200], Train Loss: 0.020694, Val Loss: 0.942553\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [62/200], Train Loss: 0.014236, Val Loss: 1.000905\n",
      "Accuracy: 89.34%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9892\n",
      "Epoch [63/200], Train Loss: 0.015996, Val Loss: 0.993671\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [64/200], Train Loss: 0.018286, Val Loss: 1.030219\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9891\n",
      "Epoch [65/200], Train Loss: 0.017656, Val Loss: 1.074456\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9892\n",
      "Epoch [66/200], Train Loss: 0.015866, Val Loss: 1.009974\n",
      "Accuracy: 89.58%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [67/200], Train Loss: 0.018134, Val Loss: 1.028827\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9895\n",
      "Epoch [68/200], Train Loss: 0.014492, Val Loss: 1.062332\n",
      "Accuracy: 88.61%, Micro ROC AUC: 0.9891, Macro ROC AUC: 0.9888\n",
      "Epoch [69/200], Train Loss: 0.015489, Val Loss: 1.080426\n",
      "Accuracy: 88.92%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9891\n",
      "Epoch [70/200], Train Loss: 0.013327, Val Loss: 1.083257\n",
      "Accuracy: 88.78%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [71/200], Train Loss: 0.018079, Val Loss: 1.061006\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9896\n",
      "Epoch [72/200], Train Loss: 0.014997, Val Loss: 1.054948\n",
      "Accuracy: 89.35%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9892\n",
      "Epoch [73/200], Train Loss: 0.015510, Val Loss: 1.107562\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9892\n",
      "Epoch [74/200], Train Loss: 0.009184, Val Loss: 1.124084\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9892\n",
      "Epoch [75/200], Train Loss: 0.015120, Val Loss: 1.121189\n",
      "Accuracy: 89.40%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9895\n",
      "Epoch [76/200], Train Loss: 0.013409, Val Loss: 1.093631\n",
      "Accuracy: 89.54%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9892\n",
      "Epoch [77/200], Train Loss: 0.010553, Val Loss: 1.201005\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9892\n",
      "Epoch [78/200], Train Loss: 0.016260, Val Loss: 1.166518\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [79/200], Train Loss: 0.008266, Val Loss: 1.132055\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9892\n",
      "Epoch [80/200], Train Loss: 0.016914, Val Loss: 1.115266\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9893\n",
      "Epoch [81/200], Train Loss: 0.010161, Val Loss: 1.172184\n",
      "Accuracy: 88.87%, Micro ROC AUC: 0.9895, Macro ROC AUC: 0.9892\n",
      "Epoch [82/200], Train Loss: 0.013058, Val Loss: 1.188204\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [83/200], Train Loss: 0.010811, Val Loss: 1.137846\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [84/200], Train Loss: 0.014033, Val Loss: 1.120263\n",
      "Accuracy: 89.63%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9895\n",
      "Epoch [85/200], Train Loss: 0.007865, Val Loss: 1.149680\n",
      "Accuracy: 89.52%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [86/200], Train Loss: 0.016318, Val Loss: 1.135493\n",
      "Accuracy: 89.57%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [87/200], Train Loss: 0.008175, Val Loss: 1.138062\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9894\n",
      "Epoch [88/200], Train Loss: 0.015016, Val Loss: 1.179931\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9893\n",
      "Epoch [89/200], Train Loss: 0.008668, Val Loss: 1.199844\n",
      "Accuracy: 89.21%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9891\n",
      "Epoch [90/200], Train Loss: 0.008646, Val Loss: 1.251239\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9896\n",
      "Epoch [91/200], Train Loss: 0.018094, Val Loss: 1.154229\n",
      "Accuracy: 89.49%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9894\n",
      "Epoch [92/200], Train Loss: 0.004189, Val Loss: 1.230219\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9895\n",
      "Epoch [93/200], Train Loss: 0.015683, Val Loss: 1.173985\n",
      "Accuracy: 89.76%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9898\n",
      "Epoch [94/200], Train Loss: 0.006401, Val Loss: 1.194958\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9895\n",
      "Epoch [95/200], Train Loss: 0.014285, Val Loss: 1.214928\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9900\n",
      "Epoch [96/200], Train Loss: 0.009281, Val Loss: 1.152418\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9892\n",
      "Epoch [97/200], Train Loss: 0.010130, Val Loss: 1.171614\n",
      "Accuracy: 89.46%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [98/200], Train Loss: 0.010867, Val Loss: 1.177627\n",
      "Accuracy: 89.53%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [99/200], Train Loss: 0.008886, Val Loss: 1.188579\n",
      "Accuracy: 89.63%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [100/200], Train Loss: 0.014138, Val Loss: 1.163731\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9897\n",
      "Epoch [101/200], Train Loss: 0.007521, Val Loss: 1.279524\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [102/200], Train Loss: 0.006402, Val Loss: 1.179067\n",
      "Accuracy: 89.42%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [103/200], Train Loss: 0.013575, Val Loss: 1.169382\n",
      "Accuracy: 89.62%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9897\n",
      "Epoch [104/200], Train Loss: 0.009069, Val Loss: 1.176734\n",
      "Accuracy: 88.92%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9894\n",
      "Epoch [105/200], Train Loss: 0.009098, Val Loss: 1.231109\n",
      "Accuracy: 89.63%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9898\n",
      "Epoch [106/200], Train Loss: 0.011298, Val Loss: 1.179018\n",
      "Accuracy: 89.44%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [107/200], Train Loss: 0.006959, Val Loss: 1.214218\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [108/200], Train Loss: 0.012699, Val Loss: 1.213633\n",
      "Accuracy: 89.58%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [109/200], Train Loss: 0.006315, Val Loss: 1.247411\n",
      "Accuracy: 89.42%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [110/200], Train Loss: 0.010694, Val Loss: 1.246225\n",
      "Accuracy: 89.09%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9891\n",
      "Epoch [111/200], Train Loss: 0.006255, Val Loss: 1.227600\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [112/200], Train Loss: 0.011967, Val Loss: 1.246033\n",
      "Accuracy: 89.76%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [113/200], Train Loss: 0.005687, Val Loss: 1.235696\n",
      "Accuracy: 89.39%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [114/200], Train Loss: 0.007102, Val Loss: 1.246825\n",
      "Accuracy: 89.49%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [115/200], Train Loss: 0.012698, Val Loss: 1.265148\n",
      "Accuracy: 89.87%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [116/200], Train Loss: 0.007192, Val Loss: 1.198106\n",
      "Accuracy: 89.28%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [117/200], Train Loss: 0.009279, Val Loss: 1.202225\n",
      "Accuracy: 89.47%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [118/200], Train Loss: 0.006199, Val Loss: 1.252293\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9898\n",
      "Epoch [119/200], Train Loss: 0.009777, Val Loss: 1.254550\n",
      "Accuracy: 89.52%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [120/200], Train Loss: 0.006733, Val Loss: 1.168793\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [121/200], Train Loss: 0.006936, Val Loss: 1.260916\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [122/200], Train Loss: 0.008993, Val Loss: 1.275343\n",
      "Accuracy: 89.40%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [123/200], Train Loss: 0.007805, Val Loss: 1.257750\n",
      "Accuracy: 89.35%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9896\n",
      "Epoch [124/200], Train Loss: 0.008038, Val Loss: 1.282131\n",
      "Accuracy: 89.60%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [125/200], Train Loss: 0.009079, Val Loss: 1.295797\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9895\n",
      "Epoch [126/200], Train Loss: 0.005829, Val Loss: 1.322661\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [127/200], Train Loss: 0.007707, Val Loss: 1.296029\n",
      "Accuracy: 90.06%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [128/200], Train Loss: 0.011407, Val Loss: 1.262753\n",
      "Accuracy: 90.14%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [129/200], Train Loss: 0.004110, Val Loss: 1.276848\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [130/200], Train Loss: 0.010912, Val Loss: 1.279101\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [131/200], Train Loss: 0.005365, Val Loss: 1.297797\n",
      "Accuracy: 89.65%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [132/200], Train Loss: 0.007172, Val Loss: 1.276920\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [133/200], Train Loss: 0.006295, Val Loss: 1.292868\n",
      "Accuracy: 89.43%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [134/200], Train Loss: 0.009893, Val Loss: 1.282286\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9895\n",
      "Epoch [135/200], Train Loss: 0.009565, Val Loss: 1.249301\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [136/200], Train Loss: 0.006177, Val Loss: 1.217194\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [137/200], Train Loss: 0.004945, Val Loss: 1.283922\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [138/200], Train Loss: 0.008493, Val Loss: 1.231855\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [139/200], Train Loss: 0.007918, Val Loss: 1.301405\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [140/200], Train Loss: 0.007991, Val Loss: 1.284156\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [141/200], Train Loss: 0.006373, Val Loss: 1.319686\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [142/200], Train Loss: 0.007207, Val Loss: 1.301303\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [143/200], Train Loss: 0.005999, Val Loss: 1.339098\n",
      "Accuracy: 89.82%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [144/200], Train Loss: 0.008271, Val Loss: 1.276308\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9901\n",
      "Epoch [145/200], Train Loss: 0.007314, Val Loss: 1.256463\n",
      "Accuracy: 89.43%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9900\n",
      "Epoch [146/200], Train Loss: 0.008438, Val Loss: 1.214114\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [147/200], Train Loss: 0.004751, Val Loss: 1.272916\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [148/200], Train Loss: 0.007462, Val Loss: 1.284029\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [149/200], Train Loss: 0.005697, Val Loss: 1.342568\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [150/200], Train Loss: 0.008238, Val Loss: 1.239236\n",
      "Accuracy: 89.03%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9892\n",
      "Epoch [151/200], Train Loss: 0.003842, Val Loss: 1.372599\n",
      "Accuracy: 88.99%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9895\n",
      "Epoch [152/200], Train Loss: 0.008536, Val Loss: 1.385333\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9896\n",
      "Epoch [153/200], Train Loss: 0.008241, Val Loss: 1.294268\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [154/200], Train Loss: 0.005450, Val Loss: 1.287552\n",
      "Accuracy: 89.89%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [155/200], Train Loss: 0.005578, Val Loss: 1.264692\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [156/200], Train Loss: 0.008691, Val Loss: 1.244097\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [157/200], Train Loss: 0.002751, Val Loss: 1.280206\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [158/200], Train Loss: 0.007667, Val Loss: 1.341538\n",
      "Accuracy: 89.39%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9898\n",
      "Epoch [159/200], Train Loss: 0.009128, Val Loss: 1.346258\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [160/200], Train Loss: 0.003924, Val Loss: 1.285942\n",
      "Accuracy: 90.02%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9903\n",
      "Epoch [161/200], Train Loss: 0.002828, Val Loss: 1.369932\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9903\n",
      "Epoch [162/200], Train Loss: 0.008691, Val Loss: 1.320310\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [163/200], Train Loss: 0.008458, Val Loss: 1.356160\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9904\n",
      "Epoch [164/200], Train Loss: 0.002735, Val Loss: 1.308190\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [165/200], Train Loss: 0.005113, Val Loss: 1.325289\n",
      "Accuracy: 90.01%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [166/200], Train Loss: 0.009130, Val Loss: 1.347373\n",
      "Accuracy: 89.92%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [167/200], Train Loss: 0.005752, Val Loss: 1.304761\n",
      "Accuracy: 89.43%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9897\n",
      "Epoch [168/200], Train Loss: 0.008168, Val Loss: 1.407013\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9898\n",
      "Epoch [169/200], Train Loss: 0.004758, Val Loss: 1.303591\n",
      "Accuracy: 89.98%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [170/200], Train Loss: 0.003988, Val Loss: 1.337876\n",
      "Accuracy: 89.94%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [171/200], Train Loss: 0.008028, Val Loss: 1.338377\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [172/200], Train Loss: 0.005633, Val Loss: 1.325590\n",
      "Accuracy: 89.76%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [173/200], Train Loss: 0.006355, Val Loss: 1.317126\n",
      "Accuracy: 89.84%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [174/200], Train Loss: 0.005333, Val Loss: 1.308323\n",
      "Accuracy: 89.20%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9892\n",
      "Epoch [175/200], Train Loss: 0.004468, Val Loss: 1.404306\n",
      "Accuracy: 90.27%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [176/200], Train Loss: 0.007662, Val Loss: 1.321499\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [177/200], Train Loss: 0.004995, Val Loss: 1.331998\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9905\n",
      "Epoch [178/200], Train Loss: 0.004297, Val Loss: 1.325821\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [179/200], Train Loss: 0.003711, Val Loss: 1.407566\n",
      "Accuracy: 90.13%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [180/200], Train Loss: 0.006871, Val Loss: 1.353770\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [181/200], Train Loss: 0.005702, Val Loss: 1.365228\n",
      "Accuracy: 90.11%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [182/200], Train Loss: 0.007434, Val Loss: 1.356988\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9906\n",
      "Epoch [183/200], Train Loss: 0.001810, Val Loss: 1.338916\n",
      "Accuracy: 89.42%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9896\n",
      "Epoch [184/200], Train Loss: 0.008361, Val Loss: 1.416851\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9904\n",
      "Epoch [185/200], Train Loss: 0.005239, Val Loss: 1.332042\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [186/200], Train Loss: 0.003495, Val Loss: 1.340372\n",
      "Accuracy: 89.58%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [187/200], Train Loss: 0.006180, Val Loss: 1.342789\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [188/200], Train Loss: 0.007139, Val Loss: 1.349816\n",
      "Accuracy: 89.73%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [189/200], Train Loss: 0.005680, Val Loss: 1.384444\n",
      "Accuracy: 90.38%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9905\n",
      "Epoch [190/200], Train Loss: 0.001180, Val Loss: 1.327602\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [191/200], Train Loss: 0.008009, Val Loss: 1.246770\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [192/200], Train Loss: 0.006688, Val Loss: 1.298062\n",
      "Accuracy: 90.17%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [193/200], Train Loss: 0.003743, Val Loss: 1.367496\n",
      "Accuracy: 90.21%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [194/200], Train Loss: 0.005864, Val Loss: 1.296980\n",
      "Accuracy: 90.02%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [195/200], Train Loss: 0.003249, Val Loss: 1.374785\n",
      "Accuracy: 89.85%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [196/200], Train Loss: 0.006613, Val Loss: 1.353107\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9904\n",
      "Epoch [197/200], Train Loss: 0.005761, Val Loss: 1.353054\n",
      "Accuracy: 90.06%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9904\n",
      "Epoch [198/200], Train Loss: 0.002003, Val Loss: 1.368262\n",
      "Accuracy: 90.38%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [199/200], Train Loss: 0.007124, Val Loss: 1.339261\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [200/200], Train Loss: 0.004167, Val Loss: 1.348696\n",
      "Accuracy: 89.05%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9884\n",
      "Training with lr=0.0001, batch_size=128, epochs=10\n",
      "Accuracy: 31.46%, Micro ROC AUC: 0.7479, Macro ROC AUC: 0.7315\n",
      "Epoch [1/10], Train Loss: 2.214923, Val Loss: 1.967106\n",
      "Accuracy: 70.41%, Micro ROC AUC: 0.9465, Macro ROC AUC: 0.9407\n",
      "Epoch [2/10], Train Loss: 1.426432, Val Loss: 0.973950\n",
      "Accuracy: 76.02%, Micro ROC AUC: 0.9649, Macro ROC AUC: 0.9627\n",
      "Epoch [3/10], Train Loss: 0.870495, Val Loss: 0.770584\n",
      "Accuracy: 79.97%, Micro ROC AUC: 0.9741, Macro ROC AUC: 0.9719\n",
      "Epoch [4/10], Train Loss: 0.719617, Val Loss: 0.658096\n",
      "Accuracy: 81.20%, Micro ROC AUC: 0.9769, Macro ROC AUC: 0.9757\n",
      "Epoch [5/10], Train Loss: 0.642833, Val Loss: 0.619271\n",
      "Accuracy: 82.53%, Micro ROC AUC: 0.9797, Macro ROC AUC: 0.9784\n",
      "Epoch [6/10], Train Loss: 0.588617, Val Loss: 0.576983\n",
      "Accuracy: 84.00%, Micro ROC AUC: 0.9828, Macro ROC AUC: 0.9812\n",
      "Epoch [7/10], Train Loss: 0.550120, Val Loss: 0.527395\n",
      "Accuracy: 84.55%, Micro ROC AUC: 0.9837, Macro ROC AUC: 0.9827\n",
      "Epoch [8/10], Train Loss: 0.516531, Val Loss: 0.513372\n",
      "Accuracy: 85.18%, Micro ROC AUC: 0.9849, Macro ROC AUC: 0.9837\n",
      "Epoch [9/10], Train Loss: 0.489671, Val Loss: 0.494063\n",
      "Accuracy: 85.99%, Micro ROC AUC: 0.9862, Macro ROC AUC: 0.9850\n",
      "Epoch [10/10], Train Loss: 0.470361, Val Loss: 0.469309\n",
      "Accuracy: 85.18%, Micro ROC AUC: 0.9836, Macro ROC AUC: 0.9817\n",
      "Training with lr=0.0001, batch_size=128, epochs=50\n",
      "Accuracy: 20.52%, Micro ROC AUC: 0.6672, Macro ROC AUC: 0.6609\n",
      "Epoch [1/50], Train Loss: 2.233180, Val Loss: 2.140405\n",
      "Accuracy: 62.09%, Micro ROC AUC: 0.9202, Macro ROC AUC: 0.9127\n",
      "Epoch [2/50], Train Loss: 1.629730, Val Loss: 1.182521\n",
      "Accuracy: 72.32%, Micro ROC AUC: 0.9539, Macro ROC AUC: 0.9496\n",
      "Epoch [3/50], Train Loss: 0.995856, Val Loss: 0.895698\n",
      "Accuracy: 76.52%, Micro ROC AUC: 0.9660, Macro ROC AUC: 0.9625\n",
      "Epoch [4/50], Train Loss: 0.797623, Val Loss: 0.760291\n",
      "Accuracy: 78.97%, Micro ROC AUC: 0.9718, Macro ROC AUC: 0.9689\n",
      "Epoch [5/50], Train Loss: 0.698542, Val Loss: 0.689503\n",
      "Accuracy: 80.91%, Micro ROC AUC: 0.9759, Macro ROC AUC: 0.9736\n",
      "Epoch [6/50], Train Loss: 0.633750, Val Loss: 0.632318\n",
      "Accuracy: 82.68%, Micro ROC AUC: 0.9791, Macro ROC AUC: 0.9767\n",
      "Epoch [7/50], Train Loss: 0.585102, Val Loss: 0.586188\n",
      "Accuracy: 83.40%, Micro ROC AUC: 0.9806, Macro ROC AUC: 0.9790\n",
      "Epoch [8/50], Train Loss: 0.548570, Val Loss: 0.562722\n",
      "Accuracy: 83.82%, Micro ROC AUC: 0.9816, Macro ROC AUC: 0.9806\n",
      "Epoch [9/50], Train Loss: 0.517962, Val Loss: 0.545925\n",
      "Accuracy: 84.53%, Micro ROC AUC: 0.9831, Macro ROC AUC: 0.9819\n",
      "Epoch [10/50], Train Loss: 0.491137, Val Loss: 0.522053\n",
      "Accuracy: 85.62%, Micro ROC AUC: 0.9851, Macro ROC AUC: 0.9837\n",
      "Epoch [11/50], Train Loss: 0.467205, Val Loss: 0.485823\n",
      "Accuracy: 86.08%, Micro ROC AUC: 0.9858, Macro ROC AUC: 0.9845\n",
      "Epoch [12/50], Train Loss: 0.449235, Val Loss: 0.474293\n",
      "Accuracy: 86.13%, Micro ROC AUC: 0.9861, Macro ROC AUC: 0.9853\n",
      "Epoch [13/50], Train Loss: 0.430307, Val Loss: 0.468072\n",
      "Accuracy: 86.48%, Micro ROC AUC: 0.9868, Macro ROC AUC: 0.9859\n",
      "Epoch [14/50], Train Loss: 0.412041, Val Loss: 0.460271\n",
      "Accuracy: 86.83%, Micro ROC AUC: 0.9873, Macro ROC AUC: 0.9863\n",
      "Epoch [15/50], Train Loss: 0.396167, Val Loss: 0.449266\n",
      "Accuracy: 87.54%, Micro ROC AUC: 0.9883, Macro ROC AUC: 0.9873\n",
      "Epoch [16/50], Train Loss: 0.385231, Val Loss: 0.429083\n",
      "Accuracy: 87.61%, Micro ROC AUC: 0.9883, Macro ROC AUC: 0.9876\n",
      "Epoch [17/50], Train Loss: 0.371315, Val Loss: 0.424858\n",
      "Accuracy: 87.69%, Micro ROC AUC: 0.9885, Macro ROC AUC: 0.9878\n",
      "Epoch [18/50], Train Loss: 0.360441, Val Loss: 0.421563\n",
      "Accuracy: 88.34%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9886\n",
      "Epoch [19/50], Train Loss: 0.350341, Val Loss: 0.400381\n",
      "Accuracy: 87.93%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9888\n",
      "Epoch [20/50], Train Loss: 0.337787, Val Loss: 0.408593\n",
      "Accuracy: 88.41%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9891\n",
      "Epoch [21/50], Train Loss: 0.329094, Val Loss: 0.401611\n",
      "Accuracy: 88.69%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9895\n",
      "Epoch [22/50], Train Loss: 0.320292, Val Loss: 0.393041\n",
      "Accuracy: 89.00%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9899\n",
      "Epoch [23/50], Train Loss: 0.311534, Val Loss: 0.376115\n",
      "Accuracy: 88.90%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [24/50], Train Loss: 0.301494, Val Loss: 0.381366\n",
      "Accuracy: 88.73%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [25/50], Train Loss: 0.290943, Val Loss: 0.388241\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9903\n",
      "Epoch [26/50], Train Loss: 0.285549, Val Loss: 0.378682\n",
      "Accuracy: 89.54%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9906\n",
      "Epoch [27/50], Train Loss: 0.276998, Val Loss: 0.364164\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907\n",
      "Epoch [28/50], Train Loss: 0.267340, Val Loss: 0.373881\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9905\n",
      "Epoch [29/50], Train Loss: 0.260003, Val Loss: 0.376976\n",
      "Accuracy: 89.47%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9908\n",
      "Epoch [30/50], Train Loss: 0.253689, Val Loss: 0.367611\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910\n",
      "Epoch [31/50], Train Loss: 0.246073, Val Loss: 0.374113\n",
      "Accuracy: 89.82%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9913\n",
      "Epoch [32/50], Train Loss: 0.239927, Val Loss: 0.366212\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9911\n",
      "Epoch [33/50], Train Loss: 0.233600, Val Loss: 0.369056\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [34/50], Train Loss: 0.226203, Val Loss: 0.358767\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9913\n",
      "Epoch [35/50], Train Loss: 0.220568, Val Loss: 0.355898\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9914\n",
      "Epoch [36/50], Train Loss: 0.215119, Val Loss: 0.357518\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [37/50], Train Loss: 0.206645, Val Loss: 0.361547\n",
      "Accuracy: 90.15%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [38/50], Train Loss: 0.200649, Val Loss: 0.367714\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9913\n",
      "Epoch [39/50], Train Loss: 0.195808, Val Loss: 0.363450\n",
      "Accuracy: 89.85%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9915\n",
      "Epoch [40/50], Train Loss: 0.189595, Val Loss: 0.375058\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9916\n",
      "Epoch [41/50], Train Loss: 0.183602, Val Loss: 0.378374\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9916\n",
      "Epoch [42/50], Train Loss: 0.179609, Val Loss: 0.367019\n",
      "Accuracy: 90.15%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [43/50], Train Loss: 0.171266, Val Loss: 0.374219\n",
      "Accuracy: 90.29%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [44/50], Train Loss: 0.166371, Val Loss: 0.379633\n",
      "Accuracy: 89.81%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [45/50], Train Loss: 0.161887, Val Loss: 0.396365\n",
      "Accuracy: 89.52%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9911\n",
      "Epoch [46/50], Train Loss: 0.155022, Val Loss: 0.402360\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9915\n",
      "Epoch [47/50], Train Loss: 0.149488, Val Loss: 0.393361\n",
      "Accuracy: 89.81%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9915\n",
      "Epoch [48/50], Train Loss: 0.144375, Val Loss: 0.413244\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914\n",
      "Epoch [49/50], Train Loss: 0.139343, Val Loss: 0.412877\n",
      "Accuracy: 89.81%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9910\n",
      "Epoch [50/50], Train Loss: 0.133981, Val Loss: 0.412316\n",
      "Accuracy: 88.73%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9887\n",
      "Training with lr=0.0001, batch_size=128, epochs=200\n",
      "Accuracy: 19.54%, Micro ROC AUC: 0.6516, Macro ROC AUC: 0.6262\n",
      "Epoch [1/200], Train Loss: 2.237925, Val Loss: 2.178095\n",
      "Accuracy: 59.15%, Micro ROC AUC: 0.9075, Macro ROC AUC: 0.9013\n",
      "Epoch [2/200], Train Loss: 1.695234, Val Loss: 1.266424\n",
      "Accuracy: 73.38%, Micro ROC AUC: 0.9575, Macro ROC AUC: 0.9535\n",
      "Epoch [3/200], Train Loss: 1.014579, Val Loss: 0.854612\n",
      "Accuracy: 77.95%, Micro ROC AUC: 0.9690, Macro ROC AUC: 0.9654\n",
      "Epoch [4/200], Train Loss: 0.776785, Val Loss: 0.724979\n",
      "Accuracy: 80.15%, Micro ROC AUC: 0.9746, Macro ROC AUC: 0.9714\n",
      "Epoch [5/200], Train Loss: 0.686334, Val Loss: 0.652653\n",
      "Accuracy: 81.03%, Micro ROC AUC: 0.9768, Macro ROC AUC: 0.9749\n",
      "Epoch [6/200], Train Loss: 0.627047, Val Loss: 0.622531\n",
      "Accuracy: 82.62%, Micro ROC AUC: 0.9801, Macro ROC AUC: 0.9779\n",
      "Epoch [7/200], Train Loss: 0.583168, Val Loss: 0.572798\n",
      "Accuracy: 83.19%, Micro ROC AUC: 0.9812, Macro ROC AUC: 0.9794\n",
      "Epoch [8/200], Train Loss: 0.547350, Val Loss: 0.555689\n",
      "Accuracy: 84.55%, Micro ROC AUC: 0.9833, Macro ROC AUC: 0.9816\n",
      "Epoch [9/200], Train Loss: 0.520706, Val Loss: 0.520844\n",
      "Accuracy: 84.98%, Micro ROC AUC: 0.9846, Macro ROC AUC: 0.9830\n",
      "Epoch [10/200], Train Loss: 0.490931, Val Loss: 0.498408\n",
      "Accuracy: 85.80%, Micro ROC AUC: 0.9858, Macro ROC AUC: 0.9842\n",
      "Epoch [11/200], Train Loss: 0.468857, Val Loss: 0.476569\n",
      "Accuracy: 86.06%, Micro ROC AUC: 0.9864, Macro ROC AUC: 0.9853\n",
      "Epoch [12/200], Train Loss: 0.448378, Val Loss: 0.465641\n",
      "Accuracy: 86.72%, Micro ROC AUC: 0.9875, Macro ROC AUC: 0.9862\n",
      "Epoch [13/200], Train Loss: 0.430584, Val Loss: 0.444252\n",
      "Accuracy: 86.71%, Micro ROC AUC: 0.9877, Macro ROC AUC: 0.9866\n",
      "Epoch [14/200], Train Loss: 0.412802, Val Loss: 0.441664\n",
      "Accuracy: 87.07%, Micro ROC AUC: 0.9883, Macro ROC AUC: 0.9872\n",
      "Epoch [15/200], Train Loss: 0.398467, Val Loss: 0.429723\n",
      "Accuracy: 87.65%, Micro ROC AUC: 0.9889, Macro ROC AUC: 0.9879\n",
      "Epoch [16/200], Train Loss: 0.385254, Val Loss: 0.417058\n",
      "Accuracy: 87.77%, Micro ROC AUC: 0.9889, Macro ROC AUC: 0.9881\n",
      "Epoch [17/200], Train Loss: 0.372602, Val Loss: 0.415854\n",
      "Accuracy: 88.02%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9885\n",
      "Epoch [18/200], Train Loss: 0.362009, Val Loss: 0.410248\n",
      "Accuracy: 87.69%, Micro ROC AUC: 0.9893, Macro ROC AUC: 0.9886\n",
      "Epoch [19/200], Train Loss: 0.351175, Val Loss: 0.412622\n",
      "Accuracy: 87.89%, Micro ROC AUC: 0.9893, Macro ROC AUC: 0.9890\n",
      "Epoch [20/200], Train Loss: 0.339258, Val Loss: 0.409622\n",
      "Accuracy: 88.22%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9895\n",
      "Epoch [21/200], Train Loss: 0.329935, Val Loss: 0.396019\n",
      "Accuracy: 88.76%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9897\n",
      "Epoch [22/200], Train Loss: 0.322978, Val Loss: 0.382353\n",
      "Accuracy: 88.68%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [23/200], Train Loss: 0.312129, Val Loss: 0.378232\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [24/200], Train Loss: 0.304162, Val Loss: 0.373585\n",
      "Accuracy: 88.75%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9903\n",
      "Epoch [25/200], Train Loss: 0.297270, Val Loss: 0.380659\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9908\n",
      "Epoch [26/200], Train Loss: 0.292290, Val Loss: 0.361602\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9908\n",
      "Epoch [27/200], Train Loss: 0.282919, Val Loss: 0.370272\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9909\n",
      "Epoch [28/200], Train Loss: 0.276986, Val Loss: 0.356518\n",
      "Accuracy: 89.70%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9912\n",
      "Epoch [29/200], Train Loss: 0.270054, Val Loss: 0.358833\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9913\n",
      "Epoch [30/200], Train Loss: 0.262863, Val Loss: 0.357952\n",
      "Accuracy: 89.80%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [31/200], Train Loss: 0.257064, Val Loss: 0.352690\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [32/200], Train Loss: 0.250141, Val Loss: 0.353418\n",
      "Accuracy: 89.97%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [33/200], Train Loss: 0.245252, Val Loss: 0.355264\n",
      "Accuracy: 89.84%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [34/200], Train Loss: 0.238809, Val Loss: 0.359900\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [35/200], Train Loss: 0.232799, Val Loss: 0.357002\n",
      "Accuracy: 89.99%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917\n",
      "Epoch [36/200], Train Loss: 0.227728, Val Loss: 0.353266\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [37/200], Train Loss: 0.222313, Val Loss: 0.356856\n",
      "Accuracy: 89.93%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [38/200], Train Loss: 0.214821, Val Loss: 0.360813\n",
      "Accuracy: 89.88%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [39/200], Train Loss: 0.211819, Val Loss: 0.360385\n",
      "Accuracy: 89.63%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9917\n",
      "Epoch [40/200], Train Loss: 0.203868, Val Loss: 0.363664\n",
      "Accuracy: 90.05%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [41/200], Train Loss: 0.199128, Val Loss: 0.363281\n",
      "Accuracy: 89.81%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [42/200], Train Loss: 0.195423, Val Loss: 0.361414\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9915\n",
      "Epoch [43/200], Train Loss: 0.188688, Val Loss: 0.360902\n",
      "Accuracy: 90.02%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [44/200], Train Loss: 0.183147, Val Loss: 0.372602\n",
      "Accuracy: 89.83%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [45/200], Train Loss: 0.178801, Val Loss: 0.379407\n",
      "Accuracy: 89.92%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [46/200], Train Loss: 0.173675, Val Loss: 0.370268\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9917\n",
      "Epoch [47/200], Train Loss: 0.168430, Val Loss: 0.365040\n",
      "Accuracy: 90.16%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9918\n",
      "Epoch [48/200], Train Loss: 0.162558, Val Loss: 0.383051\n",
      "Accuracy: 89.88%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [49/200], Train Loss: 0.159840, Val Loss: 0.381594\n",
      "Accuracy: 89.86%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9917\n",
      "Epoch [50/200], Train Loss: 0.154984, Val Loss: 0.391160\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [51/200], Train Loss: 0.148995, Val Loss: 0.392167\n",
      "Accuracy: 90.08%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9917\n",
      "Epoch [52/200], Train Loss: 0.143556, Val Loss: 0.398339\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [53/200], Train Loss: 0.139191, Val Loss: 0.399632\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9916\n",
      "Epoch [54/200], Train Loss: 0.134933, Val Loss: 0.419654\n",
      "Accuracy: 90.02%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [55/200], Train Loss: 0.130314, Val Loss: 0.417664\n",
      "Accuracy: 89.68%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [56/200], Train Loss: 0.124962, Val Loss: 0.417230\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9913\n",
      "Epoch [57/200], Train Loss: 0.122517, Val Loss: 0.436071\n",
      "Accuracy: 89.95%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915\n",
      "Epoch [58/200], Train Loss: 0.117729, Val Loss: 0.431016\n",
      "Accuracy: 89.30%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9906\n",
      "Epoch [59/200], Train Loss: 0.111674, Val Loss: 0.448826\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911\n",
      "Epoch [60/200], Train Loss: 0.108049, Val Loss: 0.451647\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9909\n",
      "Epoch [61/200], Train Loss: 0.104323, Val Loss: 0.463732\n",
      "Accuracy: 89.74%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9911\n",
      "Epoch [62/200], Train Loss: 0.098557, Val Loss: 0.461791\n",
      "Accuracy: 89.53%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9909\n",
      "Epoch [63/200], Train Loss: 0.095428, Val Loss: 0.489991\n",
      "Accuracy: 89.52%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9907\n",
      "Epoch [64/200], Train Loss: 0.093527, Val Loss: 0.477748\n",
      "Accuracy: 89.62%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9909\n",
      "Epoch [65/200], Train Loss: 0.090460, Val Loss: 0.484131\n",
      "Accuracy: 89.35%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9905\n",
      "Epoch [66/200], Train Loss: 0.083362, Val Loss: 0.507271\n",
      "Accuracy: 89.13%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9904\n",
      "Epoch [67/200], Train Loss: 0.080627, Val Loss: 0.514747\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9905\n",
      "Epoch [68/200], Train Loss: 0.076814, Val Loss: 0.522057\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9905\n",
      "Epoch [69/200], Train Loss: 0.072016, Val Loss: 0.530598\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9904\n",
      "Epoch [70/200], Train Loss: 0.067939, Val Loss: 0.568979\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9905\n",
      "Epoch [71/200], Train Loss: 0.065037, Val Loss: 0.569223\n",
      "Accuracy: 89.45%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9904\n",
      "Epoch [72/200], Train Loss: 0.063719, Val Loss: 0.583767\n",
      "Accuracy: 88.90%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [73/200], Train Loss: 0.059427, Val Loss: 0.597337\n",
      "Accuracy: 88.83%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [74/200], Train Loss: 0.055649, Val Loss: 0.621063\n",
      "Accuracy: 89.30%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [75/200], Train Loss: 0.054350, Val Loss: 0.616867\n",
      "Accuracy: 88.98%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9901\n",
      "Epoch [76/200], Train Loss: 0.050410, Val Loss: 0.647786\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [77/200], Train Loss: 0.046607, Val Loss: 0.645614\n",
      "Accuracy: 88.86%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9899\n",
      "Epoch [78/200], Train Loss: 0.046074, Val Loss: 0.692203\n",
      "Accuracy: 89.05%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [79/200], Train Loss: 0.044113, Val Loss: 0.664690\n",
      "Accuracy: 89.03%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [80/200], Train Loss: 0.037999, Val Loss: 0.707307\n",
      "Accuracy: 89.05%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [81/200], Train Loss: 0.036058, Val Loss: 0.736151\n",
      "Accuracy: 89.16%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9897\n",
      "Epoch [82/200], Train Loss: 0.033311, Val Loss: 0.735216\n",
      "Accuracy: 88.95%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [83/200], Train Loss: 0.036422, Val Loss: 0.754666\n",
      "Accuracy: 89.13%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [84/200], Train Loss: 0.030810, Val Loss: 0.755665\n",
      "Accuracy: 88.60%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9892\n",
      "Epoch [85/200], Train Loss: 0.029987, Val Loss: 0.787033\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [86/200], Train Loss: 0.028011, Val Loss: 0.818276\n",
      "Accuracy: 89.20%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [87/200], Train Loss: 0.027385, Val Loss: 0.807080\n",
      "Accuracy: 87.82%, Micro ROC AUC: 0.9885, Macro ROC AUC: 0.9885\n",
      "Epoch [88/200], Train Loss: 0.030114, Val Loss: 0.876937\n",
      "Accuracy: 88.97%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9894\n",
      "Epoch [89/200], Train Loss: 0.027472, Val Loss: 0.830813\n",
      "Accuracy: 89.29%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9898\n",
      "Epoch [90/200], Train Loss: 0.025210, Val Loss: 0.817951\n",
      "Accuracy: 88.99%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [91/200], Train Loss: 0.016362, Val Loss: 0.871059\n",
      "Accuracy: 88.97%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9897\n",
      "Epoch [92/200], Train Loss: 0.031073, Val Loss: 0.842467\n",
      "Accuracy: 89.16%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [93/200], Train Loss: 0.016418, Val Loss: 0.885474\n",
      "Accuracy: 89.00%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [94/200], Train Loss: 0.019643, Val Loss: 0.892401\n",
      "Accuracy: 88.79%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9895\n",
      "Epoch [95/200], Train Loss: 0.029312, Val Loss: 0.887559\n",
      "Accuracy: 88.81%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9895\n",
      "Epoch [96/200], Train Loss: 0.023284, Val Loss: 0.943017\n",
      "Accuracy: 88.76%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9893\n",
      "Epoch [97/200], Train Loss: 0.017350, Val Loss: 0.964577\n",
      "Accuracy: 88.90%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9894\n",
      "Epoch [98/200], Train Loss: 0.015928, Val Loss: 0.967035\n",
      "Accuracy: 89.19%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [99/200], Train Loss: 0.014705, Val Loss: 0.965350\n",
      "Accuracy: 88.76%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [100/200], Train Loss: 0.025090, Val Loss: 0.959294\n",
      "Accuracy: 88.96%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9895\n",
      "Epoch [101/200], Train Loss: 0.007471, Val Loss: 0.974826\n",
      "Accuracy: 88.82%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [102/200], Train Loss: 0.020659, Val Loss: 0.979935\n",
      "Accuracy: 89.09%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9895\n",
      "Epoch [103/200], Train Loss: 0.025967, Val Loss: 1.005113\n",
      "Accuracy: 89.15%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [104/200], Train Loss: 0.006710, Val Loss: 1.027349\n",
      "Accuracy: 89.13%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [105/200], Train Loss: 0.004296, Val Loss: 1.035538\n",
      "Accuracy: 88.96%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9894\n",
      "Epoch [106/200], Train Loss: 0.042342, Val Loss: 0.955175\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [107/200], Train Loss: 0.008390, Val Loss: 0.976947\n",
      "Accuracy: 89.38%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9899\n",
      "Epoch [108/200], Train Loss: 0.004128, Val Loss: 1.018795\n",
      "Accuracy: 88.53%, Micro ROC AUC: 0.9897, Macro ROC AUC: 0.9891\n",
      "Epoch [109/200], Train Loss: 0.020086, Val Loss: 1.015958\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9898\n",
      "Epoch [110/200], Train Loss: 0.029963, Val Loss: 0.994194\n",
      "Accuracy: 89.29%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [111/200], Train Loss: 0.004258, Val Loss: 1.021832\n",
      "Accuracy: 89.58%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [112/200], Train Loss: 0.002773, Val Loss: 1.052851\n",
      "Accuracy: 88.29%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9889\n",
      "Epoch [113/200], Train Loss: 0.005452, Val Loss: 1.153556\n",
      "Accuracy: 89.18%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [114/200], Train Loss: 0.048743, Val Loss: 0.995704\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9898\n",
      "Epoch [115/200], Train Loss: 0.012044, Val Loss: 1.002970\n",
      "Accuracy: 88.94%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9894\n",
      "Epoch [116/200], Train Loss: 0.009758, Val Loss: 1.115707\n",
      "Accuracy: 88.94%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9893\n",
      "Epoch [117/200], Train Loss: 0.017905, Val Loss: 1.036604\n",
      "Accuracy: 89.53%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [118/200], Train Loss: 0.009222, Val Loss: 1.041752\n",
      "Accuracy: 88.92%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [119/200], Train Loss: 0.008077, Val Loss: 1.065556\n",
      "Accuracy: 89.23%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [120/200], Train Loss: 0.023100, Val Loss: 1.062866\n",
      "Accuracy: 89.23%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [121/200], Train Loss: 0.010450, Val Loss: 1.066071\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [122/200], Train Loss: 0.006326, Val Loss: 1.072660\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [123/200], Train Loss: 0.019805, Val Loss: 1.075719\n",
      "Accuracy: 89.43%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [124/200], Train Loss: 0.003900, Val Loss: 1.092428\n",
      "Accuracy: 89.05%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [125/200], Train Loss: 0.022090, Val Loss: 1.065559\n",
      "Accuracy: 89.03%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9894\n",
      "Epoch [126/200], Train Loss: 0.018249, Val Loss: 1.079348\n",
      "Accuracy: 89.24%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [127/200], Train Loss: 0.004247, Val Loss: 1.132974\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [128/200], Train Loss: 0.002518, Val Loss: 1.112555\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9895\n",
      "Epoch [129/200], Train Loss: 0.027378, Val Loss: 1.049839\n",
      "Accuracy: 89.25%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [130/200], Train Loss: 0.017515, Val Loss: 1.051066\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9899\n",
      "Epoch [131/200], Train Loss: 0.003640, Val Loss: 1.109456\n",
      "Accuracy: 89.08%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9898\n",
      "Epoch [132/200], Train Loss: 0.002027, Val Loss: 1.122072\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9895\n",
      "Epoch [133/200], Train Loss: 0.031419, Val Loss: 1.048285\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [134/200], Train Loss: 0.005824, Val Loss: 1.074112\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [135/200], Train Loss: 0.002100, Val Loss: 1.166598\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [136/200], Train Loss: 0.024689, Val Loss: 1.072912\n",
      "Accuracy: 89.12%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9899\n",
      "Epoch [137/200], Train Loss: 0.006319, Val Loss: 1.111745\n",
      "Accuracy: 89.07%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9896\n",
      "Epoch [138/200], Train Loss: 0.006536, Val Loss: 1.128558\n",
      "Accuracy: 88.90%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [139/200], Train Loss: 0.021470, Val Loss: 1.073726\n",
      "Accuracy: 89.03%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [140/200], Train Loss: 0.012443, Val Loss: 1.152389\n",
      "Accuracy: 89.15%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [141/200], Train Loss: 0.004006, Val Loss: 1.138559\n",
      "Accuracy: 89.23%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9900\n",
      "Epoch [142/200], Train Loss: 0.020919, Val Loss: 1.087037\n",
      "Accuracy: 89.00%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9896\n",
      "Epoch [143/200], Train Loss: 0.011461, Val Loss: 1.137268\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [144/200], Train Loss: 0.008303, Val Loss: 1.124813\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [145/200], Train Loss: 0.001728, Val Loss: 1.131727\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9903\n",
      "Epoch [146/200], Train Loss: 0.001182, Val Loss: 1.148218\n",
      "Accuracy: 88.40%, Micro ROC AUC: 0.9896, Macro ROC AUC: 0.9890\n",
      "Epoch [147/200], Train Loss: 0.004360, Val Loss: 1.211592\n",
      "Accuracy: 89.05%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9897\n",
      "Epoch [148/200], Train Loss: 0.036461, Val Loss: 1.076843\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9900\n",
      "Epoch [149/200], Train Loss: 0.005171, Val Loss: 1.120640\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [150/200], Train Loss: 0.002417, Val Loss: 1.152733\n",
      "Accuracy: 89.62%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [151/200], Train Loss: 0.002156, Val Loss: 1.151704\n",
      "Accuracy: 89.13%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [152/200], Train Loss: 0.000945, Val Loss: 1.159324\n",
      "Accuracy: 89.40%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [153/200], Train Loss: 0.039432, Val Loss: 1.073926\n",
      "Accuracy: 89.31%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [154/200], Train Loss: 0.006153, Val Loss: 1.145950\n",
      "Accuracy: 89.55%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [155/200], Train Loss: 0.002285, Val Loss: 1.136133\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9904\n",
      "Epoch [156/200], Train Loss: 0.001130, Val Loss: 1.132876\n",
      "Accuracy: 88.75%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9894\n",
      "Epoch [157/200], Train Loss: 0.004689, Val Loss: 1.245276\n",
      "Accuracy: 89.18%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [158/200], Train Loss: 0.032527, Val Loss: 1.097439\n",
      "Accuracy: 89.43%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9904\n",
      "Epoch [159/200], Train Loss: 0.004270, Val Loss: 1.142319\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901\n",
      "Epoch [160/200], Train Loss: 0.005246, Val Loss: 1.171091\n",
      "Accuracy: 89.14%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901\n",
      "Epoch [161/200], Train Loss: 0.018265, Val Loss: 1.171834\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9902\n",
      "Epoch [162/200], Train Loss: 0.006656, Val Loss: 1.140802\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [163/200], Train Loss: 0.004936, Val Loss: 1.153066\n",
      "Accuracy: 88.87%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9898\n",
      "Epoch [164/200], Train Loss: 0.013165, Val Loss: 1.160769\n",
      "Accuracy: 89.20%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9896\n",
      "Epoch [165/200], Train Loss: 0.012473, Val Loss: 1.149894\n",
      "Accuracy: 89.47%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9903\n",
      "Epoch [166/200], Train Loss: 0.012750, Val Loss: 1.101212\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9903\n",
      "Epoch [167/200], Train Loss: 0.003017, Val Loss: 1.134387\n",
      "Accuracy: 89.28%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [168/200], Train Loss: 0.003473, Val Loss: 1.180551\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9898\n",
      "Epoch [169/200], Train Loss: 0.017272, Val Loss: 1.165853\n",
      "Accuracy: 89.33%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [170/200], Train Loss: 0.005540, Val Loss: 1.184390\n",
      "Accuracy: 89.47%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [171/200], Train Loss: 0.000950, Val Loss: 1.172954\n",
      "Accuracy: 89.01%, Micro ROC AUC: 0.9902, Macro ROC AUC: 0.9897\n",
      "Epoch [172/200], Train Loss: 0.018974, Val Loss: 1.141560\n",
      "Accuracy: 89.41%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [173/200], Train Loss: 0.008588, Val Loss: 1.156926\n",
      "Accuracy: 89.39%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [174/200], Train Loss: 0.007516, Val Loss: 1.159644\n",
      "Accuracy: 89.42%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [175/200], Train Loss: 0.012407, Val Loss: 1.134729\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9904\n",
      "Epoch [176/200], Train Loss: 0.001504, Val Loss: 1.150641\n",
      "Accuracy: 89.44%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [177/200], Train Loss: 0.000694, Val Loss: 1.191395\n",
      "Accuracy: 89.18%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9897\n",
      "Epoch [178/200], Train Loss: 0.016066, Val Loss: 1.170757\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9898\n",
      "Epoch [179/200], Train Loss: 0.017902, Val Loss: 1.162726\n",
      "Accuracy: 89.40%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901\n",
      "Epoch [180/200], Train Loss: 0.005714, Val Loss: 1.173372\n",
      "Accuracy: 89.54%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [181/200], Train Loss: 0.002160, Val Loss: 1.210468\n",
      "Accuracy: 89.38%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9903\n",
      "Epoch [182/200], Train Loss: 0.001424, Val Loss: 1.185660\n",
      "Accuracy: 89.20%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9899\n",
      "Epoch [183/200], Train Loss: 0.001516, Val Loss: 1.227978\n",
      "Accuracy: 88.83%, Micro ROC AUC: 0.9900, Macro ROC AUC: 0.9894\n",
      "Epoch [184/200], Train Loss: 0.033601, Val Loss: 1.202956\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9903\n",
      "Epoch [185/200], Train Loss: 0.003892, Val Loss: 1.156873\n",
      "Accuracy: 89.82%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9904\n",
      "Epoch [186/200], Train Loss: 0.002141, Val Loss: 1.185216\n",
      "Accuracy: 89.56%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9901\n",
      "Epoch [187/200], Train Loss: 0.001017, Val Loss: 1.174422\n",
      "Accuracy: 89.36%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [188/200], Train Loss: 0.001512, Val Loss: 1.246345\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9908, Macro ROC AUC: 0.9902\n",
      "Epoch [189/200], Train Loss: 0.030497, Val Loss: 1.164340\n",
      "Accuracy: 89.52%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9904\n",
      "Epoch [190/200], Train Loss: 0.003414, Val Loss: 1.152565\n",
      "Accuracy: 89.59%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9906\n",
      "Epoch [191/200], Train Loss: 0.001270, Val Loss: 1.160991\n",
      "Accuracy: 89.67%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9907\n",
      "Epoch [192/200], Train Loss: 0.000668, Val Loss: 1.158644\n",
      "Accuracy: 89.35%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9900\n",
      "Epoch [193/200], Train Loss: 0.025616, Val Loss: 1.119155\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9904\n",
      "Epoch [194/200], Train Loss: 0.005470, Val Loss: 1.126805\n",
      "Accuracy: 89.06%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897\n",
      "Epoch [195/200], Train Loss: 0.012496, Val Loss: 1.182537\n",
      "Accuracy: 89.37%, Micro ROC AUC: 0.9909, Macro ROC AUC: 0.9902\n",
      "Epoch [196/200], Train Loss: 0.003794, Val Loss: 1.132758\n",
      "Accuracy: 89.68%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9905\n",
      "Epoch [197/200], Train Loss: 0.000957, Val Loss: 1.182322\n",
      "Accuracy: 89.65%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9905\n",
      "Epoch [198/200], Train Loss: 0.000567, Val Loss: 1.175205\n",
      "Accuracy: 89.78%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9905\n",
      "Epoch [199/200], Train Loss: 0.000913, Val Loss: 1.205457\n",
      "Accuracy: 88.93%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9893\n",
      "Epoch [200/200], Train Loss: 0.018592, Val Loss: 1.137088\n",
      "Accuracy: 88.42%, Micro ROC AUC: 0.9879, Macro ROC AUC: 0.9869\n"
     ]
    },
    {
     "ename": "PermissionError",
     "evalue": "[Errno 13] Permission denied: 'ex1\\\\hyperparameter_tuning_results.csv'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mPermissionError\u001B[0m                           Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[1], line 236\u001B[0m\n\u001B[0;32m    234\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;18m__name__\u001B[39m \u001B[38;5;241m==\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m__main__\u001B[39m\u001B[38;5;124m\"\u001B[39m:\n\u001B[0;32m    235\u001B[0m     device \u001B[38;5;241m=\u001B[39m torch\u001B[38;5;241m.\u001B[39mdevice(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mcuda\u001B[39m\u001B[38;5;124m'\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m torch\u001B[38;5;241m.\u001B[39mcuda\u001B[38;5;241m.\u001B[39mis_available() \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mcpu\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m--> 236\u001B[0m     \u001B[43mgrid_search_hyperparameters\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdevice\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdevice\u001B[49m\u001B[43m)\u001B[49m\n",
      "Cell \u001B[1;32mIn[1], line 231\u001B[0m, in \u001B[0;36mgrid_search_hyperparameters\u001B[1;34m(device)\u001B[0m\n\u001B[0;32m    229\u001B[0m \u001B[38;5;66;03m# 将结果保存到 ex1 目录中的 CSV 文件\u001B[39;00m\n\u001B[0;32m    230\u001B[0m results_csv_path \u001B[38;5;241m=\u001B[39m os\u001B[38;5;241m.\u001B[39mpath\u001B[38;5;241m.\u001B[39mjoin(output_dir, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124mhyperparameter_tuning_results.csv\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m--> 231\u001B[0m \u001B[43mmetrics_df\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mto_csv\u001B[49m\u001B[43m(\u001B[49m\u001B[43mresults_csv_path\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mindex\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m)\u001B[49m\n\u001B[0;32m    232\u001B[0m \u001B[38;5;28mprint\u001B[39m(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m结果已保存到 \u001B[39m\u001B[38;5;132;01m{\u001B[39;00mresults_csv_path\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m\"\u001B[39m)\n",
      "File \u001B[1;32mE:\\envs\\cisc3024_venv1\\Lib\\site-packages\\pandas\\util\\_decorators.py:333\u001B[0m, in \u001B[0;36mdeprecate_nonkeyword_arguments.<locals>.decorate.<locals>.wrapper\u001B[1;34m(*args, **kwargs)\u001B[0m\n\u001B[0;32m    327\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(args) \u001B[38;5;241m>\u001B[39m num_allow_args:\n\u001B[0;32m    328\u001B[0m     warnings\u001B[38;5;241m.\u001B[39mwarn(\n\u001B[0;32m    329\u001B[0m         msg\u001B[38;5;241m.\u001B[39mformat(arguments\u001B[38;5;241m=\u001B[39m_format_argument_list(allow_args)),\n\u001B[0;32m    330\u001B[0m         \u001B[38;5;167;01mFutureWarning\u001B[39;00m,\n\u001B[0;32m    331\u001B[0m         stacklevel\u001B[38;5;241m=\u001B[39mfind_stack_level(),\n\u001B[0;32m    332\u001B[0m     )\n\u001B[1;32m--> 333\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32mE:\\envs\\cisc3024_venv1\\Lib\\site-packages\\pandas\\core\\generic.py:3967\u001B[0m, in \u001B[0;36mNDFrame.to_csv\u001B[1;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001B[0m\n\u001B[0;32m   3956\u001B[0m df \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(\u001B[38;5;28mself\u001B[39m, ABCDataFrame) \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mto_frame()\n\u001B[0;32m   3958\u001B[0m formatter \u001B[38;5;241m=\u001B[39m DataFrameFormatter(\n\u001B[0;32m   3959\u001B[0m     frame\u001B[38;5;241m=\u001B[39mdf,\n\u001B[0;32m   3960\u001B[0m     header\u001B[38;5;241m=\u001B[39mheader,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m   3964\u001B[0m     decimal\u001B[38;5;241m=\u001B[39mdecimal,\n\u001B[0;32m   3965\u001B[0m )\n\u001B[1;32m-> 3967\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mDataFrameRenderer\u001B[49m\u001B[43m(\u001B[49m\u001B[43mformatter\u001B[49m\u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mto_csv\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m   3968\u001B[0m \u001B[43m    \u001B[49m\u001B[43mpath_or_buf\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3969\u001B[0m \u001B[43m    \u001B[49m\u001B[43mlineterminator\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mlineterminator\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3970\u001B[0m \u001B[43m    \u001B[49m\u001B[43msep\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43msep\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3971\u001B[0m \u001B[43m    \u001B[49m\u001B[43mencoding\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mencoding\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3972\u001B[0m \u001B[43m    \u001B[49m\u001B[43merrors\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43merrors\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3973\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcompression\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcompression\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3974\u001B[0m \u001B[43m    \u001B[49m\u001B[43mquoting\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mquoting\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3975\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcolumns\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcolumns\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3976\u001B[0m \u001B[43m    \u001B[49m\u001B[43mindex_label\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mindex_label\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3977\u001B[0m \u001B[43m    \u001B[49m\u001B[43mmode\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3978\u001B[0m \u001B[43m    \u001B[49m\u001B[43mchunksize\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mchunksize\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3979\u001B[0m \u001B[43m    \u001B[49m\u001B[43mquotechar\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mquotechar\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3980\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdate_format\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdate_format\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3981\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdoublequote\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdoublequote\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3982\u001B[0m \u001B[43m    \u001B[49m\u001B[43mescapechar\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mescapechar\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3983\u001B[0m \u001B[43m    \u001B[49m\u001B[43mstorage_options\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mstorage_options\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m   3984\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[1;32mE:\\envs\\cisc3024_venv1\\Lib\\site-packages\\pandas\\io\\formats\\format.py:1014\u001B[0m, in \u001B[0;36mDataFrameRenderer.to_csv\u001B[1;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001B[0m\n\u001B[0;32m    993\u001B[0m     created_buffer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[0;32m    995\u001B[0m csv_formatter \u001B[38;5;241m=\u001B[39m CSVFormatter(\n\u001B[0;32m    996\u001B[0m     path_or_buf\u001B[38;5;241m=\u001B[39mpath_or_buf,\n\u001B[0;32m    997\u001B[0m     lineterminator\u001B[38;5;241m=\u001B[39mlineterminator,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m   1012\u001B[0m     formatter\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mfmt,\n\u001B[0;32m   1013\u001B[0m )\n\u001B[1;32m-> 1014\u001B[0m \u001B[43mcsv_formatter\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msave\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m   1016\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m created_buffer:\n\u001B[0;32m   1017\u001B[0m     \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(path_or_buf, StringIO)\n",
      "File \u001B[1;32mE:\\envs\\cisc3024_venv1\\Lib\\site-packages\\pandas\\io\\formats\\csvs.py:251\u001B[0m, in \u001B[0;36mCSVFormatter.save\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    247\u001B[0m \u001B[38;5;250m\u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m    248\u001B[0m \u001B[38;5;124;03mCreate the writer & save.\u001B[39;00m\n\u001B[0;32m    249\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m    250\u001B[0m \u001B[38;5;66;03m# apply compression and byte/text conversion\u001B[39;00m\n\u001B[1;32m--> 251\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[43mget_handle\u001B[49m\u001B[43m(\u001B[49m\n\u001B[0;32m    252\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfilepath_or_buffer\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    253\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    254\u001B[0m \u001B[43m    \u001B[49m\u001B[43mencoding\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mencoding\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    255\u001B[0m \u001B[43m    \u001B[49m\u001B[43merrors\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43merrors\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    256\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcompression\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mcompression\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    257\u001B[0m \u001B[43m    \u001B[49m\u001B[43mstorage_options\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mstorage_options\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    258\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m \u001B[38;5;28;01mas\u001B[39;00m handles:\n\u001B[0;32m    259\u001B[0m     \u001B[38;5;66;03m# Note: self.encoding is irrelevant here\u001B[39;00m\n\u001B[0;32m    260\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mwriter \u001B[38;5;241m=\u001B[39m csvlib\u001B[38;5;241m.\u001B[39mwriter(\n\u001B[0;32m    261\u001B[0m         handles\u001B[38;5;241m.\u001B[39mhandle,\n\u001B[0;32m    262\u001B[0m         lineterminator\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mlineterminator,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    267\u001B[0m         quotechar\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mquotechar,\n\u001B[0;32m    268\u001B[0m     )\n\u001B[0;32m    270\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_save()\n",
      "File \u001B[1;32mE:\\envs\\cisc3024_venv1\\Lib\\site-packages\\pandas\\io\\common.py:873\u001B[0m, in \u001B[0;36mget_handle\u001B[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001B[0m\n\u001B[0;32m    868\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(handle, \u001B[38;5;28mstr\u001B[39m):\n\u001B[0;32m    869\u001B[0m     \u001B[38;5;66;03m# Check whether the filename is to be opened in binary mode.\u001B[39;00m\n\u001B[0;32m    870\u001B[0m     \u001B[38;5;66;03m# Binary mode does not support 'encoding' and 'newline'.\u001B[39;00m\n\u001B[0;32m    871\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m ioargs\u001B[38;5;241m.\u001B[39mencoding \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mb\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m ioargs\u001B[38;5;241m.\u001B[39mmode:\n\u001B[0;32m    872\u001B[0m         \u001B[38;5;66;03m# Encoding\u001B[39;00m\n\u001B[1;32m--> 873\u001B[0m         handle \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mopen\u001B[39;49m\u001B[43m(\u001B[49m\n\u001B[0;32m    874\u001B[0m \u001B[43m            \u001B[49m\u001B[43mhandle\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    875\u001B[0m \u001B[43m            \u001B[49m\u001B[43mioargs\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmode\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    876\u001B[0m \u001B[43m            \u001B[49m\u001B[43mencoding\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mioargs\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mencoding\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    877\u001B[0m \u001B[43m            \u001B[49m\u001B[43merrors\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43merrors\u001B[49m\u001B[43m,\u001B[49m\n\u001B[0;32m    878\u001B[0m \u001B[43m            \u001B[49m\u001B[43mnewline\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[38;5;124;43m\"\u001B[39;49m\u001B[43m,\u001B[49m\n\u001B[0;32m    879\u001B[0m \u001B[43m        \u001B[49m\u001B[43m)\u001B[49m\n\u001B[0;32m    880\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    881\u001B[0m         \u001B[38;5;66;03m# Binary mode\u001B[39;00m\n\u001B[0;32m    882\u001B[0m         handle \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mopen\u001B[39m(handle, ioargs\u001B[38;5;241m.\u001B[39mmode)\n",
      "\u001B[1;31mPermissionError\u001B[0m: [Errno 13] Permission denied: 'ex1\\\\hyperparameter_tuning_results.csv'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x600 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "c332bc365de1b908"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-08T18:29:30.687849Z",
     "start_time": "2024-11-08T18:14:53.158613Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.metrics import roc_auc_score, confusion_matrix\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "\n",
    "output_dir = 'ex2'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# SVHN Data Module\n",
    "class SVHNDataModule:\n",
    "    def __init__(self):\n",
    "        self.transform = transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=(0.4377, 0.4438, 0.4728), std=(0.1980, 0.2010, 0.1970))\n",
    "        ])\n",
    "\n",
    "    def load_data(self, batch_size):\n",
    "        train_dataset = datasets.SVHN(root='./data', split='train', download=True, transform=self.transform)\n",
    "        test_dataset = datasets.SVHN(root='./data', split='test', download=True, transform=self.transform)\n",
    "\n",
    "        train_size = int(0.8 * len(train_dataset))\n",
    "        val_size = len(train_dataset) - train_size\n",
    "        train_subset, val_subset = torch.utils.data.random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "        train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True)\n",
    "        val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "        return train_loader, val_loader, test_loader\n",
    "\n",
    "# Define Small VGG Model\n",
    "class SmallVGG(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SmallVGG, self).__init__()\n",
    "        self.conv_layers = nn.Sequential(\n",
    "            nn.Conv2d(3, 8, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(8, 16, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.fc_layers = nn.Sequential(\n",
    "            nn.Linear(32 * 4 * 4, 256),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv_layers(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc_layers(x)\n",
    "        return x\n",
    "\n",
    "# Training function with optional early stopping\n",
    "def train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs=50, patience=None, device='cuda', early_stop_start_epoch=30):\n",
    "    model.train()\n",
    "    device = torch.device(device)\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "    best_val_loss = float('inf')\n",
    "    patience_counter = 0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0\n",
    "        for images, labels in train_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "\n",
    "        train_loss = running_loss / len(train_loader)\n",
    "        train_losses.append(train_loss)\n",
    "\n",
    "        # Evaluate on validation set\n",
    "        val_loss = evaluate_model(model, val_loader, device, return_loss=True)\n",
    "        val_losses.append(val_loss)\n",
    "        print(f\"Epoch [{epoch + 1}/{num_epochs}], Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}\")\n",
    "\n",
    "        # Early stopping check (enabled if `patience` is set)\n",
    "        if patience is not None and epoch >= early_stop_start_epoch:\n",
    "            if val_loss < best_val_loss:\n",
    "                best_val_loss = val_loss\n",
    "                patience_counter = 0\n",
    "                # Save best model\n",
    "                torch.save(model.state_dict(), 'best_model.pth')\n",
    "            else:\n",
    "                patience_counter += 1\n",
    "                if patience_counter >= patience:\n",
    "                    print(\"Early stopping triggered\")\n",
    "                    break\n",
    "        elif patience is None:\n",
    "            torch.save(model.state_dict(), 'best_model_no_early_stopping.pth')\n",
    "\n",
    "    return train_losses, val_losses\n",
    "\n",
    "# Evaluation function\n",
    "def evaluate_model(model, data_loader, device='cuda', return_loss=False):\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    all_labels = []\n",
    "    all_preds = []\n",
    "    all_probs = []\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for images, labels in data_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            running_loss += loss.item()\n",
    "\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            probs = torch.softmax(outputs, dim=1)\n",
    "\n",
    "            all_labels.extend(labels.cpu().numpy())\n",
    "            all_preds.extend(predicted.cpu().numpy())\n",
    "            all_probs.extend(probs.cpu().numpy())\n",
    "\n",
    "    avg_loss = running_loss / len(data_loader)\n",
    "    accuracy = np.mean(np.array(all_labels) == np.array(all_preds))\n",
    "\n",
    "    if return_loss:\n",
    "        return avg_loss\n",
    "    else:\n",
    "        return avg_loss, accuracy, all_labels, all_preds, all_probs\n",
    "\n",
    "def main():\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    batch_size = 128\n",
    "    learning_rate = 0.0005\n",
    "    num_epochs = 50\n",
    "\n",
    "    # Load data\n",
    "    data_module = SVHNDataModule()\n",
    "    train_loader, val_loader, test_loader = data_module.load_data(batch_size)\n",
    "\n",
    "    # Model, criterion, and optimizer\n",
    "    model = SmallVGG().to(device)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "    # Train model with early stopping\n",
    "    print(\"Training with early stopping:\")\n",
    "    patience = 5\n",
    "    train_losses_es, val_losses_es = train_model(\n",
    "        model, train_loader, val_loader, criterion, optimizer, num_epochs=num_epochs, patience=patience, device=device\n",
    "    )\n",
    "\n",
    "    # Test the model trained with early stopping\n",
    "    model.load_state_dict(torch.load('best_model.pth'))\n",
    "    test_loss_es, accuracy_es, all_labels_es, all_preds_es, _ = evaluate_model(model, test_loader, device)\n",
    "\n",
    "    # Train model without early stopping\n",
    "    print(\"\\nTraining without early stopping:\")\n",
    "    model = SmallVGG().to(device)  # Reinitialize model\n",
    "    optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "    train_losses_no_es, val_losses_no_es = train_model(\n",
    "        model, train_loader, val_loader, criterion, optimizer, num_epochs=num_epochs, patience=None, device=device\n",
    "    )\n",
    "\n",
    "    # Test the model trained without early stopping\n",
    "    model.load_state_dict(torch.load('best_model_no_early_stopping.pth'))\n",
    "    test_loss_no_es, accuracy_no_es, all_labels_no_es, all_preds_no_es, _ = evaluate_model(model, test_loader, device)\n",
    "\n",
    "    # Plot training and validation losses for both cases\n",
    "    plt.figure(figsize=(10, 5))\n",
    "    plt.plot(train_losses_es, label='Train Loss (Early Stopping)')\n",
    "    plt.plot(val_losses_es, label='Validation Loss (Early Stopping)')\n",
    "    plt.plot(train_losses_no_es, label='Train Loss (No Early Stopping)')\n",
    "    plt.plot(val_losses_no_es, label='Validation Loss (No Early Stopping)')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.title('Train and Validation Loss Comparison')\n",
    "    plt.legend()\n",
    "    plt.savefig(os.path.join(output_dir, 'train_val_loss_comparison.png'))\n",
    "    plt.close()\n",
    "\n",
    "    # Print results for both cases\n",
    "    print(f\"Test Accuracy with Early Stopping: {accuracy_es * 100:.2f}%\")\n",
    "    print(f\"Test Accuracy without Early Stopping: {accuracy_no_es * 100:.2f}%\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ],
   "id": "e88d30db0f4a17bb",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training with early stopping:\n",
      "Epoch [1/50], Train Loss: 1.480955, Val Loss: 0.761919\n",
      "Epoch [2/50], Train Loss: 0.604997, Val Loss: 0.519917\n",
      "Epoch [3/50], Train Loss: 0.449691, Val Loss: 0.455123\n",
      "Epoch [4/50], Train Loss: 0.376425, Val Loss: 0.396634\n",
      "Epoch [5/50], Train Loss: 0.328962, Val Loss: 0.357455\n",
      "Epoch [6/50], Train Loss: 0.294710, Val Loss: 0.343161\n",
      "Epoch [7/50], Train Loss: 0.268193, Val Loss: 0.331345\n",
      "Epoch [8/50], Train Loss: 0.245416, Val Loss: 0.325697\n",
      "Epoch [9/50], Train Loss: 0.225722, Val Loss: 0.319676\n",
      "Epoch [10/50], Train Loss: 0.206910, Val Loss: 0.315899\n",
      "Epoch [11/50], Train Loss: 0.188776, Val Loss: 0.336337\n",
      "Epoch [12/50], Train Loss: 0.172044, Val Loss: 0.346223\n",
      "Epoch [13/50], Train Loss: 0.158769, Val Loss: 0.321149\n",
      "Epoch [14/50], Train Loss: 0.143932, Val Loss: 0.339691\n",
      "Epoch [15/50], Train Loss: 0.125809, Val Loss: 0.352704\n",
      "Epoch [16/50], Train Loss: 0.114289, Val Loss: 0.359308\n",
      "Epoch [17/50], Train Loss: 0.099919, Val Loss: 0.401707\n",
      "Epoch [18/50], Train Loss: 0.087860, Val Loss: 0.424010\n",
      "Epoch [19/50], Train Loss: 0.079081, Val Loss: 0.435253\n",
      "Epoch [20/50], Train Loss: 0.067436, Val Loss: 0.471196\n",
      "Epoch [21/50], Train Loss: 0.060311, Val Loss: 0.479669\n",
      "Epoch [22/50], Train Loss: 0.054585, Val Loss: 0.541274\n",
      "Epoch [23/50], Train Loss: 0.049119, Val Loss: 0.604477\n",
      "Epoch [24/50], Train Loss: 0.044690, Val Loss: 0.551408\n",
      "Epoch [25/50], Train Loss: 0.040776, Val Loss: 0.588067\n",
      "Epoch [26/50], Train Loss: 0.038383, Val Loss: 0.618025\n",
      "Epoch [27/50], Train Loss: 0.035265, Val Loss: 0.641862\n",
      "Epoch [28/50], Train Loss: 0.035020, Val Loss: 0.643382\n",
      "Epoch [29/50], Train Loss: 0.031092, Val Loss: 0.680427\n",
      "Epoch [30/50], Train Loss: 0.031957, Val Loss: 0.654446\n",
      "Epoch [31/50], Train Loss: 0.029320, Val Loss: 0.718591\n",
      "Epoch [32/50], Train Loss: 0.028628, Val Loss: 0.743157\n",
      "Epoch [33/50], Train Loss: 0.027571, Val Loss: 0.708313\n",
      "Epoch [34/50], Train Loss: 0.024869, Val Loss: 0.761395\n",
      "Epoch [35/50], Train Loss: 0.023544, Val Loss: 0.785116\n",
      "Epoch [36/50], Train Loss: 0.026383, Val Loss: 0.772237\n",
      "Epoch [37/50], Train Loss: 0.018731, Val Loss: 0.779605\n",
      "Epoch [38/50], Train Loss: 0.025757, Val Loss: 0.784525\n",
      "Early stopping triggered\n",
      "\n",
      "Training without early stopping:\n",
      "Epoch [1/50], Train Loss: 1.760273, Val Loss: 0.881965\n",
      "Epoch [2/50], Train Loss: 0.700532, Val Loss: 0.581070\n",
      "Epoch [3/50], Train Loss: 0.502002, Val Loss: 0.461834\n",
      "Epoch [4/50], Train Loss: 0.413909, Val Loss: 0.401444\n",
      "Epoch [5/50], Train Loss: 0.361972, Val Loss: 0.378969\n",
      "Epoch [6/50], Train Loss: 0.322468, Val Loss: 0.344151\n",
      "Epoch [7/50], Train Loss: 0.292528, Val Loss: 0.346461\n",
      "Epoch [8/50], Train Loss: 0.267618, Val Loss: 0.312789\n",
      "Epoch [9/50], Train Loss: 0.245766, Val Loss: 0.319260\n",
      "Epoch [10/50], Train Loss: 0.227749, Val Loss: 0.318196\n",
      "Epoch [11/50], Train Loss: 0.208473, Val Loss: 0.332024\n",
      "Epoch [12/50], Train Loss: 0.193410, Val Loss: 0.311025\n",
      "Epoch [13/50], Train Loss: 0.177180, Val Loss: 0.313855\n",
      "Epoch [14/50], Train Loss: 0.160914, Val Loss: 0.338402\n",
      "Epoch [15/50], Train Loss: 0.145843, Val Loss: 0.320296\n",
      "Epoch [16/50], Train Loss: 0.131961, Val Loss: 0.347220\n",
      "Epoch [17/50], Train Loss: 0.114692, Val Loss: 0.362572\n",
      "Epoch [18/50], Train Loss: 0.104820, Val Loss: 0.372838\n",
      "Epoch [19/50], Train Loss: 0.090651, Val Loss: 0.405193\n",
      "Epoch [20/50], Train Loss: 0.084787, Val Loss: 0.423087\n",
      "Epoch [21/50], Train Loss: 0.075331, Val Loss: 0.421902\n",
      "Epoch [22/50], Train Loss: 0.063403, Val Loss: 0.499344\n",
      "Epoch [23/50], Train Loss: 0.057105, Val Loss: 0.514651\n",
      "Epoch [24/50], Train Loss: 0.052234, Val Loss: 0.529853\n",
      "Epoch [25/50], Train Loss: 0.045408, Val Loss: 0.584271\n",
      "Epoch [26/50], Train Loss: 0.038450, Val Loss: 0.564201\n",
      "Epoch [27/50], Train Loss: 0.045771, Val Loss: 0.606529\n",
      "Epoch [28/50], Train Loss: 0.037284, Val Loss: 0.607537\n",
      "Epoch [29/50], Train Loss: 0.039391, Val Loss: 0.593132\n",
      "Epoch [30/50], Train Loss: 0.030130, Val Loss: 0.677361\n",
      "Epoch [31/50], Train Loss: 0.031769, Val Loss: 0.684574\n",
      "Epoch [32/50], Train Loss: 0.033027, Val Loss: 0.661246\n",
      "Epoch [33/50], Train Loss: 0.026956, Val Loss: 0.662973\n",
      "Epoch [34/50], Train Loss: 0.023692, Val Loss: 0.696937\n",
      "Epoch [35/50], Train Loss: 0.028846, Val Loss: 0.717781\n",
      "Epoch [36/50], Train Loss: 0.028361, Val Loss: 0.718385\n",
      "Epoch [37/50], Train Loss: 0.031520, Val Loss: 0.759838\n",
      "Epoch [38/50], Train Loss: 0.019650, Val Loss: 0.784961\n",
      "Epoch [39/50], Train Loss: 0.024547, Val Loss: 0.726987\n",
      "Epoch [40/50], Train Loss: 0.019613, Val Loss: 0.836054\n",
      "Epoch [41/50], Train Loss: 0.027920, Val Loss: 0.870800\n",
      "Epoch [42/50], Train Loss: 0.023534, Val Loss: 0.830978\n",
      "Epoch [43/50], Train Loss: 0.021366, Val Loss: 0.853995\n",
      "Epoch [44/50], Train Loss: 0.021627, Val Loss: 0.791999\n",
      "Epoch [45/50], Train Loss: 0.025277, Val Loss: 0.794751\n",
      "Epoch [46/50], Train Loss: 0.016891, Val Loss: 0.827533\n",
      "Epoch [47/50], Train Loss: 0.025475, Val Loss: 0.867181\n",
      "Epoch [48/50], Train Loss: 0.023250, Val Loss: 0.812327\n",
      "Epoch [49/50], Train Loss: 0.016165, Val Loss: 0.837578\n",
      "Epoch [50/50], Train Loss: 0.017541, Val Loss: 0.862370\n",
      "Test Accuracy with Early Stopping: 90.41%\n",
      "Test Accuracy without Early Stopping: 89.79%\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-08T19:11:38.637290Z",
     "start_time": "2024-11-08T18:30:20.904374Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.metrics import roc_auc_score, roc_curve, confusion_matrix\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "# SVHN 数据模块类，接受数据增强参数\n",
    "class SVHNDataModule:\n",
    "    def __init__(self, augmentation=None):\n",
    "        if augmentation is None:\n",
    "            augmentation = []\n",
    "\n",
    "        # 基础数据处理\n",
    "        transform_list = [transforms.ToTensor(),\n",
    "                          transforms.Normalize(mean=(0.4377, 0.4438, 0.4728), std=(0.1980, 0.2010, 0.1970))]\n",
    "\n",
    "        # 应用数据增强\n",
    "        transform_list = augmentation + transform_list\n",
    "        self.transform = transforms.Compose(transform_list)\n",
    "\n",
    "    def load_data(self, batch_size):\n",
    "        train_dataset = datasets.SVHN(root='./data', split='train', download=True, transform=self.transform)\n",
    "        test_dataset = datasets.SVHN(root='./data', split='test', download=True, transform=self.transform)\n",
    "\n",
    "        train_size = int(0.8 * len(train_dataset))\n",
    "        val_size = len(train_dataset) - train_size\n",
    "        train_subset, val_subset = torch.utils.data.random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "        train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True)\n",
    "        val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "        return train_loader, val_loader, test_loader\n",
    "\n",
    "\n",
    "# 定义小型 VGG 模型\n",
    "class SmallVGG(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SmallVGG, self).__init__()\n",
    "        self.conv_layers = nn.Sequential(\n",
    "            nn.Conv2d(3, 8, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(8, 16, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.fc_layers = nn.Sequential(\n",
    "            nn.Linear(32 * 4 * 4, 256),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv_layers(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc_layers(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "# 训练模型的函数\n",
    "def train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs=20, device='cuda'):\n",
    "    model.train()\n",
    "    device = torch.device(device)\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0\n",
    "        for images, labels in train_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "\n",
    "        train_loss = running_loss / len(train_loader)\n",
    "        train_losses.append(train_loss)\n",
    "\n",
    "        # 评估模型在验证集上的损失\n",
    "        val_loss = evaluate_model(model, val_loader, device, return_loss=True)\n",
    "        val_losses.append(val_loss)\n",
    "\n",
    "        print(f\"Epoch [{epoch + 1}/{num_epochs}], Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}\")\n",
    "\n",
    "    return train_losses, val_losses\n",
    "\n",
    "\n",
    "def evaluate_model(model, data_loader, device='cuda', return_loss=False):\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    all_labels = []\n",
    "    all_preds = []\n",
    "    all_probs = []\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for images, labels in data_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            running_loss += loss.item()\n",
    "\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            probs = torch.softmax(outputs, dim=1)\n",
    "\n",
    "            all_labels.extend(labels.cpu().numpy())\n",
    "            all_preds.extend(predicted.cpu().numpy())\n",
    "            all_probs.extend(probs.cpu().numpy())\n",
    "\n",
    "    avg_loss = running_loss / len(data_loader)\n",
    "    accuracy = np.mean(np.array(all_labels) == np.array(all_preds))\n",
    "\n",
    "    num_classes = 10\n",
    "    all_labels_onehot = np.eye(num_classes)[all_labels]\n",
    "    all_probs = np.array(all_probs)\n",
    "\n",
    "    micro_roc_auc = roc_auc_score(all_labels_onehot, all_probs, average='micro')\n",
    "    macro_roc_auc = roc_auc_score(all_labels_onehot, all_probs, average='macro')\n",
    "\n",
    "    precision = precision_score(all_labels, all_preds, average='weighted')\n",
    "    recall = recall_score(all_labels, all_preds, average='weighted')\n",
    "\n",
    "    print(f'Accuracy: {accuracy * 100:.2f}%, Micro ROC AUC: {micro_roc_auc:.4f}, Macro ROC AUC: {macro_roc_auc:.4f}, Precision: {precision:.4f}, Recall: {recall:.4f}')\n",
    "\n",
    "    if return_loss:\n",
    "        return avg_loss\n",
    "    else:\n",
    "        return avg_loss, accuracy, micro_roc_auc, macro_roc_auc, precision, recall, all_labels, all_probs\n",
    "\n",
    "\n",
    "def main():\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    batch_size = 128\n",
    "    learning_rate = 0.0005\n",
    "    num_epochs = 20\n",
    "\n",
    "    augmentations = {\n",
    "        'No Augmentation': [],\n",
    "        'Horizontal Flip': [transforms.RandomHorizontalFlip(p=0.5)],\n",
    "        'Random Rotation': [transforms.RandomRotation(15)],\n",
    "        'Random Crop and Flip': [transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip()],\n",
    "        'Color Jitter': [transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2, hue=0.1)],\n",
    "    }\n",
    "\n",
    "    results = {}  # Store metrics for each augmentation\n",
    "\n",
    "    for aug_name, aug_transforms in augmentations.items():\n",
    "        print(f\"\\nUsing augmentation: {aug_name}\")\n",
    "\n",
    "        data_module = SVHNDataModule(augmentation=aug_transforms)\n",
    "        train_loader, val_loader, test_loader = data_module.load_data(batch_size)\n",
    "\n",
    "        model = SmallVGG().to(device)\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        optimizer = optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "        train_losses, val_losses = train_model(model, train_loader, val_loader, criterion, optimizer,\n",
    "                                               num_epochs=num_epochs, device=device)\n",
    "\n",
    "        test_loss, accuracy, micro_roc_auc, macro_roc_auc, precision, recall, all_labels, all_probs = evaluate_model(model, test_loader, device)\n",
    "\n",
    "        results[aug_name] = {\n",
    "            \"accuracy\": accuracy,\n",
    "            \"precision\": precision,\n",
    "            \"recall\": recall,\n",
    "        }\n",
    "\n",
    "        plt.figure(figsize=(10, 5))\n",
    "        plt.plot(range(1, len(train_losses) + 1), train_losses, label='Train Loss')\n",
    "        plt.plot(range(1, len(val_losses) + 1), val_losses, label='Validation Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.title(f'Train and Validation Loss with {aug_name}')\n",
    "        plt.legend()\n",
    "        plt.savefig(f\"ex3/train_val_loss_{aug_name.replace(' ', '_')}.png\")\n",
    "        plt.close()\n",
    "\n",
    "        fpr, tpr, _ = roc_curve(np.eye(10)[all_labels].ravel(), np.array(all_probs).ravel())\n",
    "        plt.figure()\n",
    "        plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (area = {micro_roc_auc:.2f})')\n",
    "        plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "        plt.xlabel('False Positive Rate')\n",
    "        plt.ylabel('True Positive Rate')\n",
    "        plt.title(f'ROC Curve with {aug_name}')\n",
    "        plt.legend(loc=\"lower right\")\n",
    "        plt.savefig(f\"ex3/roc_curve_{aug_name.replace(' ', '_')}.png\")\n",
    "        plt.close()\n",
    "\n",
    "    # Plot and save the comparison bar chart\n",
    "    metrics = ['accuracy', 'precision', 'recall']\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    for i, metric in enumerate(metrics):\n",
    "        values = [results[aug][metric] for aug in augmentations.keys()]\n",
    "        plt.bar(np.arange(len(values)) + i*0.25, values, width=0.25, label=metric)\n",
    "\n",
    "    plt.xticks(np.arange(len(augmentations)) + 0.25, augmentations.keys(), rotation=45)\n",
    "    plt.ylabel('Scores')\n",
    "    plt.title('Comparison of Precision, Recall, and Accuracy using Different Augmentations')\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(\"ex3/comparison_bar_chart.png\")\n",
    "    plt.close()\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ],
   "id": "e9c4fa13f8c9e8f1",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Using augmentation: No Augmentation\n",
      "Accuracy: 74.56%, Micro ROC AUC: 0.9602, Macro ROC AUC: 0.9569, Precision: 0.7503, Recall: 0.7456\n",
      "Epoch [1/20], Train Loss: 1.629542, Val Loss: 0.830281\n",
      "Accuracy: 81.87%, Micro ROC AUC: 0.9798, Macro ROC AUC: 0.9797, Precision: 0.8291, Recall: 0.8187\n",
      "Epoch [2/20], Train Loss: 0.632709, Val Loss: 0.586146\n",
      "Accuracy: 86.36%, Micro ROC AUC: 0.9868, Macro ROC AUC: 0.9862, Precision: 0.8650, Recall: 0.8636\n",
      "Epoch [3/20], Train Loss: 0.467216, Val Loss: 0.456687\n",
      "Accuracy: 87.22%, Micro ROC AUC: 0.9889, Macro ROC AUC: 0.9889, Precision: 0.8764, Recall: 0.8722\n",
      "Epoch [4/20], Train Loss: 0.390242, Val Loss: 0.420422\n",
      "Accuracy: 89.16%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9907, Precision: 0.8921, Recall: 0.8916\n",
      "Epoch [5/20], Train Loss: 0.344308, Val Loss: 0.367772\n",
      "Accuracy: 89.72%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914, Precision: 0.8989, Recall: 0.8972\n",
      "Epoch [6/20], Train Loss: 0.305096, Val Loss: 0.351617\n",
      "Accuracy: 90.51%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9924, Precision: 0.9051, Recall: 0.9051\n",
      "Epoch [7/20], Train Loss: 0.277769, Val Loss: 0.321118\n",
      "Accuracy: 90.83%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9925, Precision: 0.9089, Recall: 0.9083\n",
      "Epoch [8/20], Train Loss: 0.253536, Val Loss: 0.322470\n",
      "Accuracy: 90.67%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928, Precision: 0.9079, Recall: 0.9067\n",
      "Epoch [9/20], Train Loss: 0.233654, Val Loss: 0.321775\n",
      "Accuracy: 90.45%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9927, Precision: 0.9068, Recall: 0.9045\n",
      "Epoch [10/20], Train Loss: 0.213492, Val Loss: 0.332971\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9929, Precision: 0.9098, Recall: 0.9096\n",
      "Epoch [11/20], Train Loss: 0.195946, Val Loss: 0.325479\n",
      "Accuracy: 90.90%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9927, Precision: 0.9100, Recall: 0.9090\n",
      "Epoch [12/20], Train Loss: 0.179518, Val Loss: 0.325028\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9926, Precision: 0.9057, Recall: 0.9035\n",
      "Epoch [13/20], Train Loss: 0.162851, Val Loss: 0.350972\n",
      "Accuracy: 90.81%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9931, Precision: 0.9102, Recall: 0.9081\n",
      "Epoch [14/20], Train Loss: 0.148393, Val Loss: 0.360003\n",
      "Accuracy: 90.47%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9927, Precision: 0.9065, Recall: 0.9047\n",
      "Epoch [15/20], Train Loss: 0.134913, Val Loss: 0.372376\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9928, Precision: 0.9092, Recall: 0.9087\n",
      "Epoch [16/20], Train Loss: 0.120791, Val Loss: 0.367093\n",
      "Accuracy: 90.76%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927, Precision: 0.9083, Recall: 0.9076\n",
      "Epoch [17/20], Train Loss: 0.109025, Val Loss: 0.417617\n",
      "Accuracy: 90.49%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9926, Precision: 0.9055, Recall: 0.9049\n",
      "Epoch [18/20], Train Loss: 0.099237, Val Loss: 0.401328\n",
      "Accuracy: 90.77%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9926, Precision: 0.9083, Recall: 0.9077\n",
      "Epoch [19/20], Train Loss: 0.088504, Val Loss: 0.417975\n",
      "Accuracy: 90.20%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9922, Precision: 0.9034, Recall: 0.9020\n",
      "Epoch [20/20], Train Loss: 0.075690, Val Loss: 0.448781\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9906, Precision: 0.8995, Recall: 0.8969\n",
      "\n",
      "Using augmentation: Horizontal Flip\n",
      "Accuracy: 49.71%, Micro ROC AUC: 0.8804, Macro ROC AUC: 0.8667, Precision: 0.5455, Recall: 0.4971\n",
      "Epoch [1/20], Train Loss: 2.099603, Val Loss: 1.435018\n",
      "Accuracy: 73.58%, Micro ROC AUC: 0.9617, Macro ROC AUC: 0.9569, Precision: 0.7392, Recall: 0.7358\n",
      "Epoch [2/20], Train Loss: 1.029758, Val Loss: 0.816854\n",
      "Accuracy: 79.34%, Micro ROC AUC: 0.9759, Macro ROC AUC: 0.9731, Precision: 0.7971, Recall: 0.7934\n",
      "Epoch [3/20], Train Loss: 0.710106, Val Loss: 0.640543\n",
      "Accuracy: 82.41%, Micro ROC AUC: 0.9812, Macro ROC AUC: 0.9794, Precision: 0.8268, Recall: 0.8241\n",
      "Epoch [4/20], Train Loss: 0.584357, Val Loss: 0.558899\n",
      "Accuracy: 84.89%, Micro ROC AUC: 0.9852, Macro ROC AUC: 0.9838, Precision: 0.8497, Recall: 0.8489\n",
      "Epoch [5/20], Train Loss: 0.509716, Val Loss: 0.490472\n",
      "Accuracy: 85.97%, Micro ROC AUC: 0.9869, Macro ROC AUC: 0.9858, Precision: 0.8604, Recall: 0.8597\n",
      "Epoch [6/20], Train Loss: 0.451822, Val Loss: 0.457714\n",
      "Accuracy: 86.75%, Micro ROC AUC: 0.9879, Macro ROC AUC: 0.9871, Precision: 0.8687, Recall: 0.8675\n",
      "Epoch [7/20], Train Loss: 0.412741, Val Loss: 0.441023\n",
      "Accuracy: 87.67%, Micro ROC AUC: 0.9891, Macro ROC AUC: 0.9886, Precision: 0.8787, Recall: 0.8767\n",
      "Epoch [8/20], Train Loss: 0.387527, Val Loss: 0.411416\n",
      "Accuracy: 88.42%, Micro ROC AUC: 0.9903, Macro ROC AUC: 0.9897, Precision: 0.8851, Recall: 0.8842\n",
      "Epoch [9/20], Train Loss: 0.361390, Val Loss: 0.387407\n",
      "Accuracy: 88.53%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9901, Precision: 0.8864, Recall: 0.8853\n",
      "Epoch [10/20], Train Loss: 0.339024, Val Loss: 0.382613\n",
      "Accuracy: 88.88%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9907, Precision: 0.8892, Recall: 0.8888\n",
      "Epoch [11/20], Train Loss: 0.323071, Val Loss: 0.369225\n",
      "Accuracy: 88.90%, Micro ROC AUC: 0.9911, Macro ROC AUC: 0.9906, Precision: 0.8894, Recall: 0.8890\n",
      "Epoch [12/20], Train Loss: 0.307929, Val Loss: 0.367802\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9911, Precision: 0.8941, Recall: 0.8922\n",
      "Epoch [13/20], Train Loss: 0.293959, Val Loss: 0.359512\n",
      "Accuracy: 89.69%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9915, Precision: 0.8983, Recall: 0.8969\n",
      "Epoch [14/20], Train Loss: 0.285930, Val Loss: 0.351537\n",
      "Accuracy: 89.73%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9917, Precision: 0.8978, Recall: 0.8973\n",
      "Epoch [15/20], Train Loss: 0.272891, Val Loss: 0.348045\n",
      "Accuracy: 89.62%, Micro ROC AUC: 0.9918, Macro ROC AUC: 0.9914, Precision: 0.8976, Recall: 0.8962\n",
      "Epoch [16/20], Train Loss: 0.262016, Val Loss: 0.352727\n",
      "Accuracy: 89.46%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9914, Precision: 0.8959, Recall: 0.8946\n",
      "Epoch [17/20], Train Loss: 0.253206, Val Loss: 0.372738\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919, Precision: 0.9005, Recall: 0.9004\n",
      "Epoch [18/20], Train Loss: 0.239750, Val Loss: 0.345737\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919, Precision: 0.9020, Recall: 0.9007\n",
      "Epoch [19/20], Train Loss: 0.231988, Val Loss: 0.348022\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921, Precision: 0.9025, Recall: 0.9018\n",
      "Epoch [20/20], Train Loss: 0.225617, Val Loss: 0.351163\n",
      "Accuracy: 89.79%, Micro ROC AUC: 0.9915, Macro ROC AUC: 0.9910, Precision: 0.8984, Recall: 0.8979\n",
      "\n",
      "Using augmentation: Random Rotation\n",
      "Accuracy: 70.03%, Micro ROC AUC: 0.9493, Macro ROC AUC: 0.9464, Precision: 0.7068, Recall: 0.7003\n",
      "Epoch [1/20], Train Loss: 1.864431, Val Loss: 0.936085\n",
      "Accuracy: 81.72%, Micro ROC AUC: 0.9790, Macro ROC AUC: 0.9774, Precision: 0.8193, Recall: 0.8172\n",
      "Epoch [2/20], Train Loss: 0.731724, Val Loss: 0.588670\n",
      "Accuracy: 84.07%, Micro ROC AUC: 0.9848, Macro ROC AUC: 0.9839, Precision: 0.8436, Recall: 0.8407\n",
      "Epoch [3/20], Train Loss: 0.535560, Val Loss: 0.497066\n",
      "Accuracy: 86.53%, Micro ROC AUC: 0.9882, Macro ROC AUC: 0.9874, Precision: 0.8668, Recall: 0.8653\n",
      "Epoch [4/20], Train Loss: 0.451849, Val Loss: 0.433807\n",
      "Accuracy: 87.35%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9894, Precision: 0.8771, Recall: 0.8735\n",
      "Epoch [5/20], Train Loss: 0.402140, Val Loss: 0.402502\n",
      "Accuracy: 88.75%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9906, Precision: 0.8876, Recall: 0.8875\n",
      "Epoch [6/20], Train Loss: 0.370402, Val Loss: 0.368744\n",
      "Accuracy: 89.19%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9910, Precision: 0.8927, Recall: 0.8919\n",
      "Epoch [7/20], Train Loss: 0.343557, Val Loss: 0.360469\n",
      "Accuracy: 89.61%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920, Precision: 0.8972, Recall: 0.8961\n",
      "Epoch [8/20], Train Loss: 0.323530, Val Loss: 0.342421\n",
      "Accuracy: 90.07%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9925, Precision: 0.9017, Recall: 0.9007\n",
      "Epoch [9/20], Train Loss: 0.307617, Val Loss: 0.328090\n",
      "Accuracy: 90.25%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9927, Precision: 0.9034, Recall: 0.9025\n",
      "Epoch [10/20], Train Loss: 0.291224, Val Loss: 0.324447\n",
      "Accuracy: 90.59%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9929, Precision: 0.9062, Recall: 0.9059\n",
      "Epoch [11/20], Train Loss: 0.280502, Val Loss: 0.313133\n",
      "Accuracy: 90.79%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9931, Precision: 0.9080, Recall: 0.9079\n",
      "Epoch [12/20], Train Loss: 0.269354, Val Loss: 0.311024\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9935, Macro ROC AUC: 0.9931, Precision: 0.9078, Recall: 0.9070\n",
      "Epoch [13/20], Train Loss: 0.259616, Val Loss: 0.311616\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9933, Precision: 0.9104, Recall: 0.9096\n",
      "Epoch [14/20], Train Loss: 0.250668, Val Loss: 0.304544\n",
      "Accuracy: 91.00%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9935, Precision: 0.9101, Recall: 0.9100\n",
      "Epoch [15/20], Train Loss: 0.239984, Val Loss: 0.299154\n",
      "Accuracy: 90.96%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9937, Precision: 0.9100, Recall: 0.9096\n",
      "Epoch [16/20], Train Loss: 0.230432, Val Loss: 0.304178\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9933, Precision: 0.9092, Recall: 0.9066\n",
      "Epoch [17/20], Train Loss: 0.222682, Val Loss: 0.319723\n",
      "Accuracy: 91.24%, Micro ROC AUC: 0.9940, Macro ROC AUC: 0.9936, Precision: 0.9128, Recall: 0.9124\n",
      "Epoch [18/20], Train Loss: 0.213848, Val Loss: 0.296950\n",
      "Accuracy: 91.37%, Micro ROC AUC: 0.9941, Macro ROC AUC: 0.9937, Precision: 0.9142, Recall: 0.9137\n",
      "Epoch [19/20], Train Loss: 0.207115, Val Loss: 0.297523\n",
      "Accuracy: 90.98%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9934, Precision: 0.9111, Recall: 0.9098\n",
      "Epoch [20/20], Train Loss: 0.202136, Val Loss: 0.307546\n",
      "Accuracy: 91.18%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9931, Precision: 0.9140, Recall: 0.9118\n",
      "\n",
      "Using augmentation: Random Crop and Flip\n",
      "Accuracy: 42.54%, Micro ROC AUC: 0.8336, Macro ROC AUC: 0.8350, Precision: 0.4552, Recall: 0.4254\n",
      "Epoch [1/20], Train Loss: 2.023989, Val Loss: 1.645159\n",
      "Accuracy: 61.75%, Micro ROC AUC: 0.9275, Macro ROC AUC: 0.9212, Precision: 0.6325, Recall: 0.6175\n",
      "Epoch [2/20], Train Loss: 1.298127, Val Loss: 1.126362\n",
      "Accuracy: 73.63%, Micro ROC AUC: 0.9621, Macro ROC AUC: 0.9579, Precision: 0.7357, Recall: 0.7363\n",
      "Epoch [3/20], Train Loss: 0.942516, Val Loss: 0.810451\n",
      "Accuracy: 77.14%, Micro ROC AUC: 0.9703, Macro ROC AUC: 0.9690, Precision: 0.7796, Recall: 0.7714\n",
      "Epoch [4/20], Train Loss: 0.740916, Val Loss: 0.713896\n",
      "Accuracy: 80.49%, Micro ROC AUC: 0.9769, Macro ROC AUC: 0.9755, Precision: 0.8086, Recall: 0.8049\n",
      "Epoch [5/20], Train Loss: 0.636305, Val Loss: 0.622397\n",
      "Accuracy: 82.04%, Micro ROC AUC: 0.9807, Macro ROC AUC: 0.9793, Precision: 0.8235, Recall: 0.8204\n",
      "Epoch [6/20], Train Loss: 0.575298, Val Loss: 0.566500\n",
      "Accuracy: 83.47%, Micro ROC AUC: 0.9828, Macro ROC AUC: 0.9813, Precision: 0.8362, Recall: 0.8347\n",
      "Epoch [7/20], Train Loss: 0.527678, Val Loss: 0.531762\n",
      "Accuracy: 84.50%, Micro ROC AUC: 0.9849, Macro ROC AUC: 0.9839, Precision: 0.8461, Recall: 0.8450\n",
      "Epoch [8/20], Train Loss: 0.490370, Val Loss: 0.497187\n",
      "Accuracy: 85.75%, Micro ROC AUC: 0.9859, Macro ROC AUC: 0.9849, Precision: 0.8588, Recall: 0.8575\n",
      "Epoch [9/20], Train Loss: 0.467389, Val Loss: 0.471489\n",
      "Accuracy: 85.54%, Micro ROC AUC: 0.9860, Macro ROC AUC: 0.9852, Precision: 0.8593, Recall: 0.8554\n",
      "Epoch [10/20], Train Loss: 0.439731, Val Loss: 0.471161\n",
      "Accuracy: 85.88%, Micro ROC AUC: 0.9868, Macro ROC AUC: 0.9864, Precision: 0.8640, Recall: 0.8588\n",
      "Epoch [11/20], Train Loss: 0.425663, Val Loss: 0.459016\n",
      "Accuracy: 86.76%, Micro ROC AUC: 0.9884, Macro ROC AUC: 0.9876, Precision: 0.8682, Recall: 0.8676\n",
      "Epoch [12/20], Train Loss: 0.405232, Val Loss: 0.428607\n",
      "Accuracy: 86.42%, Micro ROC AUC: 0.9877, Macro ROC AUC: 0.9873, Precision: 0.8682, Recall: 0.8642\n",
      "Epoch [13/20], Train Loss: 0.399538, Val Loss: 0.442318\n",
      "Accuracy: 87.81%, Micro ROC AUC: 0.9892, Macro ROC AUC: 0.9887, Precision: 0.8791, Recall: 0.8781\n",
      "Epoch [14/20], Train Loss: 0.382385, Val Loss: 0.417346\n",
      "Accuracy: 87.60%, Micro ROC AUC: 0.9893, Macro ROC AUC: 0.9888, Precision: 0.8777, Recall: 0.8760\n",
      "Epoch [15/20], Train Loss: 0.371748, Val Loss: 0.411219\n",
      "Accuracy: 88.47%, Micro ROC AUC: 0.9898, Macro ROC AUC: 0.9892, Precision: 0.8850, Recall: 0.8847\n",
      "Epoch [16/20], Train Loss: 0.358550, Val Loss: 0.395798\n",
      "Accuracy: 88.10%, Micro ROC AUC: 0.9899, Macro ROC AUC: 0.9896, Precision: 0.8837, Recall: 0.8810\n",
      "Epoch [17/20], Train Loss: 0.354834, Val Loss: 0.394409\n",
      "Accuracy: 88.68%, Micro ROC AUC: 0.9905, Macro ROC AUC: 0.9901, Precision: 0.8876, Recall: 0.8868\n",
      "Epoch [18/20], Train Loss: 0.345032, Val Loss: 0.380129\n",
      "Accuracy: 89.16%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9901, Precision: 0.8925, Recall: 0.8916\n",
      "Epoch [19/20], Train Loss: 0.334961, Val Loss: 0.380661\n",
      "Accuracy: 89.22%, Micro ROC AUC: 0.9910, Macro ROC AUC: 0.9907, Precision: 0.8932, Recall: 0.8922\n",
      "Epoch [20/20], Train Loss: 0.329412, Val Loss: 0.371317\n",
      "Accuracy: 89.48%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914, Precision: 0.8956, Recall: 0.8948\n",
      "\n",
      "Using augmentation: Color Jitter\n",
      "Accuracy: 74.65%, Micro ROC AUC: 0.9615, Macro ROC AUC: 0.9586, Precision: 0.7482, Recall: 0.7465\n",
      "Epoch [1/20], Train Loss: 1.671428, Val Loss: 0.818730\n",
      "Accuracy: 83.40%, Micro ROC AUC: 0.9817, Macro ROC AUC: 0.9804, Precision: 0.8379, Recall: 0.8340\n",
      "Epoch [2/20], Train Loss: 0.628014, Val Loss: 0.547006\n",
      "Accuracy: 86.21%, Micro ROC AUC: 0.9870, Macro ROC AUC: 0.9855, Precision: 0.8627, Recall: 0.8621\n",
      "Epoch [3/20], Train Loss: 0.484696, Val Loss: 0.453757\n",
      "Accuracy: 87.37%, Micro ROC AUC: 0.9886, Macro ROC AUC: 0.9878, Precision: 0.8763, Recall: 0.8737\n",
      "Epoch [4/20], Train Loss: 0.408706, Val Loss: 0.421992\n",
      "Accuracy: 88.76%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9898, Precision: 0.8879, Recall: 0.8876\n",
      "Epoch [5/20], Train Loss: 0.357405, Val Loss: 0.382773\n",
      "Accuracy: 89.50%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9910, Precision: 0.8957, Recall: 0.8950\n",
      "Epoch [6/20], Train Loss: 0.323561, Val Loss: 0.355652\n",
      "Accuracy: 89.54%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9915, Precision: 0.8960, Recall: 0.8954\n",
      "Epoch [7/20], Train Loss: 0.294760, Val Loss: 0.349825\n",
      "Accuracy: 89.91%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920, Precision: 0.9009, Recall: 0.8991\n",
      "Epoch [8/20], Train Loss: 0.268518, Val Loss: 0.342258\n",
      "Accuracy: 90.34%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9923, Precision: 0.9038, Recall: 0.9034\n",
      "Epoch [9/20], Train Loss: 0.246819, Val Loss: 0.328273\n",
      "Accuracy: 90.80%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9926, Precision: 0.9090, Recall: 0.9080\n",
      "Epoch [10/20], Train Loss: 0.229199, Val Loss: 0.329141\n",
      "Accuracy: 90.39%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9926, Precision: 0.9049, Recall: 0.9039\n",
      "Epoch [11/20], Train Loss: 0.209196, Val Loss: 0.328175\n",
      "Accuracy: 90.61%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9927, Precision: 0.9074, Recall: 0.9061\n",
      "Epoch [12/20], Train Loss: 0.196241, Val Loss: 0.342050\n",
      "Accuracy: 90.84%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9926, Precision: 0.9085, Recall: 0.9084\n",
      "Epoch [13/20], Train Loss: 0.180336, Val Loss: 0.346950\n",
      "Accuracy: 90.71%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9928, Precision: 0.9086, Recall: 0.9071\n",
      "Epoch [14/20], Train Loss: 0.164214, Val Loss: 0.340344\n",
      "Accuracy: 90.78%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9926, Precision: 0.9085, Recall: 0.9078\n",
      "Epoch [15/20], Train Loss: 0.148349, Val Loss: 0.355964\n",
      "Accuracy: 90.53%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9924, Precision: 0.9061, Recall: 0.9053\n",
      "Epoch [16/20], Train Loss: 0.132593, Val Loss: 0.380916\n",
      "Accuracy: 90.69%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9923, Precision: 0.9078, Recall: 0.9069\n",
      "Epoch [17/20], Train Loss: 0.122957, Val Loss: 0.375052\n",
      "Accuracy: 90.42%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9921, Precision: 0.9050, Recall: 0.9042\n",
      "Epoch [18/20], Train Loss: 0.110386, Val Loss: 0.382024\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9922, Precision: 0.9064, Recall: 0.9057\n",
      "Epoch [19/20], Train Loss: 0.099766, Val Loss: 0.437299\n",
      "Accuracy: 90.67%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919, Precision: 0.9075, Recall: 0.9067\n",
      "Epoch [20/20], Train Loss: 0.090059, Val Loss: 0.453450\n",
      "Accuracy: 90.57%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913, Precision: 0.9071, Recall: 0.9057\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-08T19:34:00.716089Z",
     "start_time": "2024-11-08T19:15:02.218845Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.metrics import roc_auc_score, confusion_matrix, roc_curve\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "from sklearn.metrics import precision_score, recall_score\n",
    "# 创建目录，如果不存在则自动创建\n",
    "os.makedirs(\"ex4\", exist_ok=True)\n",
    "\n",
    "# SVHN 数据模块类\n",
    "class SVHNDataModule:\n",
    "    def __init__(self):\n",
    "        self.transform = transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=(0.4377, 0.4438, 0.4728), std=(0.1980, 0.2010, 0.1970))\n",
    "        ])\n",
    "\n",
    "    def load_data(self, batch_size):\n",
    "        train_dataset = datasets.SVHN(root='./data', split='train', download=True, transform=self.transform)\n",
    "        test_dataset = datasets.SVHN(root='./data', split='test', download=True, transform=self.transform)\n",
    "\n",
    "        train_size = int(0.8 * len(train_dataset))\n",
    "        val_size = len(train_dataset) - train_size\n",
    "        train_subset, val_subset = torch.utils.data.random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "        train_loader = DataLoader(train_subset, batch_size=batch_size, shuffle=True)\n",
    "        val_loader = DataLoader(val_subset, batch_size=batch_size, shuffle=False)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "        return train_loader, val_loader, test_loader\n",
    "\n",
    "# 定义小型 VGG 模型\n",
    "class SmallVGG(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(SmallVGG, self).__init__()\n",
    "        self.conv_layers = nn.Sequential(\n",
    "            nn.Conv2d(3, 8, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(8, 16, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2)\n",
    "        )\n",
    "        self.fc_layers = nn.Sequential(\n",
    "            nn.Linear(32 * 4 * 4, 256),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(256, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv_layers(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc_layers(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "# 修改训练模型函数，移除早停法的相关代码\n",
    "def train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs=50, device='cuda'):\n",
    "    model.train()\n",
    "    device = torch.device(device)\n",
    "    train_losses = []\n",
    "    val_losses = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0\n",
    "        for images, labels in train_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "\n",
    "        train_loss = running_loss / len(train_loader)\n",
    "        train_losses.append(train_loss)\n",
    "\n",
    "        # 评估模型在验证集上的损失\n",
    "        val_loss = evaluate_model(model, val_loader, device, return_loss=True)\n",
    "        val_losses.append(val_loss)\n",
    "\n",
    "        print(f\"Epoch [{epoch + 1}/{num_epochs}], Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}\")\n",
    "\n",
    "    return train_losses, val_losses\n",
    "def evaluate_model(model, data_loader, device='cuda', return_loss=False):\n",
    "    model.eval()\n",
    "    running_loss = 0.0\n",
    "    all_labels = []\n",
    "    all_preds = []\n",
    "    all_probs = []\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for images, labels in data_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            running_loss += loss.item()\n",
    "\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            probs = torch.softmax(outputs, dim=1)\n",
    "\n",
    "            all_labels.extend(labels.cpu().numpy())\n",
    "            all_preds.extend(predicted.cpu().numpy())\n",
    "            all_probs.extend(probs.cpu().numpy())\n",
    "\n",
    "    avg_loss = running_loss / len(data_loader)\n",
    "    accuracy = np.mean(np.array(all_labels) == np.array(all_preds))\n",
    "\n",
    "    num_classes = 10\n",
    "    all_labels_onehot = np.eye(num_classes)[all_labels]\n",
    "    all_probs = np.array(all_probs)\n",
    "\n",
    "    micro_roc_auc = roc_auc_score(all_labels_onehot, all_probs, average='micro')\n",
    "    macro_roc_auc = roc_auc_score(all_labels_onehot, all_probs, average='macro')\n",
    "\n",
    "    print(f'Accuracy: {accuracy * 100:.2f}%, Micro ROC AUC: {micro_roc_auc:.4f}, Macro ROC AUC: {macro_roc_auc:.4f}')\n",
    "\n",
    "    if return_loss:\n",
    "        return avg_loss\n",
    "    else:\n",
    "        return avg_loss, accuracy, micro_roc_auc, macro_roc_auc, all_labels, all_preds, all_probs\n",
    "\n",
    "# 主函数，增加保存 Precision、Recall 和 Accuracy 柱状图的代码\n",
    "def main():\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    batch_size = 128\n",
    "    learning_rate = 0.0005\n",
    "    num_epochs = 30\n",
    "\n",
    "    # 数据加载\n",
    "    data_module = SVHNDataModule()\n",
    "    train_loader, val_loader, test_loader = data_module.load_data(batch_size)\n",
    "\n",
    "    # 定义优化器列表\n",
    "    optimizers = {\n",
    "        'SGD': optim.SGD,\n",
    "        'Adam': optim.Adam,\n",
    "        'RMSprop': optim.RMSprop,\n",
    "        'AdamW': optim.AdamW\n",
    "    }\n",
    "\n",
    "    # 存储每个优化器的评估指标\n",
    "    results = {}\n",
    "\n",
    "    # 遍历不同的优化器\n",
    "    for opt_name, opt_class in optimizers.items():\n",
    "        print(f\"\\nUsing optimizer: {opt_name}\")\n",
    "\n",
    "        # 初始化模型、损失函数和优化器\n",
    "        model = SmallVGG().to(device)\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        optimizer = opt_class(model.parameters(), lr=learning_rate)\n",
    "\n",
    "        # 训练模型\n",
    "        train_losses, val_losses = train_model(model, train_loader, val_loader, criterion, optimizer,\n",
    "                                               num_epochs=num_epochs, device=device)\n",
    "\n",
    "        # 评估模型在测试集上的表现\n",
    "        test_loss, accuracy, micro_roc_auc, macro_roc_auc, all_labels, all_preds, all_probs = evaluate_model(model, test_loader, device)\n",
    "\n",
    "        # 计算 Precision 和 Recall\n",
    "        all_labels = np.array(all_labels)\n",
    "        all_preds = np.array(all_preds)\n",
    "\n",
    "        precision = precision_score(all_labels, all_preds, average='weighted', labels=np.arange(10))\n",
    "        recall = recall_score(all_labels, all_preds, average='weighted', labels=np.arange(10))\n",
    "\n",
    "        # 保存当前优化器的指标\n",
    "        results[opt_name] = {\n",
    "            \"accuracy\": accuracy,\n",
    "            \"precision\": precision,\n",
    "            \"recall\": recall\n",
    "        }\n",
    "\n",
    "        # 绘制并保存 Train Loss 和 Validation Loss\n",
    "        plt.figure(figsize=(10, 5))\n",
    "        plt.plot(range(1, len(train_losses) + 1), train_losses, label='Train Loss')\n",
    "        plt.plot(range(1, len(val_losses) + 1), val_losses, label='Validation Loss')\n",
    "        plt.xlabel('Epoch')\n",
    "        plt.ylabel('Loss')\n",
    "        plt.title(f'Train and Validation Loss with {opt_name}')\n",
    "        plt.legend()\n",
    "        plt.savefig(f\"ex4/train_val_loss_{opt_name}.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # 绘制并保存 ROC 曲线\n",
    "        fpr, tpr, _ = roc_curve(np.eye(10)[all_labels].ravel(), np.array(all_probs).ravel())\n",
    "        plt.figure()\n",
    "        plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (area = {micro_roc_auc:.2f})')\n",
    "        plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "        plt.xlabel('False Positive Rate')\n",
    "        plt.ylabel('True Positive Rate')\n",
    "        plt.title(f'ROC Curve with {opt_name}')\n",
    "        plt.legend(loc=\"lower right\")\n",
    "        plt.savefig(f\"ex4/roc_curve_{opt_name}.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # 绘制并保存混淆矩阵\n",
    "        cm = confusion_matrix(all_labels, all_preds)\n",
    "        plt.figure(figsize=(10, 8))\n",
    "        sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "        plt.xlabel('Predicted')\n",
    "        plt.ylabel('True')\n",
    "        plt.title(f'Confusion Matrix with {opt_name}')\n",
    "        plt.savefig(f\"ex4/confusion_matrix_{opt_name}.png\")\n",
    "        plt.close()\n",
    "\n",
    "    # 绘制并保存 Precision、Recall 和 Accuracy 的柱状对比图\n",
    "    metrics = ['accuracy', 'precision', 'recall']\n",
    "    plt.figure(figsize=(12, 8))\n",
    "    for i, metric in enumerate(metrics):\n",
    "        values = [results[opt][metric] for opt in optimizers.keys()]\n",
    "        plt.bar(np.arange(len(values)) + i*0.25, values, width=0.25, label=metric)\n",
    "\n",
    "    plt.xticks(np.arange(len(optimizers)) + 0.25, optimizers.keys(), rotation=45)\n",
    "    plt.ylabel('Scores')\n",
    "    plt.title('Comparison of Precision, Recall, and Accuracy using Different Optimizers')\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(\"ex4/comparison_bar_chart.png\")\n",
    "    plt.close()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ],
   "id": "5a3febc5d5cd12ba",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Using optimizer: SGD\n",
      "Accuracy: 9.45%, Micro ROC AUC: 0.5236, Macro ROC AUC: 0.4955\n",
      "Epoch [1/30], Train Loss: 2.301729, Val Loss: 2.299449\n",
      "Accuracy: 9.45%, Micro ROC AUC: 0.5855, Macro ROC AUC: 0.4955\n",
      "Epoch [2/30], Train Loss: 2.297291, Val Loss: 2.295136\n",
      "Accuracy: 9.45%, Micro ROC AUC: 0.5877, Macro ROC AUC: 0.4976\n",
      "Epoch [3/30], Train Loss: 2.293133, Val Loss: 2.291092\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.5978, Macro ROC AUC: 0.4972\n",
      "Epoch [4/30], Train Loss: 2.289220, Val Loss: 2.287282\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6036, Macro ROC AUC: 0.4967\n",
      "Epoch [5/30], Train Loss: 2.285550, Val Loss: 2.283706\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6034, Macro ROC AUC: 0.4957\n",
      "Epoch [6/30], Train Loss: 2.282106, Val Loss: 2.280359\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6034, Macro ROC AUC: 0.4958\n",
      "Epoch [7/30], Train Loss: 2.278872, Val Loss: 2.277198\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6052, Macro ROC AUC: 0.4958\n",
      "Epoch [8/30], Train Loss: 2.275810, Val Loss: 2.274223\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6079, Macro ROC AUC: 0.4956\n",
      "Epoch [9/30], Train Loss: 2.272951, Val Loss: 2.271415\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6080, Macro ROC AUC: 0.4976\n",
      "Epoch [10/30], Train Loss: 2.270237, Val Loss: 2.268772\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6080, Macro ROC AUC: 0.4976\n",
      "Epoch [11/30], Train Loss: 2.267688, Val Loss: 2.266282\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6081, Macro ROC AUC: 0.4982\n",
      "Epoch [12/30], Train Loss: 2.265286, Val Loss: 2.263928\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6083, Macro ROC AUC: 0.4983\n",
      "Epoch [13/30], Train Loss: 2.263015, Val Loss: 2.261705\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6092, Macro ROC AUC: 0.4994\n",
      "Epoch [14/30], Train Loss: 2.260868, Val Loss: 2.259610\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6115, Macro ROC AUC: 0.5006\n",
      "Epoch [15/30], Train Loss: 2.258871, Val Loss: 2.257641\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6131, Macro ROC AUC: 0.5015\n",
      "Epoch [16/30], Train Loss: 2.256972, Val Loss: 2.255794\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6130, Macro ROC AUC: 0.5009\n",
      "Epoch [17/30], Train Loss: 2.255198, Val Loss: 2.254059\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6130, Macro ROC AUC: 0.5004\n",
      "Epoch [18/30], Train Loss: 2.253542, Val Loss: 2.252427\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6129, Macro ROC AUC: 0.5004\n",
      "Epoch [19/30], Train Loss: 2.251989, Val Loss: 2.250885\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6131, Macro ROC AUC: 0.5005\n",
      "Epoch [20/30], Train Loss: 2.250502, Val Loss: 2.249442\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6131, Macro ROC AUC: 0.5004\n",
      "Epoch [21/30], Train Loss: 2.249126, Val Loss: 2.248089\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6131, Macro ROC AUC: 0.5002\n",
      "Epoch [22/30], Train Loss: 2.247852, Val Loss: 2.246832\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6130, Macro ROC AUC: 0.4999\n",
      "Epoch [23/30], Train Loss: 2.246681, Val Loss: 2.245670\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6130, Macro ROC AUC: 0.4995\n",
      "Epoch [24/30], Train Loss: 2.245570, Val Loss: 2.244594\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6137, Macro ROC AUC: 0.4995\n",
      "Epoch [25/30], Train Loss: 2.244560, Val Loss: 2.243607\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6138, Macro ROC AUC: 0.4994\n",
      "Epoch [26/30], Train Loss: 2.243668, Val Loss: 2.242709\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6138, Macro ROC AUC: 0.4993\n",
      "Epoch [27/30], Train Loss: 2.242837, Val Loss: 2.241896\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6138, Macro ROC AUC: 0.4992\n",
      "Epoch [28/30], Train Loss: 2.242093, Val Loss: 2.241160\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6137, Macro ROC AUC: 0.4988\n",
      "Epoch [29/30], Train Loss: 2.241428, Val Loss: 2.240499\n",
      "Accuracy: 18.65%, Micro ROC AUC: 0.6137, Macro ROC AUC: 0.4987\n",
      "Epoch [30/30], Train Loss: 2.240847, Val Loss: 2.239910\n",
      "Accuracy: 19.59%, Micro ROC AUC: 0.6196, Macro ROC AUC: 0.4911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\envs\\cisc3024_venv1\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1531: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Using optimizer: Adam\n",
      "Accuracy: 66.95%, Micro ROC AUC: 0.9417, Macro ROC AUC: 0.9385\n",
      "Epoch [1/30], Train Loss: 1.811828, Val Loss: 1.009995\n",
      "Accuracy: 77.39%, Micro ROC AUC: 0.9704, Macro ROC AUC: 0.9701\n",
      "Epoch [2/30], Train Loss: 0.802329, Val Loss: 0.707372\n",
      "Accuracy: 83.81%, Micro ROC AUC: 0.9838, Macro ROC AUC: 0.9826\n",
      "Epoch [3/30], Train Loss: 0.575788, Val Loss: 0.514666\n",
      "Accuracy: 86.75%, Micro ROC AUC: 0.9882, Macro ROC AUC: 0.9873\n",
      "Epoch [4/30], Train Loss: 0.449839, Val Loss: 0.434817\n",
      "Accuracy: 87.37%, Micro ROC AUC: 0.9891, Macro ROC AUC: 0.9891\n",
      "Epoch [5/30], Train Loss: 0.383677, Val Loss: 0.412607\n",
      "Accuracy: 88.41%, Micro ROC AUC: 0.9906, Macro ROC AUC: 0.9903\n",
      "Epoch [6/30], Train Loss: 0.341747, Val Loss: 0.380017\n",
      "Accuracy: 89.75%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9914\n",
      "Epoch [7/30], Train Loss: 0.310378, Val Loss: 0.347556\n",
      "Accuracy: 90.04%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9919\n",
      "Epoch [8/30], Train Loss: 0.285270, Val Loss: 0.343686\n",
      "Accuracy: 90.18%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [9/30], Train Loss: 0.262597, Val Loss: 0.333133\n",
      "Accuracy: 90.99%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9928\n",
      "Epoch [10/30], Train Loss: 0.244399, Val Loss: 0.310045\n",
      "Accuracy: 90.30%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9926\n",
      "Epoch [11/30], Train Loss: 0.227690, Val Loss: 0.332608\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9926\n",
      "Epoch [12/30], Train Loss: 0.211558, Val Loss: 0.325317\n",
      "Accuracy: 90.98%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9931\n",
      "Epoch [13/30], Train Loss: 0.196953, Val Loss: 0.305063\n",
      "Accuracy: 91.37%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9932\n",
      "Epoch [14/30], Train Loss: 0.184527, Val Loss: 0.317662\n",
      "Accuracy: 91.69%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9934\n",
      "Epoch [15/30], Train Loss: 0.169389, Val Loss: 0.318204\n",
      "Accuracy: 91.30%, Micro ROC AUC: 0.9938, Macro ROC AUC: 0.9934\n",
      "Epoch [16/30], Train Loss: 0.156415, Val Loss: 0.319652\n",
      "Accuracy: 91.50%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9933\n",
      "Epoch [17/30], Train Loss: 0.143159, Val Loss: 0.320789\n",
      "Accuracy: 91.06%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9929\n",
      "Epoch [18/30], Train Loss: 0.129547, Val Loss: 0.349141\n",
      "Accuracy: 91.22%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9932\n",
      "Epoch [19/30], Train Loss: 0.120780, Val Loss: 0.355989\n",
      "Accuracy: 91.29%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9933\n",
      "Epoch [20/30], Train Loss: 0.108253, Val Loss: 0.372878\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9925\n",
      "Epoch [21/30], Train Loss: 0.096903, Val Loss: 0.395466\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [22/30], Train Loss: 0.091073, Val Loss: 0.416863\n",
      "Accuracy: 91.09%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9928\n",
      "Epoch [23/30], Train Loss: 0.080953, Val Loss: 0.407535\n",
      "Accuracy: 91.16%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9927\n",
      "Epoch [24/30], Train Loss: 0.069905, Val Loss: 0.453001\n",
      "Accuracy: 90.99%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9926\n",
      "Epoch [25/30], Train Loss: 0.065765, Val Loss: 0.479979\n",
      "Accuracy: 90.60%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9922\n",
      "Epoch [26/30], Train Loss: 0.056666, Val Loss: 0.508338\n",
      "Accuracy: 90.78%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9923\n",
      "Epoch [27/30], Train Loss: 0.051940, Val Loss: 0.560134\n",
      "Accuracy: 90.69%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9918\n",
      "Epoch [28/30], Train Loss: 0.048638, Val Loss: 0.558254\n",
      "Accuracy: 90.95%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9924\n",
      "Epoch [29/30], Train Loss: 0.044696, Val Loss: 0.541292\n",
      "Accuracy: 90.72%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9922\n",
      "Epoch [30/30], Train Loss: 0.039960, Val Loss: 0.572787\n",
      "Accuracy: 91.05%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9913\n",
      "\n",
      "Using optimizer: RMSprop\n",
      "Accuracy: 75.07%, Micro ROC AUC: 0.9634, Macro ROC AUC: 0.9610\n",
      "Epoch [1/30], Train Loss: 1.489027, Val Loss: 0.789427\n",
      "Accuracy: 82.55%, Micro ROC AUC: 0.9809, Macro ROC AUC: 0.9811\n",
      "Epoch [2/30], Train Loss: 0.600956, Val Loss: 0.572333\n",
      "Accuracy: 86.95%, Micro ROC AUC: 0.9890, Macro ROC AUC: 0.9889\n",
      "Epoch [3/30], Train Loss: 0.434562, Val Loss: 0.417106\n",
      "Accuracy: 88.30%, Micro ROC AUC: 0.9907, Macro ROC AUC: 0.9903\n",
      "Epoch [4/30], Train Loss: 0.357326, Val Loss: 0.379953\n",
      "Accuracy: 90.06%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9921\n",
      "Epoch [5/30], Train Loss: 0.311076, Val Loss: 0.333759\n",
      "Accuracy: 90.36%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9925\n",
      "Epoch [6/30], Train Loss: 0.273837, Val Loss: 0.327788\n",
      "Accuracy: 91.02%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9930\n",
      "Epoch [7/30], Train Loss: 0.245675, Val Loss: 0.314602\n",
      "Accuracy: 90.87%, Micro ROC AUC: 0.9937, Macro ROC AUC: 0.9934\n",
      "Epoch [8/30], Train Loss: 0.222292, Val Loss: 0.312610\n",
      "Accuracy: 90.70%, Micro ROC AUC: 0.9930, Macro ROC AUC: 0.9930\n",
      "Epoch [9/30], Train Loss: 0.200098, Val Loss: 0.324634\n",
      "Accuracy: 91.28%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9933\n",
      "Epoch [10/30], Train Loss: 0.179216, Val Loss: 0.318162\n",
      "Accuracy: 91.37%, Micro ROC AUC: 0.9936, Macro ROC AUC: 0.9935\n",
      "Epoch [11/30], Train Loss: 0.160523, Val Loss: 0.316244\n",
      "Accuracy: 90.98%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9932\n",
      "Epoch [12/30], Train Loss: 0.143638, Val Loss: 0.337427\n",
      "Accuracy: 91.72%, Micro ROC AUC: 0.9939, Macro ROC AUC: 0.9935\n",
      "Epoch [13/30], Train Loss: 0.124407, Val Loss: 0.343428\n",
      "Accuracy: 91.03%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9929\n",
      "Epoch [14/30], Train Loss: 0.109841, Val Loss: 0.375074\n",
      "Accuracy: 91.13%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9932\n",
      "Epoch [15/30], Train Loss: 0.093462, Val Loss: 0.370447\n",
      "Accuracy: 91.05%, Micro ROC AUC: 0.9932, Macro ROC AUC: 0.9928\n",
      "Epoch [16/30], Train Loss: 0.082061, Val Loss: 0.401554\n",
      "Accuracy: 91.28%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9927\n",
      "Epoch [17/30], Train Loss: 0.069097, Val Loss: 0.414786\n",
      "Accuracy: 91.45%, Micro ROC AUC: 0.9933, Macro ROC AUC: 0.9929\n",
      "Epoch [18/30], Train Loss: 0.058804, Val Loss: 0.474031\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9920, Macro ROC AUC: 0.9917\n",
      "Epoch [19/30], Train Loss: 0.052420, Val Loss: 0.505739\n",
      "Accuracy: 90.58%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9923\n",
      "Epoch [20/30], Train Loss: 0.046141, Val Loss: 0.537685\n",
      "Accuracy: 90.77%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9919\n",
      "Epoch [21/30], Train Loss: 0.040120, Val Loss: 0.544952\n",
      "Accuracy: 90.95%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [22/30], Train Loss: 0.036891, Val Loss: 0.632377\n",
      "Accuracy: 91.05%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9924\n",
      "Epoch [23/30], Train Loss: 0.033674, Val Loss: 0.630042\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9920\n",
      "Epoch [24/30], Train Loss: 0.032774, Val Loss: 0.704222\n",
      "Accuracy: 91.08%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9923\n",
      "Epoch [25/30], Train Loss: 0.029956, Val Loss: 0.661135\n",
      "Accuracy: 91.07%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9923\n",
      "Epoch [26/30], Train Loss: 0.028345, Val Loss: 0.651998\n",
      "Accuracy: 91.03%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [27/30], Train Loss: 0.026187, Val Loss: 0.677365\n",
      "Accuracy: 91.21%, Micro ROC AUC: 0.9927, Macro ROC AUC: 0.9923\n",
      "Epoch [28/30], Train Loss: 0.027947, Val Loss: 0.696357\n",
      "Accuracy: 91.18%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9922\n",
      "Epoch [29/30], Train Loss: 0.023432, Val Loss: 0.760410\n",
      "Accuracy: 91.09%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9922\n",
      "Epoch [30/30], Train Loss: 0.024334, Val Loss: 0.734524\n",
      "Accuracy: 91.50%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9917\n",
      "\n",
      "Using optimizer: AdamW\n",
      "Accuracy: 72.58%, Micro ROC AUC: 0.9560, Macro ROC AUC: 0.9545\n",
      "Epoch [1/30], Train Loss: 1.806601, Val Loss: 0.871128\n",
      "Accuracy: 84.58%, Micro ROC AUC: 0.9837, Macro ROC AUC: 0.9820\n",
      "Epoch [2/30], Train Loss: 0.627654, Val Loss: 0.508840\n",
      "Accuracy: 87.04%, Micro ROC AUC: 0.9876, Macro ROC AUC: 0.9870\n",
      "Epoch [3/30], Train Loss: 0.453406, Val Loss: 0.439739\n",
      "Accuracy: 88.47%, Micro ROC AUC: 0.9901, Macro ROC AUC: 0.9893\n",
      "Epoch [4/30], Train Loss: 0.377768, Val Loss: 0.391487\n",
      "Accuracy: 89.42%, Micro ROC AUC: 0.9913, Macro ROC AUC: 0.9908\n",
      "Epoch [5/30], Train Loss: 0.327155, Val Loss: 0.369043\n",
      "Accuracy: 89.11%, Micro ROC AUC: 0.9912, Macro ROC AUC: 0.9913\n",
      "Epoch [6/30], Train Loss: 0.292033, Val Loss: 0.365173\n",
      "Accuracy: 90.43%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9921\n",
      "Epoch [7/30], Train Loss: 0.264097, Val Loss: 0.332797\n",
      "Accuracy: 90.69%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9924\n",
      "Epoch [8/30], Train Loss: 0.240212, Val Loss: 0.323891\n",
      "Accuracy: 90.35%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9924\n",
      "Epoch [9/30], Train Loss: 0.218200, Val Loss: 0.348052\n",
      "Accuracy: 90.93%, Micro ROC AUC: 0.9931, Macro ROC AUC: 0.9927\n",
      "Epoch [10/30], Train Loss: 0.197503, Val Loss: 0.322981\n",
      "Accuracy: 91.50%, Micro ROC AUC: 0.9934, Macro ROC AUC: 0.9929\n",
      "Epoch [11/30], Train Loss: 0.182074, Val Loss: 0.324514\n",
      "Accuracy: 90.88%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9926\n",
      "Epoch [12/30], Train Loss: 0.164242, Val Loss: 0.337436\n",
      "Accuracy: 91.10%, Micro ROC AUC: 0.9929, Macro ROC AUC: 0.9926\n",
      "Epoch [13/30], Train Loss: 0.148521, Val Loss: 0.365840\n",
      "Accuracy: 91.00%, Micro ROC AUC: 0.9926, Macro ROC AUC: 0.9922\n",
      "Epoch [14/30], Train Loss: 0.132353, Val Loss: 0.355455\n",
      "Accuracy: 90.85%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9921\n",
      "Epoch [15/30], Train Loss: 0.119894, Val Loss: 0.365751\n",
      "Accuracy: 90.88%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9921\n",
      "Epoch [16/30], Train Loss: 0.106955, Val Loss: 0.401043\n",
      "Accuracy: 91.24%, Micro ROC AUC: 0.9928, Macro ROC AUC: 0.9924\n",
      "Epoch [17/30], Train Loss: 0.095927, Val Loss: 0.423767\n",
      "Accuracy: 90.88%, Micro ROC AUC: 0.9925, Macro ROC AUC: 0.9922\n",
      "Epoch [18/30], Train Loss: 0.081500, Val Loss: 0.435450\n",
      "Accuracy: 91.07%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9920\n",
      "Epoch [19/30], Train Loss: 0.073864, Val Loss: 0.472191\n",
      "Accuracy: 90.97%, Micro ROC AUC: 0.9924, Macro ROC AUC: 0.9919\n",
      "Epoch [20/30], Train Loss: 0.065967, Val Loss: 0.464219\n",
      "Accuracy: 90.78%, Micro ROC AUC: 0.9922, Macro ROC AUC: 0.9919\n",
      "Epoch [21/30], Train Loss: 0.054260, Val Loss: 0.539824\n",
      "Accuracy: 90.77%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9916\n",
      "Epoch [22/30], Train Loss: 0.055161, Val Loss: 0.503471\n",
      "Accuracy: 90.62%, Micro ROC AUC: 0.9919, Macro ROC AUC: 0.9916\n",
      "Epoch [23/30], Train Loss: 0.044978, Val Loss: 0.587123\n",
      "Accuracy: 90.19%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [24/30], Train Loss: 0.046536, Val Loss: 0.588199\n",
      "Accuracy: 90.55%, Micro ROC AUC: 0.9921, Macro ROC AUC: 0.9917\n",
      "Epoch [25/30], Train Loss: 0.037327, Val Loss: 0.610827\n",
      "Accuracy: 90.71%, Micro ROC AUC: 0.9923, Macro ROC AUC: 0.9918\n",
      "Epoch [26/30], Train Loss: 0.037042, Val Loss: 0.625918\n",
      "Accuracy: 90.66%, Micro ROC AUC: 0.9917, Macro ROC AUC: 0.9913\n",
      "Epoch [27/30], Train Loss: 0.034966, Val Loss: 0.642638\n",
      "Accuracy: 90.06%, Micro ROC AUC: 0.9914, Macro ROC AUC: 0.9910\n",
      "Epoch [28/30], Train Loss: 0.035704, Val Loss: 0.748303\n",
      "Accuracy: 90.15%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9913\n",
      "Epoch [29/30], Train Loss: 0.033854, Val Loss: 0.687891\n",
      "Accuracy: 90.22%, Micro ROC AUC: 0.9916, Macro ROC AUC: 0.9912\n",
      "Epoch [30/30], Train Loss: 0.034882, Val Loss: 0.658231\n",
      "Accuracy: 89.71%, Micro ROC AUC: 0.9904, Macro ROC AUC: 0.9898\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-08T20:41:17.684400Z",
     "start_time": "2024-11-08T19:39:38.593310Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "from torch.utils.data import DataLoader\n",
    "from sklearn.metrics import roc_auc_score, precision_score, recall_score, confusion_matrix, roc_curve\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "os.makedirs(\"ex5\", exist_ok=True)\n",
    "\n",
    "\n",
    "class SVHNDataModule:\n",
    "    def __init__(self):\n",
    "        self.transform = transforms.Compose([\n",
    "            transforms.ToTensor(),\n",
    "            transforms.Normalize(mean=(0.4377, 0.4438, 0.4728), std=(0.1980, 0.2010, 0.1970))\n",
    "        ])\n",
    "\n",
    "    def load_data(self):\n",
    "        train_dataset = datasets.SVHN(root='./data', split='train', download=True, transform=self.transform)\n",
    "        test_dataset = datasets.SVHN(root='./data', split='test', download=True, transform=self.transform)\n",
    "        train_loader = DataLoader(train_dataset, batch_size=128, shuffle=True)\n",
    "        test_loader = DataLoader(test_dataset, batch_size=128, shuffle=False)\n",
    "        return train_loader, test_loader\n",
    "\n",
    "# 定义小型 VGG 模型，支持不同的 Dropout 率\n",
    "class SmallVGG(nn.Module):\n",
    "    def __init__(self, dropout_rate_conv=0.25, dropout_rate_fc=0.5):\n",
    "        super(SmallVGG, self).__init__()\n",
    "        self.conv_layers = nn.Sequential(\n",
    "            nn.Conv2d(3, 8, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(8, 16, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Dropout(dropout_rate_conv),\n",
    "            nn.Conv2d(16, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Dropout(dropout_rate_conv),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(32, 32, kernel_size=3, padding=1),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2, stride=2),\n",
    "            nn.Dropout(dropout_rate_conv)\n",
    "        )\n",
    "        self.fc_layers = nn.Sequential(\n",
    "            nn.Linear(32 * 4 * 4, 256),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(dropout_rate_fc),\n",
    "            nn.Linear(256, 10)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv_layers(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc_layers(x)\n",
    "        return x\n",
    "\n",
    "# 训练模型的函数，返回训练中的所有损失值\n",
    "def train_model(model, train_loader, criterion, optimizer, num_epochs=10, device='cuda'):\n",
    "    model.train()\n",
    "    device = torch.device(device)\n",
    "    train_losses = []\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        running_loss = 0.0\n",
    "        for images, labels in train_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            running_loss += loss.item()\n",
    "\n",
    "        epoch_loss = running_loss / len(train_loader)\n",
    "        train_losses.append(epoch_loss)\n",
    "        print(f'Epoch [{epoch + 1}/{num_epochs}], Loss: {epoch_loss:.6f}')\n",
    "\n",
    "    return train_losses\n",
    "\n",
    "# 评估模型的函数\n",
    "def evaluate_model_metrics(model, test_loader, criterion, device='cuda'):\n",
    "    model.eval()\n",
    "    all_labels = []\n",
    "    all_preds = []\n",
    "    all_probs = []\n",
    "    test_loss = 0.0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for images, labels in test_loader:\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "            outputs = model(images)\n",
    "            loss = criterion(outputs, labels)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = torch.max(outputs, 1)\n",
    "            probs = torch.softmax(outputs, dim=1)\n",
    "\n",
    "            all_labels.extend(labels.cpu().numpy())\n",
    "            all_preds.extend(predicted.cpu().numpy())\n",
    "            all_probs.extend(probs.cpu().numpy())\n",
    "\n",
    "    test_loss /= len(test_loader)\n",
    "    accuracy = np.mean(np.array(all_labels) == np.array(all_preds))\n",
    "    precision = precision_score(all_labels, all_preds, average='weighted')\n",
    "    recall = recall_score(all_labels, all_preds, average='weighted')\n",
    "    cm = confusion_matrix(all_labels, all_preds)\n",
    "\n",
    "    return test_loss, accuracy, precision, recall, cm, all_labels, all_probs\n",
    "\n",
    "# 生成 ROC 曲线的函数\n",
    "def plot_roc_curve(all_labels, all_probs, dropout):\n",
    "    plt.figure()\n",
    "    for i in range(10):\n",
    "        fpr, tpr, _ = roc_curve(np.array(all_labels) == i, np.array(all_probs)[:, i])\n",
    "        auc_score = roc_auc_score(np.array(all_labels) == i, np.array(all_probs)[:, i])\n",
    "        plt.plot(fpr, tpr, label=f'Class {i} (AUC = {auc_score:.2f})')\n",
    "\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title(f'ROC Curve (Dropout {dropout})')\n",
    "    plt.legend()\n",
    "    plt.savefig(f'ex5/roc_curve_dropout_{dropout}.png')\n",
    "    plt.close()\n",
    "\n",
    "# 主函数，测试不同 Dropout 率并保存图表\n",
    "def main():\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    data_module = SVHNDataModule()\n",
    "    train_loader, test_loader = data_module.load_data()\n",
    "    dropout_rates = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]\n",
    "\n",
    "    results = {}\n",
    "\n",
    "    for dropout in dropout_rates:\n",
    "        print(f\"\\nUsing Dropout Rate - Conv & FC: {dropout}\")\n",
    "        model = SmallVGG(dropout_rate_conv=dropout, dropout_rate_fc=dropout).to(device)\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        optimizer = optim.Adam(model.parameters(), lr=0.0005)\n",
    "\n",
    "        # 训练模型并记录损失\n",
    "        train_losses = train_model(model, train_loader, criterion, optimizer, num_epochs=50, device=device)\n",
    "\n",
    "        # 评估模型并获得所有预测的概率\n",
    "        test_loss, accuracy, precision, recall, cm, all_labels, all_probs = evaluate_model_metrics(model, test_loader, criterion, device)\n",
    "\n",
    "        results[dropout] = {\n",
    "            \"train_loss\": train_losses[-1],\n",
    "            \"test_loss\": test_loss,\n",
    "            \"accuracy\": accuracy,\n",
    "            \"precision\": precision,\n",
    "            \"recall\": recall,\n",
    "            \"cm\": cm\n",
    "        }\n",
    "\n",
    "        # 绘制并保存训练过程的 Loss 曲线\n",
    "        plt.figure()\n",
    "        plt.plot(range(1, len(train_losses) + 1), train_losses, label=\"Train Loss\")\n",
    "        plt.xlabel(\"Epochs\")\n",
    "        plt.ylabel(\"Loss\")\n",
    "        plt.title(f\"Loss Curve (Dropout {dropout})\")\n",
    "        plt.legend()\n",
    "        plt.savefig(f\"ex5/loss_curve_dropout_{dropout}.png\")\n",
    "        plt.close()\n",
    "\n",
    "        # 绘制并保存 ROC 曲线\n",
    "        plot_roc_curve(all_labels, all_probs, dropout)\n",
    "\n",
    "        # 保存混淆矩阵\n",
    "        plt.figure(figsize=(8, 6))\n",
    "        sns.heatmap(cm, annot=True, fmt=\"d\", cmap=\"Blues\", cbar=False, xticklabels=range(10), yticklabels=range(10))\n",
    "        plt.xlabel(\"Predicted Label\")\n",
    "        plt.ylabel(\"True Label\")\n",
    "        plt.title(f\"Confusion Matrix (Dropout {dropout})\")\n",
    "        plt.savefig(f\"ex5/confusion_matrix_dropout_{dropout}.png\")\n",
    "        plt.close()\n",
    "\n",
    "    # 绘制并保存不同 Dropout 率的指标柱状图\n",
    "    metrics = ['train_loss', 'test_loss', 'accuracy', 'precision', 'recall']\n",
    "    plt.figure(figsize=(15, 8))\n",
    "    for i, metric in enumerate(metrics):\n",
    "        values = [results[dropout][metric] for dropout in dropout_rates]\n",
    "        plt.bar(np.arange(len(dropout_rates)) + i * 0.15, values, width=0.15, label=metric)\n",
    "\n",
    "    plt.xticks(np.arange(len(dropout_rates)) + 0.3, [f\"Dropout {d}\" for d in dropout_rates], rotation=45)\n",
    "    plt.ylabel('Scores')\n",
    "    plt.title('Comparison of Metrics with Different Dropout Rates')\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(\"ex5/metrics_comparison_bar_chart.png\")\n",
    "    plt.close()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()\n"
   ],
   "id": "f015a623c9784125",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Using Dropout Rate - Conv & FC: 0.1\n",
      "Epoch [1/50], Loss: 1.562087\n",
      "Epoch [2/50], Loss: 0.637070\n",
      "Epoch [3/50], Loss: 0.473867\n",
      "Epoch [4/50], Loss: 0.402071\n",
      "Epoch [5/50], Loss: 0.361342\n",
      "Epoch [6/50], Loss: 0.326549\n",
      "Epoch [7/50], Loss: 0.304580\n",
      "Epoch [8/50], Loss: 0.282998\n",
      "Epoch [9/50], Loss: 0.267809\n",
      "Epoch [10/50], Loss: 0.253569\n",
      "Epoch [11/50], Loss: 0.242066\n",
      "Epoch [12/50], Loss: 0.229694\n",
      "Epoch [13/50], Loss: 0.218907\n",
      "Epoch [14/50], Loss: 0.210744\n",
      "Epoch [15/50], Loss: 0.200336\n",
      "Epoch [16/50], Loss: 0.191889\n",
      "Epoch [17/50], Loss: 0.182928\n",
      "Epoch [18/50], Loss: 0.177726\n",
      "Epoch [19/50], Loss: 0.171618\n",
      "Epoch [20/50], Loss: 0.161420\n",
      "Epoch [21/50], Loss: 0.158741\n",
      "Epoch [22/50], Loss: 0.151599\n",
      "Epoch [23/50], Loss: 0.146770\n",
      "Epoch [24/50], Loss: 0.141287\n",
      "Epoch [25/50], Loss: 0.140585\n",
      "Epoch [26/50], Loss: 0.136619\n",
      "Epoch [27/50], Loss: 0.129486\n",
      "Epoch [28/50], Loss: 0.126266\n",
      "Epoch [29/50], Loss: 0.124861\n",
      "Epoch [30/50], Loss: 0.121121\n",
      "Epoch [31/50], Loss: 0.115890\n",
      "Epoch [32/50], Loss: 0.113719\n",
      "Epoch [33/50], Loss: 0.114965\n",
      "Epoch [34/50], Loss: 0.109241\n",
      "Epoch [35/50], Loss: 0.105862\n",
      "Epoch [36/50], Loss: 0.105540\n",
      "Epoch [37/50], Loss: 0.102920\n",
      "Epoch [38/50], Loss: 0.101128\n",
      "Epoch [39/50], Loss: 0.097961\n",
      "Epoch [40/50], Loss: 0.096438\n",
      "Epoch [41/50], Loss: 0.094586\n",
      "Epoch [42/50], Loss: 0.093880\n",
      "Epoch [43/50], Loss: 0.093400\n",
      "Epoch [44/50], Loss: 0.089226\n",
      "Epoch [45/50], Loss: 0.087370\n",
      "Epoch [46/50], Loss: 0.086622\n",
      "Epoch [47/50], Loss: 0.086993\n",
      "Epoch [48/50], Loss: 0.085534\n",
      "Epoch [49/50], Loss: 0.082985\n",
      "Epoch [50/50], Loss: 0.081798\n",
      "\n",
      "Using Dropout Rate - Conv & FC: 0.2\n",
      "Epoch [1/50], Loss: 1.600837\n",
      "Epoch [2/50], Loss: 0.714349\n",
      "Epoch [3/50], Loss: 0.536294\n",
      "Epoch [4/50], Loss: 0.459193\n",
      "Epoch [5/50], Loss: 0.416858\n",
      "Epoch [6/50], Loss: 0.379810\n",
      "Epoch [7/50], Loss: 0.358429\n",
      "Epoch [8/50], Loss: 0.338489\n",
      "Epoch [9/50], Loss: 0.322877\n",
      "Epoch [10/50], Loss: 0.309717\n",
      "Epoch [11/50], Loss: 0.301655\n",
      "Epoch [12/50], Loss: 0.287142\n",
      "Epoch [13/50], Loss: 0.279140\n",
      "Epoch [14/50], Loss: 0.266683\n",
      "Epoch [15/50], Loss: 0.263083\n",
      "Epoch [16/50], Loss: 0.254177\n",
      "Epoch [17/50], Loss: 0.247627\n",
      "Epoch [18/50], Loss: 0.240222\n",
      "Epoch [19/50], Loss: 0.236332\n",
      "Epoch [20/50], Loss: 0.231285\n",
      "Epoch [21/50], Loss: 0.225689\n",
      "Epoch [22/50], Loss: 0.221612\n",
      "Epoch [23/50], Loss: 0.217467\n",
      "Epoch [24/50], Loss: 0.212925\n",
      "Epoch [25/50], Loss: 0.208166\n",
      "Epoch [26/50], Loss: 0.206112\n",
      "Epoch [27/50], Loss: 0.201512\n",
      "Epoch [28/50], Loss: 0.198819\n",
      "Epoch [29/50], Loss: 0.196478\n",
      "Epoch [30/50], Loss: 0.193488\n",
      "Epoch [31/50], Loss: 0.189730\n",
      "Epoch [32/50], Loss: 0.189950\n",
      "Epoch [33/50], Loss: 0.184637\n",
      "Epoch [34/50], Loss: 0.183288\n",
      "Epoch [35/50], Loss: 0.177184\n",
      "Epoch [36/50], Loss: 0.181371\n",
      "Epoch [37/50], Loss: 0.175933\n",
      "Epoch [38/50], Loss: 0.173506\n",
      "Epoch [39/50], Loss: 0.175494\n",
      "Epoch [40/50], Loss: 0.172810\n",
      "Epoch [41/50], Loss: 0.166872\n",
      "Epoch [42/50], Loss: 0.167504\n",
      "Epoch [43/50], Loss: 0.164572\n",
      "Epoch [44/50], Loss: 0.165772\n",
      "Epoch [45/50], Loss: 0.162838\n",
      "Epoch [46/50], Loss: 0.159557\n",
      "Epoch [47/50], Loss: 0.157564\n",
      "Epoch [48/50], Loss: 0.157630\n",
      "Epoch [49/50], Loss: 0.158194\n",
      "Epoch [50/50], Loss: 0.158091\n",
      "\n",
      "Using Dropout Rate - Conv & FC: 0.3\n",
      "Epoch [1/50], Loss: 1.900688\n",
      "Epoch [2/50], Loss: 0.794467\n",
      "Epoch [3/50], Loss: 0.580460\n",
      "Epoch [4/50], Loss: 0.505064\n",
      "Epoch [5/50], Loss: 0.460275\n",
      "Epoch [6/50], Loss: 0.429417\n",
      "Epoch [7/50], Loss: 0.402332\n",
      "Epoch [8/50], Loss: 0.384653\n",
      "Epoch [9/50], Loss: 0.371551\n",
      "Epoch [10/50], Loss: 0.354913\n",
      "Epoch [11/50], Loss: 0.343708\n",
      "Epoch [12/50], Loss: 0.340274\n",
      "Epoch [13/50], Loss: 0.326709\n",
      "Epoch [14/50], Loss: 0.323950\n",
      "Epoch [15/50], Loss: 0.315443\n",
      "Epoch [16/50], Loss: 0.311143\n",
      "Epoch [17/50], Loss: 0.304922\n",
      "Epoch [18/50], Loss: 0.297125\n",
      "Epoch [19/50], Loss: 0.294936\n",
      "Epoch [20/50], Loss: 0.286272\n",
      "Epoch [21/50], Loss: 0.285697\n",
      "Epoch [22/50], Loss: 0.281155\n",
      "Epoch [23/50], Loss: 0.276884\n",
      "Epoch [24/50], Loss: 0.276675\n",
      "Epoch [25/50], Loss: 0.272452\n",
      "Epoch [26/50], Loss: 0.266545\n",
      "Epoch [27/50], Loss: 0.262004\n",
      "Epoch [28/50], Loss: 0.260792\n",
      "Epoch [29/50], Loss: 0.259759\n",
      "Epoch [30/50], Loss: 0.255091\n",
      "Epoch [31/50], Loss: 0.254749\n",
      "Epoch [32/50], Loss: 0.250474\n",
      "Epoch [33/50], Loss: 0.251855\n",
      "Epoch [34/50], Loss: 0.246006\n",
      "Epoch [35/50], Loss: 0.246256\n",
      "Epoch [36/50], Loss: 0.243503\n",
      "Epoch [37/50], Loss: 0.243466\n",
      "Epoch [38/50], Loss: 0.241436\n",
      "Epoch [39/50], Loss: 0.239532\n",
      "Epoch [40/50], Loss: 0.237419\n",
      "Epoch [41/50], Loss: 0.238995\n",
      "Epoch [42/50], Loss: 0.233039\n",
      "Epoch [43/50], Loss: 0.231970\n",
      "Epoch [44/50], Loss: 0.230159\n",
      "Epoch [45/50], Loss: 0.231125\n",
      "Epoch [46/50], Loss: 0.229435\n",
      "Epoch [47/50], Loss: 0.227864\n",
      "Epoch [48/50], Loss: 0.228405\n",
      "Epoch [49/50], Loss: 0.225063\n",
      "Epoch [50/50], Loss: 0.223311\n",
      "\n",
      "Using Dropout Rate - Conv & FC: 0.4\n",
      "Epoch [1/50], Loss: 2.014615\n",
      "Epoch [2/50], Loss: 0.915281\n",
      "Epoch [3/50], Loss: 0.694298\n",
      "Epoch [4/50], Loss: 0.605686\n",
      "Epoch [5/50], Loss: 0.555323\n",
      "Epoch [6/50], Loss: 0.525018\n",
      "Epoch [7/50], Loss: 0.488137\n",
      "Epoch [8/50], Loss: 0.472306\n",
      "Epoch [9/50], Loss: 0.448737\n",
      "Epoch [10/50], Loss: 0.439162\n",
      "Epoch [11/50], Loss: 0.426194\n",
      "Epoch [12/50], Loss: 0.416244\n",
      "Epoch [13/50], Loss: 0.408285\n",
      "Epoch [14/50], Loss: 0.395800\n",
      "Epoch [15/50], Loss: 0.388706\n",
      "Epoch [16/50], Loss: 0.384206\n",
      "Epoch [17/50], Loss: 0.374253\n",
      "Epoch [18/50], Loss: 0.369813\n",
      "Epoch [19/50], Loss: 0.365163\n",
      "Epoch [20/50], Loss: 0.358097\n",
      "Epoch [21/50], Loss: 0.357355\n",
      "Epoch [22/50], Loss: 0.354432\n",
      "Epoch [23/50], Loss: 0.347837\n",
      "Epoch [24/50], Loss: 0.345958\n",
      "Epoch [25/50], Loss: 0.338824\n",
      "Epoch [26/50], Loss: 0.339741\n",
      "Epoch [27/50], Loss: 0.335103\n",
      "Epoch [28/50], Loss: 0.333673\n",
      "Epoch [29/50], Loss: 0.329401\n",
      "Epoch [30/50], Loss: 0.326166\n",
      "Epoch [31/50], Loss: 0.324266\n",
      "Epoch [32/50], Loss: 0.322566\n",
      "Epoch [33/50], Loss: 0.321952\n",
      "Epoch [34/50], Loss: 0.317681\n",
      "Epoch [35/50], Loss: 0.314006\n",
      "Epoch [36/50], Loss: 0.314829\n",
      "Epoch [37/50], Loss: 0.314181\n",
      "Epoch [38/50], Loss: 0.312036\n",
      "Epoch [39/50], Loss: 0.309494\n",
      "Epoch [40/50], Loss: 0.309143\n",
      "Epoch [41/50], Loss: 0.307335\n",
      "Epoch [42/50], Loss: 0.302194\n",
      "Epoch [43/50], Loss: 0.303380\n",
      "Epoch [44/50], Loss: 0.303163\n",
      "Epoch [45/50], Loss: 0.301726\n",
      "Epoch [46/50], Loss: 0.296671\n",
      "Epoch [47/50], Loss: 0.296539\n",
      "Epoch [48/50], Loss: 0.297965\n",
      "Epoch [49/50], Loss: 0.296836\n",
      "Epoch [50/50], Loss: 0.297045\n",
      "\n",
      "Using Dropout Rate - Conv & FC: 0.5\n",
      "Epoch [1/50], Loss: 1.854902\n",
      "Epoch [2/50], Loss: 1.047309\n",
      "Epoch [3/50], Loss: 0.872403\n",
      "Epoch [4/50], Loss: 0.775199\n",
      "Epoch [5/50], Loss: 0.702780\n",
      "Epoch [6/50], Loss: 0.641026\n",
      "Epoch [7/50], Loss: 0.603627\n",
      "Epoch [8/50], Loss: 0.576201\n",
      "Epoch [9/50], Loss: 0.547508\n",
      "Epoch [10/50], Loss: 0.522146\n",
      "Epoch [11/50], Loss: 0.508406\n",
      "Epoch [12/50], Loss: 0.489614\n",
      "Epoch [13/50], Loss: 0.482146\n",
      "Epoch [14/50], Loss: 0.470023\n",
      "Epoch [15/50], Loss: 0.458354\n",
      "Epoch [16/50], Loss: 0.453586\n",
      "Epoch [17/50], Loss: 0.440977\n",
      "Epoch [18/50], Loss: 0.436045\n",
      "Epoch [19/50], Loss: 0.428437\n",
      "Epoch [20/50], Loss: 0.423533\n",
      "Epoch [21/50], Loss: 0.425299\n",
      "Epoch [22/50], Loss: 0.418908\n",
      "Epoch [23/50], Loss: 0.413937\n",
      "Epoch [24/50], Loss: 0.411112\n",
      "Epoch [25/50], Loss: 0.404647\n",
      "Epoch [26/50], Loss: 0.398642\n",
      "Epoch [27/50], Loss: 0.398046\n",
      "Epoch [28/50], Loss: 0.393338\n",
      "Epoch [29/50], Loss: 0.393235\n",
      "Epoch [30/50], Loss: 0.387044\n",
      "Epoch [31/50], Loss: 0.387493\n",
      "Epoch [32/50], Loss: 0.381347\n",
      "Epoch [33/50], Loss: 0.378355\n",
      "Epoch [34/50], Loss: 0.378169\n",
      "Epoch [35/50], Loss: 0.376187\n",
      "Epoch [36/50], Loss: 0.372527\n",
      "Epoch [37/50], Loss: 0.372197\n",
      "Epoch [38/50], Loss: 0.375276\n",
      "Epoch [39/50], Loss: 0.368850\n",
      "Epoch [40/50], Loss: 0.370993\n",
      "Epoch [41/50], Loss: 0.365991\n",
      "Epoch [42/50], Loss: 0.366235\n",
      "Epoch [43/50], Loss: 0.365342\n",
      "Epoch [44/50], Loss: 0.363999\n",
      "Epoch [45/50], Loss: 0.362109\n",
      "Epoch [46/50], Loss: 0.359113\n",
      "Epoch [47/50], Loss: 0.359214\n",
      "Epoch [48/50], Loss: 0.359247\n",
      "Epoch [49/50], Loss: 0.359499\n",
      "Epoch [50/50], Loss: 0.356333\n",
      "\n",
      "Using Dropout Rate - Conv & FC: 0.6\n",
      "Epoch [1/50], Loss: 2.143599\n",
      "Epoch [2/50], Loss: 1.313263\n",
      "Epoch [3/50], Loss: 1.057822\n",
      "Epoch [4/50], Loss: 0.949468\n",
      "Epoch [5/50], Loss: 0.874795\n",
      "Epoch [6/50], Loss: 0.817487\n",
      "Epoch [7/50], Loss: 0.780506\n",
      "Epoch [8/50], Loss: 0.737187\n",
      "Epoch [9/50], Loss: 0.710896\n",
      "Epoch [10/50], Loss: 0.685356\n",
      "Epoch [11/50], Loss: 0.666025\n",
      "Epoch [12/50], Loss: 0.651595\n",
      "Epoch [13/50], Loss: 0.635921\n",
      "Epoch [14/50], Loss: 0.615028\n",
      "Epoch [15/50], Loss: 0.613637\n",
      "Epoch [16/50], Loss: 0.591500\n",
      "Epoch [17/50], Loss: 0.589060\n",
      "Epoch [18/50], Loss: 0.577896\n",
      "Epoch [19/50], Loss: 0.567899\n",
      "Epoch [20/50], Loss: 0.560582\n",
      "Epoch [21/50], Loss: 0.554301\n",
      "Epoch [22/50], Loss: 0.547333\n",
      "Epoch [23/50], Loss: 0.536554\n",
      "Epoch [24/50], Loss: 0.532011\n",
      "Epoch [25/50], Loss: 0.526184\n",
      "Epoch [26/50], Loss: 0.524203\n",
      "Epoch [27/50], Loss: 0.516816\n",
      "Epoch [28/50], Loss: 0.513780\n",
      "Epoch [29/50], Loss: 0.516159\n",
      "Epoch [30/50], Loss: 0.511562\n",
      "Epoch [31/50], Loss: 0.504994\n",
      "Epoch [32/50], Loss: 0.502313\n",
      "Epoch [33/50], Loss: 0.499761\n",
      "Epoch [34/50], Loss: 0.495692\n",
      "Epoch [35/50], Loss: 0.494144\n",
      "Epoch [36/50], Loss: 0.493807\n",
      "Epoch [37/50], Loss: 0.490368\n",
      "Epoch [38/50], Loss: 0.486192\n",
      "Epoch [39/50], Loss: 0.480601\n",
      "Epoch [40/50], Loss: 0.481026\n",
      "Epoch [41/50], Loss: 0.482668\n",
      "Epoch [42/50], Loss: 0.480031\n",
      "Epoch [43/50], Loss: 0.475887\n",
      "Epoch [44/50], Loss: 0.472171\n",
      "Epoch [45/50], Loss: 0.468790\n",
      "Epoch [46/50], Loss: 0.466797\n",
      "Epoch [47/50], Loss: 0.468823\n",
      "Epoch [48/50], Loss: 0.468023\n",
      "Epoch [49/50], Loss: 0.463406\n",
      "Epoch [50/50], Loss: 0.466483\n"
     ]
    }
   ],
   "execution_count": 5
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}