signate_stu_v23.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "signate_stu_v23.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP+VzCxorYW+mlOO1inIjKf",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/t-nakatani/signate_stu22/blob/exp/signate_stu_v23.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "v20: \"allenai/scibert_scivocab_uncased\""
      ],
      "metadata": {
        "id": "GDti4y2d6yb_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ofvp5QVbEagJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e5c9fa0-18e1-4fb8-a363-415625fffe0a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import os\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "ROOT = \"/content/drive/MyDrive/Colab Notebooks/signate/\"\n",
        "\n",
        "#学習用データと評価用データの読み込み\n",
        "train = pd.read_csv(os.path.join(ROOT, \"train_cleaned.csv\")).drop_duplicates(subset='description')\n",
        "test = pd.read_csv(os.path.join(ROOT, \"test_cleaned.csv\"))\n",
        "\n",
        "jobdic = dict(zip([1, 2, 3, 4], ['DataScientist', 'ML Engineer', 'Software Engineer', 'Consultant']))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q transformers==3"
      ],
      "metadata": {
        "id": "YisF3KcxVqbT"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import transformers\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "# from transformers import RobertaTokenizer, RobertaModel\n",
        "# from transformers import BertModel, AutoTokenizer \n",
        "\n",
        "from transformers import AutoTokenizer, AutoModel, AutoConfig\n",
        "# from transformers import BertTokenizer, BertModel\n",
        "from torch import optim\n",
        "from torch import cuda\n",
        "import time\n",
        "from matplotlib import pyplot as plt\n",
        "from tqdm import tqdm\n",
        "from sklearn.metrics import f1_score"
      ],
      "metadata": {
        "id": "IrNv3cPZOec-"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# seeds\n",
        "SEED = 2022\n",
        "def seed_everything(seed):\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    if torch.cuda.is_available():\n",
        "        torch.cuda.manual_seed(seed)\n",
        "        torch.cuda.manual_seed_all(seed)\n",
        "\n",
        "seed_everything(SEED)\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False\n",
        "    current_device = torch.cuda.current_device()\n",
        "    print(\"Device:\", torch.cuda.get_device_name(current_device))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qnz-PWYD9S92",
        "outputId": "708b7565-56ac-4a1d-fa4e-554cbd66b15a"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Device: Tesla P100-PCIE-16GB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Datasetの定義\n",
        "class CreateDataset(Dataset):\n",
        "  def __init__(self, X, y, tokenizer, max_len):\n",
        "    self.X = X\n",
        "    self.y = y\n",
        "    self.tokenizer = tokenizer\n",
        "    self.max_len = max_len\n",
        "\n",
        "  def __len__(self):  # len(Dataset)で返す値を指定\n",
        "    return len(self.X)\n",
        "\n",
        "  def __getitem__(self, index):  # Dataset[index]で返す値を指定\n",
        "    text = self.X[index]\n",
        "    inputs = self.tokenizer.encode_plus(\n",
        "      text,\n",
        "      add_special_tokens=True,\n",
        "      max_length=self.max_len,\n",
        "      pad_to_max_length=True,\n",
        "      truncation=True\n",
        "    )\n",
        "    ids = inputs['input_ids']\n",
        "    mask = inputs['attention_mask']\n",
        "    labels = self.y[index]\n",
        "\n",
        "    return {\n",
        "      'ids': torch.LongTensor(ids),\n",
        "      'mask': torch.LongTensor(mask),\n",
        "      'labels': torch.Tensor(labels)\n",
        "    }"
      ],
      "metadata": {
        "id": "bTbjxDE60aFs"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "1. from_pretrainedの文字列でconfig.from_pretrainedを呼び出し\n",
        "2. config.from_pretrainedの結果をconfigにセット"
      ],
      "metadata": {
        "id": "yYsvP_-hf-pQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pandas._config import config\n",
        "# BERT分類モデルの定義\n",
        "class BERTClass(torch.nn.Module):\n",
        "  def __init__(self, drop_rate, otuput_size):\n",
        "    super().__init__()\n",
        "    # self.config = AutoConfig.from_pretrained(\"allenai/scibert_scivocab_uncased\", num_labels=4)\n",
        "    self.config = AutoConfig.from_pretrained(\"allenai/scibert_scivocab_uncased\")\n",
        "    self.bert = AutoModel.from_config(config=self.config)\n",
        "    self.drop = torch.nn.Dropout(drop_rate)\n",
        "    self.fc = torch.nn.Linear(768, otuput_size)  # BERTの出力に合わせて768次元を指定\n",
        "    torch.nn.init.normal_(self.fc.weight, std=0.02)\n",
        "    torch.nn.init.zeros_(self.fc.bias)\n",
        "\n",
        "  def forward(self, ids, mask):\n",
        "    _, out = self.bert(ids, attention_mask=mask)\n",
        "    out = self.fc(self.drop(out))\n",
        "    # out = self.bert(ids, attention_mask=mask)\n",
        "    # out = self.fc(self.drop(out[:, 0, :]))\n",
        "    return out"
      ],
      "metadata": {
        "id": "M38JBNnv7QA4"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_loss_and_f1_score(model, criterion, loader, device, epoch, max_epoch):\n",
        "  \"\"\" 損失・正解率を計算\"\"\"\n",
        "  model.eval()\n",
        "  loss = 0.0\n",
        "  total = 0\n",
        "  correct = 0\n",
        "  all_labels = []\n",
        "  all_preds = []\n",
        "\n",
        "  with torch.no_grad():\n",
        "    with tqdm(loader) as pbar:\n",
        "      pbar.set_description('valid')\n",
        "      for i, data in enumerate(pbar):\n",
        "        ids = data['ids'].to(device)\n",
        "        mask = data['mask'].to(device)\n",
        "        labels = data['labels'].to(device)\n",
        "\n",
        "\n",
        "        outputs = model(ids, mask)\n",
        "        loss += criterion(outputs, labels).item()\n",
        "\n",
        "        preds = torch.argmax(outputs, dim=-1).cpu().numpy() # バッチサイズの長さの予測ラベル配列\n",
        "        labels = torch.argmax(labels, dim=-1).cpu().numpy()  # バッチサイズの長さの正解ラベル配列\n",
        "        all_labels += labels.tolist()\n",
        "        all_preds += preds.tolist()\n",
        "        total += len(labels)\n",
        "        correct += (preds == labels).sum().item()\n",
        "        pbar.set_postfix(loss=loss/(i+1), f1=f1_score(all_labels, all_preds, average=\"macro\"))\n",
        "\n",
        "  return loss / len(loader), f1_score(all_labels, all_preds, average=\"macro\")"
      ],
      "metadata": {
        "id": "KFQSOOn7SbPp"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def train_val(train_loader, val_loader, model_path):\n",
        "  device = 'cuda' if cuda.is_available() else 'cpu'\n",
        "  model = BERTClass(DROP_RATE, OUTPUT_SIZE).to(device)\n",
        "  criterion = torch.nn.CrossEntropyLoss()\n",
        "  optimizer = torch.optim.AdamW(params=model.parameters(), lr=LEARNING_RATE)\n",
        "  \n",
        "  log_valid = []\n",
        "  best_acc = 0\n",
        "  all_labels = []\n",
        "  all_preds = []\n",
        "\n",
        "  # train\n",
        "  for epoch in range(NUM_EPOCHS):\n",
        "    print('\\nepoch:', epoch+1)\n",
        "    loss_train = correct = total = 0\n",
        "    model.train()\n",
        "    with tqdm(train_loader) as pbar:\n",
        "      pbar.set_description('train')\n",
        "      for i, data in enumerate(pbar):\n",
        "        ids = data['ids'].to(device)\n",
        "        mask = data['mask'].to(device)\n",
        "        labels = data['labels'].to(device)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "\n",
        "        outputs = model(ids, mask)\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        \n",
        "        loss_train += loss.item()\n",
        "        preds = torch.argmax(outputs, dim=-1).cpu().numpy()\n",
        "        labels = torch.argmax(labels, dim=-1).cpu().numpy()\n",
        "\n",
        "        all_labels += labels.tolist()\n",
        "        all_preds += preds.tolist()\n",
        "\n",
        "        total += len(labels)\n",
        "        correct += (preds == labels).sum().item()\n",
        "        pbar.set_postfix(loss=loss_train/(i+1), f1=f1_score(all_labels, all_preds, average=\"macro\"))\n",
        "\n",
        "    loss_valid, f1_valid = calculate_loss_and_f1_score(model, criterion, val_loader, device, epoch, NUM_EPOCHS)\n",
        "    log_valid.append([loss_valid, f1_valid])\n",
        "    if best_acc < f1_valid:\n",
        "      best_acc = f1_valid\n",
        "      \n",
        "      torch.save(model, os.path.join(ROOT, model_path))\n",
        "      print('==== model saved ====')\n",
        "\n",
        "  return log_valid"
      ],
      "metadata": {
        "id": "T9Hy83GPWD1C"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "DROP_RATE = 0.1\n",
        "OUTPUT_SIZE = 4\n",
        "BATCH_SIZE = 16\n",
        "NUM_EPOCHS = 20\n",
        "LEARNING_RATE = 2e-5\n",
        "N_CV = 3\n",
        "VERSION = 'v23'"
      ],
      "metadata": {
        "id": "XfF5KLQJWJCG"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.metrics\n",
        "def train_val_predict(\n",
        "        df_train,       # 学習用のデータ\n",
        "        text_column,    # 対象のカラム名\n",
        "        target_column,  # 目的変数のカラム名\n",
        "        df_valid=None,  # 検証用データ\n",
        "        df_val_test=None,       # 予測用データ\n",
        "        model_file_prefix=\"\",  # 保存時のファイル名識別子\n",
        "        epochs=NUM_EPOCHS,\n",
        "        batch_size=BATCH_SIZE,\n",
        "    ):\n",
        "  \n",
        "    # model\n",
        "    model_path = \"{}_{}.pth\".format(model_file_prefix, VERSION)\n",
        "    model = BERTClass(DROP_RATE, OUTPUT_SIZE)\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.AdamW(params=model.parameters(), lr=LEARNING_RATE)\n",
        "    device = 'cuda' if cuda.is_available() else 'cpu'\n",
        "\n",
        "    # dataset\n",
        "    df_val_test[target_column].fillna(0)\n",
        "\n",
        "    max_len = 128\n",
        "    tokenizer = AutoTokenizer.from_pretrained(\"allenai/scibert_scivocab_uncased\")\n",
        "    dataset_train = CreateDataset(df_train[text_column].values, pd.get_dummies(df_train[target_column]).values, tokenizer, max_len)\n",
        "    dataset_val = CreateDataset(df_valid[text_column].values, pd.get_dummies(df_valid[target_column]).values, tokenizer, max_len)\n",
        "    dataset_test = CreateDataset(df_val_test[text_column].values, pd.get_dummies(df_val_test[target_column]).values, tokenizer, max_len)\n",
        "\n",
        "    dataloader_train = DataLoader(dataset_train, batch_size=BATCH_SIZE, shuffle=True)\n",
        "    dataloader_valid = DataLoader(dataset_val, batch_size=BATCH_SIZE, shuffle=True)\n",
        "    dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, shuffle=False)\n",
        "\n",
        "    # train\n",
        "    print('\\n================  start train  ================')\n",
        "    metric = train_val(dataloader_train, dataloader_valid, model_path)\n",
        "    print('================  end train  ================\\n')\n",
        "\n",
        "    # predict\n",
        "    model = torch.load(os.path.join(ROOT, model_path))\n",
        "    model.eval()\n",
        "    pred_y_list = []\n",
        "    emb_list = []\n",
        "    with torch.no_grad():\n",
        "      with tqdm(dataloader_test) as pbar:\n",
        "        pbar.set_description('test')\n",
        "        for data in pbar:\n",
        "          ids = data['ids'].to(device)\n",
        "          mask = data['mask'].to(device)\n",
        "          labels = data['labels'].to(device)\n",
        "\n",
        "          output = model.forward(ids, mask)\n",
        "          pred = torch.argmax(output, dim=-1).cpu().numpy()\n",
        "\n",
        "          pred_y_list.extend(pred)\n",
        "          emb_list.extend(output.cpu().numpy())\n",
        "    return metric, pred_y_list, emb_list"
      ],
      "metadata": {
        "id": "La2mPphur18q"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import sklearn.model_selection\n",
        "\n",
        "text_column, target_column, n_splits = ('description', 'jobflag', N_CV)\n",
        "\n",
        "df = pd.concat([train, test], ignore_index=True, sort=False)\n",
        "df_train = df[df[target_column].notnull()]\n",
        "df_test = df[df[target_column].isnull()]\n",
        "\n",
        "df_train_idx = df_train.index\n",
        "\n",
        "# store result\n",
        "df_pred = pd.DataFrame(df.index, columns=[\"index\"]).set_index(\"index\")\n",
        "df_emb = pd.DataFrame(df.index, columns=[\"index\"]).set_index(\"index\")\n",
        "df_emb_pred = None\n",
        "metric_list = []\n",
        "all_emb = []\n",
        "\n",
        "# cross validation\n",
        "kf = sklearn.model_selection.StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED)\n",
        "for i, (train_idx, test_idx) in enumerate(kf.split(df_train, df_train[target_column])):\n",
        "  df_train_sub = df_train.iloc[train_idx]\n",
        "  df_test_sub = df_train.iloc[test_idx]\n",
        "\n",
        "  df_val_test = pd.concat([df_test_sub, df_test], ignore_index=True, sort=False)\n",
        "\n",
        "  model_file_prefix = \"cv_{}\".format(i)\n",
        "\n",
        "  # train\n",
        "  metric, pred_y_list, emb_list = train_val_predict(\n",
        "      df_train=df_train_sub, \n",
        "      text_column=text_column,\n",
        "      target_column=target_column, \n",
        "      df_valid=df_test_sub,\n",
        "      df_val_test=df_val_test,\n",
        "      model_file_prefix=model_file_prefix,\n",
        "  )\n",
        "  metric_list.append(metric)\n",
        "  all_emb.append(emb_list)\n",
        "  # break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QB2vixNEo7_B",
        "outputId": "0c698557-3cd1-456f-8404-ab60c2982a62"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:18<00:00,  3.33it/s, f1=0.25, loss=1.28]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.90it/s, f1=0.116, loss=1.25]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.30it/s, f1=0.257, loss=1.25]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.03it/s, f1=0.269, loss=1.18]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.29it/s, f1=0.278, loss=1.23]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.99it/s, f1=0.305, loss=1.16]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.29it/s, f1=0.327, loss=0.956]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.88it/s, f1=0.464, loss=0.858]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.26it/s, f1=0.364, loss=0.82]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.92it/s, f1=0.46, loss=0.888]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.30it/s, f1=0.405, loss=0.622]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.05it/s, f1=0.559, loss=0.849]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.12it/s, f1=0.445, loss=0.473]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.02it/s, f1=0.58, loss=0.816]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.18it/s, f1=0.497, loss=0.334]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.93it/s, f1=0.619, loss=1.04]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.04it/s, f1=0.532, loss=0.47]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.04it/s, f1=0.503, loss=0.958]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.22it/s, f1=0.568, loss=0.268]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.97it/s, f1=0.611, loss=0.957]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 11\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.21it/s, f1=0.603, loss=0.193]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.99it/s, f1=0.644, loss=0.979]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 12\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.07it/s, f1=0.631, loss=0.183]\n",
            "valid: 100%|██████████| 32/32 [00:03<00:00, 10.34it/s, f1=0.529, loss=1.24]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 13\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.17it/s, f1=0.656, loss=0.131]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.96it/s, f1=0.616, loss=1.17]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.15it/s, f1=0.681, loss=0.0536]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.91it/s, f1=0.627, loss=1.13]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.13it/s, f1=0.703, loss=0.0644]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.00it/s, f1=0.625, loss=1.26]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.10it/s, f1=0.722, loss=0.04]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.92it/s, f1=0.624, loss=1.21]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 17\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.09it/s, f1=0.74, loss=0.0244]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.05it/s, f1=0.61, loss=1.46]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 18\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  3.97it/s, f1=0.754, loss=0.0392]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.01it/s, f1=0.645, loss=1.38]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 19\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  3.97it/s, f1=0.767, loss=0.0508]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.91it/s, f1=0.601, loss=1.71]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 20\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.03it/s, f1=0.773, loss=0.266]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.93it/s, f1=0.632, loss=1.03]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 126/126 [00:10<00:00, 12.51it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.40it/s, f1=0.253, loss=1.29]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.06it/s, f1=0.182, loss=1.26]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.34it/s, f1=0.281, loss=1.23]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.84it/s, f1=0.247, loss=1.28]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.25it/s, f1=0.298, loss=1.2]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.44it/s, f1=0.244, loss=1.22]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.30it/s, f1=0.337, loss=1.01]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.86it/s, f1=0.348, loss=0.923]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.29it/s, f1=0.379, loss=0.748]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.87it/s, f1=0.431, loss=1.14]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.26it/s, f1=0.422, loss=0.577]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.74it/s, f1=0.517, loss=0.893]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.24it/s, f1=0.47, loss=0.43]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.81it/s, f1=0.55, loss=0.838]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.17it/s, f1=0.51, loss=0.322]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.08it/s, f1=0.629, loss=1.05]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.06it/s, f1=0.548, loss=0.236]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.93it/s, f1=0.598, loss=1.25]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.21it/s, f1=0.588, loss=0.159]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.97it/s, f1=0.541, loss=1.37]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 11\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.18it/s, f1=0.62, loss=0.153]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.07it/s, f1=0.524, loss=1.72]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 12\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.18it/s, f1=0.637, loss=0.238]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.09it/s, f1=0.597, loss=1.19]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 13\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.15it/s, f1=0.663, loss=0.0943]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.09it/s, f1=0.602, loss=1.26]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.13it/s, f1=0.688, loss=0.0516]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.08it/s, f1=0.581, loss=1.49]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.01it/s, f1=0.711, loss=0.0291]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.08it/s, f1=0.583, loss=1.63]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.09it/s, f1=0.726, loss=0.0792]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.09it/s, f1=0.587, loss=1.39]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 17\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.07it/s, f1=0.742, loss=0.0453]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.01it/s, f1=0.596, loss=1.43]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 18\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.05it/s, f1=0.758, loss=0.0158]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.10it/s, f1=0.573, loss=1.58]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 19\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.04it/s, f1=0.771, loss=0.0413]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.95it/s, f1=0.556, loss=1.78]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 20\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.02it/s, f1=0.78, loss=0.165]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.03it/s, f1=0.605, loss=1.51]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 126/126 [00:10<00:00, 11.84it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "================  start train  ================\n",
            "\n",
            "epoch: 1\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.37it/s, f1=0.237, loss=1.29]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.02it/s, f1=0.228, loss=1.26]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 2\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.28it/s, f1=0.251, loss=1.26]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.10it/s, f1=0.257, loss=1.21]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 3\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.27it/s, f1=0.295, loss=1.15]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.98it/s, f1=0.476, loss=0.989]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 4\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.25it/s, f1=0.349, loss=0.842]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.00it/s, f1=0.386, loss=1]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 5\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.29it/s, f1=0.386, loss=0.758]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.07it/s, f1=0.384, loss=1.13]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 6\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.19it/s, f1=0.417, loss=0.621]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.04it/s, f1=0.425, loss=1.16]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 7\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.26it/s, f1=0.452, loss=0.499]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.97it/s, f1=0.545, loss=1.05]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 8\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.15it/s, f1=0.493, loss=0.361]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.01it/s, f1=0.477, loss=1.26]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 9\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:14<00:00,  4.23it/s, f1=0.539, loss=0.232]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.08it/s, f1=0.52, loss=1.56]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 10\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.05it/s, f1=0.574, loss=0.248]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.06it/s, f1=0.545, loss=1.28]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 11\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.03it/s, f1=0.6, loss=0.259]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.26it/s, f1=0.576, loss=1.15]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 12\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.08it/s, f1=0.625, loss=0.173]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.01it/s, f1=0.564, loss=1.33]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 13\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.13it/s, f1=0.647, loss=0.124]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.14it/s, f1=0.569, loss=1.48]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 14\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.12it/s, f1=0.674, loss=0.052]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.06it/s, f1=0.588, loss=1.41]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.03it/s, f1=0.697, loss=0.0676]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.05it/s, f1=0.595, loss=1.55]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  3.99it/s, f1=0.715, loss=0.104]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.07it/s, f1=0.571, loss=1.77]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 17\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  3.97it/s, f1=0.731, loss=0.09]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.00it/s, f1=0.571, loss=1.6]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 18\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.05it/s, f1=0.743, loss=0.168]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.99it/s, f1=0.526, loss=1.49]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "epoch: 19\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  4.04it/s, f1=0.756, loss=0.0723]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 12.04it/s, f1=0.596, loss=1.78]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "\n",
            "epoch: 20\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "train: 100%|██████████| 63/63 [00:15<00:00,  3.97it/s, f1=0.769, loss=0.0194]\n",
            "valid: 100%|██████████| 32/32 [00:02<00:00, 11.54it/s, f1=0.607, loss=1.73]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "==== model saved ====\n",
            "================  end train  ================\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "test: 100%|██████████| 126/126 [00:10<00:00, 12.11it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "all_emb_ = np.array([np.array(emb)[-len(df_test):] for emb in all_emb])"
      ],
      "metadata": {
        "id": "eK-UI_G3wzIG"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mean_df = pd.DataFrame([[np.mean(arr_val) for arr_val in arr_vec] for arr_vec in np.moveaxis(all_emb_, 0, 2)])\n",
        "preds = [np.argmax(arr)+1 for arr in mean_df.to_numpy()]\n",
        "df_submit = pd.DataFrame([df_test.id.to_list(), preds]).T\n",
        "df_submit.to_csv(os.path.join(ROOT, f'{VERSION}_ensemble.csv'), index=None, header=None)"
      ],
      "metadata": {
        "id": "zHARK-Z-lvL8"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "f1_list = []\n",
        "for item in metric_list:\n",
        "  f1_list.append((np.max([acc for loss, acc in item])))\n",
        "print(np.mean(f1_list))"
      ],
      "metadata": {
        "id": "JVEj-oYop9as",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ddd72497-7afa-41b9-989f-fae285ac1f26"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.6271266229588782\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "chcdmZd0W5_K"
      },
      "execution_count": 14,
      "outputs": []
    }
  ]
}