notebooks/training-results-fp16-batched.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0a04b5c-80bb-413c-998f-2f442c96e456",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/tk/repos/erc/results\n"
     ]
    }
   ],
   "source": [
    "%cd ../results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2619a3d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import pprint\n",
    "\n",
    "\n",
    "def parse_path(path):\n",
    "    if \"val\" in os.path.basename(path):\n",
    "        SPLIT = \"val\"\n",
    "    elif \"test\" in os.path.basename(path):\n",
    "        SPLIT = \"test\"\n",
    "    else:\n",
    "        raise ValueError\n",
    "    splits = path.split(\"/\")\n",
    "    DATASET = splits[0]\n",
    "    roberta = splits[1]\n",
    "    seed_details = splits[3].split(\"-\")\n",
    "\n",
    "    timestamp = \"-\".join(seed_details[:6])\n",
    "\n",
    "    speaker_mode = seed_details[7]\n",
    "    assert speaker_mode in [\"upper\", \"None\", \"title\"]\n",
    "    num_past_utterances = int(seed_details[9])\n",
    "    num_future_utterances = int(seed_details[11])\n",
    "    bsz = int(seed_details[13])\n",
    "    seed = int(seed_details[15])\n",
    "\n",
    "    to_return = {\n",
    "        \"DATASET\": DATASET,\n",
    "        \"SPLIT\": SPLIT,\n",
    "        \"roberta\": roberta,\n",
    "        \"speaker_mode\": speaker_mode,\n",
    "        \"num_past_utterances\": num_past_utterances,\n",
    "        \"num_future_utterances\": num_future_utterances,\n",
    "        \"bsz\": bsz,\n",
    "        \"seed\": seed,\n",
    "        \"timestamp\": timestamp,\n",
    "    }\n",
    "    return to_return\n",
    "\n",
    "\n",
    "def read_json(path):\n",
    "    with open(path, \"r\") as stream:\n",
    "        foo = json.load(stream)\n",
    "    return foo\n",
    "\n",
    "\n",
    "def merge_two_dicts(a, b):\n",
    "    c = {}\n",
    "    for key, val in a.items():\n",
    "        c[key] = val\n",
    "    for key, val in b.items():\n",
    "        c[key] = val\n",
    "\n",
    "    return c\n",
    "\n",
    "\n",
    "def return_by_timestamp(seed_results):\n",
    "    timestamps = set([foo[\"timestamp\"] for foo in seed_results])\n",
    "    seed_results = {\n",
    "        timestamp: [foo for foo in seed_results if foo[\"timestamp\"] == timestamp]\n",
    "        for timestamp in timestamps\n",
    "    }\n",
    "\n",
    "    return seed_results\n",
    "\n",
    "\n",
    "def return_metrics(seed_results):\n",
    "\n",
    "    to_return = {}\n",
    "    for timestamp, foo in seed_results.items():\n",
    "        stats_val = [bar for bar in foo if bar[\"SPLIT\"] == \"val\"]\n",
    "        stats_test = [bar for bar in foo if bar[\"SPLIT\"] == \"test\"]\n",
    "\n",
    "        for metric in [\n",
    "            \"DATASET\",\n",
    "            \"roberta\",\n",
    "            \"speaker_mode\",\n",
    "            \"num_past_utterances\",\n",
    "            \"num_future_utterances\",\n",
    "        ]:\n",
    "            assert len(set([bar[metric] for bar in foo]))\n",
    "\n",
    "        keyname = \", \".join(\n",
    "            [\n",
    "                f\"{metric}: {foo[0][metric]}\"\n",
    "                for metric in [\n",
    "                    \"DATASET\",\n",
    "                    \"roberta\",\n",
    "                    \"speaker_mode\",\n",
    "                    \"num_past_utterances\",\n",
    "                    \"num_future_utterances\",\n",
    "                ]\n",
    "            ]\n",
    "        )\n",
    "\n",
    "        keyname += f\", {str(timestamp)}\"\n",
    "        stats_test_ = {}\n",
    "        for metric in [\n",
    "            \"test_loss\",\n",
    "            \"test_f1_weighted\",\n",
    "            \"test_f1_micro\",\n",
    "            \"test_f1_macro\",\n",
    "        ]:\n",
    "            stats_test_[metric] = (\n",
    "                np.mean([foo[metric] for foo in stats_test]),\n",
    "                np.std([foo[metric] for foo in stats_test]),\n",
    "                len([foo[metric] for foo in stats_test]),\n",
    "            )\n",
    "\n",
    "        stats_val_ = {}\n",
    "        for metric in [\n",
    "            \"eval_loss\",\n",
    "            \"eval_f1_weighted\",\n",
    "            \"eval_f1_micro\",\n",
    "            \"eval_f1_macro\",\n",
    "        ]:\n",
    "            stats_val_[metric] = (\n",
    "                np.mean([foo[metric] for foo in stats_val]),\n",
    "                np.std([foo[metric] for foo in stats_val]),\n",
    "                len([foo[metric] for foo in stats_val]),\n",
    "            )\n",
    "\n",
    "        to_return[keyname] = (stats_val_, stats_test_)\n",
    "\n",
    "    to_return = {foo: to_return[foo] for foo in sorted(list(to_return.keys()))}\n",
    "\n",
    "    return to_return\n",
    "\n",
    "\n",
    "def get_average(seed_results):\n",
    "    timestamps = set(list(seed_results.keys()))\n",
    "\n",
    "    to_return = {}\n",
    "    for timestamp in timestamps:\n",
    "        to_return[timestamp] = {\n",
    "            key: (np.mean(list(val)), len(list(val)))\n",
    "            for key, val in seed_results[timestamp].items()\n",
    "        }\n",
    "\n",
    "    return to_return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a10c8838",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DATASET: IEMOCAP, roberta: roberta-large, speaker_mode: None, num_past_utterances: 0, num_future_utterances: 0, 2021-05-09-11-21-35\n",
      "({'eval_f1_macro': (0.5522587758713577, 0.005700503695160694, 5),\n",
      "  'eval_f1_micro': (0.5687755102040817, 0.006505052020616037, 5),\n",
      "  'eval_f1_weighted': (0.575727996274017, 0.005460606873912623, 5),\n",
      "  'eval_loss': (1.2397764921188354, 0.06936472873560774, 5)},\n",
      " {'test_f1_macro': (0.5378345315121387, 0.006522106517741987, 5),\n",
      "  'test_f1_micro': (0.5617755856966707, 0.0073301425142795786, 5),\n",
      "  'test_f1_weighted': (0.5587721595281857, 0.006840271384498524, 5),\n",
      "  'test_loss': (1.2600319862365723, 0.03722152681838191, 5)})\n",
      "\n",
      "DATASET: IEMOCAP, roberta: roberta-large, speaker_mode: None, num_past_utterances: 1000, num_future_utterances: 1000, 2021-05-17-22-02-20\n",
      "({'eval_f1_macro': (0.5918548396452185, 0.013224703460383593, 5),\n",
      "  'eval_f1_micro': (0.6218367346938776, 0.006848147647168361, 5),\n",
      "  'eval_f1_weighted': (0.6177202952197651, 0.008984988114688916, 5),\n",
      "  'eval_loss': (0.978229534626007, 0.017938782802415995, 5)},\n",
      " {'test_f1_macro': (0.6253152837177338, 0.026139324899799712, 5),\n",
      "  'test_f1_micro': (0.6404438964241678, 0.020586323524049276, 5),\n",
      "  'test_f1_weighted': (0.6401577334598076, 0.0211100773399199, 5),\n",
      "  'test_loss': (0.9373827576637268, 0.0210231178570917, 5)})\n",
      "\n",
      "DATASET: IEMOCAP, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 0, num_future_utterances: 0, 2021-05-09-11-49-51\n",
      "({'eval_f1_macro': (0.5269594714657198, 0.010379405536027687, 5),\n",
      "  'eval_f1_micro': (0.5724489795918368, 0.007717416490281227, 5),\n",
      "  'eval_f1_weighted': (0.564236788122536, 0.005601506278689179, 5),\n",
      "  'eval_loss': (1.2878952741622924, 0.04621741931503711, 5)},\n",
      " {'test_f1_macro': (0.5431934834368952, 0.005246401178079798, 5),\n",
      "  'test_f1_micro': (0.5637484586929716, 0.007100458717084058, 5),\n",
      "  'test_f1_weighted': (0.5608727948849529, 0.006838565818616568, 5),\n",
      "  'test_loss': (1.3399529695510863, 0.02710581041443506, 5)})\n",
      "\n",
      "DATASET: IEMOCAP, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 0, num_future_utterances: 1000, 2021-05-09-15-16-25\n",
      "({'eval_f1_macro': (0.6396472234317585, 0.013680590763494559, 5),\n",
      "  'eval_f1_micro': (0.6495918367346938, 0.01109573650536179, 5),\n",
      "  'eval_f1_weighted': (0.6504960283305034, 0.01020846277487317, 5),\n",
      "  'eval_loss': (1.8104423761367798, 0.3882764954185471, 5)},\n",
      " {'test_f1_macro': (0.6453711024980999, 0.006587585239162013, 5),\n",
      "  'test_f1_micro': (0.6674475955610357, 0.0064849291207421614, 5),\n",
      "  'test_f1_weighted': (0.6656272077776554, 0.004839493882345372, 5),\n",
      "  'test_loss': (1.6912848472595214, 0.3341220930571711, 5)})\n",
      "\n",
      "DATASET: IEMOCAP, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 1000, num_future_utterances: 0, 2021-05-09-12-19-54\n",
      "({'eval_f1_macro': (0.6597466866669253, 0.010837695056091235, 5),\n",
      "  'eval_f1_micro': (0.6693877551020408, 0.008264676216863906, 5),\n",
      "  'eval_f1_weighted': (0.6666148556427564, 0.007426962361417682, 5),\n",
      "  'eval_loss': (1.3905875444412232, 0.3761005271923305, 5)},\n",
      " {'test_f1_macro': (0.6689570035188008, 0.029427809685156278, 5),\n",
      "  'test_f1_micro': (0.6871763255240444, 0.020472270992572445, 5),\n",
      "  'test_f1_weighted': (0.6856809139267018, 0.022639715810633135, 5),\n",
      "  'test_loss': (1.259392774105072, 0.2332461248339133, 5)})\n",
      "\n",
      "DATASET: IEMOCAP, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 1000, num_future_utterances: 1000, 2021-05-08-16-15-19\n",
      "({'eval_f1_macro': (0.62225173556929, 0.013124909180567024, 5),\n",
      "  'eval_f1_micro': (0.6393877551020408, 0.013668899632359474, 5),\n",
      "  'eval_f1_weighted': (0.6393675494445887, 0.013131825801780794, 5),\n",
      "  'eval_loss': (1.0893967151641846, 0.0536085487879549, 5)},\n",
      " {'test_f1_macro': (0.6593415963373445, 0.01478072671367545, 5),\n",
      "  'test_f1_micro': (0.6743526510480887, 0.012558919237002432, 5),\n",
      "  'test_f1_weighted': (0.6741956238807851, 0.01290999036130419, 5),\n",
      "  'test_loss': (0.9940088391304016, 0.054569210131915014, 5)})\n",
      "\n",
      "DATASET: MELD, roberta: roberta-large, speaker_mode: None, num_past_utterances: 0, num_future_utterances: 0, 2021-05-09-10-34-30\n",
      "({'eval_f1_macro': (0.49769530133977274, 0.010085638843148228, 5),\n",
      "  'eval_f1_micro': (0.6445446348061317, 0.004088650693870024, 5),\n",
      "  'eval_f1_weighted': (0.6310382873181728, 0.003771889253918822, 5),\n",
      "  'eval_loss': (1.0687896013259888, 0.022409618237624752, 5)},\n",
      " {'test_f1_macro': (0.4711702840026922, 0.009756345772609967, 5),\n",
      "  'test_f1_micro': (0.6472030651340996, 0.00887500567289878, 5),\n",
      "  'test_f1_weighted': (0.6374340068650503, 0.007576161077227481, 5),\n",
      "  'test_loss': (1.0612706422805787, 0.03533038359378681, 5)})\n",
      "\n",
      "DATASET: MELD, roberta: roberta-large, speaker_mode: None, num_past_utterances: 1000, num_future_utterances: 1000, 2021-05-17-18-24-48\n",
      "({'eval_f1_macro': (0.5434059812259459, 0.013941075682803367, 5),\n",
      "  'eval_f1_micro': (0.6694319206492335, 0.0010820559062218126, 5),\n",
      "  'eval_f1_weighted': (0.6622215604046072, 0.0035632579942559955, 5),\n",
      "  'eval_loss': (1.5122009754180907, 0.30039934069850555, 5)},\n",
      " {'test_f1_macro': (0.48847725837417116, 0.01354289031387888, 5),\n",
      "  'test_f1_micro': (0.657471264367816, 0.010174560170697124, 5),\n",
      "  'test_f1_weighted': (0.6507094787731461, 0.008282916120317254, 5),\n",
      "  'test_loss': (1.611864447593689, 0.3325411893150695, 5)})\n",
      "\n",
      "DATASET: MELD, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 0, num_future_utterances: 0, 2021-05-09-09-46-30\n",
      "({'eval_f1_macro': (0.48847254387259903, 0.008495853701994283, 5),\n",
      "  'eval_f1_micro': (0.6371505861136159, 0.010769340758761875, 5),\n",
      "  'eval_f1_weighted': (0.623769957569515, 0.005859307103685082, 5),\n",
      "  'eval_loss': (1.170855164527893, 0.08526505672547598, 5)},\n",
      " {'test_f1_macro': (0.46928286476559455, 0.011713978274065371, 5),\n",
      "  'test_f1_micro': (0.6440613026819924, 0.010337729933511139, 5),\n",
      "  'test_f1_weighted': (0.6345787122115116, 0.005355029472872085, 5),\n",
      "  'test_loss': (1.149923324584961, 0.08976748597921823, 5)})\n",
      "\n",
      "DATASET: MELD, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 0, num_future_utterances: 1000, 2021-05-10-09-49-06\n",
      "({'eval_f1_macro': (0.4965479575059077, 0.011459399507116217, 5),\n",
      "  'eval_f1_micro': (0.6503155996393147, 0.008272205548190945, 5),\n",
      "  'eval_f1_weighted': (0.6355222708985134, 0.004283409125717582, 5),\n",
      "  'eval_loss': (1.2762720584869385, 0.23882561122396728, 5)},\n",
      " {'test_f1_macro': (0.4683521430057107, 0.012352855406170054, 5),\n",
      "  'test_f1_micro': (0.653256704980843, 0.007776928402369506, 5),\n",
      "  'test_f1_weighted': (0.6422649707423742, 0.0027429970350398053, 5),\n",
      "  'test_loss': (1.2563609838485719, 0.23830774197874788, 5)})\n",
      "\n",
      "DATASET: MELD, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 1000, num_future_utterances: 0, 2021-05-09-18-15-01\n",
      "({'eval_f1_macro': (0.5335582959109558, 0.013736411856708508, 5),\n",
      "  'eval_f1_micro': (0.6643823264201985, 0.004576583423065661, 5),\n",
      "  'eval_f1_weighted': (0.6567062929470665, 0.005184104960563743, 5),\n",
      "  'eval_loss': (1.695301079750061, 0.34121903295795375, 5)},\n",
      " {'test_f1_macro': (0.4805377507237064, 0.017029589352614217, 5),\n",
      "  'test_f1_micro': (0.651800766283525, 0.00980244070065738, 5),\n",
      "  'test_f1_weighted': (0.6455350975434817, 0.008393357993359898, 5),\n",
      "  'test_loss': (1.773275661468506, 0.36145308383427205, 5)})\n",
      "\n",
      "DATASET: MELD, roberta: roberta-large, speaker_mode: upper, num_past_utterances: 1000, num_future_utterances: 1000, 2021-05-08-19-57-31\n",
      "({'eval_f1_macro': (0.5556886436897545, 0.01376117449263144, 5),\n",
      "  'eval_f1_micro': (0.6815148782687106, 0.006537265630438883, 5),\n",
      "  'eval_f1_weighted': (0.6739214836802828, 0.0060834391042476664, 5),\n",
      "  'eval_loss': (1.2840325355529785, 0.11457416568222668, 5)},\n",
      " {'test_f1_macro': (0.48879753943586934, 0.00665075670382589, 5),\n",
      "  'test_f1_micro': (0.6626819923371647, 0.0043455775873587335, 5),\n",
      "  'test_f1_weighted': (0.6561223421668305, 0.0041761291012733515, 5),\n",
      "  'test_loss': (1.3610458850860596, 0.12069501090284863, 5)})\n",
      "\n"
     ]
    }
   ],
   "source": [
    "seed_results = glob(\"*/*/final/*/*-results.json\")\n",
    "seed_results = [foo for foo in seed_results if \"MELD\" in foo or \"IEMOCAP\" in foo]\n",
    "seed_results = [(parse_path(foo), read_json(foo)) for foo in seed_results]\n",
    "seed_results = [merge_two_dicts(foo[0], foo[1]) for foo in seed_results]\n",
    "seed_results = return_by_timestamp(seed_results)\n",
    "seed_results = return_metrics(seed_results)\n",
    "for key, val in seed_results.items():\n",
    "    print(key)\n",
    "    pprint.pprint(val)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4751bed3-f9f6-431c-af64-c5d23109ca68",
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_results = glob(\"*/*/final/*/*-results.json\")\n",
    "seed_results = [foo for foo in seed_results if \"MELD\" in foo or \"IEMOCAP\" in foo]\n",
    "seed_results = [(parse_path(foo), read_json(foo)) for foo in seed_results]\n",
    "seed_results = [merge_two_dicts(foo[0], foo[1]) for foo in seed_results]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8b6622b8-cc78-4ced-b79e-c81d474b85b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_by_filter(\n",
    "    seed_results,\n",
    "    DATASET,\n",
    "    SPLIT,\n",
    "    num_past_utterances,\n",
    "    num_future_utterances,\n",
    "    speaker_mode,\n",
    "):\n",
    "    seed_results = [\n",
    "        foo\n",
    "        for foo in seed_results\n",
    "        if foo[\"DATASET\"] == DATASET\n",
    "        and foo[\"SPLIT\"] == SPLIT\n",
    "        and foo[\"num_past_utterances\"] == num_past_utterances\n",
    "        and foo[\"num_future_utterances\"] == num_future_utterances\n",
    "        and foo[\"speaker_mode\"] == speaker_mode\n",
    "    ]\n",
    "\n",
    "    if SPLIT == \"test\":\n",
    "        seed_results = sorted(seed_results, key=lambda k: k[\"test_f1_weighted\"])\n",
    "    else:\n",
    "        seed_results = sorted(seed_results, key=lambda k: k[\"eval_f1_weighted\"])\n",
    "    return seed_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c6535c95-9846-439d-848f-bf2448236b0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'MELD',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 0,\n",
       " 'num_future_utterances': 0,\n",
       " 'bsz': 16,\n",
       " 'seed': 3,\n",
       " 'timestamp': '2021-05-09-09-46-30',\n",
       " 'test_loss': 1.0628687143325806,\n",
       " 'test_f1_weighted': 0.6427050027040432,\n",
       " 'test_f1_micro': 0.6540229885057471,\n",
       " 'test_f1_macro': 0.48005284601944886,\n",
       " 'test_runtime': 5.8908,\n",
       " 'test_samples_per_second': 443.067}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"MELD\", \"test\", 0, 0, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d75ba782-3e9b-4193-a71f-e2f292869371",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'IEMOCAP',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 0,\n",
       " 'num_future_utterances': 0,\n",
       " 'bsz': 16,\n",
       " 'seed': 0,\n",
       " 'timestamp': '2021-05-09-11-49-51',\n",
       " 'test_loss': 1.3592785596847534,\n",
       " 'test_f1_weighted': 0.5717553437280344,\n",
       " 'test_f1_micro': 0.5752157829839704,\n",
       " 'test_f1_macro': 0.5521338637986823,\n",
       " 'test_runtime': 5.8949,\n",
       " 'test_samples_per_second': 275.151}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"IEMOCAP\", \"test\", 0, 0, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d234fdb9-bcbf-43b3-8852-1407e314592d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'MELD',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 1000,\n",
       " 'num_future_utterances': 0,\n",
       " 'bsz': 4,\n",
       " 'seed': 4,\n",
       " 'timestamp': '2021-05-09-18-15-01',\n",
       " 'test_loss': 1.3221737146377563,\n",
       " 'test_f1_weighted': 0.6557133000420554,\n",
       " 'test_f1_micro': 0.6647509578544061,\n",
       " 'test_f1_macro': 0.5012291200656293,\n",
       " 'test_runtime': 28.7962,\n",
       " 'test_samples_per_second': 90.637}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"MELD\", \"test\", 1000, 0, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f6353b6b-67b6-47c9-b538-03c5696cdf37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'IEMOCAP',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 1000,\n",
       " 'num_future_utterances': 0,\n",
       " 'bsz': 4,\n",
       " 'seed': 4,\n",
       " 'timestamp': '2021-05-09-12-19-54',\n",
       " 'test_loss': 1.578665018081665,\n",
       " 'test_f1_weighted': 0.7119324785334752,\n",
       " 'test_f1_micro': 0.7120838471023428,\n",
       " 'test_f1_macro': 0.6983733181480981,\n",
       " 'test_runtime': 44.1226,\n",
       " 'test_samples_per_second': 36.761}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"IEMOCAP\", \"test\", 1000, 0, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "33bbe4e4-1186-4927-a8aa-b102b92b4321",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'MELD',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 0,\n",
       " 'num_future_utterances': 1000,\n",
       " 'bsz': 4,\n",
       " 'seed': 2,\n",
       " 'timestamp': '2021-05-10-09-49-06',\n",
       " 'test_loss': 1.064085841178894,\n",
       " 'test_f1_weighted': 0.6466501132599567,\n",
       " 'test_f1_micro': 0.6628352490421456,\n",
       " 'test_f1_macro': 0.46634103620119893,\n",
       " 'test_runtime': 28.7181,\n",
       " 'test_samples_per_second': 90.883}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"MELD\", \"test\", 0, 1000, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "33879f0c-219d-4aad-9eb8-970a69d898a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'IEMOCAP',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 0,\n",
       " 'num_future_utterances': 1000,\n",
       " 'bsz': 4,\n",
       " 'seed': 1,\n",
       " 'timestamp': '2021-05-09-15-16-25',\n",
       " 'test_loss': 1.7993016242980957,\n",
       " 'test_f1_weighted': 0.673464603187326,\n",
       " 'test_f1_micro': 0.6738594327990136,\n",
       " 'test_f1_macro': 0.6561482679271721,\n",
       " 'test_runtime': 43.9186,\n",
       " 'test_samples_per_second': 36.932}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"IEMOCAP\", \"test\", 0, 1000, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4708f949-87ee-4bb4-9559-1a6ee0a5a3b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'MELD',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 1000,\n",
       " 'num_future_utterances': 1000,\n",
       " 'bsz': 4,\n",
       " 'seed': 4,\n",
       " 'timestamp': '2021-05-08-19-57-31',\n",
       " 'test_loss': 1.4145874977111816,\n",
       " 'test_f1_weighted': 0.6622864224519749,\n",
       " 'test_f1_micro': 0.6674329501915709,\n",
       " 'test_f1_macro': 0.49737579178046865,\n",
       " 'test_runtime': 37.6047,\n",
       " 'test_samples_per_second': 69.406}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"MELD\", \"test\", 1000, 1000, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5b5d459a-9bef-4bb4-a590-276a601d1f21",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'IEMOCAP',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'upper',\n",
       " 'num_past_utterances': 1000,\n",
       " 'num_future_utterances': 1000,\n",
       " 'bsz': 4,\n",
       " 'seed': 2,\n",
       " 'timestamp': '2021-05-08-16-15-19',\n",
       " 'test_loss': 0.9692837595939636,\n",
       " 'test_f1_weighted': 0.6920397733913108,\n",
       " 'test_f1_micro': 0.6911220715166461,\n",
       " 'test_f1_macro': 0.6808140109741574,\n",
       " 'test_runtime': 47.1837,\n",
       " 'test_samples_per_second': 34.376}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"IEMOCAP\", \"test\", 1000, 1000, \"upper\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a3d0dd09-3fff-4e98-827e-513af0d0fd33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'MELD',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'None',\n",
       " 'num_past_utterances': 1000,\n",
       " 'num_future_utterances': 1000,\n",
       " 'bsz': 4,\n",
       " 'seed': 4,\n",
       " 'timestamp': '2021-05-17-18-24-48',\n",
       " 'test_loss': 1.2838237285614014,\n",
       " 'test_f1_weighted': 0.6642043485642859,\n",
       " 'test_f1_micro': 0.6739463601532567,\n",
       " 'test_f1_macro': 0.501331044903872,\n",
       " 'test_runtime': 29.0112,\n",
       " 'test_samples_per_second': 89.965}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"MELD\", \"test\", 1000, 1000, \"None\")[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4af7367a-d50b-43d5-b13f-8a8374e04f9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'DATASET': 'IEMOCAP',\n",
       " 'SPLIT': 'test',\n",
       " 'roberta': 'roberta-large',\n",
       " 'speaker_mode': 'None',\n",
       " 'num_past_utterances': 1000,\n",
       " 'num_future_utterances': 1000,\n",
       " 'bsz': 4,\n",
       " 'seed': 4,\n",
       " 'timestamp': '2021-05-17-22-02-20',\n",
       " 'test_loss': 0.951850950717926,\n",
       " 'test_f1_weighted': 0.6565862745691114,\n",
       " 'test_f1_micro': 0.655980271270037,\n",
       " 'test_f1_macro': 0.6398377711685964,\n",
       " 'test_runtime': 46.2573,\n",
       " 'test_samples_per_second': 35.065}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_by_filter(seed_results, \"IEMOCAP\", \"test\", 1000, 1000, \"None\")[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.11"
  },
  "metadata": {
   "interpreter": {
    "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}