experiments_new.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from get_datasets import get_data\n",
    "train_dataset, val_dataset, test_dataset, vocab = get_data('SentimentClassifier/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "\n",
    "def sorted_collate(batch):\n",
    "    x, y = zip(*batch)\n",
    "    order = np.argsort([len(xi) for xi in x])[::-1]\n",
    "    x = [torch.LongTensor(x[i]) for i in order]\n",
    "    y = torch.stack([y[i] for i in order])[:, None]\n",
    "    return x,y\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=512, shuffle=True, collate_fn=sorted_collate)\n",
    "val_loader = DataLoader(val_dataset, batch_size=1024, shuffle=False, collate_fn=sorted_collate)\n",
    "test_loader = DataLoader(test_dataset, batch_size=1024, shuffle=False, collate_fn=sorted_collate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch import jit\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class SkipGRUCell(jit.ScriptModule):\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        \"\"\"Simple SkipGRU cell.\n",
    "        Args:\n",
    "            input_size (int): size of input space of GRU\n",
    "            hidden_size (int): size of hidden space of GRU\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.inner_gru = nn.GRUCell(input_size, hidden_size)\n",
    "\n",
    "    @jit.script_method\n",
    "    def forward(self, input, hidden, mix):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            input (torch.FloatTensor): input vector for the current step\n",
    "            hidden (torch.FloatTensor): hidden vector for the previous step\n",
    "            mix (torch.FloatTensor): mixture vector which defines proportion of mix between previous step and current.\n",
    "        Returns:\n",
    "            hidden (torch.FloatTensor): hidden vetor for the current step\n",
    "        \"\"\"\n",
    "        # type: (Tensor, Tensor, Tensor) -> Tensor\n",
    "\n",
    "        output = self.inner_gru(input, hidden)\n",
    "        output = output * mix + hidden * (1 - mix)\n",
    "\n",
    "        return output\n",
    "\n",
    "from typing import List, Tuple\n",
    "\n",
    "class SkipGRULayer(jit.ScriptModule):\n",
    "    __constants__ = ['hidden_size']\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        \"\"\"\n",
    "        Layer consisted of SkipGRU cells.\n",
    "        Args:\n",
    "            input_size (int): size of input space of GRU\n",
    "            hidden_size (int): size of hidden space of GRU\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        self.cell = SkipGRUCell(input_size, hidden_size)\n",
    "        self.hidden_size = hidden_size\n",
    "\n",
    "    @jit.script_method\n",
    "    def forward(self, input, mix):\n",
    "        \"\"\"\n",
    "        Forward pass on SkipGRU layer.\n",
    "        Args:\n",
    "            input (torch.FloatTensor): inputs for the SkipGRU layer of form [BxTxN]\n",
    "            mix (torch.FloatTensor): mixture coefficients for thehidden steps of form [BxTx1]\n",
    "        Returns:\n",
    "            h (torch.FloatTensor): history of outputs by steps. Size [BxTxH]\n",
    "            o (torch.FloatTensor): last hidden step of size [BxH]\n",
    "        \"\"\"\n",
    "        # type: (Tensor, Tensor) -> Tuple[Tensor, Tensor]\n",
    "        hidden = torch.zeros(input.shape[0], self.hidden_size,\n",
    "                             device=self.cell.inner_gru.weight_ih.device)\n",
    "        inputs = input.unbind(1)\n",
    "        mixtures = mix.unbind(1)\n",
    "        outputs = jit.annotate(List[Tensor], [])\n",
    "        for i in range(len(inputs)):\n",
    "            hidden = self.cell(inputs[i], hidden, mixtures[i])\n",
    "            outputs += [hidden]\n",
    "        return torch.stack(outputs, 1), hidden\n",
    "\n",
    "\n",
    "class SkipGRU(nn.Module):\n",
    "    def __init__(self, input_size=None, hidden_size=None, layer=None, do_copy_weights=False):\n",
    "        \"\"\"\n",
    "        SkipRNN layer with trivial binarizer (All betas are directly parametrized.)\n",
    "        Args:\n",
    "            input_size (int): size of input space for SkipGRU\n",
    "            hidden_size (int): size of hidden space for SkipGRU\n",
    "            layer (nn.GRU): original GRU layers to copy sizes and make a link to weights\n",
    "        \"\"\"\n",
    "        super().__init__()\n",
    "        if layer is not None:\n",
    "            self.layer = SkipGRULayer(layer.input_size, layer.hidden_size)\n",
    "            self.impute_weights(layer, do_copy_weights)\n",
    "        elif (input_size is not None) and (hidden_size is not None):\n",
    "            self.layer = SkipGRULayer(input_size, hidden_size)\n",
    "        else:\n",
    "            raise Exception('Either layer or input_size & hidden_size are required')\n",
    "\n",
    "    def impute_weights(self, donor_layer, copy=False):\n",
    "        \"\"\"\n",
    "        Get weights from instance of nn.GRU.\n",
    "        Args:\n",
    "            donor_layer (nn.GRU): pretrained layer to get weights from.\n",
    "            copy (bool): if True, will make copy of weights instead of linkage.\n",
    "        \"\"\"\n",
    "        assert isinstance(donor_layer, nn.GRU), 'Wrong type of donor layer. GRU required!'\n",
    "\n",
    "        if copy:\n",
    "            self.layer.cell.inner_gru.weight_ih.data = donor_layer.weight_ih_l0.clone()\n",
    "            self.layer.cell.inner_gru.weight_hh.data = donor_layer.weight_hh_l0.clone()\n",
    "            self.layer.cell.inner_gru.bias_ih.data = donor_layer.bias_ih_l0.clone()\n",
    "            self.layer.cell.inner_gru.bias_hh.data = donor_layer.bias_hh_l0.clone()\n",
    "        else:\n",
    "            self.layer.cell.inner_gru.weight_ih = donor_layer.weight_ih_l0\n",
    "            self.layer.cell.inner_gru.weight_hh = donor_layer.weight_hh_l0\n",
    "            self.layer.cell.inner_gru.bias_ih = donor_layer.bias_ih_l0\n",
    "            self.layer.cell.inner_gru.bias_hh = donor_layer.bias_hh_l0\n",
    "\n",
    "    def forward(self, x, b, l=None):\n",
    "        \"\"\"\n",
    "        Make forward SkipGRU pass.\n",
    "        Args:\n",
    "            x (torch.FloatTensor): input of size [BxTxN]. NB! sequences should be padded from the end, not from the start position.\n",
    "            u (torch.FloatTensor): mixture coefficients of size [BxTx1]\n",
    "            l (torch.LongTensor): lengths of the padded sequences. If not provided, output will contain last elements of sequences.\n",
    "        Returns:\n",
    "            h (torch.FloatTensor, nn.utils.rnn.PackedSequence): history of hidden states of size [BxTxH]\n",
    "            o (torch.FloatTensor): last hidden state of size [BxH]\n",
    "        \"\"\"\n",
    "        h, o = self.layer(x, b)\n",
    "        if l is not None:\n",
    "            o = h[torch.arange(h.shape[0]), l-1]\n",
    "        return h, o"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "#from skiprnn import SkipGRU\n",
    "import torch.functional as F\n",
    "\n",
    "class IMDbRNN(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_size, hidden_size, num_classes):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = SkipGRU(embedding_size, hidden_size)\n",
    "        self.cls = nn.Linear(hidden_size, num_classes)\n",
    "        self.emb = nn.Embedding(vocab_size, embedding_size)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        #на данном этапе учим с маской из одних единиц\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)\n",
    "        return self.cls(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IMDbRNN(\n",
       "  (rnn): SkipGRU(\n",
       "    (layer): SkipGRULayer(\n",
       "      original_name=SkipGRULayer\n",
       "      (cell): SkipGRUCell(\n",
       "        original_name=SkipGRUCell\n",
       "        (inner_gru): ScriptModule(original_name=GRUCell)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (cls): Linear(in_features=16, out_features=1, bias=True)\n",
       "  (emb): Embedding(2000, 32)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cpu')\n",
    "model = IMDbRNN(2000, 32, 16, 1)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ayunts/anaconda3/lib/python3.7/site-packages/dask/config.py:168: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  data = yaml.load(f.read()) or {}\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from catalyst.dl import SupervisedRunner\n",
    "from catalyst.dl.callbacks import AUCCallback, F1ScoreCallback, CheckpointCallback, CriterionCallback, CriterionAggregatorCallback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100\n",
    "logdir = \"./logs/exp3\"\n",
    "loaders = {\n",
    "    \"train\": train_loader,\n",
    "    \"valid\": val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"bce\": nn.BCEWithLogitsLoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model.parameters(), 'lr': 3e-4}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 13:52:43,204] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=738.5268 | _timers/batch_time=0.7703 | _timers/data_time=0.0107 | _timers/model_time=0.7595 | auc/_mean=0.5220 | auc/class_0=0.5220 | f1_score=0.5011 | loss=0.6922 | loss_bce=0.6922\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=1178.0794 | _timers/batch_time=0.9482 | _timers/data_time=0.0192 | _timers/model_time=0.9290 | auc/_mean=0.5301 | auc/class_0=0.5301 | f1_score=0.3597 | loss=0.6911 | loss_bce=0.6911\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-2d282f0c9483>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0mnum_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_epochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;31m#     fp16={\"opt_level\": \"O1\"},\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m )\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/runner/supervised.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, model, criterion, optimizer, loaders, logdir, callbacks, scheduler, resume, num_epochs, valid_loader, main_metric, minimize_metric, verbose, state_kwargs, checkpoint_data, fp16, monitoring_params, check)\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0mmonitoring_params\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmonitoring_params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m         )\n\u001b[0;32m--> 204\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_experiment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcheck\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheck\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m     def infer(\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36mrun_experiment\u001b[0;34m(self, experiment, check)\u001b[0m\n\u001b[1;32m    350\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mException\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 352\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"exception\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    353\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    354\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36m_run_event\u001b[0;34m(self, event, moment)\u001b[0m\n\u001b[1;32m    121\u001b[0m                 \u001b[0;34m(\u001b[0m\u001b[0mmoment\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"end\"\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mmoment\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# for on_exception case\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mlogger\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloggers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m                 \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/callbacks/misc.py\u001b[0m in \u001b[0;36mon_exception\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m    153\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneed_reraise_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36mrun_experiment\u001b[0;34m(self, experiment, check)\u001b[0m\n\u001b[1;32m    347\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mstage\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperiment\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstages\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 349\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_stage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    350\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mException\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    351\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexception\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mex\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36m_run_stage\u001b[0;34m(self, stage)\u001b[0m\n\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"epoch\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"start\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_epoch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloaders\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    320\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"epoch\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"end\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36m_run_epoch\u001b[0;34m(self, loaders)\u001b[0m\n\u001b[1;32m    289\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"loader\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"start\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mneed_backward\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 291\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_loader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    292\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"loader\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"end\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    293\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36m_run_loader\u001b[0;34m(self, loader)\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 253\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    254\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36m_run_batch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    233\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"_timers/model_time\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"_timers/batch_time\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"batch\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoment\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"end\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_loader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/core/runner.py\u001b[0m in \u001b[0;36m_run_event\u001b[0;34m(self, event, moment)\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m                 \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    119\u001b[0m         \u001b[0;31m# after callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/catalyst/dl/callbacks/optimizer.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m    158\u001b[0m                 \u001b[0mscaled_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    159\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 160\u001b[0;31m             \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    161\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    162\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mneed_gradient_step\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    164\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m         \"\"\"\n\u001b[0;32m--> 166\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     98\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_bce\",\n",
    "                     criterion_key='bce', multiplier=1.),\n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_bce']),\n",
    "               AUCCallback(num_classes=1), \n",
    "                F1ScoreCallback(),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='auc/_mean',\n",
    "    minimize_metric=False,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state = torch.load(\"best_skiprnn.pth\",map_location='cpu')\n",
    "\n",
    "model.load_state_dict(state['model_state_dict'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IMDbRNN(\n",
       "  (rnn): SkipGRU(\n",
       "    (layer): SkipGRULayer(\n",
       "      original_name=SkipGRULayer\n",
       "      (cell): SkipGRUCell(\n",
       "        original_name=SkipGRUCell\n",
       "        (inner_gru): ScriptModule(original_name=GRUCell)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (cls): Linear(in_features=16, out_features=1, bias=True)\n",
       "  (emb): Embedding(2000, 32)\n",
       ")"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25/25 [00:21<00:00,  1.14it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7049218304"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions, labels = [], []\n",
    "with torch.no_grad():\n",
    "    for x,y in tqdm(test_loader):\n",
    "        predictions.append(model(x).detach().cpu().numpy())\n",
    "        labels.append(y.detach().cpu().numpy())\n",
    "predictions = np.concatenate(predictions)\n",
    "labels = np.concatenate(labels)\n",
    "\n",
    "roc_auc_score(labels, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## HintonBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from binarizers import HintonBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hb = HintonBinarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x,y in train_loader:\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1306])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)\n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        b_new = torch.repeat_interleave(hb(self.u), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Делаем датасет из предложения x[0] размножив его несколько раз"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset as BaseDataset\n",
    "\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/Hinton\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:42:39,393] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=4.5196 | _timers/batch_time=0.4474 | _timers/data_time=0.0003 | _timers/model_time=0.4470 | f1_score=7.940e-07 | loss=0.2718 | loss_dist=0.2263 | loss_mean=0.0455\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=9.4056 | _timers/batch_time=0.2171 | _timers/data_time=0.0004 | _timers/model_time=0.2167 | f1_score=8.099e-07 | loss=0.2710 | loss_dist=0.2257 | loss_mean=0.0453\n",
      "[2019-12-06 17:42:44,066] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3819 | _timers/batch_time=0.3745 | _timers/data_time=0.0004 | _timers/model_time=0.3741 | f1_score=8.095e-07 | loss=0.2708 | loss_dist=0.2257 | loss_mean=0.0450\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2587 | _timers/batch_time=0.1992 | _timers/data_time=0.0003 | _timers/model_time=0.1989 | f1_score=8.131e-07 | loss=0.2705 | loss_dist=0.2257 | loss_mean=0.0448\n",
      "[2019-12-06 17:42:48,508] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5728 | _timers/batch_time=0.3605 | _timers/data_time=0.0004 | _timers/model_time=0.3601 | f1_score=8.137e-07 | loss=0.2703 | loss_dist=0.2257 | loss_mean=0.0446\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.3312 | _timers/batch_time=0.1977 | _timers/data_time=0.0003 | _timers/model_time=0.1974 | f1_score=9.103e-07 | loss=0.2730 | loss_dist=0.2291 | loss_mean=0.0439\n",
      "[2019-12-06 17:42:52,996] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5664 | _timers/batch_time=0.3604 | _timers/data_time=0.0004 | _timers/model_time=0.3599 | f1_score=9.940e-07 | loss=0.2536 | loss_dist=0.2103 | loss_mean=0.0433\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1167 | _timers/batch_time=0.2018 | _timers/data_time=0.0002 | _timers/model_time=0.2015 | f1_score=1.251e-06 | loss=0.1971 | loss_dist=0.1545 | loss_mean=0.0426\n",
      "[2019-12-06 17:42:57,507] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4933 | _timers/batch_time=0.3653 | _timers/data_time=0.0004 | _timers/model_time=0.3649 | f1_score=1.259e-06 | loss=0.1974 | loss_dist=0.1555 | loss_mean=0.0419\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2508 | _timers/batch_time=0.1991 | _timers/data_time=0.0004 | _timers/model_time=0.1986 | f1_score=1.266e-06 | loss=0.1974 | loss_dist=0.1564 | loss_mean=0.0410\n",
      "[2019-12-06 17:43:01,984] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6415 | _timers/batch_time=0.3561 | _timers/data_time=0.0003 | _timers/model_time=0.3557 | f1_score=1.242e-06 | loss=0.1934 | loss_dist=0.1528 | loss_mean=0.0406\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1641 | _timers/batch_time=0.2004 | _timers/data_time=0.0003 | _timers/model_time=0.2000 | f1_score=1.174e-06 | loss=0.1827 | loss_dist=0.1424 | loss_mean=0.0403\n",
      "[2019-12-06 17:43:06,483] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5884 | _timers/batch_time=0.3595 | _timers/data_time=0.0003 | _timers/model_time=0.3591 | f1_score=1.159e-06 | loss=0.1809 | loss_dist=0.1404 | loss_mean=0.0405\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7255 | _timers/batch_time=0.2100 | _timers/data_time=0.0002 | _timers/model_time=0.2097 | f1_score=1.273e-06 | loss=0.1269 | loss_dist=0.0868 | loss_mean=0.0401\n",
      "[2019-12-06 17:43:11,559] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.9798 | _timers/batch_time=0.4041 | _timers/data_time=0.0004 | _timers/model_time=0.4037 | f1_score=1.277e-06 | loss=0.1274 | loss_dist=0.0874 | loss_mean=0.0399\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4766 | _timers/batch_time=0.2156 | _timers/data_time=0.0004 | _timers/model_time=0.2152 | f1_score=1.272e-06 | loss=0.1264 | loss_dist=0.0869 | loss_mean=0.0395\n",
      "[2019-12-06 17:43:16,293] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3071 | _timers/batch_time=0.3774 | _timers/data_time=0.0005 | _timers/model_time=0.3768 | f1_score=1.272e-06 | loss=0.1261 | loss_dist=0.0868 | loss_mean=0.0393\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.2920 | _timers/batch_time=0.2247 | _timers/data_time=0.0004 | _timers/model_time=0.2243 | f1_score=1.270e-06 | loss=0.1255 | loss_dist=0.0867 | loss_mean=0.0388\n",
      "[2019-12-06 17:43:21,489] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.8782 | _timers/batch_time=0.4128 | _timers/data_time=0.0004 | _timers/model_time=0.4124 | f1_score=1.239e-06 | loss=0.1218 | loss_dist=0.0830 | loss_mean=0.0388\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.9886 | _timers/batch_time=0.2279 | _timers/data_time=0.0003 | _timers/model_time=0.2276 | f1_score=1.150e-06 | loss=0.1120 | loss_dist=0.0736 | loss_mean=0.0384\n",
      "[2019-12-06 17:43:26,473] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1389 | _timers/batch_time=0.3900 | _timers/data_time=0.0003 | _timers/model_time=0.3897 | f1_score=1.125e-06 | loss=0.1093 | loss_dist=0.0710 | loss_mean=0.0384\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4908 | _timers/batch_time=0.2149 | _timers/data_time=0.0004 | _timers/model_time=0.2145 | f1_score=1.078e-06 | loss=0.1049 | loss_dist=0.0667 | loss_mean=0.0382\n",
      "[2019-12-06 17:43:31,194] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2928 | _timers/batch_time=0.3791 | _timers/data_time=0.0003 | _timers/model_time=0.3787 | f1_score=1.068e-06 | loss=0.1035 | loss_dist=0.0656 | loss_mean=0.0379\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.6141 | _timers/batch_time=0.2125 | _timers/data_time=0.0003 | _timers/model_time=0.2121 | f1_score=9.633e-07 | loss=0.0679 | loss_dist=0.0301 | loss_mean=0.0378\n",
      "[2019-12-06 17:43:35,945] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2020 | _timers/batch_time=0.3850 | _timers/data_time=0.0002 | _timers/model_time=0.3847 | f1_score=8.916e-07 | loss=0.0665 | loss_dist=0.0292 | loss_mean=0.0373\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4798 | _timers/batch_time=0.2145 | _timers/data_time=0.0003 | _timers/model_time=0.2142 | f1_score=8.097e-07 | loss=0.0645 | loss_dist=0.0279 | loss_mean=0.0365\n",
      "[2019-12-06 17:43:40,641] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3254 | _timers/batch_time=0.3763 | _timers/data_time=0.0003 | _timers/model_time=0.3760 | f1_score=8.079e-07 | loss=0.0644 | loss_dist=0.0280 | loss_mean=0.0364\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0010 | _timers/batch_time=0.2047 | _timers/data_time=0.0003 | _timers/model_time=0.2043 | f1_score=7.080e-07 | loss=0.0680 | loss_dist=0.0321 | loss_mean=0.0359\n",
      "[2019-12-06 17:43:45,177] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5635 | _timers/batch_time=0.3606 | _timers/data_time=0.0004 | _timers/model_time=0.3602 | f1_score=5.247e-07 | loss=0.1177 | loss_dist=0.0816 | loss_mean=0.0361\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2745 | _timers/batch_time=0.1990 | _timers/data_time=0.0004 | _timers/model_time=0.1986 | f1_score=5.155e-07 | loss=0.1064 | loss_dist=0.0701 | loss_mean=0.0363\n",
      "[2019-12-06 17:43:49,619] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6157 | _timers/batch_time=0.3577 | _timers/data_time=0.0004 | _timers/model_time=0.3573 | f1_score=6.143e-07 | loss=0.0858 | loss_dist=0.0496 | loss_mean=0.0363\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2161 | _timers/batch_time=0.1994 | _timers/data_time=0.0003 | _timers/model_time=0.1990 | f1_score=7.042e-07 | loss=0.0645 | loss_dist=0.0282 | loss_mean=0.0363\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:43:54,046] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6214 | _timers/batch_time=0.3564 | _timers/data_time=0.0003 | _timers/model_time=0.3560 | f1_score=7.030e-07 | loss=0.0643 | loss_dist=0.0283 | loss_mean=0.0361\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.4071 | _timers/batch_time=0.1962 | _timers/data_time=0.0002 | _timers/model_time=0.1960 | f1_score=7.785e-07 | loss=0.0597 | loss_dist=0.0238 | loss_mean=0.0359\n",
      "[2019-12-06 17:43:58,409] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7331 | _timers/batch_time=0.3503 | _timers/data_time=0.0004 | _timers/model_time=0.3498 | f1_score=7.897e-07 | loss=0.0592 | loss_dist=0.0235 | loss_mean=0.0357\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.3656 | _timers/batch_time=0.1969 | _timers/data_time=0.0004 | _timers/model_time=0.1965 | f1_score=8.861e-07 | loss=0.0579 | loss_dist=0.0220 | loss_mean=0.0359\n",
      "[2019-12-06 17:44:02,816] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7051 | _timers/batch_time=0.3516 | _timers/data_time=0.0003 | _timers/model_time=0.3513 | f1_score=8.861e-07 | loss=0.0579 | loss_dist=0.0220 | loss_mean=0.0359\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2983 | _timers/batch_time=0.1989 | _timers/data_time=0.0005 | _timers/model_time=0.1984 | f1_score=8.861e-07 | loss=0.0579 | loss_dist=0.0220 | loss_mean=0.0359\n",
      "[2019-12-06 17:44:07,613] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3899 | _timers/batch_time=0.3749 | _timers/data_time=0.0003 | _timers/model_time=0.3745 | f1_score=8.979e-07 | loss=0.0575 | loss_dist=0.0221 | loss_mean=0.0354\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.6559 | _timers/batch_time=0.2366 | _timers/data_time=0.0004 | _timers/model_time=0.2362 | f1_score=9.019e-07 | loss=0.0570 | loss_dist=0.0221 | loss_mean=0.0349\n",
      "[2019-12-06 17:44:13,125] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.6013 | _timers/batch_time=0.4391 | _timers/data_time=0.0003 | _timers/model_time=0.4388 | f1_score=9.082e-07 | loss=0.0565 | loss_dist=0.0221 | loss_mean=0.0343\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4079 | _timers/batch_time=0.2167 | _timers/data_time=0.0003 | _timers/model_time=0.2164 | f1_score=9.526e-07 | loss=0.0369 | loss_dist=0.0020 | loss_mean=0.0349\n",
      "[2019-12-06 17:44:18,612] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.7189 | _timers/batch_time=0.4253 | _timers/data_time=0.0004 | _timers/model_time=0.4249 | f1_score=9.555e-07 | loss=0.0363 | loss_dist=0.0020 | loss_mean=0.0343\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.9874 | _timers/batch_time=0.2255 | _timers/data_time=0.0003 | _timers/model_time=0.2252 | f1_score=9.666e-07 | loss=0.0358 | loss_dist=0.0021 | loss_mean=0.0337\n",
      "[2019-12-06 17:44:24,694] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.6274 | _timers/batch_time=0.4356 | _timers/data_time=0.0003 | _timers/model_time=0.4353 | f1_score=9.648e-07 | loss=0.0354 | loss_dist=0.0022 | loss_mean=0.0332\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=7.0797 | _timers/batch_time=0.2963 | _timers/data_time=0.0006 | _timers/model_time=0.2957 | f1_score=1.032e-06 | loss=0.0352 | loss_dist=0.0029 | loss_mean=0.0323\n",
      "[2019-12-06 17:44:29,588] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1410 | _timers/batch_time=0.3899 | _timers/data_time=0.0004 | _timers/model_time=0.3895 | f1_score=1.039e-06 | loss=0.0352 | loss_dist=0.0030 | loss_mean=0.0322\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7866 | _timers/batch_time=0.2086 | _timers/data_time=0.0003 | _timers/model_time=0.2083 | f1_score=1.052e-06 | loss=0.0352 | loss_dist=0.0033 | loss_mean=0.0319\n",
      "[2019-12-06 17:44:34,145] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4367 | _timers/batch_time=0.3691 | _timers/data_time=0.0005 | _timers/model_time=0.3685 | f1_score=1.045e-06 | loss=0.0350 | loss_dist=0.0032 | loss_mean=0.0318\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1636 | _timers/batch_time=0.2004 | _timers/data_time=0.0004 | _timers/model_time=0.2000 | f1_score=1.054e-06 | loss=0.0352 | loss_dist=0.0033 | loss_mean=0.0319\n",
      "[2019-12-06 17:44:38,843] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2616 | _timers/batch_time=0.3824 | _timers/data_time=0.0003 | _timers/model_time=0.3820 | f1_score=1.041e-06 | loss=0.0347 | loss_dist=0.0031 | loss_mean=0.0315\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7199 | _timers/batch_time=0.2101 | _timers/data_time=0.0003 | _timers/model_time=0.2098 | f1_score=1.043e-06 | loss=0.0345 | loss_dist=0.0032 | loss_mean=0.0313\n",
      "[2019-12-06 17:44:43,468] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3982 | _timers/batch_time=0.3722 | _timers/data_time=0.0003 | _timers/model_time=0.3718 | f1_score=1.043e-06 | loss=0.0341 | loss_dist=0.0032 | loss_mean=0.0309\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1432 | _timers/batch_time=0.2020 | _timers/data_time=0.0004 | _timers/model_time=0.2015 | f1_score=1.043e-06 | loss=0.0332 | loss_dist=0.0032 | loss_mean=0.0300\n",
      "[2019-12-06 17:44:47,955] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5659 | _timers/batch_time=0.3607 | _timers/data_time=0.0004 | _timers/model_time=0.3603 | f1_score=1.067e-06 | loss=0.0333 | loss_dist=0.0035 | loss_mean=0.0298\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1165 | _timers/batch_time=0.2014 | _timers/data_time=0.0003 | _timers/model_time=0.2011 | f1_score=1.090e-06 | loss=0.0333 | loss_dist=0.0038 | loss_mean=0.0295\n",
      "[2019-12-06 17:44:52,669] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3721 | _timers/batch_time=0.3744 | _timers/data_time=0.0003 | _timers/model_time=0.3740 | f1_score=1.090e-06 | loss=0.0332 | loss_dist=0.0038 | loss_mean=0.0294\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.5560 | _timers/batch_time=0.2140 | _timers/data_time=0.0004 | _timers/model_time=0.2136 | f1_score=1.094e-06 | loss=0.0334 | loss_dist=0.0039 | loss_mean=0.0295\n",
      "[2019-12-06 17:44:57,269] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4065 | _timers/batch_time=0.3707 | _timers/data_time=0.0004 | _timers/model_time=0.3703 | f1_score=1.054e-06 | loss=0.0330 | loss_dist=0.0034 | loss_mean=0.0296\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1394 | _timers/batch_time=0.2017 | _timers/data_time=0.0004 | _timers/model_time=0.2013 | f1_score=1.026e-06 | loss=0.0325 | loss_dist=0.0031 | loss_mean=0.0295\n",
      "[2019-12-06 17:45:01,777] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6136 | _timers/batch_time=0.3580 | _timers/data_time=0.0003 | _timers/model_time=0.3576 | f1_score=1.026e-06 | loss=0.0322 | loss_dist=0.0031 | loss_mean=0.0292\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8941 | _timers/batch_time=0.2074 | _timers/data_time=0.0005 | _timers/model_time=0.2069 | f1_score=1.016e-06 | loss=0.0317 | loss_dist=0.0030 | loss_mean=0.0287\n",
      "[2019-12-06 17:45:06,372] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6289 | _timers/batch_time=0.3566 | _timers/data_time=0.0004 | _timers/model_time=0.3562 | f1_score=1.016e-06 | loss=0.0313 | loss_dist=0.0030 | loss_mean=0.0283\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.9753 | _timers/batch_time=0.2282 | _timers/data_time=0.0004 | _timers/model_time=0.2278 | f1_score=1.016e-06 | loss=0.0309 | loss_dist=0.0030 | loss_mean=0.0280\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:45:11,015] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5506 | _timers/batch_time=0.3622 | _timers/data_time=0.0004 | _timers/model_time=0.3618 | f1_score=1.015e-06 | loss=0.0307 | loss_dist=0.0030 | loss_mean=0.0277\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8237 | _timers/batch_time=0.2080 | _timers/data_time=0.0004 | _timers/model_time=0.2076 | f1_score=1.016e-06 | loss=0.0299 | loss_dist=0.0030 | loss_mean=0.0269\n",
      "[2019-12-06 17:45:15,801] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3750 | _timers/batch_time=0.3741 | _timers/data_time=0.0004 | _timers/model_time=0.3737 | f1_score=1.016e-06 | loss=0.0298 | loss_dist=0.0030 | loss_mean=0.0269\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.1978 | _timers/batch_time=0.2218 | _timers/data_time=0.0005 | _timers/model_time=0.2212 | f1_score=1.016e-06 | loss=0.0297 | loss_dist=0.0030 | loss_mean=0.0267\n",
      "[2019-12-06 17:45:20,510] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2082 | _timers/batch_time=0.3849 | _timers/data_time=0.0004 | _timers/model_time=0.3845 | f1_score=9.874e-07 | loss=0.0288 | loss_dist=0.0025 | loss_mean=0.0263\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2193 | _timers/batch_time=0.2006 | _timers/data_time=0.0004 | _timers/model_time=0.2001 | f1_score=9.742e-07 | loss=0.0280 | loss_dist=0.0023 | loss_mean=0.0257\n",
      "[2019-12-06 17:45:25,051] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5905 | _timers/batch_time=0.3595 | _timers/data_time=0.0004 | _timers/model_time=0.3590 | f1_score=9.581e-07 | loss=0.0275 | loss_dist=0.0021 | loss_mean=0.0254\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8441 | _timers/batch_time=0.2064 | _timers/data_time=0.0004 | _timers/model_time=0.2060 | f1_score=9.125e-07 | loss=0.0266 | loss_dist=0.0014 | loss_mean=0.0252\n",
      "[2019-12-06 17:45:29,607] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5256 | _timers/batch_time=0.3626 | _timers/data_time=0.0003 | _timers/model_time=0.3622 | f1_score=9.274e-07 | loss=0.0263 | loss_dist=0.0014 | loss_mean=0.0248\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8377 | _timers/batch_time=0.2067 | _timers/data_time=0.0003 | _timers/model_time=0.2063 | f1_score=9.457e-07 | loss=0.0263 | loss_dist=0.0015 | loss_mean=0.0248\n",
      "[2019-12-06 17:45:34,424] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2882 | _timers/batch_time=0.3816 | _timers/data_time=0.0002 | _timers/model_time=0.3813 | f1_score=9.535e-07 | loss=0.0263 | loss_dist=0.0015 | loss_mean=0.0248\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4704 | _timers/batch_time=0.2174 | _timers/data_time=0.0005 | _timers/model_time=0.2169 | f1_score=9.560e-07 | loss=0.0262 | loss_dist=0.0015 | loss_mean=0.0247\n",
      "[2019-12-06 17:45:39,126] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3832 | _timers/batch_time=0.3732 | _timers/data_time=0.0004 | _timers/model_time=0.3728 | f1_score=9.565e-07 | loss=0.0259 | loss_dist=0.0015 | loss_mean=0.0244\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.8439 | _timers/batch_time=0.2289 | _timers/data_time=0.0004 | _timers/model_time=0.2285 | f1_score=9.828e-07 | loss=0.0255 | loss_dist=0.0017 | loss_mean=0.0238\n",
      "[2019-12-06 17:45:43,876] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3432 | _timers/batch_time=0.3744 | _timers/data_time=0.0003 | _timers/model_time=0.3741 | f1_score=9.840e-07 | loss=0.0249 | loss_dist=0.0017 | loss_mean=0.0233\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2489 | _timers/batch_time=0.1991 | _timers/data_time=0.0003 | _timers/model_time=0.1988 | f1_score=9.879e-07 | loss=0.0245 | loss_dist=0.0017 | loss_mean=0.0228\n",
      "[2019-12-06 17:45:48,437] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5390 | _timers/batch_time=0.3629 | _timers/data_time=0.0004 | _timers/model_time=0.3625 | f1_score=1.019e-06 | loss=0.0249 | loss_dist=0.0019 | loss_mean=0.0229\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2536 | _timers/batch_time=0.1997 | _timers/data_time=0.0003 | _timers/model_time=0.1993 | f1_score=1.043e-06 | loss=0.0251 | loss_dist=0.0021 | loss_mean=0.0230\n",
      "[2019-12-06 17:45:53,083] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5082 | _timers/batch_time=0.3651 | _timers/data_time=0.0004 | _timers/model_time=0.3647 | f1_score=1.043e-06 | loss=0.0247 | loss_dist=0.0021 | loss_mean=0.0227\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8748 | _timers/batch_time=0.2071 | _timers/data_time=0.0003 | _timers/model_time=0.2068 | f1_score=9.231e-07 | loss=0.0230 | loss_dist=0.0010 | loss_mean=0.0220\n",
      "[2019-12-06 17:45:57,615] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5598 | _timers/batch_time=0.3612 | _timers/data_time=0.0004 | _timers/model_time=0.3608 | f1_score=9.228e-07 | loss=0.0228 | loss_dist=0.0010 | loss_mean=0.0218\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0962 | _timers/batch_time=0.2017 | _timers/data_time=0.0005 | _timers/model_time=0.2012 | f1_score=9.222e-07 | loss=0.0223 | loss_dist=0.0010 | loss_mean=0.0214\n",
      "[2019-12-06 17:46:02,087] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5882 | _timers/batch_time=0.3598 | _timers/data_time=0.0002 | _timers/model_time=0.3595 | f1_score=9.168e-07 | loss=0.0220 | loss_dist=0.0010 | loss_mean=0.0210\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1132 | _timers/batch_time=0.2018 | _timers/data_time=0.0002 | _timers/model_time=0.2016 | f1_score=8.999e-07 | loss=0.0213 | loss_dist=0.0010 | loss_mean=0.0203\n",
      "[2019-12-06 17:46:06,610] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5614 | _timers/batch_time=0.3615 | _timers/data_time=0.0003 | _timers/model_time=0.3611 | f1_score=9.000e-07 | loss=0.0207 | loss_dist=0.0010 | loss_mean=0.0197\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0970 | _timers/batch_time=0.2037 | _timers/data_time=0.0003 | _timers/model_time=0.2034 | f1_score=9.000e-07 | loss=0.0204 | loss_dist=0.0010 | loss_mean=0.0194\n",
      "[2019-12-06 17:46:11,127] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6192 | _timers/batch_time=0.3572 | _timers/data_time=0.0003 | _timers/model_time=0.3568 | f1_score=9.000e-07 | loss=0.0204 | loss_dist=0.0010 | loss_mean=0.0194\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.3764 | _timers/batch_time=0.2160 | _timers/data_time=0.0005 | _timers/model_time=0.2155 | f1_score=9.000e-07 | loss=0.0199 | loss_dist=0.0010 | loss_mean=0.0189\n",
      "[2019-12-06 17:46:15,712] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5117 | _timers/batch_time=0.3633 | _timers/data_time=0.0002 | _timers/model_time=0.3631 | f1_score=8.574e-07 | loss=0.0198 | loss_dist=0.0012 | loss_mean=0.0186\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0812 | _timers/batch_time=0.2032 | _timers/data_time=0.0003 | _timers/model_time=0.2028 | f1_score=7.922e-07 | loss=0.0198 | loss_dist=0.0014 | loss_mean=0.0185\n",
      "[2019-12-06 17:46:20,499] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3269 | _timers/batch_time=0.3788 | _timers/data_time=0.0004 | _timers/model_time=0.3784 | f1_score=7.742e-07 | loss=0.0193 | loss_dist=0.0009 | loss_mean=0.0184\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9869 | _timers/batch_time=0.2048 | _timers/data_time=0.0005 | _timers/model_time=0.2043 | f1_score=8.558e-07 | loss=0.0183 | loss_dist=0.0003 | loss_mean=0.0180\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:46:25,088] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5588 | _timers/batch_time=0.3607 | _timers/data_time=0.0004 | _timers/model_time=0.3602 | f1_score=8.604e-07 | loss=0.0183 | loss_dist=0.0003 | loss_mean=0.0180\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.6411 | _timers/batch_time=0.2112 | _timers/data_time=0.0005 | _timers/model_time=0.2107 | f1_score=8.702e-07 | loss=0.0182 | loss_dist=0.0003 | loss_mean=0.0179\n",
      "[2019-12-06 17:46:29,627] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4898 | _timers/batch_time=0.3654 | _timers/data_time=0.0003 | _timers/model_time=0.3650 | f1_score=8.745e-07 | loss=0.0181 | loss_dist=0.0003 | loss_mean=0.0178\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1425 | _timers/batch_time=0.2008 | _timers/data_time=0.0003 | _timers/model_time=0.2005 | f1_score=8.760e-07 | loss=0.0179 | loss_dist=0.0003 | loss_mean=0.0176\n",
      "[2019-12-06 17:46:34,195] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5332 | _timers/batch_time=0.3629 | _timers/data_time=0.0004 | _timers/model_time=0.3625 | f1_score=8.760e-07 | loss=0.0177 | loss_dist=0.0003 | loss_mean=0.0174\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7347 | _timers/batch_time=0.2095 | _timers/data_time=0.0003 | _timers/model_time=0.2092 | f1_score=8.759e-07 | loss=0.0174 | loss_dist=0.0003 | loss_mean=0.0172\n",
      "[2019-12-06 17:46:38,864] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4221 | _timers/batch_time=0.3693 | _timers/data_time=0.0006 | _timers/model_time=0.3687 | f1_score=8.444e-07 | loss=0.0174 | loss_dist=0.0004 | loss_mean=0.0171\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8702 | _timers/batch_time=0.2059 | _timers/data_time=0.0003 | _timers/model_time=0.2055 | f1_score=7.774e-07 | loss=0.0175 | loss_dist=0.0006 | loss_mean=0.0169\n",
      "[2019-12-06 17:46:43,540] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3093 | _timers/batch_time=0.3791 | _timers/data_time=0.0004 | _timers/model_time=0.3786 | f1_score=7.774e-07 | loss=0.0175 | loss_dist=0.0006 | loss_mean=0.0169\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1937 | _timers/batch_time=0.2002 | _timers/data_time=0.0002 | _timers/model_time=0.1999 | f1_score=8.141e-07 | loss=0.0173 | loss_dist=0.0006 | loss_mean=0.0167\n",
      "[2019-12-06 17:46:48,177] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4288 | _timers/batch_time=0.3711 | _timers/data_time=0.0004 | _timers/model_time=0.3706 | f1_score=7.946e-07 | loss=0.0174 | loss_dist=0.0007 | loss_mean=0.0167\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9753 | _timers/batch_time=0.2066 | _timers/data_time=0.0003 | _timers/model_time=0.2063 | f1_score=7.883e-07 | loss=0.0170 | loss_dist=0.0007 | loss_mean=0.0163\n",
      "[2019-12-06 17:46:52,843] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4865 | _timers/batch_time=0.3682 | _timers/data_time=0.0004 | _timers/model_time=0.3677 | f1_score=7.883e-07 | loss=0.0168 | loss_dist=0.0007 | loss_mean=0.0161\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.3754 | _timers/batch_time=0.2166 | _timers/data_time=0.0003 | _timers/model_time=0.2163 | f1_score=7.883e-07 | loss=0.0164 | loss_dist=0.0007 | loss_mean=0.0156\n",
      "[2019-12-06 17:46:57,643] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1291 | _timers/batch_time=0.3907 | _timers/data_time=0.0003 | _timers/model_time=0.3903 | f1_score=7.909e-07 | loss=0.0160 | loss_dist=0.0007 | loss_mean=0.0153\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7206 | _timers/batch_time=0.2101 | _timers/data_time=0.0003 | _timers/model_time=0.2097 | f1_score=7.932e-07 | loss=0.0155 | loss_dist=0.0007 | loss_mean=0.0148\n",
      "[2019-12-06 17:47:02,422] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3255 | _timers/batch_time=0.3777 | _timers/data_time=0.0003 | _timers/model_time=0.3773 | f1_score=7.927e-07 | loss=0.0154 | loss_dist=0.0007 | loss_mean=0.0147\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.3576 | _timers/batch_time=0.2180 | _timers/data_time=0.0005 | _timers/model_time=0.2175 | f1_score=7.992e-07 | loss=0.0153 | loss_dist=0.0006 | loss_mean=0.0147\n",
      "[2019-12-06 17:47:07,101] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2817 | _timers/batch_time=0.3791 | _timers/data_time=0.0003 | _timers/model_time=0.3788 | f1_score=7.988e-07 | loss=0.0153 | loss_dist=0.0006 | loss_mean=0.0147\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1243 | _timers/batch_time=0.2016 | _timers/data_time=0.0005 | _timers/model_time=0.2011 | f1_score=7.978e-07 | loss=0.0152 | loss_dist=0.0006 | loss_mean=0.0146\n",
      "[2019-12-06 17:47:11,868] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3889 | _timers/batch_time=0.3746 | _timers/data_time=0.0004 | _timers/model_time=0.3741 | f1_score=7.978e-07 | loss=0.0151 | loss_dist=0.0006 | loss_mean=0.0145\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4983 | _timers/batch_time=0.2169 | _timers/data_time=0.0004 | _timers/model_time=0.2165 | f1_score=7.987e-07 | loss=0.0150 | loss_dist=0.0006 | loss_mean=0.0143\n",
      "[2019-12-06 17:47:16,589] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4081 | _timers/batch_time=0.3706 | _timers/data_time=0.0003 | _timers/model_time=0.3702 | f1_score=7.988e-07 | loss=0.0146 | loss_dist=0.0006 | loss_mean=0.0140\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.3371 | _timers/batch_time=0.2204 | _timers/data_time=0.0003 | _timers/model_time=0.2201 | f1_score=7.988e-07 | loss=0.0142 | loss_dist=0.0006 | loss_mean=0.0136\n",
      "[2019-12-06 17:47:21,286] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2580 | _timers/batch_time=0.3806 | _timers/data_time=0.0003 | _timers/model_time=0.3802 | f1_score=7.988e-07 | loss=0.0140 | loss_dist=0.0006 | loss_mean=0.0134\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8927 | _timers/batch_time=0.2069 | _timers/data_time=0.0004 | _timers/model_time=0.2065 | f1_score=8.000e-07 | loss=0.0137 | loss_dist=0.0006 | loss_mean=0.0131\n",
      "[2019-12-06 17:47:26,137] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2782 | _timers/batch_time=0.3818 | _timers/data_time=0.0003 | _timers/model_time=0.3815 | f1_score=8.000e-07 | loss=0.0136 | loss_dist=0.0006 | loss_mean=0.0129\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.6760 | _timers/batch_time=0.2108 | _timers/data_time=0.0004 | _timers/model_time=0.2104 | f1_score=8.000e-07 | loss=0.0133 | loss_dist=0.0006 | loss_mean=0.0127\n",
      "[2019-12-06 17:47:30,747] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4237 | _timers/batch_time=0.3707 | _timers/data_time=0.0003 | _timers/model_time=0.3703 | f1_score=7.959e-07 | loss=0.0132 | loss_dist=0.0007 | loss_mean=0.0126\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7271 | _timers/batch_time=0.2101 | _timers/data_time=0.0003 | _timers/model_time=0.2097 | f1_score=7.956e-07 | loss=0.0127 | loss_dist=0.0007 | loss_mean=0.0121\n",
      "[2019-12-06 17:47:35,428] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3459 | _timers/batch_time=0.3762 | _timers/data_time=0.0004 | _timers/model_time=0.3758 | f1_score=7.956e-07 | loss=0.0127 | loss_dist=0.0007 | loss_mean=0.0120\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.0261 | _timers/batch_time=0.2241 | _timers/data_time=0.0004 | _timers/model_time=0.2237 | f1_score=1.037e-06 | loss=0.0123 | loss_dist=0.0005 | loss_mean=0.0117\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:47:40,243] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1639 | _timers/batch_time=0.3884 | _timers/data_time=0.0005 | _timers/model_time=0.3879 | f1_score=1.036e-06 | loss=0.0121 | loss_dist=0.0005 | loss_mean=0.0116\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8428 | _timers/batch_time=0.2077 | _timers/data_time=0.0004 | _timers/model_time=0.2073 | f1_score=1.031e-06 | loss=0.0118 | loss_dist=0.0005 | loss_mean=0.0113\n",
      "[2019-12-06 17:47:44,964] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4891 | _timers/batch_time=0.3665 | _timers/data_time=0.0003 | _timers/model_time=0.3661 | f1_score=1.030e-06 | loss=0.0117 | loss_dist=0.0005 | loss_mean=0.0112\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8360 | _timers/batch_time=0.2084 | _timers/data_time=0.0004 | _timers/model_time=0.2080 | f1_score=1.030e-06 | loss=0.0117 | loss_dist=0.0005 | loss_mean=0.0111\n",
      "[2019-12-06 17:47:49,672] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3252 | _timers/batch_time=0.3785 | _timers/data_time=0.0004 | _timers/model_time=0.3781 | f1_score=1.024e-06 | loss=0.0115 | loss_dist=0.0005 | loss_mean=0.0110\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.5087 | _timers/batch_time=0.2141 | _timers/data_time=0.0004 | _timers/model_time=0.2136 | f1_score=1.019e-06 | loss=0.0114 | loss_dist=0.0005 | loss_mean=0.0109\n",
      "[2019-12-06 17:47:54,290] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3734 | _timers/batch_time=0.3741 | _timers/data_time=0.0003 | _timers/model_time=0.3737 | f1_score=1.019e-06 | loss=0.0114 | loss_dist=0.0005 | loss_mean=0.0109\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0912 | _timers/batch_time=0.2035 | _timers/data_time=0.0003 | _timers/model_time=0.2032 | f1_score=9.698e-07 | loss=0.0110 | loss_dist=0.0004 | loss_mean=0.0106\n",
      "[2019-12-06 17:47:59,436] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1931 | _timers/batch_time=0.3954 | _timers/data_time=0.0006 | _timers/model_time=0.3947 | f1_score=9.704e-07 | loss=0.0109 | loss_dist=0.0004 | loss_mean=0.0106\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.0072 | _timers/batch_time=0.2265 | _timers/data_time=0.0002 | _timers/model_time=0.2262 | f1_score=9.579e-07 | loss=0.0107 | loss_dist=0.0004 | loss_mean=0.0104\n",
      "[2019-12-06 17:48:04,553] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.9332 | _timers/batch_time=0.4073 | _timers/data_time=0.0003 | _timers/model_time=0.4070 | f1_score=9.900e-07 | loss=0.0108 | loss_dist=0.0005 | loss_mean=0.0103\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.1042 | _timers/batch_time=0.2247 | _timers/data_time=0.0002 | _timers/model_time=0.2245 | f1_score=1.086e-06 | loss=0.0112 | loss_dist=0.0010 | loss_mean=0.0101\n",
      "[2019-12-06 17:48:09,507] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1442 | _timers/batch_time=0.3905 | _timers/data_time=0.0004 | _timers/model_time=0.3900 | f1_score=1.086e-06 | loss=0.0112 | loss_dist=0.0010 | loss_mean=0.0101\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.3043 | _timers/batch_time=0.2196 | _timers/data_time=0.0002 | _timers/model_time=0.2194 | f1_score=1.098e-06 | loss=0.0111 | loss_dist=0.0012 | loss_mean=0.0099\n",
      "[2019-12-06 17:48:14,505] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1501 | _timers/batch_time=0.3900 | _timers/data_time=0.0003 | _timers/model_time=0.3896 | f1_score=1.098e-06 | loss=0.0110 | loss_dist=0.0012 | loss_mean=0.0098\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.8990 | _timers/batch_time=0.2282 | _timers/data_time=0.0003 | _timers/model_time=0.2278 | f1_score=1.098e-06 | loss=0.0110 | loss_dist=0.0012 | loss_mean=0.0098\n",
      "[2019-12-06 17:48:19,523] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.0762 | _timers/batch_time=0.3959 | _timers/data_time=0.0003 | _timers/model_time=0.3955 | f1_score=1.098e-06 | loss=0.0109 | loss_dist=0.0012 | loss_mean=0.0098\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.7517 | _timers/batch_time=0.2327 | _timers/data_time=0.0004 | _timers/model_time=0.2322 | f1_score=1.102e-06 | loss=0.0107 | loss_dist=0.0012 | loss_mean=0.0095\n",
      "[2019-12-06 17:48:24,509] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.0573 | _timers/batch_time=0.3964 | _timers/data_time=0.0003 | _timers/model_time=0.3960 | f1_score=1.095e-06 | loss=0.0106 | loss_dist=0.0011 | loss_mean=0.0094\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.2708 | _timers/batch_time=0.2205 | _timers/data_time=0.0002 | _timers/model_time=0.2202 | f1_score=1.079e-06 | loss=0.0103 | loss_dist=0.0010 | loss_mean=0.0093\n",
      "[2019-12-06 17:48:29,458] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.0865 | _timers/batch_time=0.3950 | _timers/data_time=0.0004 | _timers/model_time=0.3946 | f1_score=1.079e-06 | loss=0.0102 | loss_dist=0.0010 | loss_mean=0.0092\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.2650 | _timers/batch_time=0.2205 | _timers/data_time=0.0003 | _timers/model_time=0.2202 | f1_score=1.079e-06 | loss=0.0100 | loss_dist=0.0010 | loss_mean=0.0090\n",
      "[2019-12-06 17:48:34,467] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.0894 | _timers/batch_time=0.3943 | _timers/data_time=0.0003 | _timers/model_time=0.3939 | f1_score=1.079e-06 | loss=0.0099 | loss_dist=0.0010 | loss_mean=0.0089\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.5354 | _timers/batch_time=0.2378 | _timers/data_time=0.0003 | _timers/model_time=0.2374 | f1_score=1.079e-06 | loss=0.0098 | loss_dist=0.0010 | loss_mean=0.0088\n",
      "[2019-12-06 17:48:39,416] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=4.9765 | _timers/batch_time=0.4030 | _timers/data_time=0.0002 | _timers/model_time=0.4028 | f1_score=1.080e-06 | loss=0.0097 | loss_dist=0.0010 | loss_mean=0.0087\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1611 | _timers/batch_time=0.2018 | _timers/data_time=0.0003 | _timers/model_time=0.2014 | f1_score=1.085e-06 | loss=0.0097 | loss_dist=0.0010 | loss_mean=0.0087\n",
      "[2019-12-06 17:48:43,951] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5723 | _timers/batch_time=0.3611 | _timers/data_time=0.0004 | _timers/model_time=0.3606 | f1_score=1.085e-06 | loss=0.0097 | loss_dist=0.0010 | loss_mean=0.0086\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9988 | _timers/batch_time=0.2039 | _timers/data_time=0.0005 | _timers/model_time=0.2034 | f1_score=1.085e-06 | loss=0.0096 | loss_dist=0.0010 | loss_mean=0.0086\n",
      "[2019-12-06 17:48:48,554] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6114 | _timers/batch_time=0.3578 | _timers/data_time=0.0003 | _timers/model_time=0.3574 | f1_score=1.088e-06 | loss=0.0095 | loss_dist=0.0011 | loss_mean=0.0085\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4120 | _timers/batch_time=0.2183 | _timers/data_time=0.0002 | _timers/model_time=0.2180 | f1_score=1.089e-06 | loss=0.0094 | loss_dist=0.0011 | loss_mean=0.0084\n",
      "[2019-12-06 17:48:53,081] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5647 | _timers/batch_time=0.3610 | _timers/data_time=0.0004 | _timers/model_time=0.3605 | f1_score=1.089e-06 | loss=0.0093 | loss_dist=0.0011 | loss_mean=0.0082\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0922 | _timers/batch_time=0.2023 | _timers/data_time=0.0003 | _timers/model_time=0.2020 | f1_score=1.089e-06 | loss=0.0091 | loss_dist=0.0011 | loss_mean=0.0080\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:48:57,603] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5849 | _timers/batch_time=0.3593 | _timers/data_time=0.0004 | _timers/model_time=0.3588 | f1_score=1.089e-06 | loss=0.0091 | loss_dist=0.0011 | loss_mean=0.0080\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2098 | _timers/batch_time=0.2010 | _timers/data_time=0.0003 | _timers/model_time=0.2007 | f1_score=1.020e-06 | loss=0.0087 | loss_dist=0.0006 | loss_mean=0.0081\n",
      "[2019-12-06 17:49:02,288] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4742 | _timers/batch_time=0.3679 | _timers/data_time=0.0004 | _timers/model_time=0.3675 | f1_score=9.896e-07 | loss=0.0086 | loss_dist=0.0005 | loss_mean=0.0080\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7907 | _timers/batch_time=0.2093 | _timers/data_time=0.0003 | _timers/model_time=0.2089 | f1_score=9.588e-07 | loss=0.0083 | loss_dist=0.0004 | loss_mean=0.0079\n",
      "[2019-12-06 17:49:06,936] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4669 | _timers/batch_time=0.3670 | _timers/data_time=0.0003 | _timers/model_time=0.3666 | f1_score=9.592e-07 | loss=0.0082 | loss_dist=0.0004 | loss_mean=0.0078\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.8415 | _timers/batch_time=0.2077 | _timers/data_time=0.0003 | _timers/model_time=0.2073 | f1_score=9.595e-07 | loss=0.0080 | loss_dist=0.0004 | loss_mean=0.0076\n",
      "[2019-12-06 17:49:11,564] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4732 | _timers/batch_time=0.3668 | _timers/data_time=0.0004 | _timers/model_time=0.3664 | f1_score=9.663e-07 | loss=0.0080 | loss_dist=0.0004 | loss_mean=0.0076\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9523 | _timers/batch_time=0.2053 | _timers/data_time=0.0003 | _timers/model_time=0.2049 | f1_score=9.867e-07 | loss=0.0080 | loss_dist=0.0005 | loss_mean=0.0075\n",
      "[2019-12-06 17:49:16,172] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5500 | _timers/batch_time=0.3613 | _timers/data_time=0.0003 | _timers/model_time=0.3610 | f1_score=9.945e-07 | loss=0.0079 | loss_dist=0.0005 | loss_mean=0.0074\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.4661 | _timers/batch_time=0.2150 | _timers/data_time=0.0002 | _timers/model_time=0.2147 | f1_score=1.001e-06 | loss=0.0079 | loss_dist=0.0005 | loss_mean=0.0074\n",
      "[2019-12-06 17:49:20,951] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2374 | _timers/batch_time=0.3833 | _timers/data_time=0.0007 | _timers/model_time=0.3826 | f1_score=1.001e-06 | loss=0.0079 | loss_dist=0.0005 | loss_mean=0.0074\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7994 | _timers/batch_time=0.2085 | _timers/data_time=0.0003 | _timers/model_time=0.2082 | f1_score=1.001e-06 | loss=0.0079 | loss_dist=0.0005 | loss_mean=0.0074\n",
      "[2019-12-06 17:49:25,735] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4473 | _timers/batch_time=0.3695 | _timers/data_time=0.0005 | _timers/model_time=0.3690 | f1_score=1.001e-06 | loss=0.0078 | loss_dist=0.0005 | loss_mean=0.0073\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.2144 | _timers/batch_time=0.2207 | _timers/data_time=0.0004 | _timers/model_time=0.2203 | f1_score=1.001e-06 | loss=0.0077 | loss_dist=0.0005 | loss_mean=0.0071\n",
      "[2019-12-06 17:49:30,423] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2780 | _timers/batch_time=0.3804 | _timers/data_time=0.0004 | _timers/model_time=0.3799 | f1_score=1.001e-06 | loss=0.0077 | loss_dist=0.0005 | loss_mean=0.0071\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9877 | _timers/batch_time=0.2051 | _timers/data_time=0.0003 | _timers/model_time=0.2048 | f1_score=1.001e-06 | loss=0.0077 | loss_dist=0.0005 | loss_mean=0.0071\n",
      "[2019-12-06 17:49:34,981] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5423 | _timers/batch_time=0.3623 | _timers/data_time=0.0003 | _timers/model_time=0.3619 | f1_score=9.175e-07 | loss=0.0076 | loss_dist=0.0004 | loss_mean=0.0072\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9626 | _timers/batch_time=0.2046 | _timers/data_time=0.0004 | _timers/model_time=0.2042 | f1_score=1.034e-06 | loss=0.0076 | loss_dist=0.0004 | loss_mean=0.0071\n",
      "[2019-12-06 17:49:39,523] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5830 | _timers/batch_time=0.3599 | _timers/data_time=0.0004 | _timers/model_time=0.3595 | f1_score=1.097e-06 | loss=0.0086 | loss_dist=0.0015 | loss_mean=0.0071\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2510 | _timers/batch_time=0.1991 | _timers/data_time=0.0004 | _timers/model_time=0.1987 | f1_score=1.159e-06 | loss=0.0096 | loss_dist=0.0026 | loss_mean=0.0070\n",
      "[2019-12-06 17:49:44,156] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.3782 | _timers/batch_time=0.3753 | _timers/data_time=0.0003 | _timers/model_time=0.3749 | f1_score=1.159e-06 | loss=0.0095 | loss_dist=0.0026 | loss_mean=0.0070\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9941 | _timers/batch_time=0.2041 | _timers/data_time=0.0004 | _timers/model_time=0.2037 | f1_score=1.159e-06 | loss=0.0094 | loss_dist=0.0026 | loss_mean=0.0069\n",
      "[2019-12-06 17:49:48,670] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6589 | _timers/batch_time=0.3547 | _timers/data_time=0.0004 | _timers/model_time=0.3542 | f1_score=1.159e-06 | loss=0.0094 | loss_dist=0.0026 | loss_mean=0.0068\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1066 | _timers/batch_time=0.2024 | _timers/data_time=0.0004 | _timers/model_time=0.2020 | f1_score=1.159e-06 | loss=0.0092 | loss_dist=0.0026 | loss_mean=0.0066\n",
      "[2019-12-06 17:49:53,532] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4349 | _timers/batch_time=0.3704 | _timers/data_time=0.0004 | _timers/model_time=0.3699 | f1_score=1.159e-06 | loss=0.0091 | loss_dist=0.0026 | loss_mean=0.0066\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.6356 | _timers/batch_time=0.2121 | _timers/data_time=0.0004 | _timers/model_time=0.2116 | f1_score=1.159e-06 | loss=0.0091 | loss_dist=0.0026 | loss_mean=0.0065\n",
      "[2019-12-06 17:49:58,154] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4439 | _timers/batch_time=0.3692 | _timers/data_time=0.0005 | _timers/model_time=0.3687 | f1_score=1.159e-06 | loss=0.0090 | loss_dist=0.0026 | loss_mean=0.0065\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9488 | _timers/batch_time=0.2064 | _timers/data_time=0.0004 | _timers/model_time=0.2060 | f1_score=1.159e-06 | loss=0.0090 | loss_dist=0.0026 | loss_mean=0.0064\n",
      "[2019-12-06 17:50:02,654] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.6038 | _timers/batch_time=0.3584 | _timers/data_time=0.0004 | _timers/model_time=0.3580 | f1_score=1.161e-06 | loss=0.0090 | loss_dist=0.0026 | loss_mean=0.0064\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.0989 | _timers/batch_time=0.2018 | _timers/data_time=0.0003 | _timers/model_time=0.2014 | f1_score=1.168e-06 | loss=0.0091 | loss_dist=0.0027 | loss_mean=0.0064\n",
      "[2019-12-06 17:50:07,159] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.5516 | _timers/batch_time=0.3622 | _timers/data_time=0.0004 | _timers/model_time=0.3618 | f1_score=1.168e-06 | loss=0.0092 | loss_dist=0.0027 | loss_mean=0.0064\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.1770 | _timers/batch_time=0.2000 | _timers/data_time=0.0003 | _timers/model_time=0.1997 | f1_score=1.168e-06 | loss=0.0092 | loss_dist=0.0027 | loss_mean=0.0065\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 17:50:11,637] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.6063 | _timers/batch_time=0.3583 | _timers/data_time=0.0004 | _timers/model_time=0.3578 | f1_score=1.168e-06 | loss=0.0092 | loss_dist=0.0027 | loss_mean=0.0065\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.2885 | _timers/batch_time=0.1979 | _timers/data_time=0.0003 | _timers/model_time=0.1976 | f1_score=1.168e-06 | loss=0.0091 | loss_dist=0.0027 | loss_mean=0.0064\n",
      "[2019-12-06 17:50:16,135] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.6234 | _timers/batch_time=0.3572 | _timers/data_time=0.0003 | _timers/model_time=0.3568 | f1_score=1.168e-06 | loss=0.0090 | loss_dist=0.0027 | loss_mean=0.0063\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.1119 | _timers/batch_time=0.2020 | _timers/data_time=0.0003 | _timers/model_time=0.2016 | f1_score=1.169e-06 | loss=0.0090 | loss_dist=0.0027 | loss_mean=0.0062\n",
      "[2019-12-06 17:50:20,660] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.5984 | _timers/batch_time=0.3587 | _timers/data_time=0.0003 | _timers/model_time=0.3584 | f1_score=1.169e-06 | loss=0.0090 | loss_dist=0.0027 | loss_mean=0.0062\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.0655 | _timers/batch_time=0.2023 | _timers/data_time=0.0004 | _timers/model_time=0.2019 | f1_score=1.169e-06 | loss=0.0090 | loss_dist=0.0027 | loss_mean=0.0062\n",
      "[2019-12-06 17:50:25,172] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.6083 | _timers/batch_time=0.3582 | _timers/data_time=0.0004 | _timers/model_time=0.3577 | f1_score=1.169e-06 | loss=0.0090 | loss_dist=0.0027 | loss_mean=0.0062\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=9.9026 | _timers/batch_time=0.2061 | _timers/data_time=0.0003 | _timers/model_time=0.2058 | f1_score=1.169e-06 | loss=0.0089 | loss_dist=0.0027 | loss_mean=0.0062\n",
      "Top best models:\n",
      "logs/Hinton/checkpoints/train.89.pth\t0.0076\n",
      "logs/Hinton/checkpoints/train.87.pth\t0.0077\n",
      "logs/Hinton/checkpoints/train.88.pth\t0.0077\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25/25 [00:43<00:00,  1.75s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5299492057635427"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_bin.eval()\n",
    "predictions, labels = [], []\n",
    "with torch.no_grad():\n",
    "    for x,y in tqdm(test_loader):\n",
    "        try:\n",
    "        \n",
    "            predictions.append(model_bin(x)['cls'].detach().cpu().numpy())\n",
    "            labels.append(y.detach().cpu().numpy())\n",
    "        except: pass\n",
    "predictions = np.concatenate(predictions)\n",
    "labels = np.concatenate(labels)\n",
    "\n",
    "roc_auc_score(labels, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ConcreteBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "from binarizers import ConcreteBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "cb = ConcreteBinarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "        \n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)\n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        b_new = torch.repeat_interleave(cb(self.u), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import Dataset as BaseDataset\n",
    "\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/Concrete\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:14:22,067] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.3213 | _timers/batch_time=0.3169 | _timers/data_time=0.0007 | _timers/model_time=0.3160 | f1_score=8.577e-07 | loss=3.2292 | loss_dist=3.1015 | loss_mean=0.1277\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.5877 | _timers/batch_time=0.1756 | _timers/data_time=0.0003 | _timers/model_time=0.1752 | f1_score=8.689e-07 | loss=3.1823 | loss_dist=3.0550 | loss_mean=0.1273\n",
      "[2019-12-06 18:14:25,970] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.3997 | _timers/batch_time=0.3137 | _timers/data_time=0.0004 | _timers/model_time=0.3132 | f1_score=1.236e-06 | loss=0.6186 | loss_dist=0.4880 | loss_mean=0.1306\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.7097 | _timers/batch_time=0.1746 | _timers/data_time=0.0003 | _timers/model_time=0.1742 | f1_score=1.251e-06 | loss=0.6099 | loss_dist=0.4797 | loss_mean=0.1302\n",
      "[2019-12-06 18:14:29,905] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.3845 | _timers/batch_time=0.3140 | _timers/data_time=0.0004 | _timers/model_time=0.3136 | f1_score=7.402e-07 | loss=5.1984 | loss_dist=5.0672 | loss_mean=0.1313\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.4465 | _timers/batch_time=0.1786 | _timers/data_time=0.0002 | _timers/model_time=0.1783 | f1_score=7.679e-07 | loss=4.3442 | loss_dist=4.2131 | loss_mean=0.1311\n",
      "[2019-12-06 18:14:33,809] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4434 | _timers/batch_time=0.3116 | _timers/data_time=0.0004 | _timers/model_time=0.3112 | f1_score=8.201e-07 | loss=2.6085 | loss_dist=2.4783 | loss_mean=0.1303\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.8177 | _timers/batch_time=0.1726 | _timers/data_time=0.0002 | _timers/model_time=0.1723 | f1_score=8.901e-07 | loss=1.9794 | loss_dist=1.8494 | loss_mean=0.1301\n",
      "[2019-12-06 18:14:37,655] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4918 | _timers/batch_time=0.3089 | _timers/data_time=0.0004 | _timers/model_time=0.3085 | f1_score=6.903e-07 | loss=3.5761 | loss_dist=3.4523 | loss_mean=0.1238\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.7476 | _timers/batch_time=0.1737 | _timers/data_time=0.0003 | _timers/model_time=0.1734 | f1_score=7.659e-07 | loss=3.2746 | loss_dist=3.1509 | loss_mean=0.1237\n",
      "[2019-12-06 18:14:41,541] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.5341 | _timers/batch_time=0.3068 | _timers/data_time=0.0004 | _timers/model_time=0.3064 | f1_score=9.138e-07 | loss=0.4441 | loss_dist=0.3158 | loss_mean=0.1283\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.3845 | _timers/batch_time=0.1780 | _timers/data_time=0.0003 | _timers/model_time=0.1776 | f1_score=9.321e-07 | loss=0.3992 | loss_dist=0.2712 | loss_mean=0.1280\n",
      "[2019-12-06 18:14:45,422] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4043 | _timers/batch_time=0.3131 | _timers/data_time=0.0004 | _timers/model_time=0.3126 | f1_score=1.020e-06 | loss=8.3092 | loss_dist=8.1820 | loss_mean=0.1271\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.7678 | _timers/batch_time=0.1731 | _timers/data_time=0.0002 | _timers/model_time=0.1728 | f1_score=1.034e-06 | loss=8.1947 | loss_dist=8.0677 | loss_mean=0.1270\n",
      "[2019-12-06 18:14:49,280] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4457 | _timers/batch_time=0.3108 | _timers/data_time=0.0005 | _timers/model_time=0.3103 | f1_score=8.124e-07 | loss=0.6348 | loss_dist=0.5086 | loss_mean=0.1262\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.7228 | _timers/batch_time=0.1737 | _timers/data_time=0.0003 | _timers/model_time=0.1734 | f1_score=8.376e-07 | loss=0.5756 | loss_dist=0.4497 | loss_mean=0.1260\n",
      "[2019-12-06 18:14:53,723] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.0160 | _timers/batch_time=0.3355 | _timers/data_time=0.0004 | _timers/model_time=0.3350 | f1_score=6.670e-07 | loss=10.2089 | loss_dist=10.0921 | loss_mean=0.1169\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2073 | _timers/batch_time=0.1989 | _timers/data_time=0.0003 | _timers/model_time=0.1985 | f1_score=6.959e-07 | loss=9.3557 | loss_dist=9.2390 | loss_mean=0.1167\n",
      "[2019-12-06 18:14:58,115] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7925 | _timers/batch_time=0.3458 | _timers/data_time=0.0004 | _timers/model_time=0.3453 | f1_score=9.956e-07 | loss=0.8051 | loss_dist=0.6875 | loss_mean=0.1176\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1532 | _timers/batch_time=0.1851 | _timers/data_time=0.0003 | _timers/model_time=0.1848 | f1_score=1.014e-06 | loss=0.7835 | loss_dist=0.6662 | loss_mean=0.1173\n",
      "[2019-12-06 18:15:02,220] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2081 | _timers/batch_time=0.3234 | _timers/data_time=0.0003 | _timers/model_time=0.3232 | f1_score=1.025e-06 | loss=10.2706 | loss_dist=10.1512 | loss_mean=0.1194\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1676 | _timers/batch_time=0.1844 | _timers/data_time=0.0005 | _timers/model_time=0.1839 | f1_score=1.029e-06 | loss=9.9282 | loss_dist=9.8091 | loss_mean=0.1191\n",
      "[2019-12-06 18:15:06,402] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.2849 | _timers/batch_time=0.3196 | _timers/data_time=0.0003 | _timers/model_time=0.3193 | f1_score=1.092e-06 | loss=0.3922 | loss_dist=0.2730 | loss_mean=0.1193\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.4632 | _timers/batch_time=0.1947 | _timers/data_time=0.0002 | _timers/model_time=0.1944 | f1_score=1.128e-06 | loss=0.2799 | loss_dist=0.1608 | loss_mean=0.1191\n",
      "[2019-12-06 18:15:10,833] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.7518 | _timers/batch_time=0.3572 | _timers/data_time=0.0003 | _timers/model_time=0.3569 | f1_score=9.935e-07 | loss=0.6402 | loss_dist=0.5211 | loss_mean=0.1191\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.3561 | _timers/batch_time=0.1957 | _timers/data_time=0.0002 | _timers/model_time=0.1955 | f1_score=1.002e-06 | loss=0.6212 | loss_dist=0.5022 | loss_mean=0.1189\n",
      "[2019-12-06 18:15:15,039] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.8502 | _timers/batch_time=0.3421 | _timers/data_time=0.0004 | _timers/model_time=0.3416 | f1_score=1.728e-06 | loss=0.2500 | loss_dist=0.1275 | loss_mean=0.1225\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.4815 | _timers/batch_time=0.1773 | _timers/data_time=0.0004 | _timers/model_time=0.1769 | f1_score=1.740e-06 | loss=0.2497 | loss_dist=0.1274 | loss_mean=0.1223\n",
      "[2019-12-06 18:15:18,984] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.3595 | _timers/batch_time=0.3151 | _timers/data_time=0.0003 | _timers/model_time=0.3147 | f1_score=1.370e-06 | loss=0.3256 | loss_dist=0.2008 | loss_mean=0.1248\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6119 | _timers/batch_time=0.1751 | _timers/data_time=0.0003 | _timers/model_time=0.1748 | f1_score=1.374e-06 | loss=0.3226 | loss_dist=0.1980 | loss_mean=0.1247\n",
      "[2019-12-06 18:15:22,902] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.4053 | _timers/batch_time=0.3130 | _timers/data_time=0.0005 | _timers/model_time=0.3124 | f1_score=1.015e-06 | loss=0.4573 | loss_dist=0.3291 | loss_mean=0.1282\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6581 | _timers/batch_time=0.1749 | _timers/data_time=0.0004 | _timers/model_time=0.1745 | f1_score=1.027e-06 | loss=0.4508 | loss_dist=0.3228 | loss_mean=0.1280\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:15:27,177] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.1392 | _timers/batch_time=0.3276 | _timers/data_time=0.0004 | _timers/model_time=0.3272 | f1_score=1.250e-06 | loss=0.3591 | loss_dist=0.2376 | loss_mean=0.1216\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.2200 | _timers/batch_time=0.2003 | _timers/data_time=0.0004 | _timers/model_time=0.1999 | f1_score=1.253e-06 | loss=0.3565 | loss_dist=0.2350 | loss_mean=0.1215\n",
      "[2019-12-06 18:15:31,570] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.7391 | _timers/batch_time=0.3519 | _timers/data_time=0.0003 | _timers/model_time=0.3515 | f1_score=1.081e-06 | loss=0.3549 | loss_dist=0.2339 | loss_mean=0.1209\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.6587 | _timers/batch_time=0.1932 | _timers/data_time=0.0004 | _timers/model_time=0.1928 | f1_score=1.082e-06 | loss=0.3529 | loss_dist=0.2321 | loss_mean=0.1208\n",
      "[2019-12-06 18:15:35,722] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0723 | _timers/batch_time=0.3313 | _timers/data_time=0.0003 | _timers/model_time=0.3310 | f1_score=1.252e-06 | loss=0.6382 | loss_dist=0.5175 | loss_mean=0.1207\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.9953 | _timers/batch_time=0.1850 | _timers/data_time=0.0003 | _timers/model_time=0.1847 | f1_score=1.271e-06 | loss=0.6281 | loss_dist=0.5076 | loss_mean=0.1205\n",
      "[2019-12-06 18:15:39,981] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0449 | _timers/batch_time=0.3326 | _timers/data_time=0.0003 | _timers/model_time=0.3323 | f1_score=1.070e-06 | loss=0.3637 | loss_dist=0.2452 | loss_mean=0.1184\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.7604 | _timers/batch_time=0.1897 | _timers/data_time=0.0003 | _timers/model_time=0.1894 | f1_score=1.076e-06 | loss=0.3598 | loss_dist=0.2416 | loss_mean=0.1182\n",
      "[2019-12-06 18:15:44,766] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.5132 | _timers/batch_time=0.3709 | _timers/data_time=0.0005 | _timers/model_time=0.3704 | f1_score=1.455e-06 | loss=0.6359 | loss_dist=0.5146 | loss_mean=0.1213\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=8.2126 | _timers/batch_time=0.2627 | _timers/data_time=0.0004 | _timers/model_time=0.2622 | f1_score=1.463e-06 | loss=0.6359 | loss_dist=0.5148 | loss_mean=0.1211\n",
      "[2019-12-06 18:15:49,629] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.1272 | _timers/batch_time=0.3934 | _timers/data_time=0.0005 | _timers/model_time=0.3928 | f1_score=1.760e-06 | loss=0.3025 | loss_dist=0.1813 | loss_mean=0.1212\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=9.9048 | _timers/batch_time=0.2086 | _timers/data_time=0.0003 | _timers/model_time=0.2083 | f1_score=1.773e-06 | loss=0.2990 | loss_dist=0.1781 | loss_mean=0.1209\n",
      "[2019-12-06 18:15:53,962] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.9866 | _timers/batch_time=0.3344 | _timers/data_time=0.0003 | _timers/model_time=0.3340 | f1_score=1.171e-06 | loss=0.5443 | loss_dist=0.4237 | loss_mean=0.1206\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=9.6522 | _timers/batch_time=0.2125 | _timers/data_time=0.0003 | _timers/model_time=0.2122 | f1_score=1.237e-06 | loss=0.3811 | loss_dist=0.2606 | loss_mean=0.1204\n",
      "[2019-12-06 18:15:58,266] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.8359 | _timers/batch_time=0.3447 | _timers/data_time=0.0002 | _timers/model_time=0.3444 | f1_score=9.273e-07 | loss=0.5283 | loss_dist=0.4095 | loss_mean=0.1189\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.7322 | _timers/batch_time=0.1896 | _timers/data_time=0.0003 | _timers/model_time=0.1893 | f1_score=9.312e-07 | loss=0.5182 | loss_dist=0.3995 | loss_mean=0.1187\n",
      "[2019-12-06 18:16:02,317] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2966 | _timers/batch_time=0.3189 | _timers/data_time=0.0003 | _timers/model_time=0.3185 | f1_score=1.425e-06 | loss=0.5144 | loss_dist=0.3981 | loss_mean=0.1163\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.6524 | _timers/batch_time=0.1772 | _timers/data_time=0.0003 | _timers/model_time=0.1769 | f1_score=1.428e-06 | loss=0.5114 | loss_dist=0.3952 | loss_mean=0.1162\n",
      "[2019-12-06 18:16:06,294] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2897 | _timers/batch_time=0.3219 | _timers/data_time=0.0004 | _timers/model_time=0.3214 | f1_score=1.389e-06 | loss=0.1875 | loss_dist=0.0677 | loss_mean=0.1198\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.6892 | _timers/batch_time=0.1753 | _timers/data_time=0.0004 | _timers/model_time=0.1749 | f1_score=1.392e-06 | loss=0.1871 | loss_dist=0.0675 | loss_mean=0.1196\n",
      "[2019-12-06 18:16:10,448] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.1080 | _timers/batch_time=0.3350 | _timers/data_time=0.0003 | _timers/model_time=0.3346 | f1_score=1.323e-06 | loss=0.4791 | loss_dist=0.3662 | loss_mean=0.1129\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.2444 | _timers/batch_time=0.1815 | _timers/data_time=0.0003 | _timers/model_time=0.1811 | f1_score=1.330e-06 | loss=0.4777 | loss_dist=0.3649 | loss_mean=0.1127\n",
      "[2019-12-06 18:16:14,639] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0748 | _timers/batch_time=0.3330 | _timers/data_time=0.0004 | _timers/model_time=0.3326 | f1_score=1.520e-06 | loss=0.2257 | loss_dist=0.1081 | loss_mean=0.1175\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.0966 | _timers/batch_time=0.1827 | _timers/data_time=0.0002 | _timers/model_time=0.1825 | f1_score=1.525e-06 | loss=0.2242 | loss_dist=0.1068 | loss_mean=0.1174\n",
      "[2019-12-06 18:16:19,649] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.1475 | _timers/batch_time=0.3918 | _timers/data_time=0.0004 | _timers/model_time=0.3914 | f1_score=1.305e-06 | loss=0.8226 | loss_dist=0.7017 | loss_mean=0.1209\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=9.6947 | _timers/batch_time=0.2128 | _timers/data_time=0.0002 | _timers/model_time=0.2125 | f1_score=1.312e-06 | loss=0.8052 | loss_dist=0.6845 | loss_mean=0.1207\n",
      "[2019-12-06 18:16:24,072] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.6284 | _timers/batch_time=0.3591 | _timers/data_time=0.0004 | _timers/model_time=0.3587 | f1_score=1.463e-06 | loss=0.5128 | loss_dist=0.3928 | loss_mean=0.1199\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.0100 | _timers/batch_time=0.1850 | _timers/data_time=0.0003 | _timers/model_time=0.1846 | f1_score=1.466e-06 | loss=0.5081 | loss_dist=0.3884 | loss_mean=0.1197\n",
      "[2019-12-06 18:16:28,170] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2787 | _timers/batch_time=0.3201 | _timers/data_time=0.0004 | _timers/model_time=0.3197 | f1_score=1.384e-06 | loss=0.3943 | loss_dist=0.2802 | loss_mean=0.1141\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.5671 | _timers/batch_time=0.1762 | _timers/data_time=0.0005 | _timers/model_time=0.1757 | f1_score=1.387e-06 | loss=0.3885 | loss_dist=0.2746 | loss_mean=0.1139\n",
      "[2019-12-06 18:16:32,479] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.9318 | _timers/batch_time=0.3375 | _timers/data_time=0.0004 | _timers/model_time=0.3371 | f1_score=1.693e-06 | loss=0.1661 | loss_dist=0.0462 | loss_mean=0.1199\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.8379 | _timers/batch_time=0.1888 | _timers/data_time=0.0003 | _timers/model_time=0.1885 | f1_score=1.690e-06 | loss=0.1649 | loss_dist=0.0451 | loss_mean=0.1198\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:16:36,804] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.8860 | _timers/batch_time=0.3417 | _timers/data_time=0.0004 | _timers/model_time=0.3413 | f1_score=1.610e-06 | loss=0.2352 | loss_dist=0.1171 | loss_mean=0.1181\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.5727 | _timers/batch_time=0.1930 | _timers/data_time=0.0002 | _timers/model_time=0.1928 | f1_score=1.617e-06 | loss=0.2343 | loss_dist=0.1164 | loss_mean=0.1179\n",
      "[2019-12-06 18:16:40,962] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.0177 | _timers/batch_time=0.3330 | _timers/data_time=0.0003 | _timers/model_time=0.3327 | f1_score=1.056e-06 | loss=0.4652 | loss_dist=0.3510 | loss_mean=0.1141\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.4361 | _timers/batch_time=0.1943 | _timers/data_time=0.0002 | _timers/model_time=0.1941 | f1_score=1.061e-06 | loss=0.4639 | loss_dist=0.3500 | loss_mean=0.1140\n",
      "[2019-12-06 18:16:45,110] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.0235 | _timers/batch_time=0.3329 | _timers/data_time=0.0004 | _timers/model_time=0.3325 | f1_score=1.663e-06 | loss=0.2804 | loss_dist=0.1639 | loss_mean=0.1165\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.5314 | _timers/batch_time=0.1770 | _timers/data_time=0.0003 | _timers/model_time=0.1767 | f1_score=1.668e-06 | loss=0.2800 | loss_dist=0.1636 | loss_mean=0.1164\n",
      "[2019-12-06 18:16:49,210] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.2615 | _timers/batch_time=0.3240 | _timers/data_time=0.0004 | _timers/model_time=0.3236 | f1_score=1.701e-06 | loss=0.1988 | loss_dist=0.0859 | loss_mean=0.1129\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.6707 | _timers/batch_time=0.1920 | _timers/data_time=0.0005 | _timers/model_time=0.1914 | f1_score=1.706e-06 | loss=0.1985 | loss_dist=0.0858 | loss_mean=0.1127\n",
      "[2019-12-06 18:16:53,410] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.8972 | _timers/batch_time=0.3393 | _timers/data_time=0.0004 | _timers/model_time=0.3389 | f1_score=1.272e-06 | loss=0.3711 | loss_dist=0.2574 | loss_mean=0.1137\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.7886 | _timers/batch_time=0.1924 | _timers/data_time=0.0002 | _timers/model_time=0.1922 | f1_score=1.272e-06 | loss=0.3643 | loss_dist=0.2507 | loss_mean=0.1136\n",
      "[2019-12-06 18:16:57,406] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.4327 | _timers/batch_time=0.3114 | _timers/data_time=0.0005 | _timers/model_time=0.3109 | f1_score=1.573e-06 | loss=0.2722 | loss_dist=0.1582 | loss_mean=0.1141\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.0644 | _timers/batch_time=0.1854 | _timers/data_time=0.0003 | _timers/model_time=0.1851 | f1_score=1.576e-06 | loss=0.2714 | loss_dist=0.1575 | loss_mean=0.1139\n",
      "[2019-12-06 18:17:01,433] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.1844 | _timers/batch_time=0.3240 | _timers/data_time=0.0003 | _timers/model_time=0.3236 | f1_score=1.583e-06 | loss=0.2802 | loss_dist=0.1669 | loss_mean=0.1132\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.8559 | _timers/batch_time=0.1723 | _timers/data_time=0.0003 | _timers/model_time=0.1720 | f1_score=1.585e-06 | loss=0.2794 | loss_dist=0.1662 | loss_mean=0.1131\n",
      "[2019-12-06 18:17:05,334] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.3954 | _timers/batch_time=0.3138 | _timers/data_time=0.0004 | _timers/model_time=0.3133 | f1_score=1.580e-06 | loss=0.2471 | loss_dist=0.1302 | loss_mean=0.1170\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.2160 | _timers/batch_time=0.1816 | _timers/data_time=0.0003 | _timers/model_time=0.1813 | f1_score=1.585e-06 | loss=0.2446 | loss_dist=0.1278 | loss_mean=0.1168\n",
      "[2019-12-06 18:17:09,157] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5829 | _timers/batch_time=0.3051 | _timers/data_time=0.0004 | _timers/model_time=0.3047 | f1_score=1.706e-06 | loss=0.4514 | loss_dist=0.3351 | loss_mean=0.1163\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=12.0107 | _timers/batch_time=0.1693 | _timers/data_time=0.0003 | _timers/model_time=0.1690 | f1_score=1.707e-06 | loss=0.4445 | loss_dist=0.3284 | loss_mean=0.1162\n",
      "[2019-12-06 18:17:12,952] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5938 | _timers/batch_time=0.3048 | _timers/data_time=0.0004 | _timers/model_time=0.3043 | f1_score=1.336e-06 | loss=0.4956 | loss_dist=0.3784 | loss_mean=0.1172\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.9136 | _timers/batch_time=0.1713 | _timers/data_time=0.0003 | _timers/model_time=0.1709 | f1_score=1.339e-06 | loss=0.4943 | loss_dist=0.3772 | loss_mean=0.1170\n",
      "[2019-12-06 18:17:17,045] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.2669 | _timers/batch_time=0.3222 | _timers/data_time=0.0004 | _timers/model_time=0.3217 | f1_score=1.837e-06 | loss=0.2311 | loss_dist=0.1172 | loss_mean=0.1139\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.9998 | _timers/batch_time=0.1864 | _timers/data_time=0.0003 | _timers/model_time=0.1860 | f1_score=1.837e-06 | loss=0.2302 | loss_dist=0.1165 | loss_mean=0.1137\n",
      "[2019-12-06 18:17:21,083] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.2508 | _timers/batch_time=0.3215 | _timers/data_time=0.0003 | _timers/model_time=0.3212 | f1_score=1.759e-06 | loss=0.4265 | loss_dist=0.3106 | loss_mean=0.1160\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.0722 | _timers/batch_time=0.1841 | _timers/data_time=0.0003 | _timers/model_time=0.1838 | f1_score=1.765e-06 | loss=0.4258 | loss_dist=0.3100 | loss_mean=0.1158\n",
      "[2019-12-06 18:17:25,174] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.2142 | _timers/batch_time=0.3235 | _timers/data_time=0.0003 | _timers/model_time=0.3231 | f1_score=1.140e-06 | loss=0.2741 | loss_dist=0.1623 | loss_mean=0.1119\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.9007 | _timers/batch_time=0.1881 | _timers/data_time=0.0003 | _timers/model_time=0.1878 | f1_score=1.145e-06 | loss=0.2729 | loss_dist=0.1612 | loss_mean=0.1117\n",
      "[2019-12-06 18:17:29,136] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.2462 | _timers/batch_time=0.3209 | _timers/data_time=0.0002 | _timers/model_time=0.3206 | f1_score=2.130e-06 | loss=0.4224 | loss_dist=0.3142 | loss_mean=0.1081\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.5933 | _timers/batch_time=0.1761 | _timers/data_time=0.0005 | _timers/model_time=0.1756 | f1_score=2.132e-06 | loss=0.4212 | loss_dist=0.3133 | loss_mean=0.1080\n",
      "[2019-12-06 18:17:33,037] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.4521 | _timers/batch_time=0.3122 | _timers/data_time=0.0003 | _timers/model_time=0.3118 | f1_score=1.616e-06 | loss=0.1533 | loss_dist=0.0447 | loss_mean=0.1086\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.7499 | _timers/batch_time=0.1730 | _timers/data_time=0.0004 | _timers/model_time=0.1725 | f1_score=1.619e-06 | loss=0.1510 | loss_dist=0.0426 | loss_mean=0.1085\n",
      "[2019-12-06 18:17:36,843] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5457 | _timers/batch_time=0.3071 | _timers/data_time=0.0003 | _timers/model_time=0.3068 | f1_score=1.503e-06 | loss=0.4408 | loss_dist=0.3328 | loss_mean=0.1080\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.8623 | _timers/batch_time=0.1707 | _timers/data_time=0.0004 | _timers/model_time=0.1702 | f1_score=1.503e-06 | loss=0.4384 | loss_dist=0.3306 | loss_mean=0.1078\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:17:40,722] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5369 | _timers/batch_time=0.3073 | _timers/data_time=0.0004 | _timers/model_time=0.3069 | f1_score=1.292e-06 | loss=0.2648 | loss_dist=0.1536 | loss_mean=0.1111\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.8972 | _timers/batch_time=0.1707 | _timers/data_time=0.0005 | _timers/model_time=0.1702 | f1_score=1.295e-06 | loss=0.2628 | loss_dist=0.1518 | loss_mean=0.1110\n",
      "[2019-12-06 18:17:44,494] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.6405 | _timers/batch_time=0.3019 | _timers/data_time=0.0003 | _timers/model_time=0.3015 | f1_score=1.274e-06 | loss=0.2102 | loss_dist=0.0979 | loss_mean=0.1122\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.8715 | _timers/batch_time=0.1709 | _timers/data_time=0.0004 | _timers/model_time=0.1705 | f1_score=1.278e-06 | loss=0.2096 | loss_dist=0.0976 | loss_mean=0.1121\n",
      "[2019-12-06 18:17:48,330] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.6094 | _timers/batch_time=0.3039 | _timers/data_time=0.0004 | _timers/model_time=0.3034 | f1_score=1.671e-06 | loss=0.2181 | loss_dist=0.1038 | loss_mean=0.1143\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.8423 | _timers/batch_time=0.1718 | _timers/data_time=0.0005 | _timers/model_time=0.1713 | f1_score=1.679e-06 | loss=0.2171 | loss_dist=0.1030 | loss_mean=0.1141\n",
      "[2019-12-06 18:17:52,456] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.2711 | _timers/batch_time=0.3211 | _timers/data_time=0.0003 | _timers/model_time=0.3208 | f1_score=1.543e-06 | loss=0.2850 | loss_dist=0.1763 | loss_mean=0.1087\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.3132 | _timers/batch_time=0.1812 | _timers/data_time=0.0002 | _timers/model_time=0.1810 | f1_score=1.544e-06 | loss=0.2875 | loss_dist=0.1789 | loss_mean=0.1086\n",
      "[2019-12-06 18:17:56,619] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.0047 | _timers/batch_time=0.3384 | _timers/data_time=0.0003 | _timers/model_time=0.3380 | f1_score=1.649e-06 | loss=0.5743 | loss_dist=0.4572 | loss_mean=0.1171\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.2040 | _timers/batch_time=0.2004 | _timers/data_time=0.0002 | _timers/model_time=0.2001 | f1_score=1.651e-06 | loss=0.5737 | loss_dist=0.4568 | loss_mean=0.1169\n",
      "[2019-12-06 18:18:00,759] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.1379 | _timers/batch_time=0.3260 | _timers/data_time=0.0003 | _timers/model_time=0.3257 | f1_score=1.397e-06 | loss=0.2326 | loss_dist=0.1220 | loss_mean=0.1106\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.9682 | _timers/batch_time=0.1867 | _timers/data_time=0.0003 | _timers/model_time=0.1864 | f1_score=1.398e-06 | loss=0.2300 | loss_dist=0.1195 | loss_mean=0.1105\n",
      "[2019-12-06 18:18:04,761] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.3979 | _timers/batch_time=0.3154 | _timers/data_time=0.0003 | _timers/model_time=0.3151 | f1_score=1.454e-06 | loss=0.3914 | loss_dist=0.2791 | loss_mean=0.1124\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.9837 | _timers/batch_time=0.1853 | _timers/data_time=0.0003 | _timers/model_time=0.1850 | f1_score=1.457e-06 | loss=0.3905 | loss_dist=0.2783 | loss_mean=0.1122\n",
      "[2019-12-06 18:18:08,727] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.3796 | _timers/batch_time=0.3157 | _timers/data_time=0.0004 | _timers/model_time=0.3152 | f1_score=1.460e-06 | loss=0.2524 | loss_dist=0.1444 | loss_mean=0.1080\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.2226 | _timers/batch_time=0.1840 | _timers/data_time=0.0003 | _timers/model_time=0.1838 | f1_score=1.462e-06 | loss=0.2521 | loss_dist=0.1443 | loss_mean=0.1079\n",
      "[2019-12-06 18:18:12,533] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.6273 | _timers/batch_time=0.3029 | _timers/data_time=0.0004 | _timers/model_time=0.3025 | f1_score=1.623e-06 | loss=0.5467 | loss_dist=0.4383 | loss_mean=0.1084\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=12.0559 | _timers/batch_time=0.1698 | _timers/data_time=0.0003 | _timers/model_time=0.1695 | f1_score=1.625e-06 | loss=0.5460 | loss_dist=0.4377 | loss_mean=0.1083\n",
      "[2019-12-06 18:18:16,391] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5924 | _timers/batch_time=0.3051 | _timers/data_time=0.0004 | _timers/model_time=0.3046 | f1_score=1.342e-06 | loss=0.2554 | loss_dist=0.1473 | loss_mean=0.1080\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.8124 | _timers/batch_time=0.1729 | _timers/data_time=0.0002 | _timers/model_time=0.1726 | f1_score=1.344e-06 | loss=0.2548 | loss_dist=0.1468 | loss_mean=0.1079\n",
      "[2019-12-06 18:18:20,226] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5923 | _timers/batch_time=0.3048 | _timers/data_time=0.0004 | _timers/model_time=0.3043 | f1_score=1.416e-06 | loss=0.4252 | loss_dist=0.3165 | loss_mean=0.1087\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.8275 | _timers/batch_time=0.1725 | _timers/data_time=0.0003 | _timers/model_time=0.1722 | f1_score=1.418e-06 | loss=0.4244 | loss_dist=0.3158 | loss_mean=0.1086\n",
      "[2019-12-06 18:18:24,064] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.6047 | _timers/batch_time=0.3041 | _timers/data_time=0.0004 | _timers/model_time=0.3037 | f1_score=1.157e-06 | loss=0.2362 | loss_dist=0.1269 | loss_mean=0.1093\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.4647 | _timers/batch_time=0.1771 | _timers/data_time=0.0002 | _timers/model_time=0.1768 | f1_score=1.158e-06 | loss=0.2351 | loss_dist=0.1259 | loss_mean=0.1092\n",
      "[2019-12-06 18:18:27,977] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4940 | _timers/batch_time=0.3090 | _timers/data_time=0.0003 | _timers/model_time=0.3086 | f1_score=1.508e-06 | loss=0.2904 | loss_dist=0.1804 | loss_mean=0.1101\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.8104 | _timers/batch_time=0.1739 | _timers/data_time=0.0003 | _timers/model_time=0.1736 | f1_score=1.514e-06 | loss=0.2794 | loss_dist=0.1694 | loss_mean=0.1100\n",
      "[2019-12-06 18:18:31,978] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4859 | _timers/batch_time=0.3104 | _timers/data_time=0.0003 | _timers/model_time=0.3100 | f1_score=1.602e-06 | loss=0.2007 | loss_dist=0.0885 | loss_mean=0.1121\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.5891 | _timers/batch_time=0.1904 | _timers/data_time=0.0002 | _timers/model_time=0.1901 | f1_score=1.604e-06 | loss=0.2003 | loss_dist=0.0883 | loss_mean=0.1120\n",
      "[2019-12-06 18:18:36,058] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=5.9083 | _timers/batch_time=0.3391 | _timers/data_time=0.0003 | _timers/model_time=0.3388 | f1_score=1.573e-06 | loss=0.2380 | loss_dist=0.1304 | loss_mean=0.1076\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.9245 | _timers/batch_time=0.1711 | _timers/data_time=0.0002 | _timers/model_time=0.1708 | f1_score=1.575e-06 | loss=0.2375 | loss_dist=0.1300 | loss_mean=0.1075\n",
      "[2019-12-06 18:18:40,090] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3451 | _timers/batch_time=0.3187 | _timers/data_time=0.0003 | _timers/model_time=0.3184 | f1_score=1.674e-06 | loss=0.1117 | loss_dist=0.0046 | loss_mean=0.1072\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.8140 | _timers/batch_time=0.1723 | _timers/data_time=0.0004 | _timers/model_time=0.1718 | f1_score=1.675e-06 | loss=0.1116 | loss_dist=0.0045 | loss_mean=0.1071\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:18:43,996] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5917 | _timers/batch_time=0.3049 | _timers/data_time=0.0004 | _timers/model_time=0.3045 | f1_score=1.356e-06 | loss=0.4229 | loss_dist=0.3153 | loss_mean=0.1076\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.7263 | _timers/batch_time=0.1728 | _timers/data_time=0.0003 | _timers/model_time=0.1725 | f1_score=1.357e-06 | loss=0.4227 | loss_dist=0.3152 | loss_mean=0.1075\n",
      "[2019-12-06 18:18:47,871] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5135 | _timers/batch_time=0.3088 | _timers/data_time=0.0003 | _timers/model_time=0.3085 | f1_score=1.132e-06 | loss=0.2220 | loss_dist=0.1130 | loss_mean=0.1089\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.7317 | _timers/batch_time=0.1736 | _timers/data_time=0.0003 | _timers/model_time=0.1733 | f1_score=1.137e-06 | loss=0.2208 | loss_dist=0.1120 | loss_mean=0.1088\n",
      "[2019-12-06 18:18:51,886] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1873 | _timers/batch_time=0.3245 | _timers/data_time=0.0003 | _timers/model_time=0.3242 | f1_score=1.431e-06 | loss=0.3420 | loss_dist=0.2353 | loss_mean=0.1067\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.5882 | _timers/batch_time=0.1757 | _timers/data_time=0.0002 | _timers/model_time=0.1754 | f1_score=1.433e-06 | loss=0.3413 | loss_dist=0.2346 | loss_mean=0.1066\n",
      "[2019-12-06 18:18:55,708] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5745 | _timers/batch_time=0.3047 | _timers/data_time=0.0004 | _timers/model_time=0.3043 | f1_score=1.085e-06 | loss=0.2128 | loss_dist=0.1094 | loss_mean=0.1034\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.9596 | _timers/batch_time=0.1706 | _timers/data_time=0.0002 | _timers/model_time=0.1703 | f1_score=1.086e-06 | loss=0.2125 | loss_dist=0.1092 | loss_mean=0.1033\n",
      "[2019-12-06 18:18:59,691] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3790 | _timers/batch_time=0.3171 | _timers/data_time=0.0004 | _timers/model_time=0.3166 | f1_score=1.332e-06 | loss=0.1944 | loss_dist=0.0860 | loss_mean=0.1084\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.0342 | _timers/batch_time=0.1844 | _timers/data_time=0.0004 | _timers/model_time=0.1840 | f1_score=1.333e-06 | loss=0.1938 | loss_dist=0.0855 | loss_mean=0.1083\n",
      "[2019-12-06 18:19:03,710] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.2027 | _timers/batch_time=0.3232 | _timers/data_time=0.0003 | _timers/model_time=0.3229 | f1_score=1.170e-06 | loss=0.1544 | loss_dist=0.0475 | loss_mean=0.1069\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.3164 | _timers/batch_time=0.1808 | _timers/data_time=0.0003 | _timers/model_time=0.1804 | f1_score=1.171e-06 | loss=0.1537 | loss_dist=0.0469 | loss_mean=0.1068\n",
      "[2019-12-06 18:19:07,757] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.4057 | _timers/batch_time=0.3149 | _timers/data_time=0.0003 | _timers/model_time=0.3145 | f1_score=1.510e-06 | loss=0.2414 | loss_dist=0.1326 | loss_mean=0.1087\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.9093 | _timers/batch_time=0.1873 | _timers/data_time=0.0003 | _timers/model_time=0.1870 | f1_score=1.513e-06 | loss=0.2412 | loss_dist=0.1325 | loss_mean=0.1086\n",
      "[2019-12-06 18:19:11,925] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.3416 | _timers/batch_time=0.3165 | _timers/data_time=0.0003 | _timers/model_time=0.3161 | f1_score=1.899e-06 | loss=0.3691 | loss_dist=0.2562 | loss_mean=0.1129\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.2448 | _timers/batch_time=0.1812 | _timers/data_time=0.0003 | _timers/model_time=0.1808 | f1_score=1.903e-06 | loss=0.3686 | loss_dist=0.2558 | loss_mean=0.1128\n",
      "[2019-12-06 18:19:15,901] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.3285 | _timers/batch_time=0.3161 | _timers/data_time=0.0002 | _timers/model_time=0.3159 | f1_score=1.179e-06 | loss=0.4662 | loss_dist=0.3542 | loss_mean=0.1120\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.4646 | _timers/batch_time=0.1770 | _timers/data_time=0.0003 | _timers/model_time=0.1766 | f1_score=1.180e-06 | loss=0.4649 | loss_dist=0.3531 | loss_mean=0.1118\n",
      "[2019-12-06 18:19:19,801] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6331 | _timers/batch_time=0.3023 | _timers/data_time=0.0003 | _timers/model_time=0.3020 | f1_score=1.947e-06 | loss=0.1512 | loss_dist=0.0482 | loss_mean=0.1031\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.6851 | _timers/batch_time=0.1925 | _timers/data_time=0.0003 | _timers/model_time=0.1922 | f1_score=1.953e-06 | loss=0.1511 | loss_dist=0.0481 | loss_mean=0.1030\n",
      "[2019-12-06 18:19:23,673] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5099 | _timers/batch_time=0.3078 | _timers/data_time=0.0003 | _timers/model_time=0.3075 | f1_score=1.770e-06 | loss=0.1602 | loss_dist=0.0488 | loss_mean=0.1114\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.3163 | _timers/batch_time=0.1794 | _timers/data_time=0.0004 | _timers/model_time=0.1790 | f1_score=1.772e-06 | loss=0.1600 | loss_dist=0.0487 | loss_mean=0.1113\n",
      "[2019-12-06 18:19:27,662] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.2525 | _timers/batch_time=0.3233 | _timers/data_time=0.0003 | _timers/model_time=0.3230 | f1_score=1.694e-06 | loss=0.1976 | loss_dist=0.0864 | loss_mean=0.1112\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.8249 | _timers/batch_time=0.1727 | _timers/data_time=0.0004 | _timers/model_time=0.1722 | f1_score=1.696e-06 | loss=0.1974 | loss_dist=0.0863 | loss_mean=0.1111\n",
      "[2019-12-06 18:19:31,710] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.2355 | _timers/batch_time=0.3244 | _timers/data_time=0.0003 | _timers/model_time=0.3241 | f1_score=1.390e-06 | loss=0.3355 | loss_dist=0.2279 | loss_mean=0.1076\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.7502 | _timers/batch_time=0.1738 | _timers/data_time=0.0004 | _timers/model_time=0.1733 | f1_score=1.390e-06 | loss=0.3351 | loss_dist=0.2276 | loss_mean=0.1075\n",
      "[2019-12-06 18:19:35,661] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5490 | _timers/batch_time=0.3069 | _timers/data_time=0.0003 | _timers/model_time=0.3065 | f1_score=1.558e-06 | loss=0.2110 | loss_dist=0.1044 | loss_mean=0.1067\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.1469 | _timers/batch_time=0.1874 | _timers/data_time=0.0003 | _timers/model_time=0.1870 | f1_score=1.560e-06 | loss=0.2109 | loss_dist=0.1044 | loss_mean=0.1066\n",
      "[2019-12-06 18:19:39,511] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4632 | _timers/batch_time=0.3097 | _timers/data_time=0.0003 | _timers/model_time=0.3093 | f1_score=1.626e-06 | loss=0.1255 | loss_dist=0.0177 | loss_mean=0.1078\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.4839 | _timers/batch_time=0.1774 | _timers/data_time=0.0003 | _timers/model_time=0.1771 | f1_score=1.629e-06 | loss=0.1253 | loss_dist=0.0176 | loss_mean=0.1077\n",
      "[2019-12-06 18:19:43,433] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4280 | _timers/batch_time=0.3132 | _timers/data_time=0.0004 | _timers/model_time=0.3128 | f1_score=1.727e-06 | loss=0.1702 | loss_dist=0.0669 | loss_mean=0.1033\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5171 | _timers/batch_time=0.1794 | _timers/data_time=0.0004 | _timers/model_time=0.1790 | f1_score=1.728e-06 | loss=0.1698 | loss_dist=0.0666 | loss_mean=0.1032\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:19:47,320] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5210 | _timers/batch_time=0.3070 | _timers/data_time=0.0004 | _timers/model_time=0.3066 | f1_score=1.667e-06 | loss=0.3028 | loss_dist=0.1985 | loss_mean=0.1042\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8073 | _timers/batch_time=0.1724 | _timers/data_time=0.0002 | _timers/model_time=0.1722 | f1_score=1.668e-06 | loss=0.3023 | loss_dist=0.1982 | loss_mean=0.1041\n",
      "[2019-12-06 18:19:51,106] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.6520 | _timers/batch_time=0.3021 | _timers/data_time=0.0004 | _timers/model_time=0.3016 | f1_score=1.797e-06 | loss=0.1549 | loss_dist=0.0452 | loss_mean=0.1097\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.9542 | _timers/batch_time=0.1707 | _timers/data_time=0.0003 | _timers/model_time=0.1704 | f1_score=1.798e-06 | loss=0.1547 | loss_dist=0.0451 | loss_mean=0.1096\n",
      "[2019-12-06 18:19:55,081] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5655 | _timers/batch_time=0.3059 | _timers/data_time=0.0003 | _timers/model_time=0.3055 | f1_score=1.508e-06 | loss=0.2373 | loss_dist=0.1346 | loss_mean=0.1028\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.4030 | _timers/batch_time=0.1804 | _timers/data_time=0.0003 | _timers/model_time=0.1801 | f1_score=1.509e-06 | loss=0.2367 | loss_dist=0.1340 | loss_mean=0.1027\n",
      "[2019-12-06 18:19:59,047] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4785 | _timers/batch_time=0.3092 | _timers/data_time=0.0003 | _timers/model_time=0.3088 | f1_score=2.048e-06 | loss=0.1684 | loss_dist=0.0630 | loss_mean=0.1054\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.1666 | _timers/batch_time=0.2003 | _timers/data_time=0.0003 | _timers/model_time=0.2000 | f1_score=2.049e-06 | loss=0.1683 | loss_dist=0.0630 | loss_mean=0.1053\n",
      "[2019-12-06 18:20:02,949] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5413 | _timers/batch_time=0.3071 | _timers/data_time=0.0003 | _timers/model_time=0.3067 | f1_score=1.344e-06 | loss=0.2798 | loss_dist=0.1826 | loss_mean=0.0972\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8891 | _timers/batch_time=0.1719 | _timers/data_time=0.0003 | _timers/model_time=0.1715 | f1_score=1.345e-06 | loss=0.2787 | loss_dist=0.1816 | loss_mean=0.0971\n",
      "[2019-12-06 18:20:06,812] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2699 | _timers/batch_time=0.3191 | _timers/data_time=0.0003 | _timers/model_time=0.3187 | f1_score=2.028e-06 | loss=0.3892 | loss_dist=0.2856 | loss_mean=0.1036\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=12.0927 | _timers/batch_time=0.1679 | _timers/data_time=0.0003 | _timers/model_time=0.1676 | f1_score=2.030e-06 | loss=0.3886 | loss_dist=0.2851 | loss_mean=0.1035\n",
      "[2019-12-06 18:20:10,652] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5495 | _timers/batch_time=0.3069 | _timers/data_time=0.0004 | _timers/model_time=0.3065 | f1_score=1.953e-06 | loss=0.4968 | loss_dist=0.3897 | loss_mean=0.1071\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.9353 | _timers/batch_time=0.1710 | _timers/data_time=0.0002 | _timers/model_time=0.1707 | f1_score=1.954e-06 | loss=0.4964 | loss_dist=0.3894 | loss_mean=0.1070\n",
      "[2019-12-06 18:20:14,710] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2402 | _timers/batch_time=0.3250 | _timers/data_time=0.0003 | _timers/model_time=0.3247 | f1_score=1.483e-06 | loss=0.1471 | loss_dist=0.0431 | loss_mean=0.1040\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.3218 | _timers/batch_time=0.1797 | _timers/data_time=0.0005 | _timers/model_time=0.1792 | f1_score=1.485e-06 | loss=0.1468 | loss_dist=0.0429 | loss_mean=0.1040\n",
      "[2019-12-06 18:20:18,589] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4618 | _timers/batch_time=0.3117 | _timers/data_time=0.0004 | _timers/model_time=0.3112 | f1_score=1.945e-06 | loss=0.1599 | loss_dist=0.0556 | loss_mean=0.1043\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.7098 | _timers/batch_time=0.1741 | _timers/data_time=0.0003 | _timers/model_time=0.1737 | f1_score=1.947e-06 | loss=0.1596 | loss_dist=0.0555 | loss_mean=0.1042\n",
      "[2019-12-06 18:20:22,538] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4211 | _timers/batch_time=0.3147 | _timers/data_time=0.0004 | _timers/model_time=0.3142 | f1_score=1.966e-06 | loss=0.1526 | loss_dist=0.0495 | loss_mean=0.1031\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.2580 | _timers/batch_time=0.1799 | _timers/data_time=0.0003 | _timers/model_time=0.1795 | f1_score=1.968e-06 | loss=0.1522 | loss_dist=0.0492 | loss_mean=0.1030\n",
      "[2019-12-06 18:20:26,382] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.6500 | _timers/batch_time=0.3020 | _timers/data_time=0.0004 | _timers/model_time=0.3015 | f1_score=1.518e-06 | loss=0.2177 | loss_dist=0.1100 | loss_mean=0.1077\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.7285 | _timers/batch_time=0.1740 | _timers/data_time=0.0003 | _timers/model_time=0.1737 | f1_score=1.519e-06 | loss=0.2176 | loss_dist=0.1099 | loss_mean=0.1076\n",
      "[2019-12-06 18:20:30,572] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.3122 | _timers/batch_time=0.3190 | _timers/data_time=0.0004 | _timers/model_time=0.3185 | f1_score=2.042e-06 | loss=0.1772 | loss_dist=0.0749 | loss_mean=0.1023\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.0338 | _timers/batch_time=0.1878 | _timers/data_time=0.0003 | _timers/model_time=0.1874 | f1_score=2.042e-06 | loss=0.1771 | loss_dist=0.0749 | loss_mean=0.1022\n",
      "[2019-12-06 18:20:34,447] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4915 | _timers/batch_time=0.3091 | _timers/data_time=0.0004 | _timers/model_time=0.3086 | f1_score=1.983e-06 | loss=0.1987 | loss_dist=0.0941 | loss_mean=0.1046\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.9062 | _timers/batch_time=0.1708 | _timers/data_time=0.0003 | _timers/model_time=0.1705 | f1_score=1.985e-06 | loss=0.1981 | loss_dist=0.0936 | loss_mean=0.1045\n",
      "[2019-12-06 18:20:38,273] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.5103 | _timers/batch_time=0.3079 | _timers/data_time=0.0003 | _timers/model_time=0.3075 | f1_score=1.472e-06 | loss=0.1982 | loss_dist=0.0979 | loss_mean=0.1004\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.9387 | _timers/batch_time=0.1704 | _timers/data_time=0.0004 | _timers/model_time=0.1700 | f1_score=1.473e-06 | loss=0.1982 | loss_dist=0.0979 | loss_mean=0.1003\n",
      "[2019-12-06 18:20:42,412] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.3319 | _timers/batch_time=0.3189 | _timers/data_time=0.0004 | _timers/model_time=0.3184 | f1_score=1.913e-06 | loss=0.2789 | loss_dist=0.1732 | loss_mean=0.1057\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.8301 | _timers/batch_time=0.1726 | _timers/data_time=0.0003 | _timers/model_time=0.1723 | f1_score=1.914e-06 | loss=0.2788 | loss_dist=0.1732 | loss_mean=0.1056\n",
      "[2019-12-06 18:20:46,248] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.6661 | _timers/batch_time=0.3013 | _timers/data_time=0.0004 | _timers/model_time=0.3009 | f1_score=1.784e-06 | loss=0.1377 | loss_dist=0.0288 | loss_mean=0.1089\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.7288 | _timers/batch_time=0.1751 | _timers/data_time=0.0003 | _timers/model_time=0.1748 | f1_score=1.785e-06 | loss=0.1376 | loss_dist=0.0288 | loss_mean=0.1088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:20:50,280] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2791 | _timers/batch_time=0.3233 | _timers/data_time=0.0004 | _timers/model_time=0.3229 | f1_score=1.407e-06 | loss=0.1394 | loss_dist=0.0387 | loss_mean=0.1007\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.1642 | _timers/batch_time=0.1810 | _timers/data_time=0.0003 | _timers/model_time=0.1806 | f1_score=1.408e-06 | loss=0.1391 | loss_dist=0.0384 | loss_mean=0.1007\n",
      "[2019-12-06 18:20:54,417] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2374 | _timers/batch_time=0.3245 | _timers/data_time=0.0003 | _timers/model_time=0.3241 | f1_score=1.528e-06 | loss=0.1387 | loss_dist=0.0387 | loss_mean=0.1000\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.8333 | _timers/batch_time=0.1879 | _timers/data_time=0.0003 | _timers/model_time=0.1876 | f1_score=1.529e-06 | loss=0.1391 | loss_dist=0.0391 | loss_mean=0.0999\n",
      "[2019-12-06 18:20:58,725] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.6860 | _timers/batch_time=0.3536 | _timers/data_time=0.0002 | _timers/model_time=0.3533 | f1_score=1.949e-06 | loss=0.2741 | loss_dist=0.1673 | loss_mean=0.1068\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.2940 | _timers/batch_time=0.1807 | _timers/data_time=0.0003 | _timers/model_time=0.1803 | f1_score=1.951e-06 | loss=0.2739 | loss_dist=0.1672 | loss_mean=0.1067\n",
      "[2019-12-06 18:21:03,093] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.9098 | _timers/batch_time=0.3441 | _timers/data_time=0.0003 | _timers/model_time=0.3437 | f1_score=1.631e-06 | loss=0.2434 | loss_dist=0.1415 | loss_mean=0.1019\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.1571 | _timers/batch_time=0.1997 | _timers/data_time=0.0003 | _timers/model_time=0.1994 | f1_score=1.633e-06 | loss=0.2433 | loss_dist=0.1415 | loss_mean=0.1019\n",
      "Top best models:\n",
      "logs/Concrete/checkpoints/train.64.pth\t0.1116\n",
      "logs/Concrete/checkpoints/train.79.pth\t0.1253\n",
      "logs/Concrete/checkpoints/train.96.pth\t0.1376\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### HardKumaBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from binarizers import HardKumaBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "hkb = HardKumaBinarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "        self.v = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)\n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        b_new = torch.repeat_interleave(hkb(self.u, self.v), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/HardKuma\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:22:44,807] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.8000 | _timers/batch_time=0.3458 | _timers/data_time=0.0002 | _timers/model_time=0.3455 | f1_score=5.300e-07 | loss=2.6945 | loss_dist=2.5584 | loss_mean=0.1361\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.9563 | _timers/batch_time=0.1864 | _timers/data_time=0.0002 | _timers/model_time=0.1861 | f1_score=5.511e-07 | loss=2.5137 | loss_dist=2.3779 | loss_mean=0.1358\n",
      "[2019-12-06 18:22:49,198] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.8326 | _timers/batch_time=0.3468 | _timers/data_time=0.0003 | _timers/model_time=0.3465 | f1_score=6.214e-07 | loss=13.2910 | loss_dist=13.1538 | loss_mean=0.1372\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.0507 | _timers/batch_time=0.1853 | _timers/data_time=0.0002 | _timers/model_time=0.1850 | f1_score=6.311e-07 | loss=13.0239 | loss_dist=12.8869 | loss_mean=0.1371\n",
      "[2019-12-06 18:22:53,298] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1913 | _timers/batch_time=0.3235 | _timers/data_time=0.0004 | _timers/model_time=0.3231 | f1_score=4.897e-07 | loss=10.1086 | loss_dist=9.9718 | loss_mean=0.1368\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.8468 | _timers/batch_time=0.1879 | _timers/data_time=0.0003 | _timers/model_time=0.1875 | f1_score=4.965e-07 | loss=9.3994 | loss_dist=9.2628 | loss_mean=0.1366\n",
      "[2019-12-06 18:22:57,385] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1559 | _timers/batch_time=0.3257 | _timers/data_time=0.0004 | _timers/model_time=0.3253 | f1_score=7.055e-07 | loss=12.2818 | loss_dist=12.1485 | loss_mean=0.1333\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1426 | _timers/batch_time=0.1825 | _timers/data_time=0.0003 | _timers/model_time=0.1822 | f1_score=7.301e-07 | loss=11.5267 | loss_dist=11.3934 | loss_mean=0.1332\n",
      "[2019-12-06 18:23:01,421] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2894 | _timers/batch_time=0.3189 | _timers/data_time=0.0002 | _timers/model_time=0.3187 | f1_score=3.606e-07 | loss=13.1583 | loss_dist=13.0252 | loss_mean=0.1332\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.2767 | _timers/batch_time=0.1804 | _timers/data_time=0.0002 | _timers/model_time=0.1801 | f1_score=4.104e-07 | loss=11.9201 | loss_dist=11.7870 | loss_mean=0.1332\n",
      "[2019-12-06 18:23:05,516] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2386 | _timers/batch_time=0.3218 | _timers/data_time=0.0004 | _timers/model_time=0.3213 | f1_score=7.605e-07 | loss=0.6357 | loss_dist=0.5008 | loss_mean=0.1349\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.0658 | _timers/batch_time=0.1836 | _timers/data_time=0.0003 | _timers/model_time=0.1833 | f1_score=8.767e-07 | loss=0.4449 | loss_dist=0.3100 | loss_mean=0.1349\n",
      "[2019-12-06 18:23:09,642] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2065 | _timers/batch_time=0.3232 | _timers/data_time=0.0003 | _timers/model_time=0.3229 | f1_score=8.916e-07 | loss=1.1677 | loss_dist=1.0319 | loss_mean=0.1358\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.3838 | _timers/batch_time=0.1953 | _timers/data_time=0.0003 | _timers/model_time=0.1949 | f1_score=9.005e-07 | loss=1.1504 | loss_dist=1.0146 | loss_mean=0.1358\n",
      "[2019-12-06 18:23:14,372] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7545 | _timers/batch_time=0.3503 | _timers/data_time=0.0005 | _timers/model_time=0.3497 | f1_score=9.678e-07 | loss=0.5812 | loss_dist=0.4491 | loss_mean=0.1320\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=7.8193 | _timers/batch_time=0.2638 | _timers/data_time=0.0004 | _timers/model_time=0.2634 | f1_score=9.975e-07 | loss=0.5559 | loss_dist=0.4238 | loss_mean=0.1321\n",
      "[2019-12-06 18:23:19,099] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.4815 | _timers/batch_time=0.3664 | _timers/data_time=0.0003 | _timers/model_time=0.3660 | f1_score=8.239e-07 | loss=1.3658 | loss_dist=1.2342 | loss_mean=0.1315\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.7846 | _timers/batch_time=0.2305 | _timers/data_time=0.0004 | _timers/model_time=0.2300 | f1_score=8.961e-07 | loss=0.8645 | loss_dist=0.7330 | loss_mean=0.1316\n",
      "[2019-12-06 18:23:23,454] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7030 | _timers/batch_time=0.3525 | _timers/data_time=0.0004 | _timers/model_time=0.3520 | f1_score=8.927e-07 | loss=0.9867 | loss_dist=0.8543 | loss_mean=0.1324\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.3664 | _timers/batch_time=0.1965 | _timers/data_time=0.0003 | _timers/model_time=0.1962 | f1_score=9.479e-07 | loss=0.8911 | loss_dist=0.7587 | loss_mean=0.1324\n",
      "[2019-12-06 18:23:27,512] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.0821 | _timers/batch_time=0.3306 | _timers/data_time=0.0003 | _timers/model_time=0.3302 | f1_score=1.186e-06 | loss=0.4635 | loss_dist=0.3330 | loss_mean=0.1305\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.8834 | _timers/batch_time=0.1723 | _timers/data_time=0.0002 | _timers/model_time=0.1720 | f1_score=1.196e-06 | loss=0.4564 | loss_dist=0.3259 | loss_mean=0.1305\n",
      "[2019-12-06 18:23:31,438] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5894 | _timers/batch_time=0.3046 | _timers/data_time=0.0004 | _timers/model_time=0.3041 | f1_score=1.239e-06 | loss=0.3702 | loss_dist=0.2363 | loss_mean=0.1339\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.5359 | _timers/batch_time=0.1915 | _timers/data_time=0.0002 | _timers/model_time=0.1913 | f1_score=1.256e-06 | loss=0.3646 | loss_dist=0.2307 | loss_mean=0.1339\n",
      "[2019-12-06 18:23:35,991] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.3069 | _timers/batch_time=0.3819 | _timers/data_time=0.0004 | _timers/model_time=0.3815 | f1_score=1.011e-06 | loss=0.4763 | loss_dist=0.3440 | loss_mean=0.1323\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.1590 | _timers/batch_time=0.1820 | _timers/data_time=0.0002 | _timers/model_time=0.1818 | f1_score=1.025e-06 | loss=0.4695 | loss_dist=0.3372 | loss_mean=0.1323\n",
      "[2019-12-06 18:23:39,944] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.4971 | _timers/batch_time=0.3088 | _timers/data_time=0.0003 | _timers/model_time=0.3084 | f1_score=1.292e-06 | loss=0.5148 | loss_dist=0.3791 | loss_mean=0.1356\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.0465 | _timers/batch_time=0.2014 | _timers/data_time=0.0004 | _timers/model_time=0.2010 | f1_score=1.301e-06 | loss=0.5009 | loss_dist=0.3653 | loss_mean=0.1357\n",
      "[2019-12-06 18:23:44,539] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.7093 | _timers/batch_time=0.3521 | _timers/data_time=0.0003 | _timers/model_time=0.3517 | f1_score=1.345e-06 | loss=0.4485 | loss_dist=0.3104 | loss_mean=0.1381\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=9.4556 | _timers/batch_time=0.2145 | _timers/data_time=0.0003 | _timers/model_time=0.2141 | f1_score=1.354e-06 | loss=0.4411 | loss_dist=0.3030 | loss_mean=0.1381\n",
      "[2019-12-06 18:23:48,564] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0903 | _timers/batch_time=0.3286 | _timers/data_time=0.0004 | _timers/model_time=0.3281 | f1_score=1.019e-06 | loss=0.6570 | loss_dist=0.5208 | loss_mean=0.1362\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.3687 | _timers/batch_time=0.1777 | _timers/data_time=0.0003 | _timers/model_time=0.1773 | f1_score=1.050e-06 | loss=0.6089 | loss_dist=0.4727 | loss_mean=0.1362\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:23:52,440] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.3261 | _timers/batch_time=0.3163 | _timers/data_time=0.0003 | _timers/model_time=0.3160 | f1_score=1.320e-06 | loss=0.5241 | loss_dist=0.3872 | loss_mean=0.1368\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5995 | _timers/batch_time=0.1752 | _timers/data_time=0.0004 | _timers/model_time=0.1747 | f1_score=1.328e-06 | loss=0.5137 | loss_dist=0.3769 | loss_mean=0.1368\n",
      "[2019-12-06 18:23:56,342] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.5595 | _timers/batch_time=0.3056 | _timers/data_time=0.0002 | _timers/model_time=0.3053 | f1_score=1.206e-06 | loss=0.3433 | loss_dist=0.2080 | loss_mean=0.1353\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.2003 | _timers/batch_time=0.1855 | _timers/data_time=0.0005 | _timers/model_time=0.1850 | f1_score=1.218e-06 | loss=0.3396 | loss_dist=0.2043 | loss_mean=0.1353\n",
      "[2019-12-06 18:24:00,182] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.4319 | _timers/batch_time=0.3111 | _timers/data_time=0.0003 | _timers/model_time=0.3107 | f1_score=1.236e-06 | loss=0.5015 | loss_dist=0.3660 | loss_mean=0.1355\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.7570 | _timers/batch_time=0.1728 | _timers/data_time=0.0004 | _timers/model_time=0.1724 | f1_score=1.242e-06 | loss=0.4941 | loss_dist=0.3585 | loss_mean=0.1355\n",
      "[2019-12-06 18:24:04,106] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.5170 | _timers/batch_time=0.3070 | _timers/data_time=0.0002 | _timers/model_time=0.3068 | f1_score=1.183e-06 | loss=0.3931 | loss_dist=0.2553 | loss_mean=0.1377\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.5758 | _timers/batch_time=0.1915 | _timers/data_time=0.0004 | _timers/model_time=0.1911 | f1_score=1.192e-06 | loss=0.3868 | loss_dist=0.2490 | loss_mean=0.1378\n",
      "[2019-12-06 18:24:08,436] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.9700 | _timers/batch_time=0.3359 | _timers/data_time=0.0003 | _timers/model_time=0.3355 | f1_score=1.326e-06 | loss=0.4740 | loss_dist=0.3382 | loss_mean=0.1358\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.0413 | _timers/batch_time=0.2023 | _timers/data_time=0.0002 | _timers/model_time=0.2020 | f1_score=1.332e-06 | loss=0.4686 | loss_dist=0.3328 | loss_mean=0.1358\n",
      "[2019-12-06 18:24:12,603] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0973 | _timers/batch_time=0.3301 | _timers/data_time=0.0002 | _timers/model_time=0.3299 | f1_score=1.346e-06 | loss=0.4653 | loss_dist=0.3315 | loss_mean=0.1338\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.8668 | _timers/batch_time=0.1727 | _timers/data_time=0.0003 | _timers/model_time=0.1724 | f1_score=1.352e-06 | loss=0.4576 | loss_dist=0.3238 | loss_mean=0.1338\n",
      "[2019-12-06 18:24:17,059] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.9612 | _timers/batch_time=0.3418 | _timers/data_time=0.0004 | _timers/model_time=0.3413 | f1_score=1.260e-06 | loss=0.3756 | loss_dist=0.2407 | loss_mean=0.1348\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.0267 | _timers/batch_time=0.2150 | _timers/data_time=0.0003 | _timers/model_time=0.2147 | f1_score=1.265e-06 | loss=0.3725 | loss_dist=0.2377 | loss_mean=0.1348\n",
      "[2019-12-06 18:24:21,194] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2359 | _timers/batch_time=0.3247 | _timers/data_time=0.0004 | _timers/model_time=0.3242 | f1_score=1.109e-06 | loss=0.3668 | loss_dist=0.2306 | loss_mean=0.1363\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.6109 | _timers/batch_time=0.1783 | _timers/data_time=0.0002 | _timers/model_time=0.1780 | f1_score=1.113e-06 | loss=0.3637 | loss_dist=0.2274 | loss_mean=0.1363\n",
      "[2019-12-06 18:24:25,569] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.9864 | _timers/batch_time=0.3395 | _timers/data_time=0.0003 | _timers/model_time=0.3391 | f1_score=1.309e-06 | loss=0.4005 | loss_dist=0.2663 | loss_mean=0.1342\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=9.9026 | _timers/batch_time=0.2059 | _timers/data_time=0.0002 | _timers/model_time=0.2057 | f1_score=1.312e-06 | loss=0.3978 | loss_dist=0.2636 | loss_mean=0.1342\n",
      "[2019-12-06 18:24:29,740] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0217 | _timers/batch_time=0.3328 | _timers/data_time=0.0005 | _timers/model_time=0.3323 | f1_score=1.324e-06 | loss=0.3484 | loss_dist=0.2146 | loss_mean=0.1339\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.5067 | _timers/batch_time=0.1765 | _timers/data_time=0.0003 | _timers/model_time=0.1762 | f1_score=1.329e-06 | loss=0.3451 | loss_dist=0.2112 | loss_mean=0.1339\n",
      "[2019-12-06 18:24:33,612] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.4490 | _timers/batch_time=0.3108 | _timers/data_time=0.0004 | _timers/model_time=0.3104 | f1_score=1.298e-06 | loss=0.3598 | loss_dist=0.2296 | loss_mean=0.1301\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.7241 | _timers/batch_time=0.1734 | _timers/data_time=0.0002 | _timers/model_time=0.1732 | f1_score=1.302e-06 | loss=0.3575 | loss_dist=0.2274 | loss_mean=0.1301\n",
      "[2019-12-06 18:24:37,885] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.4580 | _timers/batch_time=0.3106 | _timers/data_time=0.0004 | _timers/model_time=0.3102 | f1_score=1.380e-06 | loss=0.3875 | loss_dist=0.2524 | loss_mean=0.1350\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=9.8294 | _timers/batch_time=0.2070 | _timers/data_time=0.0003 | _timers/model_time=0.2067 | f1_score=1.384e-06 | loss=0.3820 | loss_dist=0.2470 | loss_mean=0.1350\n",
      "[2019-12-06 18:24:42,325] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.6944 | _timers/batch_time=0.3624 | _timers/data_time=0.0004 | _timers/model_time=0.3619 | f1_score=1.127e-06 | loss=0.6108 | loss_dist=0.4732 | loss_mean=0.1377\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.0994 | _timers/batch_time=0.1835 | _timers/data_time=0.0004 | _timers/model_time=0.1831 | f1_score=1.131e-06 | loss=0.6028 | loss_dist=0.4652 | loss_mean=0.1376\n",
      "[2019-12-06 18:24:46,249] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.4525 | _timers/batch_time=0.3105 | _timers/data_time=0.0002 | _timers/model_time=0.3103 | f1_score=1.211e-06 | loss=0.4035 | loss_dist=0.2674 | loss_mean=0.1360\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.0342 | _timers/batch_time=0.1840 | _timers/data_time=0.0004 | _timers/model_time=0.1836 | f1_score=1.216e-06 | loss=0.3994 | loss_dist=0.2634 | loss_mean=0.1360\n",
      "[2019-12-06 18:24:50,064] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5649 | _timers/batch_time=0.3056 | _timers/data_time=0.0004 | _timers/model_time=0.3052 | f1_score=1.376e-06 | loss=0.4066 | loss_dist=0.2737 | loss_mean=0.1329\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.8235 | _timers/batch_time=0.1720 | _timers/data_time=0.0002 | _timers/model_time=0.1718 | f1_score=1.380e-06 | loss=0.4029 | loss_dist=0.2700 | loss_mean=0.1329\n",
      "[2019-12-06 18:24:54,111] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1218 | _timers/batch_time=0.3277 | _timers/data_time=0.0004 | _timers/model_time=0.3273 | f1_score=1.510e-06 | loss=0.4263 | loss_dist=0.2914 | loss_mean=0.1349\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.8717 | _timers/batch_time=0.1861 | _timers/data_time=0.0002 | _timers/model_time=0.1859 | f1_score=1.514e-06 | loss=0.4197 | loss_dist=0.2847 | loss_mean=0.1349\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:24:58,511] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.2059 | _timers/batch_time=0.3257 | _timers/data_time=0.0003 | _timers/model_time=0.3254 | f1_score=1.351e-06 | loss=0.2424 | loss_dist=0.1057 | loss_mean=0.1368\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.7376 | _timers/batch_time=0.1920 | _timers/data_time=0.0004 | _timers/model_time=0.1916 | f1_score=1.355e-06 | loss=0.2412 | loss_dist=0.1045 | loss_mean=0.1368\n",
      "[2019-12-06 18:25:02,837] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.8015 | _timers/batch_time=0.3492 | _timers/data_time=0.0004 | _timers/model_time=0.3486 | f1_score=1.125e-06 | loss=0.3071 | loss_dist=0.1724 | loss_mean=0.1347\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.6071 | _timers/batch_time=0.1759 | _timers/data_time=0.0003 | _timers/model_time=0.1756 | f1_score=1.130e-06 | loss=0.3044 | loss_dist=0.1697 | loss_mean=0.1347\n",
      "[2019-12-06 18:25:06,722] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.3532 | _timers/batch_time=0.3151 | _timers/data_time=0.0003 | _timers/model_time=0.3147 | f1_score=1.445e-06 | loss=0.2595 | loss_dist=0.1229 | loss_mean=0.1366\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.9093 | _timers/batch_time=0.1711 | _timers/data_time=0.0004 | _timers/model_time=0.1708 | f1_score=1.449e-06 | loss=0.2584 | loss_dist=0.1218 | loss_mean=0.1366\n",
      "[2019-12-06 18:25:10,973] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.0357 | _timers/batch_time=0.3345 | _timers/data_time=0.0003 | _timers/model_time=0.3341 | f1_score=1.453e-06 | loss=0.2824 | loss_dist=0.1488 | loss_mean=0.1336\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.2202 | _timers/batch_time=0.1823 | _timers/data_time=0.0003 | _timers/model_time=0.1820 | f1_score=1.455e-06 | loss=0.2807 | loss_dist=0.1471 | loss_mean=0.1336\n",
      "[2019-12-06 18:25:15,208] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.7200 | _timers/batch_time=0.3503 | _timers/data_time=0.0004 | _timers/model_time=0.3498 | f1_score=1.266e-06 | loss=0.3361 | loss_dist=0.2013 | loss_mean=0.1347\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.4809 | _timers/batch_time=0.1936 | _timers/data_time=0.0003 | _timers/model_time=0.1933 | f1_score=1.270e-06 | loss=0.3330 | loss_dist=0.1982 | loss_mean=0.1347\n",
      "[2019-12-06 18:25:19,790] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.6545 | _timers/batch_time=0.3576 | _timers/data_time=0.0004 | _timers/model_time=0.3571 | f1_score=1.513e-06 | loss=0.3029 | loss_dist=0.1659 | loss_mean=0.1370\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=9.9297 | _timers/batch_time=0.2062 | _timers/data_time=0.0003 | _timers/model_time=0.2059 | f1_score=1.516e-06 | loss=0.3003 | loss_dist=0.1633 | loss_mean=0.1370\n",
      "[2019-12-06 18:25:23,918] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0612 | _timers/batch_time=0.3319 | _timers/data_time=0.0003 | _timers/model_time=0.3316 | f1_score=1.389e-06 | loss=0.3158 | loss_dist=0.1830 | loss_mean=0.1329\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.1117 | _timers/batch_time=0.1859 | _timers/data_time=0.0002 | _timers/model_time=0.1857 | f1_score=1.393e-06 | loss=0.3141 | loss_dist=0.1812 | loss_mean=0.1329\n",
      "[2019-12-06 18:25:28,047] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0734 | _timers/batch_time=0.3325 | _timers/data_time=0.0004 | _timers/model_time=0.3321 | f1_score=1.425e-06 | loss=0.3017 | loss_dist=0.1625 | loss_mean=0.1392\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.6395 | _timers/batch_time=0.1760 | _timers/data_time=0.0002 | _timers/model_time=0.1758 | f1_score=1.428e-06 | loss=0.2997 | loss_dist=0.1605 | loss_mean=0.1392\n",
      "[2019-12-06 18:25:32,002] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.4260 | _timers/batch_time=0.3125 | _timers/data_time=0.0002 | _timers/model_time=0.3122 | f1_score=1.406e-06 | loss=0.4894 | loss_dist=0.3569 | loss_mean=0.1325\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.6311 | _timers/batch_time=0.1752 | _timers/data_time=0.0002 | _timers/model_time=0.1749 | f1_score=1.410e-06 | loss=0.4833 | loss_dist=0.3508 | loss_mean=0.1325\n",
      "[2019-12-06 18:25:35,907] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5119 | _timers/batch_time=0.3082 | _timers/data_time=0.0003 | _timers/model_time=0.3079 | f1_score=1.623e-06 | loss=0.3642 | loss_dist=0.2267 | loss_mean=0.1374\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.0289 | _timers/batch_time=0.1876 | _timers/data_time=0.0003 | _timers/model_time=0.1873 | f1_score=1.627e-06 | loss=0.3602 | loss_dist=0.2228 | loss_mean=0.1374\n",
      "[2019-12-06 18:25:40,273] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0071 | _timers/batch_time=0.3378 | _timers/data_time=0.0003 | _timers/model_time=0.3374 | f1_score=1.546e-06 | loss=0.2434 | loss_dist=0.1085 | loss_mean=0.1349\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.9294 | _timers/batch_time=0.1898 | _timers/data_time=0.0004 | _timers/model_time=0.1893 | f1_score=1.550e-06 | loss=0.2420 | loss_dist=0.1071 | loss_mean=0.1349\n",
      "[2019-12-06 18:25:44,491] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.1006 | _timers/batch_time=0.3287 | _timers/data_time=0.0005 | _timers/model_time=0.3281 | f1_score=1.513e-06 | loss=0.2414 | loss_dist=0.1043 | loss_mean=0.1371\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.2657 | _timers/batch_time=0.1992 | _timers/data_time=0.0004 | _timers/model_time=0.1988 | f1_score=1.516e-06 | loss=0.2406 | loss_dist=0.1035 | loss_mean=0.1371\n",
      "[2019-12-06 18:25:48,830] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.0421 | _timers/batch_time=0.3324 | _timers/data_time=0.0004 | _timers/model_time=0.3320 | f1_score=1.456e-06 | loss=0.2404 | loss_dist=0.1060 | loss_mean=0.1344\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.2749 | _timers/batch_time=0.1848 | _timers/data_time=0.0003 | _timers/model_time=0.1845 | f1_score=1.458e-06 | loss=0.2391 | loss_dist=0.1048 | loss_mean=0.1344\n",
      "[2019-12-06 18:25:53,228] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.7839 | _timers/batch_time=0.3523 | _timers/data_time=0.0004 | _timers/model_time=0.3519 | f1_score=1.596e-06 | loss=0.3081 | loss_dist=0.1751 | loss_mean=0.1330\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.8211 | _timers/batch_time=0.1896 | _timers/data_time=0.0003 | _timers/model_time=0.1893 | f1_score=1.600e-06 | loss=0.3054 | loss_dist=0.1724 | loss_mean=0.1330\n",
      "[2019-12-06 18:25:57,653] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.8452 | _timers/batch_time=0.3477 | _timers/data_time=0.0004 | _timers/model_time=0.3473 | f1_score=1.539e-06 | loss=0.2766 | loss_dist=0.1417 | loss_mean=0.1349\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.8869 | _timers/batch_time=0.1898 | _timers/data_time=0.0002 | _timers/model_time=0.1896 | f1_score=1.541e-06 | loss=0.2752 | loss_dist=0.1404 | loss_mean=0.1348\n",
      "[2019-12-06 18:26:02,116] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.6180 | _timers/batch_time=0.3566 | _timers/data_time=0.0003 | _timers/model_time=0.3563 | f1_score=1.451e-06 | loss=0.3156 | loss_dist=0.1854 | loss_mean=0.1302\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.1738 | _timers/batch_time=0.2025 | _timers/data_time=0.0003 | _timers/model_time=0.2021 | f1_score=1.454e-06 | loss=0.3137 | loss_dist=0.1835 | loss_mean=0.1302\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:26:06,689] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.6938 | _timers/batch_time=0.3531 | _timers/data_time=0.0004 | _timers/model_time=0.3526 | f1_score=1.558e-06 | loss=0.2376 | loss_dist=0.1026 | loss_mean=0.1350\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.0012 | _timers/batch_time=0.2033 | _timers/data_time=0.0003 | _timers/model_time=0.2030 | f1_score=1.560e-06 | loss=0.2367 | loss_dist=0.1017 | loss_mean=0.1349\n",
      "[2019-12-06 18:26:11,242] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.6107 | _timers/batch_time=0.3578 | _timers/data_time=0.0004 | _timers/model_time=0.3573 | f1_score=1.421e-06 | loss=0.2255 | loss_dist=0.0921 | loss_mean=0.1334\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.4859 | _timers/batch_time=0.1940 | _timers/data_time=0.0002 | _timers/model_time=0.1938 | f1_score=1.424e-06 | loss=0.2244 | loss_dist=0.0910 | loss_mean=0.1334\n",
      "[2019-12-06 18:26:15,478] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.9208 | _timers/batch_time=0.3384 | _timers/data_time=0.0003 | _timers/model_time=0.3381 | f1_score=1.512e-06 | loss=0.2384 | loss_dist=0.1034 | loss_mean=0.1351\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.6933 | _timers/batch_time=0.1904 | _timers/data_time=0.0003 | _timers/model_time=0.1901 | f1_score=1.515e-06 | loss=0.2377 | loss_dist=0.1026 | loss_mean=0.1351\n",
      "[2019-12-06 18:26:19,472] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3084 | _timers/batch_time=0.3172 | _timers/data_time=0.0003 | _timers/model_time=0.3168 | f1_score=1.399e-06 | loss=0.2749 | loss_dist=0.1420 | loss_mean=0.1330\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.4364 | _timers/batch_time=0.1794 | _timers/data_time=0.0003 | _timers/model_time=0.1791 | f1_score=1.403e-06 | loss=0.2730 | loss_dist=0.1400 | loss_mean=0.1330\n",
      "[2019-12-06 18:26:23,499] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.2837 | _timers/batch_time=0.3208 | _timers/data_time=0.0003 | _timers/model_time=0.3204 | f1_score=1.382e-06 | loss=0.3493 | loss_dist=0.2116 | loss_mean=0.1376\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.3850 | _timers/batch_time=0.1788 | _timers/data_time=0.0003 | _timers/model_time=0.1785 | f1_score=1.386e-06 | loss=0.3467 | loss_dist=0.2091 | loss_mean=0.1376\n",
      "[2019-12-06 18:26:27,565] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3882 | _timers/batch_time=0.3154 | _timers/data_time=0.0003 | _timers/model_time=0.3150 | f1_score=1.608e-06 | loss=0.2955 | loss_dist=0.1583 | loss_mean=0.1372\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.5320 | _timers/batch_time=0.1936 | _timers/data_time=0.0002 | _timers/model_time=0.1934 | f1_score=1.610e-06 | loss=0.2934 | loss_dist=0.1563 | loss_mean=0.1371\n",
      "[2019-12-06 18:26:31,641] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.1648 | _timers/batch_time=0.3250 | _timers/data_time=0.0003 | _timers/model_time=0.3246 | f1_score=1.436e-06 | loss=0.2725 | loss_dist=0.1353 | loss_mean=0.1373\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.2957 | _timers/batch_time=0.1808 | _timers/data_time=0.0004 | _timers/model_time=0.1803 | f1_score=1.439e-06 | loss=0.2709 | loss_dist=0.1336 | loss_mean=0.1373\n",
      "[2019-12-06 18:26:35,534] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5288 | _timers/batch_time=0.3072 | _timers/data_time=0.0004 | _timers/model_time=0.3067 | f1_score=1.448e-06 | loss=0.3083 | loss_dist=0.1744 | loss_mean=0.1339\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.6714 | _timers/batch_time=0.1750 | _timers/data_time=0.0003 | _timers/model_time=0.1746 | f1_score=1.451e-06 | loss=0.3065 | loss_dist=0.1727 | loss_mean=0.1338\n",
      "[2019-12-06 18:26:39,448] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4221 | _timers/batch_time=0.3126 | _timers/data_time=0.0004 | _timers/model_time=0.3122 | f1_score=1.523e-06 | loss=0.2657 | loss_dist=0.1317 | loss_mean=0.1340\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.7822 | _timers/batch_time=0.1729 | _timers/data_time=0.0003 | _timers/model_time=0.1726 | f1_score=1.525e-06 | loss=0.2644 | loss_dist=0.1304 | loss_mean=0.1340\n",
      "[2019-12-06 18:26:43,349] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4613 | _timers/batch_time=0.3114 | _timers/data_time=0.0004 | _timers/model_time=0.3109 | f1_score=1.448e-06 | loss=0.2288 | loss_dist=0.0969 | loss_mean=0.1319\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.6415 | _timers/batch_time=0.1750 | _timers/data_time=0.0003 | _timers/model_time=0.1746 | f1_score=1.450e-06 | loss=0.2280 | loss_dist=0.0961 | loss_mean=0.1319\n",
      "[2019-12-06 18:26:47,255] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4574 | _timers/batch_time=0.3112 | _timers/data_time=0.0004 | _timers/model_time=0.3107 | f1_score=1.372e-06 | loss=0.3126 | loss_dist=0.1800 | loss_mean=0.1325\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.7259 | _timers/batch_time=0.1734 | _timers/data_time=0.0003 | _timers/model_time=0.1731 | f1_score=1.375e-06 | loss=0.3099 | loss_dist=0.1774 | loss_mean=0.1325\n",
      "[2019-12-06 18:26:51,115] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5599 | _timers/batch_time=0.3059 | _timers/data_time=0.0004 | _timers/model_time=0.3054 | f1_score=1.490e-06 | loss=0.2691 | loss_dist=0.1374 | loss_mean=0.1317\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.7507 | _timers/batch_time=0.1735 | _timers/data_time=0.0002 | _timers/model_time=0.1733 | f1_score=1.492e-06 | loss=0.2681 | loss_dist=0.1364 | loss_mean=0.1317\n",
      "[2019-12-06 18:26:54,965] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5723 | _timers/batch_time=0.3056 | _timers/data_time=0.0004 | _timers/model_time=0.3052 | f1_score=1.531e-06 | loss=0.2785 | loss_dist=0.1409 | loss_mean=0.1376\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.8875 | _timers/batch_time=0.1714 | _timers/data_time=0.0003 | _timers/model_time=0.1711 | f1_score=1.534e-06 | loss=0.2768 | loss_dist=0.1393 | loss_mean=0.1375\n",
      "[2019-12-06 18:26:59,034] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1027 | _timers/batch_time=0.3334 | _timers/data_time=0.0004 | _timers/model_time=0.3329 | f1_score=1.521e-06 | loss=0.2119 | loss_dist=0.0761 | loss_mean=0.1357\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.3402 | _timers/batch_time=0.1808 | _timers/data_time=0.0003 | _timers/model_time=0.1805 | f1_score=1.523e-06 | loss=0.2113 | loss_dist=0.0756 | loss_mean=0.1357\n",
      "[2019-12-06 18:27:02,978] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.3519 | _timers/batch_time=0.3153 | _timers/data_time=0.0003 | _timers/model_time=0.3149 | f1_score=1.542e-06 | loss=0.2177 | loss_dist=0.0878 | loss_mean=0.1299\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.5195 | _timers/batch_time=0.1773 | _timers/data_time=0.0003 | _timers/model_time=0.1770 | f1_score=1.545e-06 | loss=0.2169 | loss_dist=0.0870 | loss_mean=0.1299\n",
      "[2019-12-06 18:27:07,121] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1626 | _timers/batch_time=0.3251 | _timers/data_time=0.0003 | _timers/model_time=0.3247 | f1_score=1.553e-06 | loss=0.2092 | loss_dist=0.0755 | loss_mean=0.1337\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.3840 | _timers/batch_time=0.1946 | _timers/data_time=0.0004 | _timers/model_time=0.1942 | f1_score=1.556e-06 | loss=0.2084 | loss_dist=0.0748 | loss_mean=0.1337\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:27:11,801] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=5.8216 | _timers/batch_time=0.3437 | _timers/data_time=0.0002 | _timers/model_time=0.3435 | f1_score=1.479e-06 | loss=0.3088 | loss_dist=0.1752 | loss_mean=0.1335\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=7.4523 | _timers/batch_time=0.2735 | _timers/data_time=0.0005 | _timers/model_time=0.2729 | f1_score=1.481e-06 | loss=0.3077 | loss_dist=0.1742 | loss_mean=0.1335\n",
      "[2019-12-06 18:27:16,659] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=5.1382 | _timers/batch_time=0.3932 | _timers/data_time=0.0004 | _timers/model_time=0.3928 | f1_score=1.421e-06 | loss=0.2137 | loss_dist=0.0807 | loss_mean=0.1331\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.2467 | _timers/batch_time=0.1813 | _timers/data_time=0.0003 | _timers/model_time=0.1810 | f1_score=1.423e-06 | loss=0.2127 | loss_dist=0.0797 | loss_mean=0.1330\n",
      "[2019-12-06 18:27:20,959] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1689 | _timers/batch_time=0.3272 | _timers/data_time=0.0004 | _timers/model_time=0.3268 | f1_score=1.541e-06 | loss=0.2095 | loss_dist=0.0747 | loss_mean=0.1348\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.0385 | _timers/batch_time=0.2045 | _timers/data_time=0.0003 | _timers/model_time=0.2042 | f1_score=1.543e-06 | loss=0.2090 | loss_dist=0.0742 | loss_mean=0.1348\n",
      "[2019-12-06 18:27:25,093] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1075 | _timers/batch_time=0.3281 | _timers/data_time=0.0002 | _timers/model_time=0.3278 | f1_score=1.405e-06 | loss=0.2321 | loss_dist=0.0985 | loss_mean=0.1336\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.0815 | _timers/batch_time=0.1848 | _timers/data_time=0.0004 | _timers/model_time=0.1843 | f1_score=1.408e-06 | loss=0.2310 | loss_dist=0.0974 | loss_mean=0.1336\n",
      "[2019-12-06 18:27:29,258] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.0707 | _timers/batch_time=0.3295 | _timers/data_time=0.0004 | _timers/model_time=0.3291 | f1_score=1.479e-06 | loss=0.2427 | loss_dist=0.1085 | loss_mean=0.1342\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.8432 | _timers/batch_time=0.1873 | _timers/data_time=0.0002 | _timers/model_time=0.1871 | f1_score=1.480e-06 | loss=0.2421 | loss_dist=0.1079 | loss_mean=0.1342\n",
      "[2019-12-06 18:27:33,459] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.0341 | _timers/batch_time=0.3334 | _timers/data_time=0.0003 | _timers/model_time=0.3331 | f1_score=1.426e-06 | loss=0.1958 | loss_dist=0.0597 | loss_mean=0.1361\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.1291 | _timers/batch_time=0.1839 | _timers/data_time=0.0003 | _timers/model_time=0.1835 | f1_score=1.428e-06 | loss=0.1953 | loss_dist=0.0592 | loss_mean=0.1361\n",
      "[2019-12-06 18:27:37,605] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.2365 | _timers/batch_time=0.3228 | _timers/data_time=0.0003 | _timers/model_time=0.3224 | f1_score=1.536e-06 | loss=0.2321 | loss_dist=0.0975 | loss_mean=0.1346\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.8582 | _timers/batch_time=0.1879 | _timers/data_time=0.0004 | _timers/model_time=0.1875 | f1_score=1.538e-06 | loss=0.2314 | loss_dist=0.0968 | loss_mean=0.1346\n",
      "[2019-12-06 18:27:41,739] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1483 | _timers/batch_time=0.3264 | _timers/data_time=0.0005 | _timers/model_time=0.3259 | f1_score=1.661e-06 | loss=0.2084 | loss_dist=0.0716 | loss_mean=0.1368\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.0181 | _timers/batch_time=0.1888 | _timers/data_time=0.0003 | _timers/model_time=0.1885 | f1_score=1.663e-06 | loss=0.2080 | loss_dist=0.0712 | loss_mean=0.1368\n",
      "[2019-12-06 18:27:45,883] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=5.9329 | _timers/batch_time=0.3389 | _timers/data_time=0.0004 | _timers/model_time=0.3385 | f1_score=1.184e-06 | loss=0.3095 | loss_dist=0.1731 | loss_mean=0.1363\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.1866 | _timers/batch_time=0.1824 | _timers/data_time=0.0003 | _timers/model_time=0.1820 | f1_score=1.185e-06 | loss=0.3086 | loss_dist=0.1722 | loss_mean=0.1363\n",
      "[2019-12-06 18:27:49,875] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3584 | _timers/batch_time=0.3156 | _timers/data_time=0.0003 | _timers/model_time=0.3152 | f1_score=1.641e-06 | loss=0.2115 | loss_dist=0.0787 | loss_mean=0.1328\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.3732 | _timers/batch_time=0.1805 | _timers/data_time=0.0003 | _timers/model_time=0.1802 | f1_score=1.644e-06 | loss=0.2106 | loss_dist=0.0779 | loss_mean=0.1328\n",
      "[2019-12-06 18:27:53,886] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3864 | _timers/batch_time=0.3150 | _timers/data_time=0.0004 | _timers/model_time=0.3145 | f1_score=1.658e-06 | loss=0.2187 | loss_dist=0.0851 | loss_mean=0.1336\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.3245 | _timers/batch_time=0.1823 | _timers/data_time=0.0003 | _timers/model_time=0.1819 | f1_score=1.661e-06 | loss=0.2178 | loss_dist=0.0841 | loss_mean=0.1336\n",
      "[2019-12-06 18:27:57,950] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5244 | _timers/batch_time=0.3078 | _timers/data_time=0.0004 | _timers/model_time=0.3073 | f1_score=1.478e-06 | loss=0.2314 | loss_dist=0.0938 | loss_mean=0.1376\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.3122 | _timers/batch_time=0.1982 | _timers/data_time=0.0004 | _timers/model_time=0.1978 | f1_score=1.481e-06 | loss=0.2307 | loss_dist=0.0932 | loss_mean=0.1376\n",
      "[2019-12-06 18:28:02,168] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.8442 | _timers/batch_time=0.3436 | _timers/data_time=0.0004 | _timers/model_time=0.3432 | f1_score=1.549e-06 | loss=0.2660 | loss_dist=0.1284 | loss_mean=0.1376\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.4175 | _timers/batch_time=0.1790 | _timers/data_time=0.0004 | _timers/model_time=0.1786 | f1_score=1.550e-06 | loss=0.2654 | loss_dist=0.1279 | loss_mean=0.1375\n",
      "[2019-12-06 18:28:06,476] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.9939 | _timers/batch_time=0.3375 | _timers/data_time=0.0004 | _timers/model_time=0.3371 | f1_score=1.639e-06 | loss=0.2384 | loss_dist=0.1020 | loss_mean=0.1364\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.0539 | _timers/batch_time=0.1877 | _timers/data_time=0.0003 | _timers/model_time=0.1874 | f1_score=1.641e-06 | loss=0.2378 | loss_dist=0.1014 | loss_mean=0.1364\n",
      "[2019-12-06 18:28:10,513] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.2205 | _timers/batch_time=0.3220 | _timers/data_time=0.0004 | _timers/model_time=0.3216 | f1_score=1.642e-06 | loss=0.2059 | loss_dist=0.0716 | loss_mean=0.1342\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.3975 | _timers/batch_time=0.1784 | _timers/data_time=0.0004 | _timers/model_time=0.1780 | f1_score=1.644e-06 | loss=0.2052 | loss_dist=0.0710 | loss_mean=0.1342\n",
      "[2019-12-06 18:28:14,385] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.3630 | _timers/batch_time=0.3152 | _timers/data_time=0.0004 | _timers/model_time=0.3148 | f1_score=1.881e-06 | loss=0.1792 | loss_dist=0.0450 | loss_mean=0.1341\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7522 | _timers/batch_time=0.1738 | _timers/data_time=0.0002 | _timers/model_time=0.1735 | f1_score=1.883e-06 | loss=0.1790 | loss_dist=0.0449 | loss_mean=0.1341\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:28:18,288] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.4692 | _timers/batch_time=0.3110 | _timers/data_time=0.0003 | _timers/model_time=0.3106 | f1_score=1.437e-06 | loss=0.3270 | loss_dist=0.1945 | loss_mean=0.1325\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7204 | _timers/batch_time=0.1740 | _timers/data_time=0.0004 | _timers/model_time=0.1736 | f1_score=1.439e-06 | loss=0.3255 | loss_dist=0.1931 | loss_mean=0.1324\n",
      "[2019-12-06 18:28:22,072] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6110 | _timers/batch_time=0.3032 | _timers/data_time=0.0003 | _timers/model_time=0.3029 | f1_score=1.640e-06 | loss=0.1962 | loss_dist=0.0603 | loss_mean=0.1358\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.8844 | _timers/batch_time=0.1712 | _timers/data_time=0.0003 | _timers/model_time=0.1708 | f1_score=1.642e-06 | loss=0.1958 | loss_dist=0.0600 | loss_mean=0.1358\n",
      "[2019-12-06 18:28:26,030] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6124 | _timers/batch_time=0.3035 | _timers/data_time=0.0004 | _timers/model_time=0.3030 | f1_score=1.420e-06 | loss=0.2705 | loss_dist=0.1393 | loss_mean=0.1312\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.9979 | _timers/batch_time=0.1839 | _timers/data_time=0.0003 | _timers/model_time=0.1836 | f1_score=1.422e-06 | loss=0.2698 | loss_dist=0.1386 | loss_mean=0.1312\n",
      "[2019-12-06 18:28:30,400] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.8392 | _timers/batch_time=0.3436 | _timers/data_time=0.0003 | _timers/model_time=0.3432 | f1_score=1.678e-06 | loss=0.2544 | loss_dist=0.1184 | loss_mean=0.1360\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=9.6348 | _timers/batch_time=0.2148 | _timers/data_time=0.0004 | _timers/model_time=0.2144 | f1_score=1.680e-06 | loss=0.2536 | loss_dist=0.1176 | loss_mean=0.1360\n",
      "[2019-12-06 18:28:34,815] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.8502 | _timers/batch_time=0.3469 | _timers/data_time=0.0002 | _timers/model_time=0.3466 | f1_score=1.590e-06 | loss=0.2659 | loss_dist=0.1353 | loss_mean=0.1306\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.2882 | _timers/batch_time=0.1964 | _timers/data_time=0.0002 | _timers/model_time=0.1961 | f1_score=1.592e-06 | loss=0.2649 | loss_dist=0.1343 | loss_mean=0.1306\n",
      "[2019-12-06 18:28:39,293] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=5.7127 | _timers/batch_time=0.3515 | _timers/data_time=0.0002 | _timers/model_time=0.3512 | f1_score=1.509e-06 | loss=0.2618 | loss_dist=0.1268 | loss_mean=0.1350\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.0745 | _timers/batch_time=0.2019 | _timers/data_time=0.0004 | _timers/model_time=0.2015 | f1_score=1.511e-06 | loss=0.2611 | loss_dist=0.1261 | loss_mean=0.1350\n",
      "[2019-12-06 18:28:43,685] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=5.8387 | _timers/batch_time=0.3451 | _timers/data_time=0.0003 | _timers/model_time=0.3448 | f1_score=1.566e-06 | loss=0.3162 | loss_dist=0.1794 | loss_mean=0.1369\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.4189 | _timers/batch_time=0.1964 | _timers/data_time=0.0003 | _timers/model_time=0.1961 | f1_score=1.568e-06 | loss=0.3153 | loss_dist=0.1785 | loss_mean=0.1368\n",
      "[2019-12-06 18:28:47,955] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.0457 | _timers/batch_time=0.3357 | _timers/data_time=0.0004 | _timers/model_time=0.3353 | f1_score=1.450e-06 | loss=0.2229 | loss_dist=0.0891 | loss_mean=0.1338\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.4826 | _timers/batch_time=0.1971 | _timers/data_time=0.0003 | _timers/model_time=0.1967 | f1_score=1.453e-06 | loss=0.2225 | loss_dist=0.0887 | loss_mean=0.1338\n",
      "[2019-12-06 18:28:52,381] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=5.5553 | _timers/batch_time=0.3610 | _timers/data_time=0.0003 | _timers/model_time=0.3606 | f1_score=1.687e-06 | loss=0.2506 | loss_dist=0.1184 | loss_mean=0.1322\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.3045 | _timers/batch_time=0.1980 | _timers/data_time=0.0002 | _timers/model_time=0.1978 | f1_score=1.689e-06 | loss=0.2499 | loss_dist=0.1176 | loss_mean=0.1322\n",
      "[2019-12-06 18:28:56,604] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.1415 | _timers/batch_time=0.3263 | _timers/data_time=0.0003 | _timers/model_time=0.3259 | f1_score=1.751e-06 | loss=0.1891 | loss_dist=0.0583 | loss_mean=0.1307\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.7706 | _timers/batch_time=0.1910 | _timers/data_time=0.0002 | _timers/model_time=0.1907 | f1_score=1.752e-06 | loss=0.1888 | loss_dist=0.0581 | loss_mean=0.1307\n",
      "[2019-12-06 18:29:00,831] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.0500 | _timers/batch_time=0.3326 | _timers/data_time=0.0004 | _timers/model_time=0.3322 | f1_score=1.507e-06 | loss=0.2501 | loss_dist=0.1163 | loss_mean=0.1339\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.2648 | _timers/batch_time=0.1810 | _timers/data_time=0.0004 | _timers/model_time=0.1806 | f1_score=1.509e-06 | loss=0.2495 | loss_dist=0.1157 | loss_mean=0.1338\n",
      "[2019-12-06 18:29:05,039] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.0752 | _timers/batch_time=0.3306 | _timers/data_time=0.0003 | _timers/model_time=0.3302 | f1_score=1.754e-06 | loss=0.1863 | loss_dist=0.0539 | loss_mean=0.1324\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.1233 | _timers/batch_time=0.1847 | _timers/data_time=0.0003 | _timers/model_time=0.1844 | f1_score=1.756e-06 | loss=0.1860 | loss_dist=0.0536 | loss_mean=0.1324\n",
      "[2019-12-06 18:29:09,125] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=5.9921 | _timers/batch_time=0.3347 | _timers/data_time=0.0004 | _timers/model_time=0.3342 | f1_score=1.841e-06 | loss=0.2408 | loss_dist=0.1054 | loss_mean=0.1354\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.2926 | _timers/batch_time=0.1802 | _timers/data_time=0.0003 | _timers/model_time=0.1798 | f1_score=1.842e-06 | loss=0.2404 | loss_dist=0.1050 | loss_mean=0.1354\n",
      "[2019-12-06 18:29:13,235] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1556 | _timers/batch_time=0.3259 | _timers/data_time=0.0004 | _timers/model_time=0.3255 | f1_score=1.655e-06 | loss=0.1907 | loss_dist=0.0607 | loss_mean=0.1300\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.2223 | _timers/batch_time=0.1822 | _timers/data_time=0.0002 | _timers/model_time=0.1820 | f1_score=1.657e-06 | loss=0.1904 | loss_dist=0.0604 | loss_mean=0.1300\n",
      "[2019-12-06 18:29:17,523] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1304 | _timers/batch_time=0.3277 | _timers/data_time=0.0005 | _timers/model_time=0.3272 | f1_score=1.636e-06 | loss=0.2309 | loss_dist=0.0989 | loss_mean=0.1320\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.0565 | _timers/batch_time=0.2041 | _timers/data_time=0.0003 | _timers/model_time=0.2037 | f1_score=1.638e-06 | loss=0.2305 | loss_dist=0.0985 | loss_mean=0.1320\n",
      "[2019-12-06 18:29:21,717] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=5.9981 | _timers/batch_time=0.3339 | _timers/data_time=0.0004 | _timers/model_time=0.3335 | f1_score=1.664e-06 | loss=0.1811 | loss_dist=0.0447 | loss_mean=0.1364\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.1067 | _timers/batch_time=0.1832 | _timers/data_time=0.0003 | _timers/model_time=0.1829 | f1_score=1.666e-06 | loss=0.1809 | loss_dist=0.0445 | loss_mean=0.1364\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:29:26,015] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=5.8393 | _timers/batch_time=0.3446 | _timers/data_time=0.0003 | _timers/model_time=0.3443 | f1_score=1.518e-06 | loss=0.1814 | loss_dist=0.0481 | loss_mean=0.1333\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.6915 | _timers/batch_time=0.1938 | _timers/data_time=0.0002 | _timers/model_time=0.1935 | f1_score=1.520e-06 | loss=0.1810 | loss_dist=0.0477 | loss_mean=0.1333\n",
      "[2019-12-06 18:29:30,058] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2063 | _timers/batch_time=0.3240 | _timers/data_time=0.0003 | _timers/model_time=0.3236 | f1_score=1.525e-06 | loss=0.2151 | loss_dist=0.0839 | loss_mean=0.1312\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.0639 | _timers/batch_time=0.1841 | _timers/data_time=0.0004 | _timers/model_time=0.1836 | f1_score=1.526e-06 | loss=0.2146 | loss_dist=0.0834 | loss_mean=0.1312\n",
      "[2019-12-06 18:29:34,210] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2821 | _timers/batch_time=0.3202 | _timers/data_time=0.0004 | _timers/model_time=0.3197 | f1_score=1.855e-06 | loss=0.2270 | loss_dist=0.0933 | loss_mean=0.1337\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.4368 | _timers/batch_time=0.1941 | _timers/data_time=0.0004 | _timers/model_time=0.1937 | f1_score=1.857e-06 | loss=0.2267 | loss_dist=0.0931 | loss_mean=0.1337\n",
      "[2019-12-06 18:29:38,538] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1253 | _timers/batch_time=0.3285 | _timers/data_time=0.0003 | _timers/model_time=0.3281 | f1_score=1.421e-06 | loss=0.2027 | loss_dist=0.0692 | loss_mean=0.1336\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.4495 | _timers/batch_time=0.1935 | _timers/data_time=0.0003 | _timers/model_time=0.1932 | f1_score=1.422e-06 | loss=0.2025 | loss_dist=0.0689 | loss_mean=0.1336\n",
      "Top best models:\n",
      "logs/HardKuma/checkpoints/train.80.pth\t0.1790\n",
      "logs/HardKuma/checkpoints/train.96.pth\t0.1809\n",
      "logs/HardKuma/checkpoints/train.97.pth\t0.1810\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bernoulli p = 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)               \n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        #torch.bernoulli(torch.tensor([1.0]),)\n",
    "        b_new = torch.bernoulli(torch.ones(x.shape[:2]),0.5).unsqueeze(2).to(self.emb.weight.device)\n",
    "        #b_new = torch.repeat_interleave(bb(0.4), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/b50\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:34:08,611] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.0348 | _timers/batch_time=0.3343 | _timers/data_time=0.0003 | _timers/model_time=0.3340 | f1_score=4.040e-07 | loss=24.2843 | loss_dist=24.1560 | loss_mean=0.1283\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.6520 | _timers/batch_time=0.1943 | _timers/data_time=0.0003 | _timers/model_time=0.1940 | f1_score=4.040e-07 | loss=24.2843 | loss_dist=24.1560 | loss_mean=0.1283\n",
      "[2019-12-06 18:34:12,822] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.9265 | _timers/batch_time=0.3413 | _timers/data_time=0.0003 | _timers/model_time=0.3409 | f1_score=7.147e-07 | loss=5.0791 | loss_dist=4.9562 | loss_mean=0.1230\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1146 | _timers/batch_time=0.1859 | _timers/data_time=0.0004 | _timers/model_time=0.1855 | f1_score=7.147e-07 | loss=5.0791 | loss_dist=4.9562 | loss_mean=0.1230\n",
      "[2019-12-06 18:34:17,032] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.0547 | _timers/batch_time=0.3321 | _timers/data_time=0.0003 | _timers/model_time=0.3318 | f1_score=6.034e-07 | loss=6.8485 | loss_dist=6.7264 | loss_mean=0.1221\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.4837 | _timers/batch_time=0.1946 | _timers/data_time=0.0004 | _timers/model_time=0.1942 | f1_score=6.034e-07 | loss=6.8485 | loss_dist=6.7264 | loss_mean=0.1221\n",
      "[2019-12-06 18:34:21,171] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1081 | _timers/batch_time=0.3292 | _timers/data_time=0.0004 | _timers/model_time=0.3288 | f1_score=2.859e-07 | loss=28.2715 | loss_dist=28.1477 | loss_mean=0.1238\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.4410 | _timers/batch_time=0.1789 | _timers/data_time=0.0003 | _timers/model_time=0.1785 | f1_score=2.859e-07 | loss=28.2715 | loss_dist=28.1477 | loss_mean=0.1238\n",
      "[2019-12-06 18:34:25,592] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.8906 | _timers/batch_time=0.3441 | _timers/data_time=0.0004 | _timers/model_time=0.3437 | f1_score=5.861e-07 | loss=4.8710 | loss_dist=4.7475 | loss_mean=0.1235\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.4110 | _timers/batch_time=0.1968 | _timers/data_time=0.0003 | _timers/model_time=0.1965 | f1_score=5.861e-07 | loss=4.8710 | loss_dist=4.7475 | loss_mean=0.1235\n",
      "[2019-12-06 18:34:29,758] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2041 | _timers/batch_time=0.3248 | _timers/data_time=0.0002 | _timers/model_time=0.3245 | f1_score=7.173e-07 | loss=2.3371 | loss_dist=2.2103 | loss_mean=0.1268\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.9843 | _timers/batch_time=0.1855 | _timers/data_time=0.0003 | _timers/model_time=0.1851 | f1_score=7.173e-07 | loss=2.3371 | loss_dist=2.2103 | loss_mean=0.1268\n",
      "[2019-12-06 18:34:33,921] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.0343 | _timers/batch_time=0.3320 | _timers/data_time=0.0004 | _timers/model_time=0.3315 | f1_score=6.858e-07 | loss=9.9591 | loss_dist=9.8328 | loss_mean=0.1263\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.9562 | _timers/batch_time=0.1861 | _timers/data_time=0.0002 | _timers/model_time=0.1859 | f1_score=6.858e-07 | loss=9.9591 | loss_dist=9.8328 | loss_mean=0.1263\n",
      "[2019-12-06 18:34:37,919] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2504 | _timers/batch_time=0.3206 | _timers/data_time=0.0002 | _timers/model_time=0.3203 | f1_score=9.869e-07 | loss=5.2169 | loss_dist=5.0937 | loss_mean=0.1231\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.0968 | _timers/batch_time=0.1841 | _timers/data_time=0.0003 | _timers/model_time=0.1837 | f1_score=9.869e-07 | loss=5.2169 | loss_dist=5.0937 | loss_mean=0.1231\n",
      "[2019-12-06 18:34:41,922] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2739 | _timers/batch_time=0.3197 | _timers/data_time=0.0004 | _timers/model_time=0.3192 | f1_score=5.243e-07 | loss=16.5942 | loss_dist=16.4706 | loss_mean=0.1235\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.6023 | _timers/batch_time=0.1765 | _timers/data_time=0.0002 | _timers/model_time=0.1763 | f1_score=5.243e-07 | loss=16.5942 | loss_dist=16.4706 | loss_mean=0.1235\n",
      "[2019-12-06 18:34:45,845] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.3791 | _timers/batch_time=0.3144 | _timers/data_time=0.0003 | _timers/model_time=0.3140 | f1_score=4.428e-07 | loss=8.8054 | loss_dist=8.6831 | loss_mean=0.1223\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.3304 | _timers/batch_time=0.1800 | _timers/data_time=0.0003 | _timers/model_time=0.1797 | f1_score=4.428e-07 | loss=8.8054 | loss_dist=8.6831 | loss_mean=0.1223\n",
      "[2019-12-06 18:34:49,947] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.3883 | _timers/batch_time=0.3139 | _timers/data_time=0.0004 | _timers/model_time=0.3134 | f1_score=1.063e-06 | loss=0.3293 | loss_dist=0.2036 | loss_mean=0.1257\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.2985 | _timers/batch_time=0.1804 | _timers/data_time=0.0004 | _timers/model_time=0.1800 | f1_score=1.063e-06 | loss=0.3293 | loss_dist=0.2036 | loss_mean=0.1257\n",
      "[2019-12-06 18:34:54,003] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2899 | _timers/batch_time=0.3196 | _timers/data_time=0.0003 | _timers/model_time=0.3193 | f1_score=8.711e-07 | loss=7.3829 | loss_dist=7.2625 | loss_mean=0.1203\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.9794 | _timers/batch_time=0.1866 | _timers/data_time=0.0003 | _timers/model_time=0.1863 | f1_score=8.711e-07 | loss=7.3829 | loss_dist=7.2625 | loss_mean=0.1203\n",
      "[2019-12-06 18:34:58,132] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1605 | _timers/batch_time=0.3261 | _timers/data_time=0.0003 | _timers/model_time=0.3258 | f1_score=9.132e-07 | loss=6.6020 | loss_dist=6.4795 | loss_mean=0.1225\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.6686 | _timers/batch_time=0.1920 | _timers/data_time=0.0003 | _timers/model_time=0.1916 | f1_score=9.132e-07 | loss=6.6020 | loss_dist=6.4795 | loss_mean=0.1225\n",
      "[2019-12-06 18:35:02,244] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1690 | _timers/batch_time=0.3253 | _timers/data_time=0.0004 | _timers/model_time=0.3249 | f1_score=8.136e-07 | loss=9.1602 | loss_dist=9.0322 | loss_mean=0.1279\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.6319 | _timers/batch_time=0.1923 | _timers/data_time=0.0003 | _timers/model_time=0.1920 | f1_score=8.136e-07 | loss=9.1602 | loss_dist=9.0322 | loss_mean=0.1279\n",
      "[2019-12-06 18:35:06,315] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2336 | _timers/batch_time=0.3221 | _timers/data_time=0.0004 | _timers/model_time=0.3216 | f1_score=1.994e-07 | loss=27.5853 | loss_dist=27.4566 | loss_mean=0.1287\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.9817 | _timers/batch_time=0.1864 | _timers/data_time=0.0002 | _timers/model_time=0.1861 | f1_score=1.994e-07 | loss=27.5853 | loss_dist=27.4566 | loss_mean=0.1287\n",
      "[2019-12-06 18:35:10,529] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1273 | _timers/batch_time=0.3287 | _timers/data_time=0.0002 | _timers/model_time=0.3284 | f1_score=8.419e-07 | loss=8.9881 | loss_dist=8.8639 | loss_mean=0.1242\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.3658 | _timers/batch_time=0.1806 | _timers/data_time=0.0005 | _timers/model_time=0.1801 | f1_score=8.419e-07 | loss=8.9881 | loss_dist=8.8639 | loss_mean=0.1242\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:35:14,407] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5094 | _timers/batch_time=0.3081 | _timers/data_time=0.0003 | _timers/model_time=0.3077 | f1_score=5.519e-07 | loss=2.5328 | loss_dist=2.4046 | loss_mean=0.1282\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.4901 | _timers/batch_time=0.1775 | _timers/data_time=0.0004 | _timers/model_time=0.1770 | f1_score=5.519e-07 | loss=2.5328 | loss_dist=2.4046 | loss_mean=0.1282\n",
      "[2019-12-06 18:35:18,234] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5154 | _timers/batch_time=0.3077 | _timers/data_time=0.0006 | _timers/model_time=0.3071 | f1_score=9.825e-07 | loss=1.0114 | loss_dist=0.8853 | loss_mean=0.1261\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6510 | _timers/batch_time=0.1750 | _timers/data_time=0.0004 | _timers/model_time=0.1745 | f1_score=9.825e-07 | loss=1.0114 | loss_dist=0.8853 | loss_mean=0.1261\n",
      "[2019-12-06 18:35:22,172] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.3832 | _timers/batch_time=0.3153 | _timers/data_time=0.0004 | _timers/model_time=0.3149 | f1_score=7.538e-07 | loss=19.1416 | loss_dist=19.0201 | loss_mean=0.1215\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5807 | _timers/batch_time=0.1756 | _timers/data_time=0.0004 | _timers/model_time=0.1752 | f1_score=7.538e-07 | loss=19.1416 | loss_dist=19.0201 | loss_mean=0.1215\n",
      "[2019-12-06 18:35:26,010] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5236 | _timers/batch_time=0.3078 | _timers/data_time=0.0004 | _timers/model_time=0.3074 | f1_score=8.980e-07 | loss=7.8561 | loss_dist=7.7282 | loss_mean=0.1280\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6651 | _timers/batch_time=0.1750 | _timers/data_time=0.0005 | _timers/model_time=0.1745 | f1_score=8.980e-07 | loss=7.8561 | loss_dist=7.7282 | loss_mean=0.1280\n",
      "[2019-12-06 18:35:30,151] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.2510 | _timers/batch_time=0.3220 | _timers/data_time=0.0003 | _timers/model_time=0.3217 | f1_score=3.893e-07 | loss=11.9059 | loss_dist=11.7806 | loss_mean=0.1253\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.8390 | _timers/batch_time=0.1901 | _timers/data_time=0.0003 | _timers/model_time=0.1898 | f1_score=3.893e-07 | loss=11.9059 | loss_dist=11.7806 | loss_mean=0.1253\n",
      "[2019-12-06 18:35:34,223] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.1625 | _timers/batch_time=0.3264 | _timers/data_time=0.0004 | _timers/model_time=0.3259 | f1_score=3.731e-07 | loss=15.6009 | loss_dist=15.4719 | loss_mean=0.1290\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.5121 | _timers/batch_time=0.1777 | _timers/data_time=0.0002 | _timers/model_time=0.1775 | f1_score=3.731e-07 | loss=15.6009 | loss_dist=15.4719 | loss_mean=0.1290\n",
      "[2019-12-06 18:35:38,135] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.4458 | _timers/batch_time=0.3105 | _timers/data_time=0.0004 | _timers/model_time=0.3101 | f1_score=4.711e-07 | loss=3.2606 | loss_dist=3.1407 | loss_mean=0.1198\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.4462 | _timers/batch_time=0.1777 | _timers/data_time=0.0002 | _timers/model_time=0.1775 | f1_score=4.711e-07 | loss=3.2606 | loss_dist=3.1407 | loss_mean=0.1198\n",
      "[2019-12-06 18:35:42,351] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0439 | _timers/batch_time=0.3336 | _timers/data_time=0.0002 | _timers/model_time=0.3333 | f1_score=5.563e-07 | loss=13.0072 | loss_dist=12.8848 | loss_mean=0.1224\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.8461 | _timers/batch_time=0.1871 | _timers/data_time=0.0002 | _timers/model_time=0.1869 | f1_score=5.563e-07 | loss=13.0072 | loss_dist=12.8848 | loss_mean=0.1224\n",
      "[2019-12-06 18:35:46,558] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0697 | _timers/batch_time=0.3313 | _timers/data_time=0.0003 | _timers/model_time=0.3309 | f1_score=9.335e-07 | loss=2.3057 | loss_dist=2.1812 | loss_mean=0.1245\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.4709 | _timers/batch_time=0.1946 | _timers/data_time=0.0003 | _timers/model_time=0.1943 | f1_score=9.335e-07 | loss=2.3057 | loss_dist=2.1812 | loss_mean=0.1245\n",
      "[2019-12-06 18:35:50,713] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.1198 | _timers/batch_time=0.3272 | _timers/data_time=0.0003 | _timers/model_time=0.3269 | f1_score=3.472e-07 | loss=10.5293 | loss_dist=10.4027 | loss_mean=0.1265\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.4101 | _timers/batch_time=0.1796 | _timers/data_time=0.0003 | _timers/model_time=0.1793 | f1_score=3.472e-07 | loss=10.5293 | loss_dist=10.4027 | loss_mean=0.1265\n",
      "[2019-12-06 18:35:54,678] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2977 | _timers/batch_time=0.3200 | _timers/data_time=0.0003 | _timers/model_time=0.3197 | f1_score=8.457e-07 | loss=15.9548 | loss_dist=15.8307 | loss_mean=0.1241\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.3020 | _timers/batch_time=0.1802 | _timers/data_time=0.0003 | _timers/model_time=0.1799 | f1_score=8.457e-07 | loss=15.9548 | loss_dist=15.8307 | loss_mean=0.1241\n",
      "[2019-12-06 18:35:58,810] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2751 | _timers/batch_time=0.3198 | _timers/data_time=0.0003 | _timers/model_time=0.3195 | f1_score=6.139e-07 | loss=7.1696 | loss_dist=7.0449 | loss_mean=0.1247\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.4388 | _timers/batch_time=0.1959 | _timers/data_time=0.0002 | _timers/model_time=0.1956 | f1_score=6.139e-07 | loss=7.1696 | loss_dist=7.0449 | loss_mean=0.1247\n",
      "[2019-12-06 18:36:03,066] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.8660 | _timers/batch_time=0.3431 | _timers/data_time=0.0003 | _timers/model_time=0.3427 | f1_score=7.427e-07 | loss=11.1736 | loss_dist=11.0504 | loss_mean=0.1232\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.5306 | _timers/batch_time=0.1928 | _timers/data_time=0.0002 | _timers/model_time=0.1926 | f1_score=7.427e-07 | loss=11.1736 | loss_dist=11.0504 | loss_mean=0.1232\n",
      "[2019-12-06 18:36:07,327] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.9748 | _timers/batch_time=0.3361 | _timers/data_time=0.0003 | _timers/model_time=0.3357 | f1_score=8.663e-07 | loss=0.6281 | loss_dist=0.5075 | loss_mean=0.1206\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.8561 | _timers/batch_time=0.1907 | _timers/data_time=0.0004 | _timers/model_time=0.1902 | f1_score=8.663e-07 | loss=0.6281 | loss_dist=0.5075 | loss_mean=0.1206\n",
      "[2019-12-06 18:36:11,537] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.9001 | _timers/batch_time=0.3407 | _timers/data_time=0.0004 | _timers/model_time=0.3403 | f1_score=8.007e-07 | loss=0.4774 | loss_dist=0.3503 | loss_mean=0.1271\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.8503 | _timers/batch_time=0.1880 | _timers/data_time=0.0004 | _timers/model_time=0.1876 | f1_score=8.007e-07 | loss=0.4774 | loss_dist=0.3503 | loss_mean=0.1271\n",
      "[2019-12-06 18:36:15,567] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.2408 | _timers/batch_time=0.3213 | _timers/data_time=0.0002 | _timers/model_time=0.3210 | f1_score=9.300e-07 | loss=2.3827 | loss_dist=2.2551 | loss_mean=0.1276\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.9169 | _timers/batch_time=0.1878 | _timers/data_time=0.0004 | _timers/model_time=0.1874 | f1_score=9.300e-07 | loss=2.3827 | loss_dist=2.2551 | loss_mean=0.1276\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:36:19,677] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.3349 | _timers/batch_time=0.3170 | _timers/data_time=0.0003 | _timers/model_time=0.3167 | f1_score=9.034e-07 | loss=1.4922 | loss_dist=1.3672 | loss_mean=0.1250\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.1426 | _timers/batch_time=0.2009 | _timers/data_time=0.0004 | _timers/model_time=0.2005 | f1_score=9.034e-07 | loss=1.4922 | loss_dist=1.3672 | loss_mean=0.1250\n",
      "[2019-12-06 18:36:23,668] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.3335 | _timers/batch_time=0.3165 | _timers/data_time=0.0004 | _timers/model_time=0.3160 | f1_score=6.886e-07 | loss=14.2114 | loss_dist=14.0850 | loss_mean=0.1265\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.0508 | _timers/batch_time=0.1835 | _timers/data_time=0.0004 | _timers/model_time=0.1831 | f1_score=6.886e-07 | loss=14.2114 | loss_dist=14.0850 | loss_mean=0.1265\n",
      "[2019-12-06 18:36:27,592] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.4510 | _timers/batch_time=0.3104 | _timers/data_time=0.0002 | _timers/model_time=0.3101 | f1_score=7.178e-07 | loss=2.2586 | loss_dist=2.1321 | loss_mean=0.1264\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.1396 | _timers/batch_time=0.1848 | _timers/data_time=0.0005 | _timers/model_time=0.1842 | f1_score=7.178e-07 | loss=2.2586 | loss_dist=2.1321 | loss_mean=0.1264\n",
      "[2019-12-06 18:36:31,855] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.0980 | _timers/batch_time=0.3323 | _timers/data_time=0.0005 | _timers/model_time=0.3318 | f1_score=4.870e-07 | loss=7.9287 | loss_dist=7.8022 | loss_mean=0.1265\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.3978 | _timers/batch_time=0.1946 | _timers/data_time=0.0002 | _timers/model_time=0.1943 | f1_score=4.870e-07 | loss=7.9287 | loss_dist=7.8022 | loss_mean=0.1265\n",
      "[2019-12-06 18:36:35,913] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0523 | _timers/batch_time=0.3313 | _timers/data_time=0.0004 | _timers/model_time=0.3308 | f1_score=4.583e-07 | loss=8.0663 | loss_dist=7.9443 | loss_mean=0.1220\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.4194 | _timers/batch_time=0.1804 | _timers/data_time=0.0004 | _timers/model_time=0.1799 | f1_score=4.583e-07 | loss=8.0663 | loss_dist=7.9443 | loss_mean=0.1220\n",
      "[2019-12-06 18:36:39,906] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.2586 | _timers/batch_time=0.3213 | _timers/data_time=0.0004 | _timers/model_time=0.3209 | f1_score=5.523e-07 | loss=11.0647 | loss_dist=10.9431 | loss_mean=0.1216\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.9996 | _timers/batch_time=0.1843 | _timers/data_time=0.0003 | _timers/model_time=0.1839 | f1_score=5.523e-07 | loss=11.0647 | loss_dist=10.9431 | loss_mean=0.1216\n",
      "[2019-12-06 18:36:43,982] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.2523 | _timers/batch_time=0.3220 | _timers/data_time=0.0003 | _timers/model_time=0.3217 | f1_score=5.939e-07 | loss=26.7417 | loss_dist=26.6182 | loss_mean=0.1235\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.0417 | _timers/batch_time=0.1849 | _timers/data_time=0.0003 | _timers/model_time=0.1846 | f1_score=5.939e-07 | loss=26.7417 | loss_dist=26.6182 | loss_mean=0.1235\n",
      "[2019-12-06 18:36:48,112] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.3811 | _timers/batch_time=0.3150 | _timers/data_time=0.0003 | _timers/model_time=0.3147 | f1_score=2.883e-07 | loss=20.4421 | loss_dist=20.3203 | loss_mean=0.1218\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.6785 | _timers/batch_time=0.1900 | _timers/data_time=0.0004 | _timers/model_time=0.1896 | f1_score=2.883e-07 | loss=20.4421 | loss_dist=20.3203 | loss_mean=0.1218\n",
      "[2019-12-06 18:36:52,211] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.3269 | _timers/batch_time=0.3179 | _timers/data_time=0.0004 | _timers/model_time=0.3174 | f1_score=3.701e-07 | loss=30.0555 | loss_dist=29.9344 | loss_mean=0.1211\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.1583 | _timers/batch_time=0.2023 | _timers/data_time=0.0003 | _timers/model_time=0.2019 | f1_score=3.701e-07 | loss=30.0555 | loss_dist=29.9344 | loss_mean=0.1211\n",
      "[2019-12-06 18:36:56,448] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.9240 | _timers/batch_time=0.3389 | _timers/data_time=0.0004 | _timers/model_time=0.3385 | f1_score=4.709e-07 | loss=7.1873 | loss_dist=7.0579 | loss_mean=0.1294\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.9217 | _timers/batch_time=0.1887 | _timers/data_time=0.0004 | _timers/model_time=0.1882 | f1_score=4.709e-07 | loss=7.1873 | loss_dist=7.0579 | loss_mean=0.1294\n",
      "[2019-12-06 18:37:00,570] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.1712 | _timers/batch_time=0.3247 | _timers/data_time=0.0002 | _timers/model_time=0.3245 | f1_score=3.826e-07 | loss=14.8426 | loss_dist=14.7169 | loss_mean=0.1257\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.0609 | _timers/batch_time=0.1846 | _timers/data_time=0.0003 | _timers/model_time=0.1843 | f1_score=3.826e-07 | loss=14.8426 | loss_dist=14.7169 | loss_mean=0.1257\n",
      "[2019-12-06 18:37:04,470] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3478 | _timers/batch_time=0.3156 | _timers/data_time=0.0004 | _timers/model_time=0.3152 | f1_score=7.662e-07 | loss=6.0064 | loss_dist=5.8821 | loss_mean=0.1243\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.6360 | _timers/batch_time=0.1759 | _timers/data_time=0.0003 | _timers/model_time=0.1756 | f1_score=7.662e-07 | loss=6.0064 | loss_dist=5.8821 | loss_mean=0.1243\n",
      "[2019-12-06 18:37:08,547] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3942 | _timers/batch_time=0.3138 | _timers/data_time=0.0004 | _timers/model_time=0.3134 | f1_score=3.264e-07 | loss=7.5395 | loss_dist=7.4148 | loss_mean=0.1247\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.4683 | _timers/batch_time=0.1957 | _timers/data_time=0.0002 | _timers/model_time=0.1954 | f1_score=3.264e-07 | loss=7.5395 | loss_dist=7.4148 | loss_mean=0.1247\n",
      "[2019-12-06 18:37:12,478] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3117 | _timers/batch_time=0.3171 | _timers/data_time=0.0003 | _timers/model_time=0.3168 | f1_score=5.340e-07 | loss=8.8773 | loss_dist=8.7463 | loss_mean=0.1311\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.6740 | _timers/batch_time=0.1747 | _timers/data_time=0.0003 | _timers/model_time=0.1743 | f1_score=5.340e-07 | loss=8.8773 | loss_dist=8.7463 | loss_mean=0.1311\n",
      "[2019-12-06 18:37:16,294] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5660 | _timers/batch_time=0.3061 | _timers/data_time=0.0003 | _timers/model_time=0.3058 | f1_score=4.407e-07 | loss=15.4163 | loss_dist=15.2910 | loss_mean=0.1253\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.7264 | _timers/batch_time=0.1734 | _timers/data_time=0.0003 | _timers/model_time=0.1731 | f1_score=4.407e-07 | loss=15.4163 | loss_dist=15.2910 | loss_mean=0.1253\n",
      "[2019-12-06 18:37:20,211] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4231 | _timers/batch_time=0.3136 | _timers/data_time=0.0003 | _timers/model_time=0.3132 | f1_score=2.165e-07 | loss=23.7823 | loss_dist=23.6629 | loss_mean=0.1194\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.7159 | _timers/batch_time=0.1891 | _timers/data_time=0.0002 | _timers/model_time=0.1889 | f1_score=2.165e-07 | loss=23.7823 | loss_dist=23.6629 | loss_mean=0.1194\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:37:24,165] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.2584 | _timers/batch_time=0.3198 | _timers/data_time=0.0004 | _timers/model_time=0.3193 | f1_score=2.646e-07 | loss=22.5015 | loss_dist=22.3776 | loss_mean=0.1239\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.3785 | _timers/batch_time=0.1789 | _timers/data_time=0.0002 | _timers/model_time=0.1787 | f1_score=2.646e-07 | loss=22.5015 | loss_dist=22.3776 | loss_mean=0.1239\n",
      "[2019-12-06 18:37:28,132] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.3551 | _timers/batch_time=0.3163 | _timers/data_time=0.0003 | _timers/model_time=0.3160 | f1_score=1.592e-07 | loss=12.5136 | loss_dist=12.3879 | loss_mean=0.1258\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.3304 | _timers/batch_time=0.1802 | _timers/data_time=0.0003 | _timers/model_time=0.1799 | f1_score=1.592e-07 | loss=12.5136 | loss_dist=12.3879 | loss_mean=0.1258\n",
      "[2019-12-06 18:37:32,199] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.2926 | _timers/batch_time=0.3192 | _timers/data_time=0.0004 | _timers/model_time=0.3188 | f1_score=6.959e-07 | loss=5.9891 | loss_dist=5.8613 | loss_mean=0.1278\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.4821 | _timers/batch_time=0.1782 | _timers/data_time=0.0004 | _timers/model_time=0.1777 | f1_score=6.959e-07 | loss=5.9891 | loss_dist=5.8613 | loss_mean=0.1278\n",
      "[2019-12-06 18:37:36,064] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4574 | _timers/batch_time=0.3110 | _timers/data_time=0.0002 | _timers/model_time=0.3107 | f1_score=6.103e-07 | loss=4.5736 | loss_dist=4.4485 | loss_mean=0.1251\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6739 | _timers/batch_time=0.1747 | _timers/data_time=0.0003 | _timers/model_time=0.1745 | f1_score=6.103e-07 | loss=4.5736 | loss_dist=4.4485 | loss_mean=0.1251\n",
      "[2019-12-06 18:37:39,992] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4721 | _timers/batch_time=0.3106 | _timers/data_time=0.0003 | _timers/model_time=0.3102 | f1_score=4.409e-07 | loss=27.6067 | loss_dist=27.4774 | loss_mean=0.1293\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6591 | _timers/batch_time=0.1760 | _timers/data_time=0.0004 | _timers/model_time=0.1756 | f1_score=4.409e-07 | loss=27.6067 | loss_dist=27.4774 | loss_mean=0.1293\n",
      "[2019-12-06 18:37:43,843] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4953 | _timers/batch_time=0.3092 | _timers/data_time=0.0003 | _timers/model_time=0.3088 | f1_score=6.571e-07 | loss=4.0433 | loss_dist=3.9182 | loss_mean=0.1252\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6694 | _timers/batch_time=0.1747 | _timers/data_time=0.0003 | _timers/model_time=0.1743 | f1_score=6.571e-07 | loss=4.0433 | loss_dist=3.9182 | loss_mean=0.1252\n",
      "[2019-12-06 18:37:47,726] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4606 | _timers/batch_time=0.3109 | _timers/data_time=0.0003 | _timers/model_time=0.3106 | f1_score=9.409e-07 | loss=1.5279 | loss_dist=1.4038 | loss_mean=0.1241\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6605 | _timers/batch_time=0.1741 | _timers/data_time=0.0004 | _timers/model_time=0.1737 | f1_score=9.409e-07 | loss=1.5279 | loss_dist=1.4038 | loss_mean=0.1241\n",
      "[2019-12-06 18:37:51,610] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5118 | _timers/batch_time=0.3081 | _timers/data_time=0.0002 | _timers/model_time=0.3079 | f1_score=1.007e-06 | loss=13.4482 | loss_dist=13.3267 | loss_mean=0.1215\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.7195 | _timers/batch_time=0.1740 | _timers/data_time=0.0004 | _timers/model_time=0.1735 | f1_score=1.007e-06 | loss=13.4482 | loss_dist=13.3267 | loss_mean=0.1215\n",
      "[2019-12-06 18:37:55,478] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5633 | _timers/batch_time=0.3058 | _timers/data_time=0.0003 | _timers/model_time=0.3055 | f1_score=5.584e-07 | loss=15.9899 | loss_dist=15.8667 | loss_mean=0.1232\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.3870 | _timers/batch_time=0.1808 | _timers/data_time=0.0004 | _timers/model_time=0.1804 | f1_score=5.584e-07 | loss=15.9899 | loss_dist=15.8667 | loss_mean=0.1232\n",
      "[2019-12-06 18:37:59,376] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5522 | _timers/batch_time=0.3066 | _timers/data_time=0.0003 | _timers/model_time=0.3062 | f1_score=4.147e-07 | loss=6.4000 | loss_dist=6.2729 | loss_mean=0.1271\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.0583 | _timers/batch_time=0.1842 | _timers/data_time=0.0004 | _timers/model_time=0.1838 | f1_score=4.147e-07 | loss=6.4000 | loss_dist=6.2729 | loss_mean=0.1271\n",
      "[2019-12-06 18:38:03,570] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1832 | _timers/batch_time=0.3271 | _timers/data_time=0.0004 | _timers/model_time=0.3267 | f1_score=9.158e-07 | loss=1.8971 | loss_dist=1.7736 | loss_mean=0.1235\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.8768 | _timers/batch_time=0.1879 | _timers/data_time=0.0003 | _timers/model_time=0.1875 | f1_score=9.158e-07 | loss=1.8971 | loss_dist=1.7736 | loss_mean=0.1235\n",
      "[2019-12-06 18:38:07,631] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.2316 | _timers/batch_time=0.3225 | _timers/data_time=0.0004 | _timers/model_time=0.3221 | f1_score=7.280e-07 | loss=1.4640 | loss_dist=1.3364 | loss_mean=0.1275\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.5066 | _timers/batch_time=0.1924 | _timers/data_time=0.0004 | _timers/model_time=0.1920 | f1_score=7.280e-07 | loss=1.4640 | loss_dist=1.3364 | loss_mean=0.1275\n",
      "[2019-12-06 18:38:11,796] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=5.9936 | _timers/batch_time=0.3349 | _timers/data_time=0.0003 | _timers/model_time=0.3346 | f1_score=6.730e-07 | loss=13.6223 | loss_dist=13.4990 | loss_mean=0.1233\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.6112 | _timers/batch_time=0.1911 | _timers/data_time=0.0003 | _timers/model_time=0.1908 | f1_score=6.730e-07 | loss=13.6223 | loss_dist=13.4990 | loss_mean=0.1233\n",
      "[2019-12-06 18:38:15,839] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.1884 | _timers/batch_time=0.3241 | _timers/data_time=0.0002 | _timers/model_time=0.3239 | f1_score=8.569e-07 | loss=2.3378 | loss_dist=2.2113 | loss_mean=0.1266\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7949 | _timers/batch_time=0.1720 | _timers/data_time=0.0002 | _timers/model_time=0.1718 | f1_score=8.569e-07 | loss=2.3378 | loss_dist=2.2113 | loss_mean=0.1266\n",
      "[2019-12-06 18:38:19,622] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5946 | _timers/batch_time=0.3041 | _timers/data_time=0.0002 | _timers/model_time=0.3038 | f1_score=9.425e-07 | loss=3.2390 | loss_dist=3.1129 | loss_mean=0.1262\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7665 | _timers/batch_time=0.1725 | _timers/data_time=0.0003 | _timers/model_time=0.1722 | f1_score=9.425e-07 | loss=3.2390 | loss_dist=3.1129 | loss_mean=0.1262\n",
      "[2019-12-06 18:38:23,471] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5808 | _timers/batch_time=0.3041 | _timers/data_time=0.0002 | _timers/model_time=0.3039 | f1_score=5.895e-07 | loss=13.6714 | loss_dist=13.5450 | loss_mean=0.1265\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.6836 | _timers/batch_time=0.1897 | _timers/data_time=0.0004 | _timers/model_time=0.1893 | f1_score=5.895e-07 | loss=13.6714 | loss_dist=13.5450 | loss_mean=0.1265\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:38:27,596] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.2262 | _timers/batch_time=0.3218 | _timers/data_time=0.0002 | _timers/model_time=0.3215 | f1_score=7.077e-07 | loss=2.9024 | loss_dist=2.7773 | loss_mean=0.1251\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.5392 | _timers/batch_time=0.1977 | _timers/data_time=0.0003 | _timers/model_time=0.1974 | f1_score=7.077e-07 | loss=2.9024 | loss_dist=2.7773 | loss_mean=0.1251\n",
      "[2019-12-06 18:38:31,881] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.9060 | _timers/batch_time=0.3389 | _timers/data_time=0.0002 | _timers/model_time=0.3386 | f1_score=2.943e-07 | loss=12.2013 | loss_dist=12.0794 | loss_mean=0.1219\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=9.8835 | _timers/batch_time=0.2057 | _timers/data_time=0.0004 | _timers/model_time=0.2053 | f1_score=2.943e-07 | loss=12.2013 | loss_dist=12.0794 | loss_mean=0.1219\n",
      "[2019-12-06 18:38:35,932] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.1700 | _timers/batch_time=0.3243 | _timers/data_time=0.0003 | _timers/model_time=0.3239 | f1_score=2.951e-07 | loss=14.6271 | loss_dist=14.4987 | loss_mean=0.1285\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.8022 | _timers/batch_time=0.1730 | _timers/data_time=0.0002 | _timers/model_time=0.1728 | f1_score=2.951e-07 | loss=14.6271 | loss_dist=14.4987 | loss_mean=0.1285\n",
      "[2019-12-06 18:38:39,746] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5715 | _timers/batch_time=0.3055 | _timers/data_time=0.0003 | _timers/model_time=0.3051 | f1_score=9.999e-07 | loss=6.3451 | loss_dist=6.2215 | loss_mean=0.1236\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.8894 | _timers/batch_time=0.1718 | _timers/data_time=0.0003 | _timers/model_time=0.1715 | f1_score=9.999e-07 | loss=6.3451 | loss_dist=6.2215 | loss_mean=0.1236\n",
      "[2019-12-06 18:38:43,578] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5492 | _timers/batch_time=0.3061 | _timers/data_time=0.0005 | _timers/model_time=0.3056 | f1_score=1.792e-07 | loss=18.9751 | loss_dist=18.8502 | loss_mean=0.1249\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.8744 | _timers/batch_time=0.1706 | _timers/data_time=0.0003 | _timers/model_time=0.1703 | f1_score=1.792e-07 | loss=18.9751 | loss_dist=18.8502 | loss_mean=0.1249\n",
      "[2019-12-06 18:38:47,437] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4757 | _timers/batch_time=0.3093 | _timers/data_time=0.0003 | _timers/model_time=0.3089 | f1_score=9.297e-07 | loss=10.1785 | loss_dist=10.0572 | loss_mean=0.1213\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.7336 | _timers/batch_time=0.1746 | _timers/data_time=0.0002 | _timers/model_time=0.1743 | f1_score=9.297e-07 | loss=10.1785 | loss_dist=10.0572 | loss_mean=0.1213\n",
      "[2019-12-06 18:38:51,289] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4551 | _timers/batch_time=0.3113 | _timers/data_time=0.0004 | _timers/model_time=0.3109 | f1_score=3.905e-07 | loss=16.4096 | loss_dist=16.2820 | loss_mean=0.1276\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.7151 | _timers/batch_time=0.1755 | _timers/data_time=0.0003 | _timers/model_time=0.1752 | f1_score=3.905e-07 | loss=16.4096 | loss_dist=16.2820 | loss_mean=0.1276\n",
      "[2019-12-06 18:38:55,118] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5342 | _timers/batch_time=0.3076 | _timers/data_time=0.0003 | _timers/model_time=0.3073 | f1_score=1.347e-06 | loss=2.0641 | loss_dist=1.9430 | loss_mean=0.1211\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.7245 | _timers/batch_time=0.1734 | _timers/data_time=0.0003 | _timers/model_time=0.1731 | f1_score=1.347e-06 | loss=2.0641 | loss_dist=1.9430 | loss_mean=0.1211\n",
      "[2019-12-06 18:38:58,934] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5672 | _timers/batch_time=0.3060 | _timers/data_time=0.0003 | _timers/model_time=0.3056 | f1_score=1.047e-06 | loss=0.7015 | loss_dist=0.5772 | loss_mean=0.1243\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8331 | _timers/batch_time=0.1720 | _timers/data_time=0.0003 | _timers/model_time=0.1717 | f1_score=1.047e-06 | loss=0.7015 | loss_dist=0.5772 | loss_mean=0.1243\n",
      "[2019-12-06 18:39:02,826] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5009 | _timers/batch_time=0.3083 | _timers/data_time=0.0004 | _timers/model_time=0.3079 | f1_score=1.027e-06 | loss=1.1824 | loss_dist=1.0587 | loss_mean=0.1237\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5730 | _timers/batch_time=0.1763 | _timers/data_time=0.0002 | _timers/model_time=0.1760 | f1_score=1.027e-06 | loss=1.1824 | loss_dist=1.0587 | loss_mean=0.1237\n",
      "[2019-12-06 18:39:06,663] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5221 | _timers/batch_time=0.3073 | _timers/data_time=0.0003 | _timers/model_time=0.3070 | f1_score=3.263e-07 | loss=23.7813 | loss_dist=23.6575 | loss_mean=0.1239\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8262 | _timers/batch_time=0.1729 | _timers/data_time=0.0003 | _timers/model_time=0.1725 | f1_score=3.263e-07 | loss=23.7813 | loss_dist=23.6575 | loss_mean=0.1239\n",
      "[2019-12-06 18:39:10,496] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5840 | _timers/batch_time=0.3053 | _timers/data_time=0.0003 | _timers/model_time=0.3050 | f1_score=3.806e-07 | loss=12.6167 | loss_dist=12.4920 | loss_mean=0.1247\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.9253 | _timers/batch_time=0.1712 | _timers/data_time=0.0003 | _timers/model_time=0.1710 | f1_score=3.806e-07 | loss=12.6167 | loss_dist=12.4920 | loss_mean=0.1247\n",
      "[2019-12-06 18:39:14,299] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5509 | _timers/batch_time=0.3068 | _timers/data_time=0.0004 | _timers/model_time=0.3064 | f1_score=1.093e-06 | loss=0.2972 | loss_dist=0.1730 | loss_mean=0.1242\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8464 | _timers/batch_time=0.1726 | _timers/data_time=0.0004 | _timers/model_time=0.1722 | f1_score=1.093e-06 | loss=0.2972 | loss_dist=0.1730 | loss_mean=0.1242\n",
      "[2019-12-06 18:39:18,123] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5622 | _timers/batch_time=0.3058 | _timers/data_time=0.0004 | _timers/model_time=0.3054 | f1_score=5.923e-07 | loss=2.1917 | loss_dist=2.0662 | loss_mean=0.1255\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8180 | _timers/batch_time=0.1731 | _timers/data_time=0.0003 | _timers/model_time=0.1727 | f1_score=5.923e-07 | loss=2.1917 | loss_dist=2.0662 | loss_mean=0.1255\n",
      "[2019-12-06 18:39:21,956] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4873 | _timers/batch_time=0.3096 | _timers/data_time=0.0003 | _timers/model_time=0.3092 | f1_score=1.631e-07 | loss=17.5265 | loss_dist=17.3981 | loss_mean=0.1284\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.9456 | _timers/batch_time=0.1707 | _timers/data_time=0.0003 | _timers/model_time=0.1704 | f1_score=1.631e-07 | loss=17.5265 | loss_dist=17.3981 | loss_mean=0.1284\n",
      "[2019-12-06 18:39:25,805] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5233 | _timers/batch_time=0.3082 | _timers/data_time=0.0004 | _timers/model_time=0.3077 | f1_score=2.796e-07 | loss=17.9899 | loss_dist=17.8664 | loss_mean=0.1235\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8249 | _timers/batch_time=0.1724 | _timers/data_time=0.0003 | _timers/model_time=0.1721 | f1_score=2.796e-07 | loss=17.9899 | loss_dist=17.8664 | loss_mean=0.1235\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:39:29,633] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5241 | _timers/batch_time=0.3072 | _timers/data_time=0.0003 | _timers/model_time=0.3068 | f1_score=9.280e-07 | loss=3.8589 | loss_dist=3.7336 | loss_mean=0.1253\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.7839 | _timers/batch_time=0.1738 | _timers/data_time=0.0003 | _timers/model_time=0.1735 | f1_score=9.280e-07 | loss=3.8589 | loss_dist=3.7336 | loss_mean=0.1253\n",
      "[2019-12-06 18:39:33,936] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3509 | _timers/batch_time=0.3187 | _timers/data_time=0.0003 | _timers/model_time=0.3183 | f1_score=7.649e-07 | loss=18.8165 | loss_dist=18.6904 | loss_mean=0.1262\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.3668 | _timers/batch_time=0.1947 | _timers/data_time=0.0002 | _timers/model_time=0.1945 | f1_score=7.649e-07 | loss=18.8165 | loss_dist=18.6904 | loss_mean=0.1262\n",
      "[2019-12-06 18:39:38,197] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.7864 | _timers/batch_time=0.3458 | _timers/data_time=0.0003 | _timers/model_time=0.3455 | f1_score=6.761e-07 | loss=13.2829 | loss_dist=13.1649 | loss_mean=0.1180\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.5366 | _timers/batch_time=0.1764 | _timers/data_time=0.0003 | _timers/model_time=0.1761 | f1_score=6.761e-07 | loss=13.2829 | loss_dist=13.1649 | loss_mean=0.1180\n",
      "[2019-12-06 18:39:42,510] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.8644 | _timers/batch_time=0.3466 | _timers/data_time=0.0004 | _timers/model_time=0.3462 | f1_score=1.016e-06 | loss=11.4049 | loss_dist=11.2774 | loss_mean=0.1275\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.1442 | _timers/batch_time=0.2015 | _timers/data_time=0.0003 | _timers/model_time=0.2011 | f1_score=1.016e-06 | loss=11.4049 | loss_dist=11.2774 | loss_mean=0.1275\n",
      "[2019-12-06 18:39:46,934] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.6798 | _timers/batch_time=0.3539 | _timers/data_time=0.0003 | _timers/model_time=0.3536 | f1_score=9.149e-07 | loss=1.4988 | loss_dist=1.3714 | loss_mean=0.1273\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.6392 | _timers/batch_time=0.1907 | _timers/data_time=0.0005 | _timers/model_time=0.1901 | f1_score=9.149e-07 | loss=1.4988 | loss_dist=1.3714 | loss_mean=0.1273\n",
      "[2019-12-06 18:39:51,231] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.9951 | _timers/batch_time=0.3378 | _timers/data_time=0.0003 | _timers/model_time=0.3375 | f1_score=5.028e-07 | loss=4.9093 | loss_dist=4.7846 | loss_mean=0.1247\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.0165 | _timers/batch_time=0.2053 | _timers/data_time=0.0004 | _timers/model_time=0.2048 | f1_score=5.028e-07 | loss=4.9093 | loss_dist=4.7846 | loss_mean=0.1247\n",
      "[2019-12-06 18:39:55,528] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.7200 | _timers/batch_time=0.3503 | _timers/data_time=0.0003 | _timers/model_time=0.3499 | f1_score=5.383e-07 | loss=22.2208 | loss_dist=22.0990 | loss_mean=0.1218\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.1297 | _timers/batch_time=0.1845 | _timers/data_time=0.0003 | _timers/model_time=0.1841 | f1_score=5.383e-07 | loss=22.2208 | loss_dist=22.0990 | loss_mean=0.1218\n",
      "[2019-12-06 18:40:00,255] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.7787 | _timers/batch_time=0.3484 | _timers/data_time=0.0006 | _timers/model_time=0.3478 | f1_score=1.130e-06 | loss=3.1545 | loss_dist=3.0289 | loss_mean=0.1256\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=8.4836 | _timers/batch_time=0.2406 | _timers/data_time=0.0003 | _timers/model_time=0.2402 | f1_score=1.130e-06 | loss=3.1545 | loss_dist=3.0289 | loss_mean=0.1256\n",
      "[2019-12-06 18:40:04,952] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=4.9667 | _timers/batch_time=0.4035 | _timers/data_time=0.0004 | _timers/model_time=0.4030 | f1_score=4.775e-07 | loss=7.3714 | loss_dist=7.2470 | loss_mean=0.1244\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.7206 | _timers/batch_time=0.1905 | _timers/data_time=0.0003 | _timers/model_time=0.1902 | f1_score=4.775e-07 | loss=7.3714 | loss_dist=7.2470 | loss_mean=0.1244\n",
      "[2019-12-06 18:40:09,319] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.7720 | _timers/batch_time=0.3500 | _timers/data_time=0.0005 | _timers/model_time=0.3494 | f1_score=4.428e-07 | loss=21.2057 | loss_dist=21.0738 | loss_mean=0.1319\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.2292 | _timers/batch_time=0.2023 | _timers/data_time=0.0003 | _timers/model_time=0.2020 | f1_score=4.428e-07 | loss=21.2057 | loss_dist=21.0738 | loss_mean=0.1319\n",
      "[2019-12-06 18:40:13,515] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.0035 | _timers/batch_time=0.3342 | _timers/data_time=0.0003 | _timers/model_time=0.3339 | f1_score=6.235e-07 | loss=1.5092 | loss_dist=1.3823 | loss_mean=0.1270\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.5803 | _timers/batch_time=0.1922 | _timers/data_time=0.0002 | _timers/model_time=0.1919 | f1_score=6.235e-07 | loss=1.5092 | loss_dist=1.3823 | loss_mean=0.1270\n",
      "[2019-12-06 18:40:18,131] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.7854 | _timers/batch_time=0.3480 | _timers/data_time=0.0002 | _timers/model_time=0.3478 | f1_score=4.959e-07 | loss=22.4538 | loss_dist=22.3292 | loss_mean=0.1246\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=7.9820 | _timers/batch_time=0.2554 | _timers/data_time=0.0004 | _timers/model_time=0.2550 | f1_score=4.959e-07 | loss=22.4538 | loss_dist=22.3292 | loss_mean=0.1246\n",
      "[2019-12-06 18:40:22,840] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.4516 | _timers/batch_time=0.3680 | _timers/data_time=0.0003 | _timers/model_time=0.3676 | f1_score=9.287e-07 | loss=5.1243 | loss_dist=4.9995 | loss_mean=0.1248\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=9.5920 | _timers/batch_time=0.2119 | _timers/data_time=0.0003 | _timers/model_time=0.2115 | f1_score=9.287e-07 | loss=5.1243 | loss_dist=4.9995 | loss_mean=0.1248\n",
      "[2019-12-06 18:40:27,401] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.3223 | _timers/batch_time=0.3775 | _timers/data_time=0.0003 | _timers/model_time=0.3771 | f1_score=6.874e-07 | loss=7.9127 | loss_dist=7.7886 | loss_mean=0.1241\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.2262 | _timers/batch_time=0.1986 | _timers/data_time=0.0003 | _timers/model_time=0.1982 | f1_score=6.874e-07 | loss=7.9127 | loss_dist=7.7886 | loss_mean=0.1241\n",
      "[2019-12-06 18:40:31,638] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.8732 | _timers/batch_time=0.3414 | _timers/data_time=0.0003 | _timers/model_time=0.3411 | f1_score=7.516e-07 | loss=4.3895 | loss_dist=4.2631 | loss_mean=0.1263\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.8393 | _timers/batch_time=0.1899 | _timers/data_time=0.0002 | _timers/model_time=0.1897 | f1_score=7.516e-07 | loss=4.3895 | loss_dist=4.2631 | loss_mean=0.1263\n",
      "[2019-12-06 18:40:35,844] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2013 | _timers/batch_time=0.3238 | _timers/data_time=0.0004 | _timers/model_time=0.3234 | f1_score=4.274e-07 | loss=21.1286 | loss_dist=21.0060 | loss_mean=0.1225\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.9487 | _timers/batch_time=0.1858 | _timers/data_time=0.0003 | _timers/model_time=0.1855 | f1_score=4.274e-07 | loss=21.1286 | loss_dist=21.0060 | loss_mean=0.1225\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 18:40:39,939] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2483 | _timers/batch_time=0.3210 | _timers/data_time=0.0004 | _timers/model_time=0.3205 | f1_score=9.544e-07 | loss=8.9184 | loss_dist=8.7955 | loss_mean=0.1230\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.7169 | _timers/batch_time=0.1910 | _timers/data_time=0.0002 | _timers/model_time=0.1907 | f1_score=9.544e-07 | loss=8.9184 | loss_dist=8.7955 | loss_mean=0.1230\n",
      "[2019-12-06 18:40:44,240] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=5.9723 | _timers/batch_time=0.3374 | _timers/data_time=0.0003 | _timers/model_time=0.3371 | f1_score=1.824e-07 | loss=17.6124 | loss_dist=17.4898 | loss_mean=0.1226\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.0376 | _timers/batch_time=0.2036 | _timers/data_time=0.0002 | _timers/model_time=0.2034 | f1_score=1.824e-07 | loss=17.6124 | loss_dist=17.4898 | loss_mean=0.1226\n",
      "[2019-12-06 18:40:48,488] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=5.8961 | _timers/batch_time=0.3395 | _timers/data_time=0.0004 | _timers/model_time=0.3391 | f1_score=5.350e-07 | loss=17.0973 | loss_dist=16.9732 | loss_mean=0.1242\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.6510 | _timers/batch_time=0.1913 | _timers/data_time=0.0003 | _timers/model_time=0.1910 | f1_score=5.350e-07 | loss=17.0973 | loss_dist=16.9732 | loss_mean=0.1242\n",
      "[2019-12-06 18:40:52,709] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=5.9913 | _timers/batch_time=0.3349 | _timers/data_time=0.0003 | _timers/model_time=0.3345 | f1_score=1.103e-06 | loss=1.4900 | loss_dist=1.3652 | loss_mean=0.1248\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.6288 | _timers/batch_time=0.1915 | _timers/data_time=0.0002 | _timers/model_time=0.1912 | f1_score=1.103e-06 | loss=1.4900 | loss_dist=1.3652 | loss_mean=0.1248\n",
      "Top best models:\n",
      "logs/b50/checkpoints/train.77.pth\t0.2972\n",
      "logs/b50/checkpoints/train.11.pth\t0.3293\n",
      "logs/b50/checkpoints/train.31.pth\t0.4774\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bernoulli p = 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)               \n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        #torch.bernoulli(torch.tensor([1.0]),)\n",
    "        b_new = torch.bernoulli(torch.ones(x.shape[:2]),p).unsqueeze(2).to(self.emb.weight.device)\n",
    "        #b_new = torch.repeat_interleave(bb(0.4), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/b90\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:03:35,055] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.4257 | _timers/batch_time=0.3689 | _timers/data_time=0.0004 | _timers/model_time=0.3685 | f1_score=1.460e-06 | loss=0.4123 | loss_dist=0.0059 | loss_mean=0.4064\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.0839 | _timers/batch_time=0.2021 | _timers/data_time=0.0003 | _timers/model_time=0.2018 | f1_score=1.460e-06 | loss=0.4123 | loss_dist=0.0059 | loss_mean=0.4064\n",
      "[2019-12-06 19:03:39,127] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4065 | _timers/batch_time=0.3126 | _timers/data_time=0.0004 | _timers/model_time=0.3122 | f1_score=1.628e-06 | loss=0.4067 | loss_dist=0.0042 | loss_mean=0.4025\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.5202 | _timers/batch_time=0.1953 | _timers/data_time=0.0003 | _timers/model_time=0.1950 | f1_score=1.628e-06 | loss=0.4067 | loss_dist=0.0042 | loss_mean=0.4025\n",
      "[2019-12-06 19:03:43,478] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.9290 | _timers/batch_time=0.3381 | _timers/data_time=0.0004 | _timers/model_time=0.3377 | f1_score=1.406e-06 | loss=0.4072 | loss_dist=0.0029 | loss_mean=0.4043\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2127 | _timers/batch_time=0.2008 | _timers/data_time=0.0003 | _timers/model_time=0.2005 | f1_score=1.406e-06 | loss=0.4072 | loss_dist=0.0029 | loss_mean=0.4043\n",
      "[2019-12-06 19:03:47,846] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7044 | _timers/batch_time=0.3511 | _timers/data_time=0.0003 | _timers/model_time=0.3508 | f1_score=1.716e-06 | loss=0.4049 | loss_dist=0.0005 | loss_mean=0.4045\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.5522 | _timers/batch_time=0.1950 | _timers/data_time=0.0003 | _timers/model_time=0.1946 | f1_score=1.716e-06 | loss=0.4049 | loss_dist=0.0005 | loss_mean=0.4045\n",
      "[2019-12-06 19:03:52,163] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.8495 | _timers/batch_time=0.3441 | _timers/data_time=0.0002 | _timers/model_time=0.3439 | f1_score=1.571e-06 | loss=0.4072 | loss_dist=0.0046 | loss_mean=0.4026\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.0483 | _timers/batch_time=0.2047 | _timers/data_time=0.0003 | _timers/model_time=0.2043 | f1_score=1.571e-06 | loss=0.4072 | loss_dist=0.0046 | loss_mean=0.4026\n",
      "[2019-12-06 19:03:56,711] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.5788 | _timers/batch_time=0.3601 | _timers/data_time=0.0004 | _timers/model_time=0.3597 | f1_score=1.521e-06 | loss=0.4100 | loss_dist=0.0062 | loss_mean=0.4038\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.9883 | _timers/batch_time=0.2033 | _timers/data_time=0.0003 | _timers/model_time=0.2030 | f1_score=1.521e-06 | loss=0.4100 | loss_dist=0.0062 | loss_mean=0.4038\n",
      "[2019-12-06 19:04:00,949] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.8733 | _timers/batch_time=0.3419 | _timers/data_time=0.0003 | _timers/model_time=0.3416 | f1_score=1.682e-06 | loss=0.4116 | loss_dist=0.0001 | loss_mean=0.4115\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.6491 | _timers/batch_time=0.2112 | _timers/data_time=0.0003 | _timers/model_time=0.2108 | f1_score=1.682e-06 | loss=0.4116 | loss_dist=0.0001 | loss_mean=0.4115\n",
      "[2019-12-06 19:04:05,241] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.8826 | _timers/batch_time=0.3405 | _timers/data_time=0.0003 | _timers/model_time=0.3401 | f1_score=1.719e-06 | loss=0.4169 | loss_dist=0.0072 | loss_mean=0.4097\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1205 | _timers/batch_time=0.1844 | _timers/data_time=0.0005 | _timers/model_time=0.1839 | f1_score=1.719e-06 | loss=0.4169 | loss_dist=0.0072 | loss_mean=0.4097\n",
      "[2019-12-06 19:04:09,268] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1986 | _timers/batch_time=0.3245 | _timers/data_time=0.0005 | _timers/model_time=0.3240 | f1_score=1.582e-06 | loss=0.4238 | loss_dist=0.0149 | loss_mean=0.4088\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1547 | _timers/batch_time=0.1843 | _timers/data_time=0.0003 | _timers/model_time=0.1839 | f1_score=1.582e-06 | loss=0.4238 | loss_dist=0.0149 | loss_mean=0.4088\n",
      "[2019-12-06 19:04:13,603] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.8347 | _timers/batch_time=0.3440 | _timers/data_time=0.0004 | _timers/model_time=0.3436 | f1_score=1.412e-06 | loss=0.4245 | loss_dist=0.0189 | loss_mean=0.4057\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.3707 | _timers/batch_time=0.1975 | _timers/data_time=0.0004 | _timers/model_time=0.1971 | f1_score=1.412e-06 | loss=0.4245 | loss_dist=0.0189 | loss_mean=0.4057\n",
      "[2019-12-06 19:04:18,079] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.7937 | _timers/batch_time=0.3469 | _timers/data_time=0.0003 | _timers/model_time=0.3466 | f1_score=1.592e-06 | loss=0.4198 | loss_dist=0.0165 | loss_mean=0.4034\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=9.9522 | _timers/batch_time=0.2046 | _timers/data_time=0.0002 | _timers/model_time=0.2043 | f1_score=1.592e-06 | loss=0.4198 | loss_dist=0.0165 | loss_mean=0.4034\n",
      "[2019-12-06 19:04:22,494] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.7936 | _timers/batch_time=0.3476 | _timers/data_time=0.0004 | _timers/model_time=0.3472 | f1_score=1.499e-06 | loss=0.4113 | loss_dist=0.0094 | loss_mean=0.4019\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.0967 | _timers/batch_time=0.2015 | _timers/data_time=0.0003 | _timers/model_time=0.2012 | f1_score=1.499e-06 | loss=0.4113 | loss_dist=0.0094 | loss_mean=0.4019\n",
      "[2019-12-06 19:04:26,537] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.1122 | _timers/batch_time=0.3281 | _timers/data_time=0.0002 | _timers/model_time=0.3278 | f1_score=1.648e-06 | loss=0.4269 | loss_dist=0.0267 | loss_mean=0.4002\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5704 | _timers/batch_time=0.1766 | _timers/data_time=0.0002 | _timers/model_time=0.1764 | f1_score=1.648e-06 | loss=0.4269 | loss_dist=0.0267 | loss_mean=0.4002\n",
      "[2019-12-06 19:04:30,784] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0400 | _timers/batch_time=0.3366 | _timers/data_time=0.0005 | _timers/model_time=0.3361 | f1_score=1.540e-06 | loss=0.4125 | loss_dist=0.0044 | loss_mean=0.4080\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.2954 | _timers/batch_time=0.1957 | _timers/data_time=0.0002 | _timers/model_time=0.1955 | f1_score=1.540e-06 | loss=0.4125 | loss_dist=0.0044 | loss_mean=0.4080\n",
      "[2019-12-06 19:04:35,022] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0026 | _timers/batch_time=0.3339 | _timers/data_time=0.0002 | _timers/model_time=0.3336 | f1_score=1.199e-06 | loss=0.5046 | loss_dist=0.0970 | loss_mean=0.4076\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.7345 | _timers/batch_time=0.1904 | _timers/data_time=0.0003 | _timers/model_time=0.1901 | f1_score=1.199e-06 | loss=0.5046 | loss_dist=0.0970 | loss_mean=0.4076\n",
      "[2019-12-06 19:04:39,296] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.9812 | _timers/batch_time=0.3401 | _timers/data_time=0.0003 | _timers/model_time=0.3398 | f1_score=1.696e-06 | loss=0.4131 | loss_dist=0.0063 | loss_mean=0.4069\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.2283 | _timers/batch_time=0.1996 | _timers/data_time=0.0003 | _timers/model_time=0.1993 | f1_score=1.696e-06 | loss=0.4131 | loss_dist=0.0063 | loss_mean=0.4069\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:04:43,426] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.1057 | _timers/batch_time=0.3285 | _timers/data_time=0.0003 | _timers/model_time=0.3282 | f1_score=1.627e-06 | loss=0.4139 | loss_dist=0.0061 | loss_mean=0.4078\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.4799 | _timers/batch_time=0.1778 | _timers/data_time=0.0003 | _timers/model_time=0.1775 | f1_score=1.627e-06 | loss=0.4139 | loss_dist=0.0061 | loss_mean=0.4078\n",
      "[2019-12-06 19:04:47,350] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.3384 | _timers/batch_time=0.3163 | _timers/data_time=0.0003 | _timers/model_time=0.3159 | f1_score=1.762e-06 | loss=0.4156 | loss_dist=0.0105 | loss_mean=0.4051\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.6235 | _timers/batch_time=0.1754 | _timers/data_time=0.0003 | _timers/model_time=0.1751 | f1_score=1.762e-06 | loss=0.4156 | loss_dist=0.0105 | loss_mean=0.4051\n",
      "[2019-12-06 19:04:51,438] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.2299 | _timers/batch_time=0.3234 | _timers/data_time=0.0003 | _timers/model_time=0.3231 | f1_score=1.424e-06 | loss=0.4215 | loss_dist=0.0205 | loss_mean=0.4010\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.8528 | _timers/batch_time=0.1878 | _timers/data_time=0.0005 | _timers/model_time=0.1873 | f1_score=1.424e-06 | loss=0.4215 | loss_dist=0.0205 | loss_mean=0.4010\n",
      "[2019-12-06 19:04:55,641] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0020 | _timers/batch_time=0.3343 | _timers/data_time=0.0003 | _timers/model_time=0.3340 | f1_score=1.592e-06 | loss=0.4316 | loss_dist=0.0263 | loss_mean=0.4053\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.0152 | _timers/batch_time=0.1851 | _timers/data_time=0.0003 | _timers/model_time=0.1848 | f1_score=1.592e-06 | loss=0.4316 | loss_dist=0.0263 | loss_mean=0.4053\n",
      "[2019-12-06 19:04:59,651] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2317 | _timers/batch_time=0.3220 | _timers/data_time=0.0005 | _timers/model_time=0.3214 | f1_score=1.571e-06 | loss=0.4216 | loss_dist=0.0130 | loss_mean=0.4087\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.6536 | _timers/batch_time=0.1748 | _timers/data_time=0.0005 | _timers/model_time=0.1743 | f1_score=1.571e-06 | loss=0.4216 | loss_dist=0.0130 | loss_mean=0.4087\n",
      "[2019-12-06 19:05:03,868] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2934 | _timers/batch_time=0.3209 | _timers/data_time=0.0002 | _timers/model_time=0.3207 | f1_score=1.641e-06 | loss=0.4071 | loss_dist=0.0003 | loss_mean=0.4068\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.7282 | _timers/batch_time=0.1893 | _timers/data_time=0.0004 | _timers/model_time=0.1889 | f1_score=1.641e-06 | loss=0.4071 | loss_dist=0.0003 | loss_mean=0.4068\n",
      "[2019-12-06 19:05:08,153] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.8418 | _timers/batch_time=0.3430 | _timers/data_time=0.0003 | _timers/model_time=0.3427 | f1_score=1.722e-06 | loss=0.4083 | loss_dist=0.0097 | loss_mean=0.3985\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.4085 | _timers/batch_time=0.1939 | _timers/data_time=0.0002 | _timers/model_time=0.1937 | f1_score=1.722e-06 | loss=0.4083 | loss_dist=0.0097 | loss_mean=0.3985\n",
      "[2019-12-06 19:05:12,300] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.9660 | _timers/batch_time=0.3371 | _timers/data_time=0.0004 | _timers/model_time=0.3367 | f1_score=1.412e-06 | loss=0.4203 | loss_dist=0.0153 | loss_mean=0.4050\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.2072 | _timers/batch_time=0.1864 | _timers/data_time=0.0002 | _timers/model_time=0.1861 | f1_score=1.412e-06 | loss=0.4203 | loss_dist=0.0153 | loss_mean=0.4050\n",
      "[2019-12-06 19:05:16,419] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.0839 | _timers/batch_time=0.3302 | _timers/data_time=0.0004 | _timers/model_time=0.3298 | f1_score=1.192e-06 | loss=0.4489 | loss_dist=0.0452 | loss_mean=0.4038\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.0944 | _timers/batch_time=0.1836 | _timers/data_time=0.0003 | _timers/model_time=0.1833 | f1_score=1.192e-06 | loss=0.4489 | loss_dist=0.0452 | loss_mean=0.4038\n",
      "[2019-12-06 19:05:20,526] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1500 | _timers/batch_time=0.3296 | _timers/data_time=0.0004 | _timers/model_time=0.3292 | f1_score=1.356e-06 | loss=0.4161 | loss_dist=0.0100 | loss_mean=0.4062\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.9166 | _timers/batch_time=0.1858 | _timers/data_time=0.0002 | _timers/model_time=0.1855 | f1_score=1.356e-06 | loss=0.4161 | loss_dist=0.0100 | loss_mean=0.4062\n",
      "[2019-12-06 19:05:24,773] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1434 | _timers/batch_time=0.3286 | _timers/data_time=0.0003 | _timers/model_time=0.3283 | f1_score=1.469e-06 | loss=0.4053 | loss_dist=0.0027 | loss_mean=0.4026\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.0121 | _timers/batch_time=0.1882 | _timers/data_time=0.0003 | _timers/model_time=0.1879 | f1_score=1.469e-06 | loss=0.4053 | loss_dist=0.0027 | loss_mean=0.4026\n",
      "[2019-12-06 19:05:28,856] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1373 | _timers/batch_time=0.3298 | _timers/data_time=0.0003 | _timers/model_time=0.3294 | f1_score=1.652e-06 | loss=0.4076 | loss_dist=9.623e-05 | loss_mean=0.4075\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.2348 | _timers/batch_time=0.1825 | _timers/data_time=0.0002 | _timers/model_time=0.1823 | f1_score=1.652e-06 | loss=0.4076 | loss_dist=9.623e-05 | loss_mean=0.4075\n",
      "[2019-12-06 19:05:32,711] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5385 | _timers/batch_time=0.3068 | _timers/data_time=0.0004 | _timers/model_time=0.3064 | f1_score=1.633e-06 | loss=0.4094 | loss_dist=0.0052 | loss_mean=0.4042\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.5915 | _timers/batch_time=0.1768 | _timers/data_time=0.0003 | _timers/model_time=0.1764 | f1_score=1.633e-06 | loss=0.4094 | loss_dist=0.0052 | loss_mean=0.4042\n",
      "[2019-12-06 19:05:36,622] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5157 | _timers/batch_time=0.3085 | _timers/data_time=0.0002 | _timers/model_time=0.3083 | f1_score=1.497e-06 | loss=0.4119 | loss_dist=0.0072 | loss_mean=0.4047\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.4774 | _timers/batch_time=0.1780 | _timers/data_time=0.0003 | _timers/model_time=0.1777 | f1_score=1.497e-06 | loss=0.4119 | loss_dist=0.0072 | loss_mean=0.4047\n",
      "[2019-12-06 19:05:40,524] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.4764 | _timers/batch_time=0.3100 | _timers/data_time=0.0002 | _timers/model_time=0.3097 | f1_score=1.357e-06 | loss=0.4182 | loss_dist=0.0108 | loss_mean=0.4074\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.5789 | _timers/batch_time=0.1765 | _timers/data_time=0.0003 | _timers/model_time=0.1762 | f1_score=1.357e-06 | loss=0.4182 | loss_dist=0.0108 | loss_mean=0.4074\n",
      "[2019-12-06 19:05:44,571] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.2549 | _timers/batch_time=0.3206 | _timers/data_time=0.0003 | _timers/model_time=0.3203 | f1_score=1.666e-06 | loss=0.4096 | loss_dist=0.0004 | loss_mean=0.4092\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.4155 | _timers/batch_time=0.1943 | _timers/data_time=0.0002 | _timers/model_time=0.1940 | f1_score=1.666e-06 | loss=0.4096 | loss_dist=0.0004 | loss_mean=0.4092\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:05:48,529] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.1598 | _timers/batch_time=0.3248 | _timers/data_time=0.0004 | _timers/model_time=0.3244 | f1_score=1.581e-06 | loss=0.4157 | loss_dist=0.0105 | loss_mean=0.4052\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.5970 | _timers/batch_time=0.1762 | _timers/data_time=0.0003 | _timers/model_time=0.1760 | f1_score=1.581e-06 | loss=0.4157 | loss_dist=0.0105 | loss_mean=0.4052\n",
      "[2019-12-06 19:05:52,708] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=5.9783 | _timers/batch_time=0.3369 | _timers/data_time=0.0004 | _timers/model_time=0.3365 | f1_score=1.640e-06 | loss=0.4096 | loss_dist=0.0078 | loss_mean=0.4018\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.3590 | _timers/batch_time=0.1977 | _timers/data_time=0.0004 | _timers/model_time=0.1973 | f1_score=1.640e-06 | loss=0.4096 | loss_dist=0.0078 | loss_mean=0.4018\n",
      "[2019-12-06 19:05:57,163] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.5983 | _timers/batch_time=0.3586 | _timers/data_time=0.0003 | _timers/model_time=0.3583 | f1_score=1.495e-06 | loss=0.4240 | loss_dist=0.0202 | loss_mean=0.4038\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=9.8869 | _timers/batch_time=0.2062 | _timers/data_time=0.0004 | _timers/model_time=0.2058 | f1_score=1.495e-06 | loss=0.4240 | loss_dist=0.0202 | loss_mean=0.4038\n",
      "[2019-12-06 19:06:01,669] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.6002 | _timers/batch_time=0.3572 | _timers/data_time=0.0003 | _timers/model_time=0.3569 | f1_score=1.269e-06 | loss=0.4256 | loss_dist=0.0189 | loss_mean=0.4067\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.0828 | _timers/batch_time=0.2019 | _timers/data_time=0.0003 | _timers/model_time=0.2016 | f1_score=1.269e-06 | loss=0.4256 | loss_dist=0.0189 | loss_mean=0.4067\n",
      "[2019-12-06 19:06:05,820] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.9539 | _timers/batch_time=0.3365 | _timers/data_time=0.0003 | _timers/model_time=0.3361 | f1_score=1.310e-06 | loss=0.4221 | loss_dist=0.0160 | loss_mean=0.4061\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.8167 | _timers/batch_time=0.1728 | _timers/data_time=0.0003 | _timers/model_time=0.1725 | f1_score=1.310e-06 | loss=0.4221 | loss_dist=0.0160 | loss_mean=0.4061\n",
      "[2019-12-06 19:06:09,944] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3617 | _timers/batch_time=0.3180 | _timers/data_time=0.0003 | _timers/model_time=0.3177 | f1_score=1.740e-06 | loss=0.4136 | loss_dist=0.0069 | loss_mean=0.4067\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.6929 | _timers/batch_time=0.1952 | _timers/data_time=0.0002 | _timers/model_time=0.1950 | f1_score=1.740e-06 | loss=0.4136 | loss_dist=0.0069 | loss_mean=0.4067\n",
      "[2019-12-06 19:06:13,999] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.1984 | _timers/batch_time=0.3258 | _timers/data_time=0.0003 | _timers/model_time=0.3254 | f1_score=1.553e-06 | loss=0.4142 | loss_dist=0.0090 | loss_mean=0.4052\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.9726 | _timers/batch_time=0.1858 | _timers/data_time=0.0003 | _timers/model_time=0.1855 | f1_score=1.553e-06 | loss=0.4142 | loss_dist=0.0090 | loss_mean=0.4052\n",
      "[2019-12-06 19:06:18,162] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.1800 | _timers/batch_time=0.3256 | _timers/data_time=0.0004 | _timers/model_time=0.3252 | f1_score=1.359e-06 | loss=0.4224 | loss_dist=0.0141 | loss_mean=0.4083\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.3952 | _timers/batch_time=0.1796 | _timers/data_time=0.0003 | _timers/model_time=0.1792 | f1_score=1.359e-06 | loss=0.4224 | loss_dist=0.0141 | loss_mean=0.4083\n",
      "[2019-12-06 19:06:22,023] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4829 | _timers/batch_time=0.3105 | _timers/data_time=0.0003 | _timers/model_time=0.3102 | f1_score=1.370e-06 | loss=0.4012 | loss_dist=0.0029 | loss_mean=0.3983\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.4402 | _timers/batch_time=0.1767 | _timers/data_time=0.0003 | _timers/model_time=0.1764 | f1_score=1.370e-06 | loss=0.4012 | loss_dist=0.0029 | loss_mean=0.3983\n",
      "[2019-12-06 19:06:25,953] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4459 | _timers/batch_time=0.3119 | _timers/data_time=0.0003 | _timers/model_time=0.3116 | f1_score=1.405e-06 | loss=0.4334 | loss_dist=0.0260 | loss_mean=0.4074\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.1057 | _timers/batch_time=0.1825 | _timers/data_time=0.0003 | _timers/model_time=0.1822 | f1_score=1.405e-06 | loss=0.4334 | loss_dist=0.0260 | loss_mean=0.4074\n",
      "[2019-12-06 19:06:30,032] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.2339 | _timers/batch_time=0.3223 | _timers/data_time=0.0004 | _timers/model_time=0.3218 | f1_score=1.537e-06 | loss=0.4043 | loss_dist=0.0027 | loss_mean=0.4016\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.4752 | _timers/batch_time=0.1785 | _timers/data_time=0.0002 | _timers/model_time=0.1783 | f1_score=1.537e-06 | loss=0.4043 | loss_dist=0.0027 | loss_mean=0.4016\n",
      "[2019-12-06 19:06:34,145] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.2514 | _timers/batch_time=0.3234 | _timers/data_time=0.0002 | _timers/model_time=0.3232 | f1_score=1.521e-06 | loss=0.4370 | loss_dist=0.0335 | loss_mean=0.4035\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.5189 | _timers/batch_time=0.1922 | _timers/data_time=0.0003 | _timers/model_time=0.1919 | f1_score=1.521e-06 | loss=0.4370 | loss_dist=0.0335 | loss_mean=0.4035\n",
      "[2019-12-06 19:06:38,447] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.0296 | _timers/batch_time=0.3329 | _timers/data_time=0.0004 | _timers/model_time=0.3325 | f1_score=1.608e-06 | loss=0.4272 | loss_dist=0.0208 | loss_mean=0.4064\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.0547 | _timers/batch_time=0.2030 | _timers/data_time=0.0002 | _timers/model_time=0.2028 | f1_score=1.608e-06 | loss=0.4272 | loss_dist=0.0208 | loss_mean=0.4064\n",
      "[2019-12-06 19:06:42,677] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=5.9729 | _timers/batch_time=0.3362 | _timers/data_time=0.0003 | _timers/model_time=0.3359 | f1_score=1.401e-06 | loss=0.4300 | loss_dist=0.0244 | loss_mean=0.4056\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.4999 | _timers/batch_time=0.1950 | _timers/data_time=0.0004 | _timers/model_time=0.1946 | f1_score=1.401e-06 | loss=0.4300 | loss_dist=0.0244 | loss_mean=0.4056\n",
      "[2019-12-06 19:06:46,756] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1051 | _timers/batch_time=0.3286 | _timers/data_time=0.0004 | _timers/model_time=0.3282 | f1_score=1.431e-06 | loss=0.4310 | loss_dist=0.0282 | loss_mean=0.4028\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6172 | _timers/batch_time=0.1750 | _timers/data_time=0.0003 | _timers/model_time=0.1747 | f1_score=1.431e-06 | loss=0.4310 | loss_dist=0.0282 | loss_mean=0.4028\n",
      "[2019-12-06 19:06:50,985] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.0702 | _timers/batch_time=0.3332 | _timers/data_time=0.0003 | _timers/model_time=0.3328 | f1_score=1.468e-06 | loss=0.4146 | loss_dist=0.0129 | loss_mean=0.4016\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.1635 | _timers/batch_time=0.1995 | _timers/data_time=0.0004 | _timers/model_time=0.1991 | f1_score=1.468e-06 | loss=0.4146 | loss_dist=0.0129 | loss_mean=0.4016\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:06:55,347] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=5.7135 | _timers/batch_time=0.3510 | _timers/data_time=0.0003 | _timers/model_time=0.3507 | f1_score=1.390e-06 | loss=0.4668 | loss_dist=0.0653 | loss_mean=0.4015\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.1566 | _timers/batch_time=0.1836 | _timers/data_time=0.0003 | _timers/model_time=0.1832 | f1_score=1.390e-06 | loss=0.4668 | loss_dist=0.0653 | loss_mean=0.4015\n",
      "[2019-12-06 19:06:59,552] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.2091 | _timers/batch_time=0.3252 | _timers/data_time=0.0003 | _timers/model_time=0.3248 | f1_score=1.427e-06 | loss=0.4353 | loss_dist=0.0297 | loss_mean=0.4056\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.1371 | _timers/batch_time=0.2006 | _timers/data_time=0.0003 | _timers/model_time=0.2002 | f1_score=1.427e-06 | loss=0.4353 | loss_dist=0.0297 | loss_mean=0.4056\n",
      "[2019-12-06 19:07:03,710] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1652 | _timers/batch_time=0.3247 | _timers/data_time=0.0002 | _timers/model_time=0.3245 | f1_score=1.416e-06 | loss=0.4207 | loss_dist=0.0179 | loss_mean=0.4028\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.0103 | _timers/batch_time=0.1851 | _timers/data_time=0.0003 | _timers/model_time=0.1848 | f1_score=1.416e-06 | loss=0.4207 | loss_dist=0.0179 | loss_mean=0.4028\n",
      "[2019-12-06 19:07:07,780] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1050 | _timers/batch_time=0.3305 | _timers/data_time=0.0003 | _timers/model_time=0.3301 | f1_score=1.413e-06 | loss=0.4169 | loss_dist=0.0117 | loss_mean=0.4052\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.6983 | _timers/batch_time=0.1746 | _timers/data_time=0.0002 | _timers/model_time=0.1743 | f1_score=1.413e-06 | loss=0.4169 | loss_dist=0.0117 | loss_mean=0.4052\n",
      "[2019-12-06 19:07:11,605] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5483 | _timers/batch_time=0.3064 | _timers/data_time=0.0004 | _timers/model_time=0.3059 | f1_score=1.235e-06 | loss=0.4261 | loss_dist=0.0163 | loss_mean=0.4098\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.7463 | _timers/batch_time=0.1735 | _timers/data_time=0.0004 | _timers/model_time=0.1731 | f1_score=1.235e-06 | loss=0.4261 | loss_dist=0.0163 | loss_mean=0.4098\n",
      "[2019-12-06 19:07:15,593] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3748 | _timers/batch_time=0.3161 | _timers/data_time=0.0004 | _timers/model_time=0.3157 | f1_score=1.722e-06 | loss=0.4028 | loss_dist=0.0009 | loss_mean=0.4019\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.8710 | _timers/batch_time=0.1884 | _timers/data_time=0.0003 | _timers/model_time=0.1881 | f1_score=1.722e-06 | loss=0.4028 | loss_dist=0.0009 | loss_mean=0.4019\n",
      "[2019-12-06 19:07:19,686] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1797 | _timers/batch_time=0.3237 | _timers/data_time=0.0004 | _timers/model_time=0.3233 | f1_score=1.452e-06 | loss=0.4538 | loss_dist=0.0501 | loss_mean=0.4038\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.0664 | _timers/batch_time=0.1843 | _timers/data_time=0.0003 | _timers/model_time=0.1839 | f1_score=1.452e-06 | loss=0.4538 | loss_dist=0.0501 | loss_mean=0.4038\n",
      "[2019-12-06 19:07:23,637] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3197 | _timers/batch_time=0.3183 | _timers/data_time=0.0002 | _timers/model_time=0.3180 | f1_score=1.756e-06 | loss=0.4149 | loss_dist=0.0126 | loss_mean=0.4023\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.5962 | _timers/batch_time=0.1762 | _timers/data_time=0.0002 | _timers/model_time=0.1760 | f1_score=1.756e-06 | loss=0.4149 | loss_dist=0.0126 | loss_mean=0.4023\n",
      "[2019-12-06 19:07:27,836] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.9155 | _timers/batch_time=0.3393 | _timers/data_time=0.0004 | _timers/model_time=0.3388 | f1_score=1.783e-06 | loss=0.4349 | loss_dist=0.0300 | loss_mean=0.4049\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.6043 | _timers/batch_time=0.1942 | _timers/data_time=0.0003 | _timers/model_time=0.1939 | f1_score=1.783e-06 | loss=0.4349 | loss_dist=0.0300 | loss_mean=0.4049\n",
      "[2019-12-06 19:07:31,972] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.1572 | _timers/batch_time=0.3280 | _timers/data_time=0.0003 | _timers/model_time=0.3276 | f1_score=1.214e-06 | loss=2.4141 | loss_dist=2.0061 | loss_mean=0.4080\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.4583 | _timers/batch_time=0.1960 | _timers/data_time=0.0003 | _timers/model_time=0.1957 | f1_score=1.214e-06 | loss=2.4141 | loss_dist=2.0061 | loss_mean=0.4080\n",
      "[2019-12-06 19:07:35,832] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5082 | _timers/batch_time=0.3088 | _timers/data_time=0.0003 | _timers/model_time=0.3085 | f1_score=1.612e-06 | loss=0.4088 | loss_dist=0.0015 | loss_mean=0.4073\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7929 | _timers/batch_time=0.1725 | _timers/data_time=0.0004 | _timers/model_time=0.1721 | f1_score=1.612e-06 | loss=0.4088 | loss_dist=0.0015 | loss_mean=0.4073\n",
      "[2019-12-06 19:07:39,681] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5894 | _timers/batch_time=0.3049 | _timers/data_time=0.0003 | _timers/model_time=0.3045 | f1_score=1.515e-06 | loss=0.4172 | loss_dist=0.0098 | loss_mean=0.4074\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.6606 | _timers/batch_time=0.1758 | _timers/data_time=0.0003 | _timers/model_time=0.1754 | f1_score=1.515e-06 | loss=0.4172 | loss_dist=0.0098 | loss_mean=0.4074\n",
      "[2019-12-06 19:07:43,468] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6152 | _timers/batch_time=0.3037 | _timers/data_time=0.0003 | _timers/model_time=0.3033 | f1_score=1.672e-06 | loss=0.4018 | loss_dist=0.0014 | loss_mean=0.4004\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.9036 | _timers/batch_time=0.1713 | _timers/data_time=0.0003 | _timers/model_time=0.1710 | f1_score=1.672e-06 | loss=0.4018 | loss_dist=0.0014 | loss_mean=0.4004\n",
      "[2019-12-06 19:07:47,258] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.6019 | _timers/batch_time=0.3044 | _timers/data_time=0.0004 | _timers/model_time=0.3039 | f1_score=1.601e-06 | loss=0.4252 | loss_dist=0.0229 | loss_mean=0.4023\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.7800 | _timers/batch_time=0.1728 | _timers/data_time=0.0004 | _timers/model_time=0.1724 | f1_score=1.601e-06 | loss=0.4252 | loss_dist=0.0229 | loss_mean=0.4023\n",
      "[2019-12-06 19:07:51,076] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5387 | _timers/batch_time=0.3076 | _timers/data_time=0.0003 | _timers/model_time=0.3072 | f1_score=1.548e-06 | loss=0.4344 | loss_dist=0.0275 | loss_mean=0.4068\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=12.0138 | _timers/batch_time=0.1692 | _timers/data_time=0.0003 | _timers/model_time=0.1689 | f1_score=1.548e-06 | loss=0.4344 | loss_dist=0.0275 | loss_mean=0.4068\n",
      "[2019-12-06 19:07:54,910] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5725 | _timers/batch_time=0.3056 | _timers/data_time=0.0003 | _timers/model_time=0.3052 | f1_score=1.471e-06 | loss=0.4235 | loss_dist=0.0175 | loss_mean=0.4060\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.8397 | _timers/batch_time=0.1726 | _timers/data_time=0.0002 | _timers/model_time=0.1724 | f1_score=1.471e-06 | loss=0.4235 | loss_dist=0.0175 | loss_mean=0.4060\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:07:59,009] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.3808 | _timers/batch_time=0.3159 | _timers/data_time=0.0003 | _timers/model_time=0.3156 | f1_score=1.685e-06 | loss=0.4139 | loss_dist=0.0064 | loss_mean=0.4075\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.8258 | _timers/batch_time=0.1887 | _timers/data_time=0.0003 | _timers/model_time=0.1884 | f1_score=1.685e-06 | loss=0.4139 | loss_dist=0.0064 | loss_mean=0.4075\n",
      "[2019-12-06 19:08:03,304] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=5.9379 | _timers/batch_time=0.3380 | _timers/data_time=0.0003 | _timers/model_time=0.3377 | f1_score=1.552e-06 | loss=0.4232 | loss_dist=0.0171 | loss_mean=0.4061\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.4001 | _timers/batch_time=0.1965 | _timers/data_time=0.0003 | _timers/model_time=0.1963 | f1_score=1.552e-06 | loss=0.4232 | loss_dist=0.0171 | loss_mean=0.4061\n",
      "[2019-12-06 19:08:07,552] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.0351 | _timers/batch_time=0.3320 | _timers/data_time=0.0004 | _timers/model_time=0.3316 | f1_score=1.470e-06 | loss=0.4156 | loss_dist=0.0089 | loss_mean=0.4067\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.3821 | _timers/batch_time=0.1972 | _timers/data_time=0.0002 | _timers/model_time=0.1969 | f1_score=1.470e-06 | loss=0.4156 | loss_dist=0.0089 | loss_mean=0.4067\n",
      "[2019-12-06 19:08:11,622] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1110 | _timers/batch_time=0.3274 | _timers/data_time=0.0002 | _timers/model_time=0.3271 | f1_score=1.622e-06 | loss=0.4185 | loss_dist=0.0121 | loss_mean=0.4064\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5232 | _timers/batch_time=0.1774 | _timers/data_time=0.0002 | _timers/model_time=0.1771 | f1_score=1.622e-06 | loss=0.4185 | loss_dist=0.0121 | loss_mean=0.4064\n",
      "[2019-12-06 19:08:15,632] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2900 | _timers/batch_time=0.3191 | _timers/data_time=0.0004 | _timers/model_time=0.3187 | f1_score=1.323e-06 | loss=0.4135 | loss_dist=0.0097 | loss_mean=0.4039\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.1660 | _timers/batch_time=0.1815 | _timers/data_time=0.0003 | _timers/model_time=0.1812 | f1_score=1.323e-06 | loss=0.4135 | loss_dist=0.0097 | loss_mean=0.4039\n",
      "[2019-12-06 19:08:19,748] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1180 | _timers/batch_time=0.3284 | _timers/data_time=0.0003 | _timers/model_time=0.3281 | f1_score=1.440e-06 | loss=0.4185 | loss_dist=0.0156 | loss_mean=0.4029\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.4419 | _timers/batch_time=0.1801 | _timers/data_time=0.0005 | _timers/model_time=0.1796 | f1_score=1.440e-06 | loss=0.4185 | loss_dist=0.0156 | loss_mean=0.4029\n",
      "[2019-12-06 19:08:23,831] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2580 | _timers/batch_time=0.3217 | _timers/data_time=0.0002 | _timers/model_time=0.3214 | f1_score=1.582e-06 | loss=0.4094 | loss_dist=0.0076 | loss_mean=0.4018\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.9765 | _timers/batch_time=0.1859 | _timers/data_time=0.0004 | _timers/model_time=0.1855 | f1_score=1.582e-06 | loss=0.4094 | loss_dist=0.0076 | loss_mean=0.4018\n",
      "[2019-12-06 19:08:27,756] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2329 | _timers/batch_time=0.3219 | _timers/data_time=0.0003 | _timers/model_time=0.3216 | f1_score=1.320e-06 | loss=0.4553 | loss_dist=0.0497 | loss_mean=0.4056\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8337 | _timers/batch_time=0.1736 | _timers/data_time=0.0003 | _timers/model_time=0.1732 | f1_score=1.320e-06 | loss=0.4553 | loss_dist=0.0497 | loss_mean=0.4056\n",
      "[2019-12-06 19:08:31,607] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5818 | _timers/batch_time=0.3050 | _timers/data_time=0.0003 | _timers/model_time=0.3047 | f1_score=1.564e-06 | loss=0.4063 | loss_dist=0.0073 | loss_mean=0.3990\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5895 | _timers/batch_time=0.1768 | _timers/data_time=0.0004 | _timers/model_time=0.1763 | f1_score=1.564e-06 | loss=0.4063 | loss_dist=0.0073 | loss_mean=0.3990\n",
      "[2019-12-06 19:08:35,490] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5073 | _timers/batch_time=0.3078 | _timers/data_time=0.0004 | _timers/model_time=0.3074 | f1_score=1.613e-06 | loss=0.4130 | loss_dist=0.0090 | loss_mean=0.4040\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.6851 | _timers/batch_time=0.1749 | _timers/data_time=0.0003 | _timers/model_time=0.1746 | f1_score=1.613e-06 | loss=0.4130 | loss_dist=0.0090 | loss_mean=0.4040\n",
      "[2019-12-06 19:08:39,325] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5632 | _timers/batch_time=0.3062 | _timers/data_time=0.0003 | _timers/model_time=0.3058 | f1_score=1.520e-06 | loss=0.4127 | loss_dist=0.0100 | loss_mean=0.4027\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.6177 | _timers/batch_time=0.1765 | _timers/data_time=0.0003 | _timers/model_time=0.1762 | f1_score=1.520e-06 | loss=0.4127 | loss_dist=0.0100 | loss_mean=0.4027\n",
      "[2019-12-06 19:08:43,148] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.6248 | _timers/batch_time=0.3032 | _timers/data_time=0.0004 | _timers/model_time=0.3027 | f1_score=1.714e-06 | loss=0.4043 | loss_dist=0.0029 | loss_mean=0.4014\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.7618 | _timers/batch_time=0.1736 | _timers/data_time=0.0002 | _timers/model_time=0.1734 | f1_score=1.714e-06 | loss=0.4043 | loss_dist=0.0029 | loss_mean=0.4014\n",
      "[2019-12-06 19:08:47,005] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.5574 | _timers/batch_time=0.3065 | _timers/data_time=0.0003 | _timers/model_time=0.3061 | f1_score=1.436e-06 | loss=0.4063 | loss_dist=0.0039 | loss_mean=0.4024\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.6026 | _timers/batch_time=0.1760 | _timers/data_time=0.0003 | _timers/model_time=0.1757 | f1_score=1.436e-06 | loss=0.4063 | loss_dist=0.0039 | loss_mean=0.4024\n",
      "[2019-12-06 19:08:50,822] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.6144 | _timers/batch_time=0.3035 | _timers/data_time=0.0003 | _timers/model_time=0.3032 | f1_score=1.359e-06 | loss=0.4196 | loss_dist=0.0142 | loss_mean=0.4054\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.6404 | _timers/batch_time=0.1758 | _timers/data_time=0.0003 | _timers/model_time=0.1755 | f1_score=1.359e-06 | loss=0.4196 | loss_dist=0.0142 | loss_mean=0.4054\n",
      "[2019-12-06 19:08:54,677] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4851 | _timers/batch_time=0.3094 | _timers/data_time=0.0004 | _timers/model_time=0.3089 | f1_score=1.236e-06 | loss=0.4146 | loss_dist=0.0061 | loss_mean=0.4085\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.7521 | _timers/batch_time=0.1745 | _timers/data_time=0.0003 | _timers/model_time=0.1741 | f1_score=1.236e-06 | loss=0.4146 | loss_dist=0.0061 | loss_mean=0.4085\n",
      "[2019-12-06 19:08:58,505] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.6265 | _timers/batch_time=0.3030 | _timers/data_time=0.0004 | _timers/model_time=0.3026 | f1_score=1.571e-06 | loss=0.4294 | loss_dist=0.0251 | loss_mean=0.4043\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.9040 | _timers/batch_time=0.1716 | _timers/data_time=0.0002 | _timers/model_time=0.1714 | f1_score=1.571e-06 | loss=0.4294 | loss_dist=0.0251 | loss_mean=0.4043\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:09:02,466] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.5761 | _timers/batch_time=0.3052 | _timers/data_time=0.0002 | _timers/model_time=0.3050 | f1_score=1.336e-06 | loss=0.4049 | loss_dist=0.0045 | loss_mean=0.4004\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=9.7261 | _timers/batch_time=0.2093 | _timers/data_time=0.0006 | _timers/model_time=0.2087 | f1_score=1.336e-06 | loss=0.4049 | loss_dist=0.0045 | loss_mean=0.4004\n",
      "[2019-12-06 19:09:06,457] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5747 | _timers/batch_time=0.3056 | _timers/data_time=0.0003 | _timers/model_time=0.3052 | f1_score=1.270e-06 | loss=0.4192 | loss_dist=0.0151 | loss_mean=0.4041\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.6705 | _timers/batch_time=0.1752 | _timers/data_time=0.0003 | _timers/model_time=0.1749 | f1_score=1.270e-06 | loss=0.4192 | loss_dist=0.0151 | loss_mean=0.4041\n",
      "[2019-12-06 19:09:10,376] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4562 | _timers/batch_time=0.3120 | _timers/data_time=0.0004 | _timers/model_time=0.3116 | f1_score=1.455e-06 | loss=0.4548 | loss_dist=0.0481 | loss_mean=0.4068\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.4337 | _timers/batch_time=0.1781 | _timers/data_time=0.0004 | _timers/model_time=0.1777 | f1_score=1.455e-06 | loss=0.4548 | loss_dist=0.0481 | loss_mean=0.4068\n",
      "[2019-12-06 19:09:14,216] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4635 | _timers/batch_time=0.3105 | _timers/data_time=0.0005 | _timers/model_time=0.3100 | f1_score=1.647e-06 | loss=0.4154 | loss_dist=0.0081 | loss_mean=0.4072\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.6946 | _timers/batch_time=0.1745 | _timers/data_time=0.0002 | _timers/model_time=0.1742 | f1_score=1.647e-06 | loss=0.4154 | loss_dist=0.0081 | loss_mean=0.4072\n",
      "[2019-12-06 19:09:18,038] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5670 | _timers/batch_time=0.3058 | _timers/data_time=0.0004 | _timers/model_time=0.3054 | f1_score=1.634e-06 | loss=0.4112 | loss_dist=0.0025 | loss_mean=0.4087\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.7467 | _timers/batch_time=0.1736 | _timers/data_time=0.0002 | _timers/model_time=0.1733 | f1_score=1.634e-06 | loss=0.4112 | loss_dist=0.0025 | loss_mean=0.4087\n",
      "[2019-12-06 19:09:21,845] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5095 | _timers/batch_time=0.3090 | _timers/data_time=0.0003 | _timers/model_time=0.3087 | f1_score=1.746e-06 | loss=0.4170 | loss_dist=0.0136 | loss_mean=0.4034\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.9171 | _timers/batch_time=0.1710 | _timers/data_time=0.0003 | _timers/model_time=0.1707 | f1_score=1.746e-06 | loss=0.4170 | loss_dist=0.0136 | loss_mean=0.4034\n",
      "[2019-12-06 19:09:25,710] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5812 | _timers/batch_time=0.3052 | _timers/data_time=0.0003 | _timers/model_time=0.3048 | f1_score=1.451e-06 | loss=0.4642 | loss_dist=0.0595 | loss_mean=0.4047\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.7675 | _timers/batch_time=0.1741 | _timers/data_time=0.0002 | _timers/model_time=0.1738 | f1_score=1.451e-06 | loss=0.4642 | loss_dist=0.0595 | loss_mean=0.4047\n",
      "[2019-12-06 19:09:29,524] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5344 | _timers/batch_time=0.3073 | _timers/data_time=0.0003 | _timers/model_time=0.3068 | f1_score=1.521e-06 | loss=0.4185 | loss_dist=0.0091 | loss_mean=0.4094\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.7272 | _timers/batch_time=0.1733 | _timers/data_time=0.0003 | _timers/model_time=0.1730 | f1_score=1.521e-06 | loss=0.4185 | loss_dist=0.0091 | loss_mean=0.4094\n",
      "[2019-12-06 19:09:33,332] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.6388 | _timers/batch_time=0.3026 | _timers/data_time=0.0003 | _timers/model_time=0.3022 | f1_score=1.491e-06 | loss=0.4118 | loss_dist=0.0086 | loss_mean=0.4033\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.8949 | _timers/batch_time=0.1709 | _timers/data_time=0.0002 | _timers/model_time=0.1707 | f1_score=1.491e-06 | loss=0.4118 | loss_dist=0.0086 | loss_mean=0.4033\n",
      "[2019-12-06 19:09:37,153] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5612 | _timers/batch_time=0.3062 | _timers/data_time=0.0004 | _timers/model_time=0.3058 | f1_score=1.350e-06 | loss=0.4420 | loss_dist=0.0354 | loss_mean=0.4066\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.7776 | _timers/batch_time=0.1735 | _timers/data_time=0.0003 | _timers/model_time=0.1732 | f1_score=1.350e-06 | loss=0.4420 | loss_dist=0.0354 | loss_mean=0.4066\n",
      "[2019-12-06 19:09:40,973] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5603 | _timers/batch_time=0.3063 | _timers/data_time=0.0004 | _timers/model_time=0.3058 | f1_score=1.368e-06 | loss=2.4341 | loss_dist=2.0280 | loss_mean=0.4061\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.7097 | _timers/batch_time=0.1739 | _timers/data_time=0.0002 | _timers/model_time=0.1736 | f1_score=1.368e-06 | loss=2.4341 | loss_dist=2.0280 | loss_mean=0.4061\n",
      "[2019-12-06 19:09:44,880] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.3438 | _timers/batch_time=0.3179 | _timers/data_time=0.0004 | _timers/model_time=0.3175 | f1_score=1.376e-06 | loss=0.4715 | loss_dist=0.0643 | loss_mean=0.4072\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=12.0136 | _timers/batch_time=0.1691 | _timers/data_time=0.0002 | _timers/model_time=0.1688 | f1_score=1.376e-06 | loss=0.4715 | loss_dist=0.0643 | loss_mean=0.4072\n",
      "[2019-12-06 19:09:48,692] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.6236 | _timers/batch_time=0.3030 | _timers/data_time=0.0003 | _timers/model_time=0.3027 | f1_score=1.530e-06 | loss=0.4156 | loss_dist=0.0086 | loss_mean=0.4069\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.7694 | _timers/batch_time=0.1734 | _timers/data_time=0.0003 | _timers/model_time=0.1731 | f1_score=1.530e-06 | loss=0.4156 | loss_dist=0.0086 | loss_mean=0.4069\n",
      "[2019-12-06 19:09:52,531] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5752 | _timers/batch_time=0.3050 | _timers/data_time=0.0003 | _timers/model_time=0.3047 | f1_score=1.804e-06 | loss=0.4095 | loss_dist=0.0031 | loss_mean=0.4064\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.5055 | _timers/batch_time=0.1773 | _timers/data_time=0.0004 | _timers/model_time=0.1769 | f1_score=1.804e-06 | loss=0.4095 | loss_dist=0.0031 | loss_mean=0.4064\n",
      "[2019-12-06 19:09:56,344] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5330 | _timers/batch_time=0.3075 | _timers/data_time=0.0004 | _timers/model_time=0.3071 | f1_score=1.599e-06 | loss=0.4215 | loss_dist=0.0160 | loss_mean=0.4055\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.8530 | _timers/batch_time=0.1722 | _timers/data_time=0.0002 | _timers/model_time=0.1719 | f1_score=1.599e-06 | loss=0.4215 | loss_dist=0.0160 | loss_mean=0.4055\n",
      "[2019-12-06 19:10:00,175] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5358 | _timers/batch_time=0.3072 | _timers/data_time=0.0003 | _timers/model_time=0.3068 | f1_score=1.795e-06 | loss=0.4014 | loss_dist=0.0007 | loss_mean=0.4006\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.6594 | _timers/batch_time=0.1749 | _timers/data_time=0.0003 | _timers/model_time=0.1746 | f1_score=1.795e-06 | loss=0.4014 | loss_dist=0.0007 | loss_mean=0.4006\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:10:04,055] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.4702 | _timers/batch_time=0.3106 | _timers/data_time=0.0003 | _timers/model_time=0.3102 | f1_score=1.421e-06 | loss=0.4277 | loss_dist=0.0202 | loss_mean=0.4074\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.6600 | _timers/batch_time=0.1756 | _timers/data_time=0.0003 | _timers/model_time=0.1752 | f1_score=1.421e-06 | loss=0.4277 | loss_dist=0.0202 | loss_mean=0.4074\n",
      "[2019-12-06 19:10:07,895] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5459 | _timers/batch_time=0.3063 | _timers/data_time=0.0003 | _timers/model_time=0.3059 | f1_score=1.317e-06 | loss=2.9827 | loss_dist=2.5786 | loss_mean=0.4041\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.7777 | _timers/batch_time=0.1730 | _timers/data_time=0.0003 | _timers/model_time=0.1727 | f1_score=1.317e-06 | loss=2.9827 | loss_dist=2.5786 | loss_mean=0.4041\n",
      "[2019-12-06 19:10:11,759] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5246 | _timers/batch_time=0.3076 | _timers/data_time=0.0005 | _timers/model_time=0.3071 | f1_score=1.499e-06 | loss=0.4134 | loss_dist=0.0138 | loss_mean=0.3996\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.7433 | _timers/batch_time=0.1735 | _timers/data_time=0.0003 | _timers/model_time=0.1732 | f1_score=1.499e-06 | loss=0.4134 | loss_dist=0.0138 | loss_mean=0.3996\n",
      "[2019-12-06 19:10:15,562] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5431 | _timers/batch_time=0.3069 | _timers/data_time=0.0004 | _timers/model_time=0.3065 | f1_score=1.528e-06 | loss=0.4109 | loss_dist=0.0061 | loss_mean=0.4048\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=12.0111 | _timers/batch_time=0.1686 | _timers/data_time=0.0003 | _timers/model_time=0.1683 | f1_score=1.528e-06 | loss=0.4109 | loss_dist=0.0061 | loss_mean=0.4048\n",
      "Top best models:\n",
      "logs/b90/checkpoints/train.41.pth\t0.4012\n",
      "logs/b90/checkpoints/train.96.pth\t0.4014\n",
      "logs/b90/checkpoints/train.61.pth\t0.4018\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bernoulli p = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)               \n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        #torch.bernoulli(torch.tensor([1.0]),)\n",
    "        b_new = torch.bernoulli(torch.ones(x.shape[:2]),p).unsqueeze(2).to(self.emb.weight.device)\n",
    "        #b_new = torch.repeat_interleave(bb(0.4), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/b20\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:10:21,998] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.4760 | _timers/batch_time=0.3105 | _timers/data_time=0.0004 | _timers/model_time=0.3101 | f1_score=8.282e-08 | loss=44.1146 | loss_dist=44.0937 | loss_mean=0.0209\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7129 | _timers/batch_time=0.1738 | _timers/data_time=0.0004 | _timers/model_time=0.1734 | f1_score=8.282e-08 | loss=44.1146 | loss_dist=44.0937 | loss_mean=0.0209\n",
      "[2019-12-06 19:10:25,875] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4616 | _timers/batch_time=0.3110 | _timers/data_time=0.0003 | _timers/model_time=0.3107 | f1_score=1.666e-07 | loss=40.4243 | loss_dist=40.4041 | loss_mean=0.0202\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.2197 | _timers/batch_time=0.1814 | _timers/data_time=0.0004 | _timers/model_time=0.1810 | f1_score=1.666e-07 | loss=40.4243 | loss_dist=40.4041 | loss_mean=0.0202\n",
      "[2019-12-06 19:10:29,689] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.5235 | _timers/batch_time=0.3073 | _timers/data_time=0.0003 | _timers/model_time=0.3069 | f1_score=3.590e-07 | loss=18.9891 | loss_dist=18.9681 | loss_mean=0.0210\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.6026 | _timers/batch_time=0.1756 | _timers/data_time=0.0004 | _timers/model_time=0.1751 | f1_score=3.590e-07 | loss=18.9891 | loss_dist=18.9681 | loss_mean=0.0210\n",
      "[2019-12-06 19:10:33,510] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.5485 | _timers/batch_time=0.3067 | _timers/data_time=0.0002 | _timers/model_time=0.3065 | f1_score=3.390e-07 | loss=29.5108 | loss_dist=29.4905 | loss_mean=0.0203\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.5723 | _timers/batch_time=0.1761 | _timers/data_time=0.0003 | _timers/model_time=0.1757 | f1_score=3.390e-07 | loss=29.5108 | loss_dist=29.4905 | loss_mean=0.0203\n",
      "[2019-12-06 19:10:37,292] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.6046 | _timers/batch_time=0.3039 | _timers/data_time=0.0003 | _timers/model_time=0.3036 | f1_score=3.234e-07 | loss=46.6784 | loss_dist=46.6598 | loss_mean=0.0186\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.7831 | _timers/batch_time=0.1722 | _timers/data_time=0.0004 | _timers/model_time=0.1718 | f1_score=3.234e-07 | loss=46.6784 | loss_dist=46.6598 | loss_mean=0.0186\n",
      "[2019-12-06 19:10:41,122] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.5531 | _timers/batch_time=0.3063 | _timers/data_time=0.0004 | _timers/model_time=0.3059 | f1_score=7.936e-08 | loss=67.6313 | loss_dist=67.6103 | loss_mean=0.0210\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.5157 | _timers/batch_time=0.1761 | _timers/data_time=0.0004 | _timers/model_time=0.1757 | f1_score=7.936e-08 | loss=67.6313 | loss_dist=67.6103 | loss_mean=0.0210\n",
      "[2019-12-06 19:10:44,910] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.5896 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3041 | f1_score=1.362e-07 | loss=30.6472 | loss_dist=30.6276 | loss_mean=0.0197\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.7008 | _timers/batch_time=0.1735 | _timers/data_time=0.0005 | _timers/model_time=0.1730 | f1_score=1.362e-07 | loss=30.6472 | loss_dist=30.6276 | loss_mean=0.0197\n",
      "[2019-12-06 19:10:48,728] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.5877 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3042 | f1_score=5.511e-07 | loss=22.2663 | loss_dist=22.2462 | loss_mean=0.0202\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.6629 | _timers/batch_time=0.1749 | _timers/data_time=0.0005 | _timers/model_time=0.1744 | f1_score=5.511e-07 | loss=22.2663 | loss_dist=22.2462 | loss_mean=0.0202\n",
      "[2019-12-06 19:10:52,554] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5525 | _timers/batch_time=0.3058 | _timers/data_time=0.0003 | _timers/model_time=0.3055 | f1_score=8.444e-08 | loss=63.2266 | loss_dist=63.2064 | loss_mean=0.0202\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5970 | _timers/batch_time=0.1761 | _timers/data_time=0.0004 | _timers/model_time=0.1757 | f1_score=8.444e-08 | loss=63.2266 | loss_dist=63.2064 | loss_mean=0.0202\n",
      "[2019-12-06 19:10:56,376] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.4979 | _timers/batch_time=0.3086 | _timers/data_time=0.0003 | _timers/model_time=0.3082 | f1_score=9.980e-08 | loss=25.0069 | loss_dist=24.9879 | loss_mean=0.0190\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6229 | _timers/batch_time=0.1755 | _timers/data_time=0.0004 | _timers/model_time=0.1750 | f1_score=9.980e-08 | loss=25.0069 | loss_dist=24.9879 | loss_mean=0.0190\n",
      "[2019-12-06 19:11:00,206] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5255 | _timers/batch_time=0.3075 | _timers/data_time=0.0003 | _timers/model_time=0.3072 | f1_score=1.152e-07 | loss=33.7071 | loss_dist=33.6874 | loss_mean=0.0196\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6391 | _timers/batch_time=0.1759 | _timers/data_time=0.0004 | _timers/model_time=0.1754 | f1_score=1.152e-07 | loss=33.7071 | loss_dist=33.6874 | loss_mean=0.0196\n",
      "[2019-12-06 19:11:04,072] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5618 | _timers/batch_time=0.3060 | _timers/data_time=0.0003 | _timers/model_time=0.3057 | f1_score=3.327e-07 | loss=41.3264 | loss_dist=41.3067 | loss_mean=0.0197\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.3821 | _timers/batch_time=0.1794 | _timers/data_time=0.0004 | _timers/model_time=0.1789 | f1_score=3.327e-07 | loss=41.3264 | loss_dist=41.3067 | loss_mean=0.0197\n",
      "[2019-12-06 19:11:07,894] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.6032 | _timers/batch_time=0.3037 | _timers/data_time=0.0002 | _timers/model_time=0.3034 | f1_score=2.474e-07 | loss=18.9535 | loss_dist=18.9342 | loss_mean=0.0193\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5323 | _timers/batch_time=0.1774 | _timers/data_time=0.0004 | _timers/model_time=0.1769 | f1_score=2.474e-07 | loss=18.9535 | loss_dist=18.9342 | loss_mean=0.0193\n",
      "[2019-12-06 19:11:11,712] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.6137 | _timers/batch_time=0.3033 | _timers/data_time=0.0003 | _timers/model_time=0.3030 | f1_score=2.154e-07 | loss=38.0094 | loss_dist=37.9888 | loss_mean=0.0207\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.4950 | _timers/batch_time=0.1781 | _timers/data_time=0.0005 | _timers/model_time=0.1775 | f1_score=2.154e-07 | loss=38.0094 | loss_dist=37.9888 | loss_mean=0.0207\n",
      "[2019-12-06 19:11:15,506] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5572 | _timers/batch_time=0.3061 | _timers/data_time=0.0002 | _timers/model_time=0.3059 | f1_score=2.128e-07 | loss=44.7988 | loss_dist=44.7788 | loss_mean=0.0200\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.7477 | _timers/batch_time=0.1734 | _timers/data_time=0.0004 | _timers/model_time=0.1729 | f1_score=2.128e-07 | loss=44.7988 | loss_dist=44.7788 | loss_mean=0.0200\n",
      "[2019-12-06 19:11:19,324] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5745 | _timers/batch_time=0.3052 | _timers/data_time=0.0003 | _timers/model_time=0.3048 | f1_score=1.491e-07 | loss=30.4162 | loss_dist=30.3967 | loss_mean=0.0195\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5257 | _timers/batch_time=0.1761 | _timers/data_time=0.0004 | _timers/model_time=0.1756 | f1_score=1.491e-07 | loss=30.4162 | loss_dist=30.3967 | loss_mean=0.0195\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:11:23,137] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5978 | _timers/batch_time=0.3040 | _timers/data_time=0.0003 | _timers/model_time=0.3036 | f1_score=3.815e-07 | loss=29.6728 | loss_dist=29.6522 | loss_mean=0.0206\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6172 | _timers/batch_time=0.1750 | _timers/data_time=0.0005 | _timers/model_time=0.1745 | f1_score=3.815e-07 | loss=29.6728 | loss_dist=29.6522 | loss_mean=0.0206\n",
      "[2019-12-06 19:11:26,991] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5125 | _timers/batch_time=0.3082 | _timers/data_time=0.0004 | _timers/model_time=0.3078 | f1_score=2.853e-07 | loss=9.7974 | loss_dist=9.7764 | loss_mean=0.0210\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5090 | _timers/batch_time=0.1767 | _timers/data_time=0.0005 | _timers/model_time=0.1762 | f1_score=2.853e-07 | loss=9.7974 | loss_dist=9.7764 | loss_mean=0.0210\n",
      "[2019-12-06 19:11:30,951] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.3633 | _timers/batch_time=0.3164 | _timers/data_time=0.0003 | _timers/model_time=0.3160 | f1_score=1.128e-07 | loss=37.2158 | loss_dist=37.1960 | loss_mean=0.0198\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.2318 | _timers/batch_time=0.1828 | _timers/data_time=0.0004 | _timers/model_time=0.1824 | f1_score=1.128e-07 | loss=37.2158 | loss_dist=37.1960 | loss_mean=0.0198\n",
      "[2019-12-06 19:11:34,783] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5197 | _timers/batch_time=0.3079 | _timers/data_time=0.0004 | _timers/model_time=0.3074 | f1_score=3.105e-07 | loss=28.6629 | loss_dist=28.6414 | loss_mean=0.0215\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6414 | _timers/batch_time=0.1744 | _timers/data_time=0.0004 | _timers/model_time=0.1740 | f1_score=3.105e-07 | loss=28.6629 | loss_dist=28.6414 | loss_mean=0.0215\n",
      "[2019-12-06 19:11:38,614] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5736 | _timers/batch_time=0.3050 | _timers/data_time=0.0003 | _timers/model_time=0.3047 | f1_score=1.012e-07 | loss=58.5510 | loss_dist=58.5310 | loss_mean=0.0200\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.4432 | _timers/batch_time=0.1790 | _timers/data_time=0.0003 | _timers/model_time=0.1786 | f1_score=1.012e-07 | loss=58.5510 | loss_dist=58.5310 | loss_mean=0.0200\n",
      "[2019-12-06 19:11:42,423] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.6006 | _timers/batch_time=0.3039 | _timers/data_time=0.0003 | _timers/model_time=0.3036 | f1_score=4.513e-07 | loss=22.5827 | loss_dist=22.5623 | loss_mean=0.0203\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.6587 | _timers/batch_time=0.1754 | _timers/data_time=0.0004 | _timers/model_time=0.1749 | f1_score=4.513e-07 | loss=22.5827 | loss_dist=22.5623 | loss_mean=0.0203\n",
      "[2019-12-06 19:11:46,217] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.6312 | _timers/batch_time=0.3029 | _timers/data_time=0.0003 | _timers/model_time=0.3026 | f1_score=1.181e-07 | loss=30.7372 | loss_dist=30.7183 | loss_mean=0.0189\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.7254 | _timers/batch_time=0.1737 | _timers/data_time=0.0004 | _timers/model_time=0.1733 | f1_score=1.181e-07 | loss=30.7372 | loss_dist=30.7183 | loss_mean=0.0189\n",
      "[2019-12-06 19:11:50,012] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.6109 | _timers/batch_time=0.3038 | _timers/data_time=0.0004 | _timers/model_time=0.3034 | f1_score=1.091e-07 | loss=51.3423 | loss_dist=51.3224 | loss_mean=0.0198\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.6298 | _timers/batch_time=0.1751 | _timers/data_time=0.0005 | _timers/model_time=0.1746 | f1_score=1.091e-07 | loss=51.3423 | loss_dist=51.3224 | loss_mean=0.0198\n",
      "[2019-12-06 19:11:53,827] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.6203 | _timers/batch_time=0.3032 | _timers/data_time=0.0003 | _timers/model_time=0.3028 | f1_score=1.540e-07 | loss=24.5427 | loss_dist=24.5227 | loss_mean=0.0200\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.5171 | _timers/batch_time=0.1768 | _timers/data_time=0.0005 | _timers/model_time=0.1763 | f1_score=1.540e-07 | loss=24.5427 | loss_dist=24.5227 | loss_mean=0.0200\n",
      "[2019-12-06 19:11:57,594] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.6483 | _timers/batch_time=0.3017 | _timers/data_time=0.0003 | _timers/model_time=0.3013 | f1_score=1.781e-07 | loss=24.1710 | loss_dist=24.1526 | loss_mean=0.0183\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.6487 | _timers/batch_time=0.1747 | _timers/data_time=0.0005 | _timers/model_time=0.1742 | f1_score=1.781e-07 | loss=24.1710 | loss_dist=24.1526 | loss_mean=0.0183\n",
      "[2019-12-06 19:12:01,390] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.6162 | _timers/batch_time=0.3035 | _timers/data_time=0.0004 | _timers/model_time=0.3031 | f1_score=4.339e-07 | loss=34.4361 | loss_dist=34.4161 | loss_mean=0.0199\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.6629 | _timers/batch_time=0.1743 | _timers/data_time=0.0004 | _timers/model_time=0.1739 | f1_score=4.339e-07 | loss=34.4361 | loss_dist=34.4161 | loss_mean=0.0199\n",
      "[2019-12-06 19:12:05,229] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.5395 | _timers/batch_time=0.3071 | _timers/data_time=0.0002 | _timers/model_time=0.3069 | f1_score=9.618e-08 | loss=45.5968 | loss_dist=45.5776 | loss_mean=0.0192\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.2035 | _timers/batch_time=0.1815 | _timers/data_time=0.0004 | _timers/model_time=0.1810 | f1_score=9.618e-08 | loss=45.5968 | loss_dist=45.5776 | loss_mean=0.0192\n",
      "[2019-12-06 19:12:09,036] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.6537 | _timers/batch_time=0.3016 | _timers/data_time=0.0004 | _timers/model_time=0.3012 | f1_score=2.140e-07 | loss=44.1562 | loss_dist=44.1366 | loss_mean=0.0196\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.5693 | _timers/batch_time=0.1771 | _timers/data_time=0.0004 | _timers/model_time=0.1766 | f1_score=2.140e-07 | loss=44.1562 | loss_dist=44.1366 | loss_mean=0.0196\n",
      "[2019-12-06 19:12:12,821] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5959 | _timers/batch_time=0.3049 | _timers/data_time=0.0004 | _timers/model_time=0.3045 | f1_score=6.025e-07 | loss=13.4573 | loss_dist=13.4379 | loss_mean=0.0194\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.6990 | _timers/batch_time=0.1738 | _timers/data_time=0.0004 | _timers/model_time=0.1734 | f1_score=6.025e-07 | loss=13.4573 | loss_dist=13.4379 | loss_mean=0.0194\n",
      "[2019-12-06 19:12:16,643] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5989 | _timers/batch_time=0.3041 | _timers/data_time=0.0003 | _timers/model_time=0.3038 | f1_score=3.136e-07 | loss=13.3528 | loss_dist=13.3314 | loss_mean=0.0214\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.5663 | _timers/batch_time=0.1761 | _timers/data_time=0.0004 | _timers/model_time=0.1756 | f1_score=3.136e-07 | loss=13.3528 | loss_dist=13.3314 | loss_mean=0.0214\n",
      "[2019-12-06 19:12:20,467] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5817 | _timers/batch_time=0.3048 | _timers/data_time=0.0003 | _timers/model_time=0.3044 | f1_score=4.698e-07 | loss=49.1210 | loss_dist=49.1004 | loss_mean=0.0206\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.5733 | _timers/batch_time=0.1778 | _timers/data_time=0.0005 | _timers/model_time=0.1773 | f1_score=4.698e-07 | loss=49.1210 | loss_dist=49.1004 | loss_mean=0.0206\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:12:24,311] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5900 | _timers/batch_time=0.3047 | _timers/data_time=0.0003 | _timers/model_time=0.3044 | f1_score=2.745e-07 | loss=23.9852 | loss_dist=23.9643 | loss_mean=0.0209\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.4609 | _timers/batch_time=0.1779 | _timers/data_time=0.0005 | _timers/model_time=0.1774 | f1_score=2.745e-07 | loss=23.9852 | loss_dist=23.9643 | loss_mean=0.0209\n",
      "[2019-12-06 19:12:28,113] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5793 | _timers/batch_time=0.3054 | _timers/data_time=0.0004 | _timers/model_time=0.3050 | f1_score=2.523e-07 | loss=23.4727 | loss_dist=23.4525 | loss_mean=0.0203\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.6502 | _timers/batch_time=0.1753 | _timers/data_time=0.0005 | _timers/model_time=0.1748 | f1_score=2.523e-07 | loss=23.4727 | loss_dist=23.4525 | loss_mean=0.0203\n",
      "[2019-12-06 19:12:31,908] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.6202 | _timers/batch_time=0.3034 | _timers/data_time=0.0003 | _timers/model_time=0.3030 | f1_score=1.293e-07 | loss=24.9092 | loss_dist=24.8895 | loss_mean=0.0197\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.6358 | _timers/batch_time=0.1752 | _timers/data_time=0.0004 | _timers/model_time=0.1748 | f1_score=1.293e-07 | loss=24.9092 | loss_dist=24.8895 | loss_mean=0.0197\n",
      "[2019-12-06 19:12:35,762] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5971 | _timers/batch_time=0.3047 | _timers/data_time=0.0003 | _timers/model_time=0.3043 | f1_score=8.830e-08 | loss=60.2303 | loss_dist=60.2098 | loss_mean=0.0205\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.5853 | _timers/batch_time=0.1764 | _timers/data_time=0.0003 | _timers/model_time=0.1761 | f1_score=8.830e-08 | loss=60.2303 | loss_dist=60.2098 | loss_mean=0.0205\n",
      "[2019-12-06 19:12:39,584] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5933 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3042 | f1_score=1.196e-07 | loss=50.2033 | loss_dist=50.1830 | loss_mean=0.0203\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.7719 | _timers/batch_time=0.1735 | _timers/data_time=0.0004 | _timers/model_time=0.1731 | f1_score=1.196e-07 | loss=50.2033 | loss_dist=50.1830 | loss_mean=0.0203\n",
      "[2019-12-06 19:12:43,420] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5646 | _timers/batch_time=0.3059 | _timers/data_time=0.0004 | _timers/model_time=0.3055 | f1_score=2.000e-07 | loss=35.3700 | loss_dist=35.3507 | loss_mean=0.0193\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.7310 | _timers/batch_time=0.1737 | _timers/data_time=0.0004 | _timers/model_time=0.1732 | f1_score=2.000e-07 | loss=35.3700 | loss_dist=35.3507 | loss_mean=0.0193\n",
      "[2019-12-06 19:12:47,239] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.6486 | _timers/batch_time=0.3020 | _timers/data_time=0.0003 | _timers/model_time=0.3017 | f1_score=8.112e-08 | loss=57.0572 | loss_dist=57.0369 | loss_mean=0.0203\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.4457 | _timers/batch_time=0.1797 | _timers/data_time=0.0003 | _timers/model_time=0.1793 | f1_score=8.112e-08 | loss=57.0572 | loss_dist=57.0369 | loss_mean=0.0203\n",
      "[2019-12-06 19:12:51,083] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5757 | _timers/batch_time=0.3055 | _timers/data_time=0.0003 | _timers/model_time=0.3052 | f1_score=8.244e-08 | loss=58.3690 | loss_dist=58.3498 | loss_mean=0.0192\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.6352 | _timers/batch_time=0.1764 | _timers/data_time=0.0003 | _timers/model_time=0.1760 | f1_score=8.244e-08 | loss=58.3690 | loss_dist=58.3498 | loss_mean=0.0192\n",
      "[2019-12-06 19:12:54,925] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5667 | _timers/batch_time=0.3056 | _timers/data_time=0.0002 | _timers/model_time=0.3053 | f1_score=3.298e-07 | loss=31.4470 | loss_dist=31.4271 | loss_mean=0.0199\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.8344 | _timers/batch_time=0.1726 | _timers/data_time=0.0003 | _timers/model_time=0.1722 | f1_score=3.298e-07 | loss=31.4470 | loss_dist=31.4271 | loss_mean=0.0199\n",
      "[2019-12-06 19:12:58,725] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.6009 | _timers/batch_time=0.3041 | _timers/data_time=0.0002 | _timers/model_time=0.3038 | f1_score=3.173e-07 | loss=31.7306 | loss_dist=31.7096 | loss_mean=0.0209\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.8803 | _timers/batch_time=0.1724 | _timers/data_time=0.0002 | _timers/model_time=0.1722 | f1_score=3.173e-07 | loss=31.7306 | loss_dist=31.7096 | loss_mean=0.0209\n",
      "[2019-12-06 19:13:02,561] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5392 | _timers/batch_time=0.3067 | _timers/data_time=0.0003 | _timers/model_time=0.3063 | f1_score=3.315e-07 | loss=27.4305 | loss_dist=27.4088 | loss_mean=0.0217\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.5840 | _timers/batch_time=0.1760 | _timers/data_time=0.0002 | _timers/model_time=0.1757 | f1_score=3.315e-07 | loss=27.4305 | loss_dist=27.4088 | loss_mean=0.0217\n",
      "[2019-12-06 19:13:06,380] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5716 | _timers/batch_time=0.3057 | _timers/data_time=0.0003 | _timers/model_time=0.3054 | f1_score=5.626e-07 | loss=11.0248 | loss_dist=11.0048 | loss_mean=0.0200\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.6797 | _timers/batch_time=0.1747 | _timers/data_time=0.0004 | _timers/model_time=0.1743 | f1_score=5.626e-07 | loss=11.0248 | loss_dist=11.0048 | loss_mean=0.0200\n",
      "[2019-12-06 19:13:10,175] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.6701 | _timers/batch_time=0.3011 | _timers/data_time=0.0003 | _timers/model_time=0.3008 | f1_score=8.749e-08 | loss=51.9625 | loss_dist=51.9434 | loss_mean=0.0191\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.7789 | _timers/batch_time=0.1730 | _timers/data_time=0.0004 | _timers/model_time=0.1726 | f1_score=8.749e-08 | loss=51.9625 | loss_dist=51.9434 | loss_mean=0.0191\n",
      "[2019-12-06 19:13:14,011] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.6674 | _timers/batch_time=0.3010 | _timers/data_time=0.0003 | _timers/model_time=0.3006 | f1_score=2.159e-07 | loss=15.2985 | loss_dist=15.2772 | loss_mean=0.0213\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.4280 | _timers/batch_time=0.1792 | _timers/data_time=0.0003 | _timers/model_time=0.1788 | f1_score=2.159e-07 | loss=15.2985 | loss_dist=15.2772 | loss_mean=0.0213\n",
      "[2019-12-06 19:13:17,820] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5720 | _timers/batch_time=0.3053 | _timers/data_time=0.0003 | _timers/model_time=0.3050 | f1_score=8.101e-08 | loss=64.3404 | loss_dist=64.3207 | loss_mean=0.0197\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.6308 | _timers/batch_time=0.1757 | _timers/data_time=0.0004 | _timers/model_time=0.1752 | f1_score=8.101e-08 | loss=64.3404 | loss_dist=64.3207 | loss_mean=0.0197\n",
      "[2019-12-06 19:13:21,636] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5047 | _timers/batch_time=0.3087 | _timers/data_time=0.0003 | _timers/model_time=0.3084 | f1_score=1.616e-07 | loss=43.7018 | loss_dist=43.6839 | loss_mean=0.0178\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.6803 | _timers/batch_time=0.1744 | _timers/data_time=0.0004 | _timers/model_time=0.1739 | f1_score=1.616e-07 | loss=43.7018 | loss_dist=43.6839 | loss_mean=0.0178\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:13:25,464] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.6157 | _timers/batch_time=0.3039 | _timers/data_time=0.0003 | _timers/model_time=0.3035 | f1_score=1.278e-07 | loss=29.1340 | loss_dist=29.1146 | loss_mean=0.0194\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.5164 | _timers/batch_time=0.1774 | _timers/data_time=0.0005 | _timers/model_time=0.1768 | f1_score=1.278e-07 | loss=29.1340 | loss_dist=29.1146 | loss_mean=0.0194\n",
      "[2019-12-06 19:13:29,280] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.6524 | _timers/batch_time=0.3017 | _timers/data_time=0.0004 | _timers/model_time=0.3013 | f1_score=1.487e-07 | loss=43.7522 | loss_dist=43.7314 | loss_mean=0.0208\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.5598 | _timers/batch_time=0.1763 | _timers/data_time=0.0004 | _timers/model_time=0.1758 | f1_score=1.487e-07 | loss=43.7522 | loss_dist=43.7314 | loss_mean=0.0208\n",
      "[2019-12-06 19:13:33,056] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5872 | _timers/batch_time=0.3047 | _timers/data_time=0.0003 | _timers/model_time=0.3043 | f1_score=2.803e-07 | loss=25.7166 | loss_dist=25.6957 | loss_mean=0.0210\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.6100 | _timers/batch_time=0.1747 | _timers/data_time=0.0003 | _timers/model_time=0.1743 | f1_score=2.803e-07 | loss=25.7166 | loss_dist=25.6957 | loss_mean=0.0210\n",
      "[2019-12-06 19:13:36,906] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.6147 | _timers/batch_time=0.3036 | _timers/data_time=0.0003 | _timers/model_time=0.3032 | f1_score=8.083e-08 | loss=46.3199 | loss_dist=46.3007 | loss_mean=0.0192\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.1377 | _timers/batch_time=0.1825 | _timers/data_time=0.0004 | _timers/model_time=0.1820 | f1_score=8.083e-08 | loss=46.3199 | loss_dist=46.3007 | loss_mean=0.0192\n",
      "[2019-12-06 19:13:40,782] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.4717 | _timers/batch_time=0.3095 | _timers/data_time=0.0003 | _timers/model_time=0.3092 | f1_score=9.033e-08 | loss=43.0293 | loss_dist=43.0091 | loss_mean=0.0203\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.6497 | _timers/batch_time=0.1751 | _timers/data_time=0.0004 | _timers/model_time=0.1746 | f1_score=9.033e-08 | loss=43.0293 | loss_dist=43.0091 | loss_mean=0.0203\n",
      "[2019-12-06 19:13:44,567] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.6054 | _timers/batch_time=0.3037 | _timers/data_time=0.0002 | _timers/model_time=0.3035 | f1_score=1.604e-07 | loss=20.6342 | loss_dist=20.6137 | loss_mean=0.0205\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6602 | _timers/batch_time=0.1747 | _timers/data_time=0.0004 | _timers/model_time=0.1743 | f1_score=1.604e-07 | loss=20.6342 | loss_dist=20.6137 | loss_mean=0.0205\n",
      "[2019-12-06 19:13:48,312] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.6690 | _timers/batch_time=0.3009 | _timers/data_time=0.0004 | _timers/model_time=0.3004 | f1_score=9.192e-08 | loss=35.3395 | loss_dist=35.3204 | loss_mean=0.0191\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.8190 | _timers/batch_time=0.1718 | _timers/data_time=0.0005 | _timers/model_time=0.1713 | f1_score=9.192e-08 | loss=35.3395 | loss_dist=35.3204 | loss_mean=0.0191\n",
      "[2019-12-06 19:13:52,131] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5617 | _timers/batch_time=0.3057 | _timers/data_time=0.0004 | _timers/model_time=0.3053 | f1_score=1.240e-07 | loss=30.2116 | loss_dist=30.1921 | loss_mean=0.0195\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6705 | _timers/batch_time=0.1746 | _timers/data_time=0.0003 | _timers/model_time=0.1743 | f1_score=1.240e-07 | loss=30.2116 | loss_dist=30.1921 | loss_mean=0.0195\n",
      "[2019-12-06 19:13:55,947] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5884 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3042 | f1_score=1.003e-07 | loss=36.1703 | loss_dist=36.1492 | loss_mean=0.0211\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.7462 | _timers/batch_time=0.1727 | _timers/data_time=0.0004 | _timers/model_time=0.1722 | f1_score=1.003e-07 | loss=36.1703 | loss_dist=36.1492 | loss_mean=0.0211\n",
      "[2019-12-06 19:13:59,795] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5945 | _timers/batch_time=0.3039 | _timers/data_time=0.0003 | _timers/model_time=0.3035 | f1_score=2.154e-07 | loss=34.2086 | loss_dist=34.1881 | loss_mean=0.0205\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.5118 | _timers/batch_time=0.1783 | _timers/data_time=0.0004 | _timers/model_time=0.1779 | f1_score=2.154e-07 | loss=34.2086 | loss_dist=34.1881 | loss_mean=0.0205\n",
      "[2019-12-06 19:14:03,631] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5931 | _timers/batch_time=0.3041 | _timers/data_time=0.0003 | _timers/model_time=0.3037 | f1_score=3.524e-07 | loss=19.8936 | loss_dist=19.8741 | loss_mean=0.0195\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.6043 | _timers/batch_time=0.1763 | _timers/data_time=0.0003 | _timers/model_time=0.1760 | f1_score=3.524e-07 | loss=19.8936 | loss_dist=19.8741 | loss_mean=0.0195\n",
      "[2019-12-06 19:14:07,419] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.6071 | _timers/batch_time=0.3035 | _timers/data_time=0.0003 | _timers/model_time=0.3032 | f1_score=1.019e-07 | loss=41.1637 | loss_dist=41.1442 | loss_mean=0.0195\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.7087 | _timers/batch_time=0.1735 | _timers/data_time=0.0004 | _timers/model_time=0.1732 | f1_score=1.019e-07 | loss=41.1637 | loss_dist=41.1442 | loss_mean=0.0195\n",
      "[2019-12-06 19:14:11,233] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.6053 | _timers/batch_time=0.3039 | _timers/data_time=0.0003 | _timers/model_time=0.3035 | f1_score=4.813e-07 | loss=17.7548 | loss_dist=17.7337 | loss_mean=0.0210\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.6182 | _timers/batch_time=0.1749 | _timers/data_time=0.0002 | _timers/model_time=0.1747 | f1_score=4.813e-07 | loss=17.7548 | loss_dist=17.7337 | loss_mean=0.0210\n",
      "[2019-12-06 19:14:15,041] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.4879 | _timers/batch_time=0.3084 | _timers/data_time=0.0003 | _timers/model_time=0.3081 | f1_score=2.008e-07 | loss=20.9460 | loss_dist=20.9256 | loss_mean=0.0204\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.6681 | _timers/batch_time=0.1744 | _timers/data_time=0.0004 | _timers/model_time=0.1740 | f1_score=2.008e-07 | loss=20.9460 | loss_dist=20.9256 | loss_mean=0.0204\n",
      "[2019-12-06 19:14:18,886] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5614 | _timers/batch_time=0.3056 | _timers/data_time=0.0004 | _timers/model_time=0.3052 | f1_score=1.563e-07 | loss=30.5310 | loss_dist=30.5109 | loss_mean=0.0201\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.5673 | _timers/batch_time=0.1761 | _timers/data_time=0.0003 | _timers/model_time=0.1759 | f1_score=1.563e-07 | loss=30.5310 | loss_dist=30.5109 | loss_mean=0.0201\n",
      "[2019-12-06 19:14:22,694] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6055 | _timers/batch_time=0.3039 | _timers/data_time=0.0003 | _timers/model_time=0.3035 | f1_score=1.291e-07 | loss=26.3913 | loss_dist=26.3718 | loss_mean=0.0196\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.8828 | _timers/batch_time=0.1720 | _timers/data_time=0.0003 | _timers/model_time=0.1717 | f1_score=1.291e-07 | loss=26.3913 | loss_dist=26.3718 | loss_mean=0.0196\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:14:26,519] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6293 | _timers/batch_time=0.3025 | _timers/data_time=0.0003 | _timers/model_time=0.3021 | f1_score=3.429e-07 | loss=8.0419 | loss_dist=8.0215 | loss_mean=0.0204\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7883 | _timers/batch_time=0.1731 | _timers/data_time=0.0003 | _timers/model_time=0.1727 | f1_score=3.429e-07 | loss=8.0419 | loss_dist=8.0215 | loss_mean=0.0204\n",
      "[2019-12-06 19:14:30,332] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6194 | _timers/batch_time=0.3035 | _timers/data_time=0.0004 | _timers/model_time=0.3030 | f1_score=1.234e-07 | loss=35.8699 | loss_dist=35.8511 | loss_mean=0.0187\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.8697 | _timers/batch_time=0.1712 | _timers/data_time=0.0002 | _timers/model_time=0.1710 | f1_score=1.234e-07 | loss=35.8699 | loss_dist=35.8511 | loss_mean=0.0187\n",
      "[2019-12-06 19:14:34,110] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.6017 | _timers/batch_time=0.3044 | _timers/data_time=0.0003 | _timers/model_time=0.3040 | f1_score=9.546e-08 | loss=42.4694 | loss_dist=42.4493 | loss_mean=0.0201\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.8693 | _timers/batch_time=0.1715 | _timers/data_time=0.0003 | _timers/model_time=0.1711 | f1_score=9.546e-08 | loss=42.4694 | loss_dist=42.4493 | loss_mean=0.0201\n",
      "[2019-12-06 19:14:37,925] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5435 | _timers/batch_time=0.3068 | _timers/data_time=0.0004 | _timers/model_time=0.3064 | f1_score=8.760e-08 | loss=42.8250 | loss_dist=42.8043 | loss_mean=0.0207\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7536 | _timers/batch_time=0.1737 | _timers/data_time=0.0002 | _timers/model_time=0.1735 | f1_score=8.760e-08 | loss=42.8250 | loss_dist=42.8043 | loss_mean=0.0207\n",
      "[2019-12-06 19:14:41,822] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5593 | _timers/batch_time=0.3058 | _timers/data_time=0.0003 | _timers/model_time=0.3055 | f1_score=1.043e-07 | loss=44.7891 | loss_dist=44.7706 | loss_mean=0.0185\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.4225 | _timers/batch_time=0.1779 | _timers/data_time=0.0002 | _timers/model_time=0.1777 | f1_score=1.043e-07 | loss=44.7891 | loss_dist=44.7706 | loss_mean=0.0185\n",
      "[2019-12-06 19:14:45,624] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5211 | _timers/batch_time=0.3079 | _timers/data_time=0.0003 | _timers/model_time=0.3074 | f1_score=1.709e-07 | loss=29.7020 | loss_dist=29.6835 | loss_mean=0.0185\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.8496 | _timers/batch_time=0.1722 | _timers/data_time=0.0002 | _timers/model_time=0.1719 | f1_score=1.709e-07 | loss=29.7020 | loss_dist=29.6835 | loss_mean=0.0185\n",
      "[2019-12-06 19:14:49,456] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5814 | _timers/batch_time=0.3051 | _timers/data_time=0.0004 | _timers/model_time=0.3046 | f1_score=1.282e-07 | loss=38.1411 | loss_dist=38.1219 | loss_mean=0.0192\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.4691 | _timers/batch_time=0.1777 | _timers/data_time=0.0003 | _timers/model_time=0.1774 | f1_score=1.282e-07 | loss=38.1411 | loss_dist=38.1219 | loss_mean=0.0192\n",
      "[2019-12-06 19:14:53,406] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4414 | _timers/batch_time=0.3125 | _timers/data_time=0.0003 | _timers/model_time=0.3122 | f1_score=9.426e-08 | loss=42.3525 | loss_dist=42.3331 | loss_mean=0.0193\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.6738 | _timers/batch_time=0.1749 | _timers/data_time=0.0003 | _timers/model_time=0.1745 | f1_score=9.426e-08 | loss=42.3525 | loss_dist=42.3331 | loss_mean=0.0193\n",
      "[2019-12-06 19:14:57,290] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.3812 | _timers/batch_time=0.3150 | _timers/data_time=0.0003 | _timers/model_time=0.3146 | f1_score=2.710e-07 | loss=21.1396 | loss_dist=21.1184 | loss_mean=0.0212\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.6251 | _timers/batch_time=0.1750 | _timers/data_time=0.0002 | _timers/model_time=0.1748 | f1_score=2.710e-07 | loss=21.1396 | loss_dist=21.1184 | loss_mean=0.0212\n",
      "[2019-12-06 19:15:01,110] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5498 | _timers/batch_time=0.3064 | _timers/data_time=0.0003 | _timers/model_time=0.3061 | f1_score=1.527e-07 | loss=18.6608 | loss_dist=18.6405 | loss_mean=0.0202\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.8989 | _timers/batch_time=0.1704 | _timers/data_time=0.0003 | _timers/model_time=0.1701 | f1_score=1.527e-07 | loss=18.6608 | loss_dist=18.6405 | loss_mean=0.0202\n",
      "[2019-12-06 19:15:04,989] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5976 | _timers/batch_time=0.3042 | _timers/data_time=0.0004 | _timers/model_time=0.3037 | f1_score=7.344e-08 | loss=53.0341 | loss_dist=53.0137 | loss_mean=0.0204\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.6386 | _timers/batch_time=0.1766 | _timers/data_time=0.0003 | _timers/model_time=0.1762 | f1_score=7.344e-08 | loss=53.0341 | loss_dist=53.0137 | loss_mean=0.0204\n",
      "[2019-12-06 19:15:08,948] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3278 | _timers/batch_time=0.3173 | _timers/data_time=0.0004 | _timers/model_time=0.3169 | f1_score=1.025e-07 | loss=43.3063 | loss_dist=43.2870 | loss_mean=0.0193\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.0262 | _timers/batch_time=0.1833 | _timers/data_time=0.0003 | _timers/model_time=0.1830 | f1_score=1.025e-07 | loss=43.3063 | loss_dist=43.2870 | loss_mean=0.0193\n",
      "[2019-12-06 19:15:13,092] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3577 | _timers/batch_time=0.3176 | _timers/data_time=0.0003 | _timers/model_time=0.3173 | f1_score=1.791e-07 | loss=22.8449 | loss_dist=22.8246 | loss_mean=0.0203\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.1176 | _timers/batch_time=0.1858 | _timers/data_time=0.0003 | _timers/model_time=0.1855 | f1_score=1.791e-07 | loss=22.8449 | loss_dist=22.8246 | loss_mean=0.0203\n",
      "[2019-12-06 19:15:17,305] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=5.9506 | _timers/batch_time=0.3371 | _timers/data_time=0.0003 | _timers/model_time=0.3368 | f1_score=2.078e-07 | loss=16.2047 | loss_dist=16.1849 | loss_mean=0.0198\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.9400 | _timers/batch_time=0.1854 | _timers/data_time=0.0002 | _timers/model_time=0.1852 | f1_score=2.078e-07 | loss=16.2047 | loss_dist=16.1849 | loss_mean=0.0198\n",
      "[2019-12-06 19:15:21,403] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.0674 | _timers/batch_time=0.3300 | _timers/data_time=0.0003 | _timers/model_time=0.3297 | f1_score=1.139e-07 | loss=35.2713 | loss_dist=35.2509 | loss_mean=0.0204\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.8655 | _timers/batch_time=0.1879 | _timers/data_time=0.0004 | _timers/model_time=0.1874 | f1_score=1.139e-07 | loss=35.2713 | loss_dist=35.2509 | loss_mean=0.0204\n",
      "[2019-12-06 19:15:25,521] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.0973 | _timers/batch_time=0.3298 | _timers/data_time=0.0003 | _timers/model_time=0.3294 | f1_score=2.528e-07 | loss=14.0017 | loss_dist=13.9816 | loss_mean=0.0202\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.0146 | _timers/batch_time=0.1846 | _timers/data_time=0.0002 | _timers/model_time=0.1844 | f1_score=2.528e-07 | loss=14.0017 | loss_dist=13.9816 | loss_mean=0.0202\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:15:29,555] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2849 | _timers/batch_time=0.3198 | _timers/data_time=0.0003 | _timers/model_time=0.3195 | f1_score=1.118e-07 | loss=25.7942 | loss_dist=25.7746 | loss_mean=0.0196\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.9493 | _timers/batch_time=0.1868 | _timers/data_time=0.0003 | _timers/model_time=0.1864 | f1_score=1.118e-07 | loss=25.7942 | loss_dist=25.7746 | loss_mean=0.0196\n",
      "[2019-12-06 19:15:33,521] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4341 | _timers/batch_time=0.3121 | _timers/data_time=0.0003 | _timers/model_time=0.3118 | f1_score=2.945e-07 | loss=29.1933 | loss_dist=29.1723 | loss_mean=0.0211\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.7569 | _timers/batch_time=0.1743 | _timers/data_time=0.0003 | _timers/model_time=0.1740 | f1_score=2.945e-07 | loss=29.1933 | loss_dist=29.1723 | loss_mean=0.0211\n",
      "[2019-12-06 19:15:37,398] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5721 | _timers/batch_time=0.3057 | _timers/data_time=0.0003 | _timers/model_time=0.3053 | f1_score=1.607e-07 | loss=40.3975 | loss_dist=40.3783 | loss_mean=0.0192\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.2241 | _timers/batch_time=0.1815 | _timers/data_time=0.0002 | _timers/model_time=0.1812 | f1_score=1.607e-07 | loss=40.3975 | loss_dist=40.3783 | loss_mean=0.0192\n",
      "[2019-12-06 19:15:41,325] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3500 | _timers/batch_time=0.3165 | _timers/data_time=0.0002 | _timers/model_time=0.3163 | f1_score=1.928e-07 | loss=26.3084 | loss_dist=26.2878 | loss_mean=0.0206\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.6738 | _timers/batch_time=0.1748 | _timers/data_time=0.0003 | _timers/model_time=0.1745 | f1_score=1.928e-07 | loss=26.3084 | loss_dist=26.2878 | loss_mean=0.0206\n",
      "[2019-12-06 19:15:45,140] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5880 | _timers/batch_time=0.3047 | _timers/data_time=0.0002 | _timers/model_time=0.3044 | f1_score=3.052e-07 | loss=8.6249 | loss_dist=8.6038 | loss_mean=0.0211\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5339 | _timers/batch_time=0.1791 | _timers/data_time=0.0003 | _timers/model_time=0.1788 | f1_score=3.052e-07 | loss=8.6249 | loss_dist=8.6038 | loss_mean=0.0211\n",
      "[2019-12-06 19:15:49,098] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.2936 | _timers/batch_time=0.3199 | _timers/data_time=0.0004 | _timers/model_time=0.3195 | f1_score=4.894e-07 | loss=16.8418 | loss_dist=16.8220 | loss_mean=0.0198\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.7178 | _timers/batch_time=0.1733 | _timers/data_time=0.0004 | _timers/model_time=0.1729 | f1_score=4.894e-07 | loss=16.8418 | loss_dist=16.8220 | loss_mean=0.0198\n",
      "[2019-12-06 19:15:53,097] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4238 | _timers/batch_time=0.3124 | _timers/data_time=0.0004 | _timers/model_time=0.3119 | f1_score=1.044e-07 | loss=41.2328 | loss_dist=41.2123 | loss_mean=0.0206\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.4773 | _timers/batch_time=0.1923 | _timers/data_time=0.0002 | _timers/model_time=0.1920 | f1_score=1.044e-07 | loss=41.2328 | loss_dist=41.2123 | loss_mean=0.0206\n",
      "[2019-12-06 19:15:57,261] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.1107 | _timers/batch_time=0.3286 | _timers/data_time=0.0002 | _timers/model_time=0.3283 | f1_score=9.604e-08 | loss=51.7879 | loss_dist=51.7683 | loss_mean=0.0196\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.5173 | _timers/batch_time=0.1959 | _timers/data_time=0.0002 | _timers/model_time=0.1956 | f1_score=9.604e-08 | loss=51.7879 | loss_dist=51.7683 | loss_mean=0.0196\n",
      "[2019-12-06 19:16:01,400] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.0981 | _timers/batch_time=0.3285 | _timers/data_time=0.0003 | _timers/model_time=0.3282 | f1_score=1.015e-07 | loss=39.3515 | loss_dist=39.3309 | loss_mean=0.0206\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.2760 | _timers/batch_time=0.1802 | _timers/data_time=0.0002 | _timers/model_time=0.1799 | f1_score=1.015e-07 | loss=39.3515 | loss_dist=39.3309 | loss_mean=0.0206\n",
      "[2019-12-06 19:16:05,628] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.9157 | _timers/batch_time=0.3387 | _timers/data_time=0.0003 | _timers/model_time=0.3383 | f1_score=1.266e-07 | loss=18.5483 | loss_dist=18.5268 | loss_mean=0.0215\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.8690 | _timers/batch_time=0.1888 | _timers/data_time=0.0003 | _timers/model_time=0.1885 | f1_score=1.266e-07 | loss=18.5483 | loss_dist=18.5268 | loss_mean=0.0215\n",
      "[2019-12-06 19:16:09,717] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.3816 | _timers/batch_time=0.3148 | _timers/data_time=0.0003 | _timers/model_time=0.3145 | f1_score=1.182e-07 | loss=48.9664 | loss_dist=48.9459 | loss_mean=0.0205\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.3053 | _timers/batch_time=0.1838 | _timers/data_time=0.0002 | _timers/model_time=0.1836 | f1_score=1.182e-07 | loss=48.9664 | loss_dist=48.9459 | loss_mean=0.0205\n",
      "[2019-12-06 19:16:13,862] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2709 | _timers/batch_time=0.3220 | _timers/data_time=0.0004 | _timers/model_time=0.3216 | f1_score=4.904e-07 | loss=26.0026 | loss_dist=25.9814 | loss_mean=0.0212\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.4302 | _timers/batch_time=0.1795 | _timers/data_time=0.0003 | _timers/model_time=0.1792 | f1_score=4.904e-07 | loss=26.0026 | loss_dist=25.9814 | loss_mean=0.0212\n",
      "[2019-12-06 19:16:18,052] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.1155 | _timers/batch_time=0.3283 | _timers/data_time=0.0004 | _timers/model_time=0.3278 | f1_score=1.215e-07 | loss=23.5133 | loss_dist=23.4928 | loss_mean=0.0205\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.2924 | _timers/batch_time=0.1985 | _timers/data_time=0.0004 | _timers/model_time=0.1981 | f1_score=1.215e-07 | loss=23.5133 | loss_dist=23.4928 | loss_mean=0.0205\n",
      "[2019-12-06 19:16:22,199] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.0974 | _timers/batch_time=0.3306 | _timers/data_time=0.0004 | _timers/model_time=0.3301 | f1_score=5.452e-07 | loss=8.4437 | loss_dist=8.4244 | loss_mean=0.0192\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.2911 | _timers/batch_time=0.1818 | _timers/data_time=0.0003 | _timers/model_time=0.1815 | f1_score=5.452e-07 | loss=8.4437 | loss_dist=8.4244 | loss_mean=0.0192\n",
      "[2019-12-06 19:16:26,537] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.7876 | _timers/batch_time=0.3473 | _timers/data_time=0.0005 | _timers/model_time=0.3468 | f1_score=1.351e-07 | loss=34.6610 | loss_dist=34.6406 | loss_mean=0.0205\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.2277 | _timers/batch_time=0.1980 | _timers/data_time=0.0004 | _timers/model_time=0.1976 | f1_score=1.351e-07 | loss=34.6610 | loss_dist=34.6406 | loss_mean=0.0205\n",
      "[2019-12-06 19:16:30,699] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.1817 | _timers/batch_time=0.3249 | _timers/data_time=0.0002 | _timers/model_time=0.3246 | f1_score=1.047e-07 | loss=27.6492 | loss_dist=27.6293 | loss_mean=0.0199\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.1333 | _timers/batch_time=0.1826 | _timers/data_time=0.0004 | _timers/model_time=0.1822 | f1_score=1.047e-07 | loss=27.6492 | loss_dist=27.6293 | loss_mean=0.0199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:16:34,545] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5857 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3041 | f1_score=2.238e-07 | loss=19.6927 | loss_dist=19.6745 | loss_mean=0.0183\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.2647 | _timers/batch_time=0.1810 | _timers/data_time=0.0004 | _timers/model_time=0.1805 | f1_score=2.238e-07 | loss=19.6927 | loss_dist=19.6745 | loss_mean=0.0183\n",
      "[2019-12-06 19:16:38,396] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4254 | _timers/batch_time=0.3118 | _timers/data_time=0.0005 | _timers/model_time=0.3113 | f1_score=1.175e-07 | loss=51.5079 | loss_dist=51.4893 | loss_mean=0.0186\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.5065 | _timers/batch_time=0.1779 | _timers/data_time=0.0003 | _timers/model_time=0.1776 | f1_score=1.175e-07 | loss=51.5079 | loss_dist=51.4893 | loss_mean=0.0186\n",
      "[2019-12-06 19:16:42,193] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5935 | _timers/batch_time=0.3040 | _timers/data_time=0.0002 | _timers/model_time=0.3037 | f1_score=8.049e-08 | loss=48.1924 | loss_dist=48.1724 | loss_mean=0.0201\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.6727 | _timers/batch_time=0.1757 | _timers/data_time=0.0004 | _timers/model_time=0.1753 | f1_score=8.049e-08 | loss=48.1924 | loss_dist=48.1724 | loss_mean=0.0201\n",
      "[2019-12-06 19:16:45,995] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.6469 | _timers/batch_time=0.3021 | _timers/data_time=0.0002 | _timers/model_time=0.3019 | f1_score=2.481e-07 | loss=32.7871 | loss_dist=32.7677 | loss_mean=0.0194\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.5896 | _timers/batch_time=0.1757 | _timers/data_time=0.0005 | _timers/model_time=0.1752 | f1_score=2.481e-07 | loss=32.7871 | loss_dist=32.7677 | loss_mean=0.0194\n",
      "Top best models:\n",
      "logs/b20/checkpoints/train.65.pth\t8.0419\n",
      "logs/b20/checkpoints/train.94.pth\t8.4437\n",
      "logs/b20/checkpoints/train.85.pth\t8.6249\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25/25 [00:40<00:00,  1.64s/it]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6008593376"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_bin.eval()\n",
    "predictions, labels = [], []\n",
    "with torch.no_grad():\n",
    "    for x,y in tqdm(test_loader):\n",
    "        try:\n",
    "        \n",
    "            predictions.append(model_bin(x)['cls'].detach().cpu().numpy())\n",
    "            labels.append(y.detach().cpu().numpy())\n",
    "        except: pass\n",
    "predictions = np.concatenate(predictions)\n",
    "labels = np.concatenate(labels)\n",
    "\n",
    "roc_auc_score(labels, predictions)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bernoulli p = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)               \n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        #torch.bernoulli(torch.tensor([1.0]),)\n",
    "        b_new = torch.bernoulli(torch.ones(x.shape[:2]),p).unsqueeze(2).to(self.emb.weight.device)\n",
    "        #b_new = torch.repeat_interleave(bb(0.4), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/b80\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:17:33,521] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.2646 | _timers/batch_time=0.3201 | _timers/data_time=0.0003 | _timers/model_time=0.3198 | f1_score=1.598e-06 | loss=0.3440 | loss_dist=0.0239 | loss_mean=0.3201\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.2860 | _timers/batch_time=0.1796 | _timers/data_time=0.0003 | _timers/model_time=0.1793 | f1_score=1.598e-06 | loss=0.3440 | loss_dist=0.0239 | loss_mean=0.3201\n",
      "[2019-12-06 19:17:37,423] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4092 | _timers/batch_time=0.3135 | _timers/data_time=0.0003 | _timers/model_time=0.3131 | f1_score=1.488e-06 | loss=0.3445 | loss_dist=0.0282 | loss_mean=0.3163\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.4898 | _timers/batch_time=0.1763 | _timers/data_time=0.0003 | _timers/model_time=0.1760 | f1_score=1.488e-06 | loss=0.3445 | loss_dist=0.0282 | loss_mean=0.3163\n",
      "[2019-12-06 19:17:41,588] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2967 | _timers/batch_time=0.3194 | _timers/data_time=0.0003 | _timers/model_time=0.3191 | f1_score=8.637e-07 | loss=1.9017 | loss_dist=1.5854 | loss_mean=0.3163\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.6611 | _timers/batch_time=0.1924 | _timers/data_time=0.0004 | _timers/model_time=0.1919 | f1_score=8.637e-07 | loss=1.9017 | loss_dist=1.5854 | loss_mean=0.3163\n",
      "[2019-12-06 19:17:45,738] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.3143 | _timers/batch_time=0.3201 | _timers/data_time=0.0002 | _timers/model_time=0.3198 | f1_score=1.545e-06 | loss=0.3380 | loss_dist=0.0211 | loss_mean=0.3169\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.2342 | _timers/batch_time=0.1981 | _timers/data_time=0.0002 | _timers/model_time=0.1979 | f1_score=1.545e-06 | loss=0.3380 | loss_dist=0.0211 | loss_mean=0.3169\n",
      "[2019-12-06 19:17:49,718] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.1681 | _timers/batch_time=0.3246 | _timers/data_time=0.0003 | _timers/model_time=0.3243 | f1_score=1.539e-06 | loss=0.3434 | loss_dist=0.0232 | loss_mean=0.3202\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.3619 | _timers/batch_time=0.1798 | _timers/data_time=0.0003 | _timers/model_time=0.1795 | f1_score=1.539e-06 | loss=0.3434 | loss_dist=0.0232 | loss_mean=0.3202\n",
      "[2019-12-06 19:17:53,693] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.3440 | _timers/batch_time=0.3155 | _timers/data_time=0.0003 | _timers/model_time=0.3152 | f1_score=9.241e-07 | loss=2.1179 | loss_dist=1.7980 | loss_mean=0.3199\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.1554 | _timers/batch_time=0.1990 | _timers/data_time=0.0003 | _timers/model_time=0.1987 | f1_score=9.241e-07 | loss=2.1179 | loss_dist=1.7980 | loss_mean=0.3199\n",
      "[2019-12-06 19:17:57,874] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.0412 | _timers/batch_time=0.3328 | _timers/data_time=0.0003 | _timers/model_time=0.3324 | f1_score=1.672e-06 | loss=0.3343 | loss_dist=0.0089 | loss_mean=0.3254\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.0944 | _timers/batch_time=0.1845 | _timers/data_time=0.0003 | _timers/model_time=0.1842 | f1_score=1.672e-06 | loss=0.3343 | loss_dist=0.0089 | loss_mean=0.3254\n",
      "[2019-12-06 19:18:02,072] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.9327 | _timers/batch_time=0.3384 | _timers/data_time=0.0003 | _timers/model_time=0.3381 | f1_score=1.384e-06 | loss=0.7015 | loss_dist=0.3804 | loss_mean=0.3211\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.8990 | _timers/batch_time=0.1876 | _timers/data_time=0.0003 | _timers/model_time=0.1873 | f1_score=1.384e-06 | loss=0.7015 | loss_dist=0.3804 | loss_mean=0.3211\n",
      "[2019-12-06 19:18:06,105] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2712 | _timers/batch_time=0.3221 | _timers/data_time=0.0003 | _timers/model_time=0.3217 | f1_score=1.232e-06 | loss=0.3943 | loss_dist=0.0723 | loss_mean=0.3221\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1103 | _timers/batch_time=0.1833 | _timers/data_time=0.0003 | _timers/model_time=0.1829 | f1_score=1.232e-06 | loss=0.3943 | loss_dist=0.0723 | loss_mean=0.3221\n",
      "[2019-12-06 19:18:10,153] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2098 | _timers/batch_time=0.3221 | _timers/data_time=0.0004 | _timers/model_time=0.3217 | f1_score=9.685e-07 | loss=1.8555 | loss_dist=1.5360 | loss_mean=0.3195\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.1830 | _timers/batch_time=0.1820 | _timers/data_time=0.0003 | _timers/model_time=0.1817 | f1_score=9.685e-07 | loss=1.8555 | loss_dist=1.5360 | loss_mean=0.3195\n",
      "[2019-12-06 19:18:14,223] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4760 | _timers/batch_time=0.3105 | _timers/data_time=0.0003 | _timers/model_time=0.3102 | f1_score=1.875e-06 | loss=0.3509 | loss_dist=0.0285 | loss_mean=0.3224\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.3013 | _timers/batch_time=0.1988 | _timers/data_time=0.0003 | _timers/model_time=0.1985 | f1_score=1.875e-06 | loss=0.3509 | loss_dist=0.0285 | loss_mean=0.3224\n",
      "[2019-12-06 19:18:18,212] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.2283 | _timers/batch_time=0.3214 | _timers/data_time=0.0003 | _timers/model_time=0.3210 | f1_score=1.483e-06 | loss=0.3654 | loss_dist=0.0530 | loss_mean=0.3124\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.8444 | _timers/batch_time=0.1860 | _timers/data_time=0.0002 | _timers/model_time=0.1857 | f1_score=1.483e-06 | loss=0.3654 | loss_dist=0.0530 | loss_mean=0.3124\n",
      "[2019-12-06 19:18:22,361] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0568 | _timers/batch_time=0.3315 | _timers/data_time=0.0005 | _timers/model_time=0.3310 | f1_score=1.625e-06 | loss=0.3763 | loss_dist=0.0590 | loss_mean=0.3173\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.1043 | _timers/batch_time=0.1837 | _timers/data_time=0.0002 | _timers/model_time=0.1834 | f1_score=1.625e-06 | loss=0.3763 | loss_dist=0.0590 | loss_mean=0.3173\n",
      "[2019-12-06 19:18:26,610] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0579 | _timers/batch_time=0.3345 | _timers/data_time=0.0004 | _timers/model_time=0.3341 | f1_score=1.547e-06 | loss=0.3245 | loss_dist=0.0033 | loss_mean=0.3213\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.4994 | _timers/batch_time=0.1957 | _timers/data_time=0.0003 | _timers/model_time=0.1953 | f1_score=1.547e-06 | loss=0.3245 | loss_dist=0.0033 | loss_mean=0.3213\n",
      "[2019-12-06 19:18:30,618] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.2400 | _timers/batch_time=0.3212 | _timers/data_time=0.0004 | _timers/model_time=0.3207 | f1_score=1.077e-06 | loss=1.6154 | loss_dist=1.2941 | loss_mean=0.3213\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.2449 | _timers/batch_time=0.1814 | _timers/data_time=0.0002 | _timers/model_time=0.1812 | f1_score=1.077e-06 | loss=1.6154 | loss_dist=1.2941 | loss_mean=0.3213\n",
      "[2019-12-06 19:18:34,736] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.1206 | _timers/batch_time=0.3295 | _timers/data_time=0.0003 | _timers/model_time=0.3292 | f1_score=1.952e-06 | loss=0.3564 | loss_dist=0.0395 | loss_mean=0.3169\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.2605 | _timers/batch_time=0.1809 | _timers/data_time=0.0003 | _timers/model_time=0.1806 | f1_score=1.952e-06 | loss=0.3564 | loss_dist=0.0395 | loss_mean=0.3169\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:18:38,774] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.2604 | _timers/batch_time=0.3209 | _timers/data_time=0.0003 | _timers/model_time=0.3206 | f1_score=1.430e-06 | loss=0.3476 | loss_dist=0.0264 | loss_mean=0.3212\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.5361 | _timers/batch_time=0.1766 | _timers/data_time=0.0002 | _timers/model_time=0.1763 | f1_score=1.430e-06 | loss=0.3476 | loss_dist=0.0264 | loss_mean=0.3212\n",
      "[2019-12-06 19:18:42,634] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.5476 | _timers/batch_time=0.3067 | _timers/data_time=0.0002 | _timers/model_time=0.3065 | f1_score=1.295e-06 | loss=1.7903 | loss_dist=1.4686 | loss_mean=0.3217\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.4291 | _timers/batch_time=0.1785 | _timers/data_time=0.0003 | _timers/model_time=0.1782 | f1_score=1.295e-06 | loss=1.7903 | loss_dist=1.4686 | loss_mean=0.3217\n",
      "[2019-12-06 19:18:46,649] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.1449 | _timers/batch_time=0.3273 | _timers/data_time=0.0003 | _timers/model_time=0.3270 | f1_score=6.564e-07 | loss=9.6048 | loss_dist=9.2888 | loss_mean=0.3160\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.2742 | _timers/batch_time=0.1815 | _timers/data_time=0.0004 | _timers/model_time=0.1811 | f1_score=6.564e-07 | loss=9.6048 | loss_dist=9.2888 | loss_mean=0.3160\n",
      "[2019-12-06 19:18:50,568] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.5100 | _timers/batch_time=0.3085 | _timers/data_time=0.0003 | _timers/model_time=0.3082 | f1_score=1.415e-06 | loss=0.3688 | loss_dist=0.0505 | loss_mean=0.3183\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.2065 | _timers/batch_time=0.1814 | _timers/data_time=0.0004 | _timers/model_time=0.1810 | f1_score=1.415e-06 | loss=0.3688 | loss_dist=0.0505 | loss_mean=0.3183\n",
      "[2019-12-06 19:18:54,718] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.1631 | _timers/batch_time=0.3268 | _timers/data_time=0.0004 | _timers/model_time=0.3264 | f1_score=1.263e-06 | loss=5.3770 | loss_dist=5.0574 | loss_mean=0.3196\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.2403 | _timers/batch_time=0.1820 | _timers/data_time=0.0002 | _timers/model_time=0.1818 | f1_score=1.263e-06 | loss=5.3770 | loss_dist=5.0574 | loss_mean=0.3196\n",
      "[2019-12-06 19:18:58,909] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.1325 | _timers/batch_time=0.3293 | _timers/data_time=0.0003 | _timers/model_time=0.3289 | f1_score=1.419e-06 | loss=0.3912 | loss_dist=0.0670 | loss_mean=0.3242\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.9393 | _timers/batch_time=0.1856 | _timers/data_time=0.0003 | _timers/model_time=0.1852 | f1_score=1.419e-06 | loss=0.3912 | loss_dist=0.0670 | loss_mean=0.3242\n",
      "[2019-12-06 19:19:03,014] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0420 | _timers/batch_time=0.3323 | _timers/data_time=0.0003 | _timers/model_time=0.3319 | f1_score=1.565e-06 | loss=0.3442 | loss_dist=0.0300 | loss_mean=0.3142\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.5478 | _timers/batch_time=0.1769 | _timers/data_time=0.0003 | _timers/model_time=0.1765 | f1_score=1.565e-06 | loss=0.3442 | loss_dist=0.0300 | loss_mean=0.3142\n",
      "[2019-12-06 19:19:06,929] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.4714 | _timers/batch_time=0.3099 | _timers/data_time=0.0004 | _timers/model_time=0.3095 | f1_score=1.287e-06 | loss=0.3511 | loss_dist=0.0321 | loss_mean=0.3190\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.3590 | _timers/batch_time=0.1815 | _timers/data_time=0.0004 | _timers/model_time=0.1810 | f1_score=1.287e-06 | loss=0.3511 | loss_dist=0.0321 | loss_mean=0.3190\n",
      "[2019-12-06 19:19:11,027] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.1495 | _timers/batch_time=0.3286 | _timers/data_time=0.0003 | _timers/model_time=0.3282 | f1_score=1.099e-06 | loss=0.3772 | loss_dist=0.0599 | loss_mean=0.3174\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.1257 | _timers/batch_time=0.1858 | _timers/data_time=0.0004 | _timers/model_time=0.1854 | f1_score=1.099e-06 | loss=0.3772 | loss_dist=0.0599 | loss_mean=0.3174\n",
      "[2019-12-06 19:19:15,243] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0895 | _timers/batch_time=0.3305 | _timers/data_time=0.0004 | _timers/model_time=0.3301 | f1_score=1.374e-06 | loss=0.3650 | loss_dist=0.0411 | loss_mean=0.3239\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.0372 | _timers/batch_time=0.2011 | _timers/data_time=0.0003 | _timers/model_time=0.2008 | f1_score=1.374e-06 | loss=0.3650 | loss_dist=0.0411 | loss_mean=0.3239\n",
      "[2019-12-06 19:19:19,465] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.9802 | _timers/batch_time=0.3362 | _timers/data_time=0.0004 | _timers/model_time=0.3357 | f1_score=1.345e-06 | loss=0.3374 | loss_dist=0.0190 | loss_mean=0.3184\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.4243 | _timers/batch_time=0.1790 | _timers/data_time=0.0002 | _timers/model_time=0.1788 | f1_score=1.345e-06 | loss=0.3374 | loss_dist=0.0190 | loss_mean=0.3184\n",
      "[2019-12-06 19:19:23,349] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5634 | _timers/batch_time=0.3064 | _timers/data_time=0.0003 | _timers/model_time=0.3061 | f1_score=1.578e-06 | loss=0.3239 | loss_dist=0.0042 | loss_mean=0.3197\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.4515 | _timers/batch_time=0.1780 | _timers/data_time=0.0004 | _timers/model_time=0.1776 | f1_score=1.578e-06 | loss=0.3239 | loss_dist=0.0042 | loss_mean=0.3197\n",
      "[2019-12-06 19:19:27,382] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.1775 | _timers/batch_time=0.3261 | _timers/data_time=0.0004 | _timers/model_time=0.3256 | f1_score=1.664e-06 | loss=0.3383 | loss_dist=0.0190 | loss_mean=0.3193\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.0564 | _timers/batch_time=0.1835 | _timers/data_time=0.0003 | _timers/model_time=0.1833 | f1_score=1.664e-06 | loss=0.3383 | loss_dist=0.0190 | loss_mean=0.3193\n",
      "[2019-12-06 19:19:31,347] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2660 | _timers/batch_time=0.3205 | _timers/data_time=0.0003 | _timers/model_time=0.3201 | f1_score=1.482e-06 | loss=0.3492 | loss_dist=0.0350 | loss_mean=0.3142\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.7449 | _timers/batch_time=0.1733 | _timers/data_time=0.0003 | _timers/model_time=0.1731 | f1_score=1.482e-06 | loss=0.3492 | loss_dist=0.0350 | loss_mean=0.3142\n",
      "[2019-12-06 19:19:35,169] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5792 | _timers/batch_time=0.3052 | _timers/data_time=0.0003 | _timers/model_time=0.3049 | f1_score=1.248e-06 | loss=0.3612 | loss_dist=0.0416 | loss_mean=0.3196\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.6835 | _timers/batch_time=0.1745 | _timers/data_time=0.0003 | _timers/model_time=0.1742 | f1_score=1.248e-06 | loss=0.3612 | loss_dist=0.0416 | loss_mean=0.3196\n",
      "[2019-12-06 19:19:39,434] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.1003 | _timers/batch_time=0.3325 | _timers/data_time=0.0004 | _timers/model_time=0.3320 | f1_score=1.662e-06 | loss=0.3392 | loss_dist=0.0177 | loss_mean=0.3215\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=9.8364 | _timers/batch_time=0.2073 | _timers/data_time=0.0003 | _timers/model_time=0.2069 | f1_score=1.662e-06 | loss=0.3392 | loss_dist=0.0177 | loss_mean=0.3215\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:19:43,738] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=5.9268 | _timers/batch_time=0.3384 | _timers/data_time=0.0004 | _timers/model_time=0.3380 | f1_score=1.364e-06 | loss=0.3457 | loss_dist=0.0267 | loss_mean=0.3190\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.1762 | _timers/batch_time=0.1991 | _timers/data_time=0.0002 | _timers/model_time=0.1988 | f1_score=1.364e-06 | loss=0.3457 | loss_dist=0.0267 | loss_mean=0.3190\n",
      "[2019-12-06 19:19:48,171] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.6590 | _timers/batch_time=0.3542 | _timers/data_time=0.0004 | _timers/model_time=0.3537 | f1_score=1.247e-06 | loss=3.3798 | loss_dist=3.0601 | loss_mean=0.3197\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=9.9862 | _timers/batch_time=0.2025 | _timers/data_time=0.0003 | _timers/model_time=0.2021 | f1_score=1.247e-06 | loss=3.3798 | loss_dist=3.0601 | loss_mean=0.3197\n",
      "[2019-12-06 19:19:52,292] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.2831 | _timers/batch_time=0.3194 | _timers/data_time=0.0003 | _timers/model_time=0.3191 | f1_score=1.473e-06 | loss=0.3624 | loss_dist=0.0384 | loss_mean=0.3240\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.7588 | _timers/batch_time=0.1729 | _timers/data_time=0.0002 | _timers/model_time=0.1726 | f1_score=1.473e-06 | loss=0.3624 | loss_dist=0.0384 | loss_mean=0.3240\n",
      "[2019-12-06 19:19:56,096] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.5207 | _timers/batch_time=0.3070 | _timers/data_time=0.0003 | _timers/model_time=0.3066 | f1_score=1.089e-06 | loss=0.3769 | loss_dist=0.0556 | loss_mean=0.3214\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.6740 | _timers/batch_time=0.1747 | _timers/data_time=0.0003 | _timers/model_time=0.1743 | f1_score=1.089e-06 | loss=0.3769 | loss_dist=0.0556 | loss_mean=0.3214\n",
      "[2019-12-06 19:20:00,393] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.8949 | _timers/batch_time=0.3405 | _timers/data_time=0.0004 | _timers/model_time=0.3401 | f1_score=1.361e-06 | loss=0.3495 | loss_dist=0.0319 | loss_mean=0.3176\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.0300 | _timers/batch_time=0.1845 | _timers/data_time=0.0003 | _timers/model_time=0.1841 | f1_score=1.361e-06 | loss=0.3495 | loss_dist=0.0319 | loss_mean=0.3176\n",
      "[2019-12-06 19:20:04,442] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.2863 | _timers/batch_time=0.3193 | _timers/data_time=0.0003 | _timers/model_time=0.3190 | f1_score=6.637e-07 | loss=3.0598 | loss_dist=2.7366 | loss_mean=0.3232\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.8284 | _timers/batch_time=0.1881 | _timers/data_time=0.0003 | _timers/model_time=0.1878 | f1_score=6.637e-07 | loss=3.0598 | loss_dist=2.7366 | loss_mean=0.3232\n",
      "[2019-12-06 19:20:08,655] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0390 | _timers/batch_time=0.3320 | _timers/data_time=0.0004 | _timers/model_time=0.3316 | f1_score=1.225e-06 | loss=0.3710 | loss_dist=0.0532 | loss_mean=0.3178\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=9.9271 | _timers/batch_time=0.2058 | _timers/data_time=0.0004 | _timers/model_time=0.2054 | f1_score=1.225e-06 | loss=0.3710 | loss_dist=0.0532 | loss_mean=0.3178\n",
      "[2019-12-06 19:20:12,912] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=5.8493 | _timers/batch_time=0.3425 | _timers/data_time=0.0003 | _timers/model_time=0.3421 | f1_score=1.199e-06 | loss=2.2850 | loss_dist=1.9648 | loss_mean=0.3202\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.5012 | _timers/batch_time=0.1936 | _timers/data_time=0.0002 | _timers/model_time=0.1934 | f1_score=1.199e-06 | loss=2.2850 | loss_dist=1.9648 | loss_mean=0.3202\n",
      "[2019-12-06 19:20:17,132] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=5.8981 | _timers/batch_time=0.3404 | _timers/data_time=0.0004 | _timers/model_time=0.3400 | f1_score=1.316e-06 | loss=0.4684 | loss_dist=0.1536 | loss_mean=0.3148\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.5242 | _timers/batch_time=0.1938 | _timers/data_time=0.0004 | _timers/model_time=0.1934 | f1_score=1.316e-06 | loss=0.4684 | loss_dist=0.1536 | loss_mean=0.3148\n",
      "[2019-12-06 19:20:21,049] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.5874 | _timers/batch_time=0.3048 | _timers/data_time=0.0004 | _timers/model_time=0.3044 | f1_score=1.023e-06 | loss=1.9439 | loss_dist=1.6214 | loss_mean=0.3225\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.0375 | _timers/batch_time=0.1849 | _timers/data_time=0.0002 | _timers/model_time=0.1847 | f1_score=1.023e-06 | loss=1.9439 | loss_dist=1.6214 | loss_mean=0.3225\n",
      "[2019-12-06 19:20:25,041] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.3409 | _timers/batch_time=0.3165 | _timers/data_time=0.0005 | _timers/model_time=0.3160 | f1_score=1.506e-06 | loss=0.3199 | loss_dist=0.0044 | loss_mean=0.3155\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.5421 | _timers/batch_time=0.1770 | _timers/data_time=0.0003 | _timers/model_time=0.1767 | f1_score=1.506e-06 | loss=0.3199 | loss_dist=0.0044 | loss_mean=0.3155\n",
      "[2019-12-06 19:20:29,253] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.4465 | _timers/batch_time=0.3127 | _timers/data_time=0.0004 | _timers/model_time=0.3123 | f1_score=1.545e-06 | loss=0.3620 | loss_dist=0.0425 | loss_mean=0.3195\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=8.8105 | _timers/batch_time=0.2319 | _timers/data_time=0.0002 | _timers/model_time=0.2317 | f1_score=1.545e-06 | loss=0.3620 | loss_dist=0.0425 | loss_mean=0.3195\n",
      "[2019-12-06 19:20:33,450] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0057 | _timers/batch_time=0.3336 | _timers/data_time=0.0002 | _timers/model_time=0.3334 | f1_score=1.433e-06 | loss=0.3816 | loss_dist=0.0636 | loss_mean=0.3180\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.8539 | _timers/batch_time=0.1881 | _timers/data_time=0.0003 | _timers/model_time=0.1877 | f1_score=1.433e-06 | loss=0.3816 | loss_dist=0.0636 | loss_mean=0.3180\n",
      "[2019-12-06 19:20:37,492] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0767 | _timers/batch_time=0.3368 | _timers/data_time=0.0003 | _timers/model_time=0.3364 | f1_score=1.319e-06 | loss=5.6139 | loss_dist=5.2962 | loss_mean=0.3177\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.4701 | _timers/batch_time=0.1781 | _timers/data_time=0.0004 | _timers/model_time=0.1777 | f1_score=1.319e-06 | loss=5.6139 | loss_dist=5.2962 | loss_mean=0.3177\n",
      "[2019-12-06 19:20:41,426] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.3214 | _timers/batch_time=0.3183 | _timers/data_time=0.0003 | _timers/model_time=0.3180 | f1_score=1.466e-06 | loss=0.3524 | loss_dist=0.0337 | loss_mean=0.3186\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.5775 | _timers/batch_time=0.1755 | _timers/data_time=0.0004 | _timers/model_time=0.1751 | f1_score=1.466e-06 | loss=0.3524 | loss_dist=0.0337 | loss_mean=0.3186\n",
      "[2019-12-06 19:20:45,308] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.4270 | _timers/batch_time=0.3119 | _timers/data_time=0.0003 | _timers/model_time=0.3116 | f1_score=1.413e-06 | loss=0.3730 | loss_dist=0.0606 | loss_mean=0.3124\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.4201 | _timers/batch_time=0.1783 | _timers/data_time=0.0004 | _timers/model_time=0.1779 | f1_score=1.413e-06 | loss=0.3730 | loss_dist=0.0606 | loss_mean=0.3124\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:20:49,223] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.4429 | _timers/batch_time=0.3114 | _timers/data_time=0.0002 | _timers/model_time=0.3112 | f1_score=1.429e-06 | loss=0.4191 | loss_dist=0.1049 | loss_mean=0.3141\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.3275 | _timers/batch_time=0.1793 | _timers/data_time=0.0002 | _timers/model_time=0.1791 | f1_score=1.429e-06 | loss=0.4191 | loss_dist=0.1049 | loss_mean=0.3141\n",
      "[2019-12-06 19:20:53,075] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5503 | _timers/batch_time=0.3060 | _timers/data_time=0.0002 | _timers/model_time=0.3057 | f1_score=1.501e-06 | loss=0.3790 | loss_dist=0.0557 | loss_mean=0.3233\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.3406 | _timers/batch_time=0.1790 | _timers/data_time=0.0004 | _timers/model_time=0.1786 | f1_score=1.501e-06 | loss=0.3790 | loss_dist=0.0557 | loss_mean=0.3233\n",
      "[2019-12-06 19:20:56,957] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.5159 | _timers/batch_time=0.3078 | _timers/data_time=0.0003 | _timers/model_time=0.3074 | f1_score=8.844e-07 | loss=5.2224 | loss_dist=4.9011 | loss_mean=0.3214\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.3548 | _timers/batch_time=0.1790 | _timers/data_time=0.0003 | _timers/model_time=0.1786 | f1_score=8.844e-07 | loss=5.2224 | loss_dist=4.9011 | loss_mean=0.3214\n",
      "[2019-12-06 19:21:00,845] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.4944 | _timers/batch_time=0.3089 | _timers/data_time=0.0002 | _timers/model_time=0.3087 | f1_score=1.473e-06 | loss=0.3317 | loss_dist=0.0088 | loss_mean=0.3229\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.4070 | _timers/batch_time=0.1792 | _timers/data_time=0.0005 | _timers/model_time=0.1787 | f1_score=1.473e-06 | loss=0.3317 | loss_dist=0.0088 | loss_mean=0.3229\n",
      "[2019-12-06 19:21:04,820] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.3683 | _timers/batch_time=0.3160 | _timers/data_time=0.0005 | _timers/model_time=0.3155 | f1_score=1.148e-06 | loss=0.3766 | loss_dist=0.0523 | loss_mean=0.3243\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.3501 | _timers/batch_time=0.1795 | _timers/data_time=0.0004 | _timers/model_time=0.1790 | f1_score=1.148e-06 | loss=0.3766 | loss_dist=0.0523 | loss_mean=0.3243\n",
      "[2019-12-06 19:21:08,686] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5307 | _timers/batch_time=0.3072 | _timers/data_time=0.0003 | _timers/model_time=0.3069 | f1_score=1.479e-06 | loss=0.3280 | loss_dist=0.0084 | loss_mean=0.3196\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.1674 | _timers/batch_time=0.1824 | _timers/data_time=0.0004 | _timers/model_time=0.1820 | f1_score=1.479e-06 | loss=0.3280 | loss_dist=0.0084 | loss_mean=0.3196\n",
      "[2019-12-06 19:21:12,560] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5025 | _timers/batch_time=0.3088 | _timers/data_time=0.0004 | _timers/model_time=0.3084 | f1_score=1.342e-06 | loss=0.3785 | loss_dist=0.0559 | loss_mean=0.3226\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.3252 | _timers/batch_time=0.1798 | _timers/data_time=0.0005 | _timers/model_time=0.1793 | f1_score=1.342e-06 | loss=0.3785 | loss_dist=0.0559 | loss_mean=0.3226\n",
      "[2019-12-06 19:21:16,399] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5668 | _timers/batch_time=0.3056 | _timers/data_time=0.0003 | _timers/model_time=0.3053 | f1_score=1.665e-06 | loss=0.3349 | loss_dist=0.0206 | loss_mean=0.3143\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.4496 | _timers/batch_time=0.1779 | _timers/data_time=0.0005 | _timers/model_time=0.1774 | f1_score=1.665e-06 | loss=0.3349 | loss_dist=0.0206 | loss_mean=0.3143\n",
      "[2019-12-06 19:21:20,270] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5656 | _timers/batch_time=0.3058 | _timers/data_time=0.0004 | _timers/model_time=0.3053 | f1_score=1.803e-06 | loss=0.3575 | loss_dist=0.0389 | loss_mean=0.3186\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.2894 | _timers/batch_time=0.1809 | _timers/data_time=0.0005 | _timers/model_time=0.1804 | f1_score=1.803e-06 | loss=0.3575 | loss_dist=0.0389 | loss_mean=0.3186\n",
      "[2019-12-06 19:21:24,147] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.5892 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3041 | f1_score=1.682e-06 | loss=0.3569 | loss_dist=0.0350 | loss_mean=0.3219\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.3837 | _timers/batch_time=0.1799 | _timers/data_time=0.0005 | _timers/model_time=0.1794 | f1_score=1.682e-06 | loss=0.3569 | loss_dist=0.0350 | loss_mean=0.3219\n",
      "[2019-12-06 19:21:28,016] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4790 | _timers/batch_time=0.3100 | _timers/data_time=0.0003 | _timers/model_time=0.3097 | f1_score=1.598e-06 | loss=0.3420 | loss_dist=0.0189 | loss_mean=0.3231\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6335 | _timers/batch_time=0.1753 | _timers/data_time=0.0004 | _timers/model_time=0.1749 | f1_score=1.598e-06 | loss=0.3420 | loss_dist=0.0189 | loss_mean=0.3231\n",
      "[2019-12-06 19:21:31,855] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5518 | _timers/batch_time=0.3064 | _timers/data_time=0.0002 | _timers/model_time=0.3061 | f1_score=1.440e-06 | loss=0.3412 | loss_dist=0.0176 | loss_mean=0.3236\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.5549 | _timers/batch_time=0.1767 | _timers/data_time=0.0002 | _timers/model_time=0.1764 | f1_score=1.440e-06 | loss=0.3412 | loss_dist=0.0176 | loss_mean=0.3236\n",
      "[2019-12-06 19:21:35,714] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4707 | _timers/batch_time=0.3102 | _timers/data_time=0.0002 | _timers/model_time=0.3099 | f1_score=1.602e-06 | loss=0.3283 | loss_dist=0.0108 | loss_mean=0.3175\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6206 | _timers/batch_time=0.1752 | _timers/data_time=0.0004 | _timers/model_time=0.1747 | f1_score=1.602e-06 | loss=0.3283 | loss_dist=0.0108 | loss_mean=0.3175\n",
      "[2019-12-06 19:21:39,682] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.4014 | _timers/batch_time=0.3153 | _timers/data_time=0.0003 | _timers/model_time=0.3149 | f1_score=1.514e-06 | loss=0.3619 | loss_dist=0.0432 | loss_mean=0.3187\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.5423 | _timers/batch_time=0.1776 | _timers/data_time=0.0003 | _timers/model_time=0.1773 | f1_score=1.514e-06 | loss=0.3619 | loss_dist=0.0432 | loss_mean=0.3187\n",
      "[2019-12-06 19:21:43,540] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.5145 | _timers/batch_time=0.3080 | _timers/data_time=0.0004 | _timers/model_time=0.3075 | f1_score=1.373e-06 | loss=0.3655 | loss_dist=0.0459 | loss_mean=0.3195\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.6316 | _timers/batch_time=0.1753 | _timers/data_time=0.0004 | _timers/model_time=0.1748 | f1_score=1.373e-06 | loss=0.3655 | loss_dist=0.0459 | loss_mean=0.3195\n",
      "[2019-12-06 19:21:47,539] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3601 | _timers/batch_time=0.3170 | _timers/data_time=0.0003 | _timers/model_time=0.3167 | f1_score=1.452e-06 | loss=0.4236 | loss_dist=0.1037 | loss_mean=0.3199\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.4968 | _timers/batch_time=0.1782 | _timers/data_time=0.0003 | _timers/model_time=0.1779 | f1_score=1.452e-06 | loss=0.4236 | loss_dist=0.1037 | loss_mean=0.3199\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:21:51,390] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5043 | _timers/batch_time=0.3088 | _timers/data_time=0.0003 | _timers/model_time=0.3085 | f1_score=1.513e-06 | loss=0.3897 | loss_dist=0.0628 | loss_mean=0.3269\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.7615 | _timers/batch_time=0.1724 | _timers/data_time=0.0003 | _timers/model_time=0.1720 | f1_score=1.513e-06 | loss=0.3897 | loss_dist=0.0628 | loss_mean=0.3269\n",
      "[2019-12-06 19:21:55,249] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5303 | _timers/batch_time=0.3074 | _timers/data_time=0.0004 | _timers/model_time=0.3070 | f1_score=1.413e-06 | loss=0.3512 | loss_dist=0.0313 | loss_mean=0.3199\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.6167 | _timers/batch_time=0.1755 | _timers/data_time=0.0003 | _timers/model_time=0.1751 | f1_score=1.413e-06 | loss=0.3512 | loss_dist=0.0313 | loss_mean=0.3199\n",
      "[2019-12-06 19:21:59,074] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5923 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3041 | f1_score=1.164e-06 | loss=3.1226 | loss_dist=2.7997 | loss_mean=0.3230\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.5442 | _timers/batch_time=0.1769 | _timers/data_time=0.0003 | _timers/model_time=0.1766 | f1_score=1.164e-06 | loss=3.1226 | loss_dist=2.7997 | loss_mean=0.3230\n",
      "[2019-12-06 19:22:02,926] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.5529 | _timers/batch_time=0.3062 | _timers/data_time=0.0003 | _timers/model_time=0.3059 | f1_score=1.575e-06 | loss=0.3377 | loss_dist=0.0172 | loss_mean=0.3206\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.7332 | _timers/batch_time=0.1741 | _timers/data_time=0.0003 | _timers/model_time=0.1737 | f1_score=1.575e-06 | loss=0.3377 | loss_dist=0.0172 | loss_mean=0.3206\n",
      "[2019-12-06 19:22:06,769] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5106 | _timers/batch_time=0.3082 | _timers/data_time=0.0004 | _timers/model_time=0.3078 | f1_score=1.343e-06 | loss=0.3346 | loss_dist=0.0170 | loss_mean=0.3175\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.7861 | _timers/batch_time=0.1728 | _timers/data_time=0.0003 | _timers/model_time=0.1726 | f1_score=1.343e-06 | loss=0.3346 | loss_dist=0.0170 | loss_mean=0.3175\n",
      "[2019-12-06 19:22:10,645] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5336 | _timers/batch_time=0.3074 | _timers/data_time=0.0003 | _timers/model_time=0.3070 | f1_score=1.238e-06 | loss=0.3713 | loss_dist=0.0565 | loss_mean=0.3149\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.6238 | _timers/batch_time=0.1763 | _timers/data_time=0.0002 | _timers/model_time=0.1761 | f1_score=1.238e-06 | loss=0.3713 | loss_dist=0.0565 | loss_mean=0.3149\n",
      "[2019-12-06 19:22:14,516] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.4132 | _timers/batch_time=0.3134 | _timers/data_time=0.0003 | _timers/model_time=0.3130 | f1_score=1.752e-06 | loss=0.3514 | loss_dist=0.0299 | loss_mean=0.3215\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.5588 | _timers/batch_time=0.1761 | _timers/data_time=0.0003 | _timers/model_time=0.1758 | f1_score=1.752e-06 | loss=0.3514 | loss_dist=0.0299 | loss_mean=0.3215\n",
      "[2019-12-06 19:22:18,364] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5177 | _timers/batch_time=0.3081 | _timers/data_time=0.0003 | _timers/model_time=0.3077 | f1_score=1.382e-06 | loss=0.4201 | loss_dist=0.1012 | loss_mean=0.3190\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.6805 | _timers/batch_time=0.1749 | _timers/data_time=0.0003 | _timers/model_time=0.1745 | f1_score=1.382e-06 | loss=0.4201 | loss_dist=0.1012 | loss_mean=0.3190\n",
      "[2019-12-06 19:22:22,294] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.5531 | _timers/batch_time=0.3065 | _timers/data_time=0.0002 | _timers/model_time=0.3062 | f1_score=1.395e-06 | loss=0.4081 | loss_dist=0.0932 | loss_mean=0.3148\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.6077 | _timers/batch_time=0.1906 | _timers/data_time=0.0003 | _timers/model_time=0.1903 | f1_score=1.395e-06 | loss=0.4081 | loss_dist=0.0932 | loss_mean=0.3148\n",
      "[2019-12-06 19:22:26,447] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.0729 | _timers/batch_time=0.3318 | _timers/data_time=0.0003 | _timers/model_time=0.3315 | f1_score=1.507e-06 | loss=0.3304 | loss_dist=0.0115 | loss_mean=0.3189\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.1077 | _timers/batch_time=0.1834 | _timers/data_time=0.0004 | _timers/model_time=0.1830 | f1_score=1.507e-06 | loss=0.3304 | loss_dist=0.0115 | loss_mean=0.3189\n",
      "[2019-12-06 19:22:30,383] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.3355 | _timers/batch_time=0.3175 | _timers/data_time=0.0003 | _timers/model_time=0.3171 | f1_score=1.080e-06 | loss=1.4101 | loss_dist=1.0910 | loss_mean=0.3190\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.7230 | _timers/batch_time=0.1883 | _timers/data_time=0.0004 | _timers/model_time=0.1879 | f1_score=1.080e-06 | loss=1.4101 | loss_dist=1.0910 | loss_mean=0.3190\n",
      "[2019-12-06 19:22:34,375] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.2888 | _timers/batch_time=0.3188 | _timers/data_time=0.0004 | _timers/model_time=0.3184 | f1_score=1.771e-06 | loss=0.3462 | loss_dist=0.0329 | loss_mean=0.3133\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.5302 | _timers/batch_time=0.1764 | _timers/data_time=0.0003 | _timers/model_time=0.1761 | f1_score=1.771e-06 | loss=0.3462 | loss_dist=0.0329 | loss_mean=0.3133\n",
      "[2019-12-06 19:22:38,330] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.5087 | _timers/batch_time=0.3087 | _timers/data_time=0.0004 | _timers/model_time=0.3083 | f1_score=1.148e-06 | loss=3.6184 | loss_dist=3.2980 | loss_mean=0.3204\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.5205 | _timers/batch_time=0.1765 | _timers/data_time=0.0003 | _timers/model_time=0.1761 | f1_score=1.148e-06 | loss=3.6184 | loss_dist=3.2980 | loss_mean=0.3204\n",
      "[2019-12-06 19:22:42,196] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4573 | _timers/batch_time=0.3110 | _timers/data_time=0.0004 | _timers/model_time=0.3106 | f1_score=1.300e-06 | loss=0.3820 | loss_dist=0.0635 | loss_mean=0.3186\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.5859 | _timers/batch_time=0.1759 | _timers/data_time=0.0002 | _timers/model_time=0.1757 | f1_score=1.300e-06 | loss=0.3820 | loss_dist=0.0635 | loss_mean=0.3186\n",
      "[2019-12-06 19:22:46,039] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5776 | _timers/batch_time=0.3053 | _timers/data_time=0.0005 | _timers/model_time=0.3047 | f1_score=9.152e-07 | loss=8.7542 | loss_dist=8.4286 | loss_mean=0.3256\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.4962 | _timers/batch_time=0.1779 | _timers/data_time=0.0003 | _timers/model_time=0.1776 | f1_score=9.152e-07 | loss=8.7542 | loss_dist=8.4286 | loss_mean=0.3256\n",
      "[2019-12-06 19:22:49,914] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4663 | _timers/batch_time=0.3108 | _timers/data_time=0.0005 | _timers/model_time=0.3103 | f1_score=1.076e-06 | loss=0.6045 | loss_dist=0.2844 | loss_mean=0.3202\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5806 | _timers/batch_time=0.1763 | _timers/data_time=0.0002 | _timers/model_time=0.1760 | f1_score=1.076e-06 | loss=0.6045 | loss_dist=0.2844 | loss_mean=0.3202\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:22:53,756] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5173 | _timers/batch_time=0.3081 | _timers/data_time=0.0004 | _timers/model_time=0.3076 | f1_score=1.144e-06 | loss=7.1780 | loss_dist=6.8574 | loss_mean=0.3206\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5806 | _timers/batch_time=0.1749 | _timers/data_time=0.0003 | _timers/model_time=0.1746 | f1_score=1.144e-06 | loss=7.1780 | loss_dist=6.8574 | loss_mean=0.3206\n",
      "[2019-12-06 19:22:57,601] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5928 | _timers/batch_time=0.3044 | _timers/data_time=0.0003 | _timers/model_time=0.3040 | f1_score=1.280e-06 | loss=0.3351 | loss_dist=0.0214 | loss_mean=0.3137\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5672 | _timers/batch_time=0.1763 | _timers/data_time=0.0003 | _timers/model_time=0.1760 | f1_score=1.280e-06 | loss=0.3351 | loss_dist=0.0214 | loss_mean=0.3137\n",
      "[2019-12-06 19:23:01,472] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5455 | _timers/batch_time=0.3068 | _timers/data_time=0.0004 | _timers/model_time=0.3064 | f1_score=1.274e-06 | loss=0.6275 | loss_dist=0.3065 | loss_mean=0.3210\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.2983 | _timers/batch_time=0.1803 | _timers/data_time=0.0003 | _timers/model_time=0.1799 | f1_score=1.274e-06 | loss=0.6275 | loss_dist=0.3065 | loss_mean=0.3210\n",
      "[2019-12-06 19:23:05,322] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5024 | _timers/batch_time=0.3087 | _timers/data_time=0.0003 | _timers/model_time=0.3083 | f1_score=1.364e-06 | loss=0.3462 | loss_dist=0.0250 | loss_mean=0.3212\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5956 | _timers/batch_time=0.1761 | _timers/data_time=0.0002 | _timers/model_time=0.1758 | f1_score=1.364e-06 | loss=0.3462 | loss_dist=0.0250 | loss_mean=0.3212\n",
      "[2019-12-06 19:23:09,351] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.4478 | _timers/batch_time=0.3119 | _timers/data_time=0.0004 | _timers/model_time=0.3115 | f1_score=1.596e-06 | loss=0.3569 | loss_dist=0.0329 | loss_mean=0.3240\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.9934 | _timers/batch_time=0.1875 | _timers/data_time=0.0003 | _timers/model_time=0.1872 | f1_score=1.596e-06 | loss=0.3569 | loss_dist=0.0329 | loss_mean=0.3240\n",
      "[2019-12-06 19:23:13,331] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3851 | _timers/batch_time=0.3161 | _timers/data_time=0.0003 | _timers/model_time=0.3157 | f1_score=1.305e-06 | loss=1.3676 | loss_dist=1.0488 | loss_mean=0.3188\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5722 | _timers/batch_time=0.1760 | _timers/data_time=0.0002 | _timers/model_time=0.1758 | f1_score=1.305e-06 | loss=1.3676 | loss_dist=1.0488 | loss_mean=0.3188\n",
      "[2019-12-06 19:23:17,223] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5013 | _timers/batch_time=0.3090 | _timers/data_time=0.0004 | _timers/model_time=0.3085 | f1_score=1.178e-06 | loss=2.9305 | loss_dist=2.6099 | loss_mean=0.3206\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.4830 | _timers/batch_time=0.1776 | _timers/data_time=0.0003 | _timers/model_time=0.1773 | f1_score=1.178e-06 | loss=2.9305 | loss_dist=2.6099 | loss_mean=0.3206\n",
      "[2019-12-06 19:23:21,167] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3656 | _timers/batch_time=0.3161 | _timers/data_time=0.0004 | _timers/model_time=0.3157 | f1_score=1.518e-06 | loss=0.3534 | loss_dist=0.0303 | loss_mean=0.3231\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.5840 | _timers/batch_time=0.1763 | _timers/data_time=0.0003 | _timers/model_time=0.1760 | f1_score=1.518e-06 | loss=0.3534 | loss_dist=0.0303 | loss_mean=0.3231\n",
      "[2019-12-06 19:23:25,098] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4788 | _timers/batch_time=0.3103 | _timers/data_time=0.0004 | _timers/model_time=0.3099 | f1_score=1.276e-06 | loss=1.9753 | loss_dist=1.6571 | loss_mean=0.3182\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.4228 | _timers/batch_time=0.1782 | _timers/data_time=0.0004 | _timers/model_time=0.1778 | f1_score=1.276e-06 | loss=1.9753 | loss_dist=1.6571 | loss_mean=0.3182\n",
      "[2019-12-06 19:23:29,186] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.1700 | _timers/batch_time=0.3276 | _timers/data_time=0.0004 | _timers/model_time=0.3272 | f1_score=1.193e-06 | loss=0.4780 | loss_dist=0.1523 | loss_mean=0.3258\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.1052 | _timers/batch_time=0.1869 | _timers/data_time=0.0002 | _timers/model_time=0.1866 | f1_score=1.193e-06 | loss=0.4780 | loss_dist=0.1523 | loss_mean=0.3258\n",
      "[2019-12-06 19:23:33,074] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4508 | _timers/batch_time=0.3107 | _timers/data_time=0.0004 | _timers/model_time=0.3102 | f1_score=1.553e-06 | loss=0.3264 | loss_dist=0.0025 | loss_mean=0.3238\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.5455 | _timers/batch_time=0.1772 | _timers/data_time=0.0002 | _timers/model_time=0.1769 | f1_score=1.553e-06 | loss=0.3264 | loss_dist=0.0025 | loss_mean=0.3238\n",
      "[2019-12-06 19:23:36,959] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.4798 | _timers/batch_time=0.3098 | _timers/data_time=0.0004 | _timers/model_time=0.3093 | f1_score=1.503e-06 | loss=0.4193 | loss_dist=0.0970 | loss_mean=0.3223\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.6518 | _timers/batch_time=0.1752 | _timers/data_time=0.0003 | _timers/model_time=0.1749 | f1_score=1.503e-06 | loss=0.4193 | loss_dist=0.0970 | loss_mean=0.3223\n",
      "[2019-12-06 19:23:40,781] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.5207 | _timers/batch_time=0.3079 | _timers/data_time=0.0004 | _timers/model_time=0.3075 | f1_score=1.512e-06 | loss=0.3268 | loss_dist=0.0082 | loss_mean=0.3186\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.5545 | _timers/batch_time=0.1763 | _timers/data_time=0.0002 | _timers/model_time=0.1760 | f1_score=1.512e-06 | loss=0.3268 | loss_dist=0.0082 | loss_mean=0.3186\n",
      "[2019-12-06 19:23:44,634] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5261 | _timers/batch_time=0.3073 | _timers/data_time=0.0003 | _timers/model_time=0.3070 | f1_score=1.543e-06 | loss=0.3767 | loss_dist=0.0534 | loss_mean=0.3233\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.6992 | _timers/batch_time=0.1736 | _timers/data_time=0.0003 | _timers/model_time=0.1732 | f1_score=1.543e-06 | loss=0.3767 | loss_dist=0.0534 | loss_mean=0.3233\n",
      "[2019-12-06 19:23:48,673] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4047 | _timers/batch_time=0.3148 | _timers/data_time=0.0004 | _timers/model_time=0.3143 | f1_score=1.720e-06 | loss=0.3491 | loss_dist=0.0329 | loss_mean=0.3162\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.4009 | _timers/batch_time=0.1957 | _timers/data_time=0.0004 | _timers/model_time=0.1953 | f1_score=1.720e-06 | loss=0.3491 | loss_dist=0.0329 | loss_mean=0.3162\n",
      "[2019-12-06 19:23:52,559] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.3182 | _timers/batch_time=0.3166 | _timers/data_time=0.0004 | _timers/model_time=0.3162 | f1_score=1.679e-06 | loss=0.3174 | loss_dist=0.0028 | loss_mean=0.3145\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.6791 | _timers/batch_time=0.1748 | _timers/data_time=0.0003 | _timers/model_time=0.1745 | f1_score=1.679e-06 | loss=0.3174 | loss_dist=0.0028 | loss_mean=0.3145\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 19:23:56,435] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4687 | _timers/batch_time=0.3102 | _timers/data_time=0.0003 | _timers/model_time=0.3099 | f1_score=1.567e-06 | loss=0.3516 | loss_dist=0.0318 | loss_mean=0.3198\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.5607 | _timers/batch_time=0.1762 | _timers/data_time=0.0002 | _timers/model_time=0.1759 | f1_score=1.567e-06 | loss=0.3516 | loss_dist=0.0318 | loss_mean=0.3198\n",
      "[2019-12-06 19:24:00,300] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4630 | _timers/batch_time=0.3105 | _timers/data_time=0.0003 | _timers/model_time=0.3102 | f1_score=1.024e-06 | loss=5.7255 | loss_dist=5.4093 | loss_mean=0.3162\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.5353 | _timers/batch_time=0.1768 | _timers/data_time=0.0003 | _timers/model_time=0.1764 | f1_score=1.024e-06 | loss=5.7255 | loss_dist=5.4093 | loss_mean=0.3162\n",
      "[2019-12-06 19:24:04,130] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5641 | _timers/batch_time=0.3057 | _timers/data_time=0.0004 | _timers/model_time=0.3052 | f1_score=1.202e-06 | loss=0.3806 | loss_dist=0.0608 | loss_mean=0.3198\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.6167 | _timers/batch_time=0.1756 | _timers/data_time=0.0003 | _timers/model_time=0.1753 | f1_score=1.202e-06 | loss=0.3806 | loss_dist=0.0608 | loss_mean=0.3198\n",
      "[2019-12-06 19:24:08,004] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.5531 | _timers/batch_time=0.3059 | _timers/data_time=0.0003 | _timers/model_time=0.3055 | f1_score=1.508e-06 | loss=0.3433 | loss_dist=0.0225 | loss_mean=0.3208\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.3144 | _timers/batch_time=0.1801 | _timers/data_time=0.0003 | _timers/model_time=0.1797 | f1_score=1.508e-06 | loss=0.3433 | loss_dist=0.0225 | loss_mean=0.3208\n",
      "Top best models:\n",
      "logs/b80/checkpoints/train.96.pth\t0.3174\n",
      "logs/b80/checkpoints/train.43.pth\t0.3199\n",
      "logs/b80/checkpoints/train.28.pth\t0.3239\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bernoulli p = 0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = 0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "class IMDbRNN_bin(nn.Module):\n",
    "    def __init__(self, sent_size):\n",
    "        super().__init__()\n",
    "#         self.rnn = nn.GRU(embedding_size, hidden_size)\n",
    "        self.rnn = model.rnn\n",
    "        self.cls = model.cls\n",
    "        self.emb = model.emb\n",
    "        \n",
    "        self.u = Variable(torch.randn(1,sent_size,1), requires_grad=True)\n",
    "    \n",
    "    def prepare_sequence(self, x):\n",
    "        l = torch.LongTensor([len(xi) for xi in x])\n",
    "        x = torch.split_with_sizes(self.emb(torch.cat(x).to(self.emb.weight.device)), l.unbind(0))\n",
    "        x = nn.utils.rnn.pad_sequence(x, batch_first=True, padding_value=5)\n",
    "        return x, l\n",
    "    \n",
    "    def forward(self, x):\n",
    "#         print(x.shape,x[0].shape)\n",
    "        \n",
    "        x = [i for i in x]\n",
    "        x, l = self.prepare_sequence(x)\n",
    "        \n",
    "        #считаем с единичной маской\n",
    "        b = torch.ones(x.shape[:2]).unsqueeze(2).to(self.emb.weight.device)\n",
    "        h,o = self.rnn(x,b,l)               \n",
    "        \n",
    "        #считаем с бинаризованной маской u\n",
    "        #torch.bernoulli(torch.tensor([1.0]),)\n",
    "        b_new = torch.bernoulli(torch.ones(x.shape[:2]),p).unsqueeze(2).to(self.emb.weight.device)\n",
    "        #b_new = torch.repeat_interleave(bb(0.4), repeats=len(x), dim=0).to(self.emb.weight.device)\n",
    "        h_new,o_new = self.rnn(x,b_new,l)\n",
    "        \n",
    "        #расстояние между выходами сети на полном и прореженом предложении\n",
    "        dist = ((o-o_new)**2).sum(1)\n",
    "        \n",
    "        #среднее значение бинарной маски\n",
    "        mean = b_new.squeeze(2).mean(1)\n",
    "        \n",
    "        #финальное предсказание класса на прореженом сообщении\n",
    "        target = self.cls(o_new)\n",
    "        \n",
    "        return {'cls': target, 'dist': dist, 'mean': mean}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_bin = IMDbRNN_bin(x[0].shape[0])\n",
    "class Dataset(BaseDataset):\n",
    "\n",
    "    \n",
    "    def __init__(\n",
    "            self, x, y, idx\n",
    "    ):\n",
    "        \n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.idx = idx\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "\n",
    "        return {'features': self.x[self.idx], 'target_cls':self.y[self.idx],\n",
    "                'target_dist': torch.tensor(0).float(), 'target_mean': torch.tensor(0).float()}\n",
    "        \n",
    "    def __len__(self):\n",
    "        return 8\n",
    "\n",
    "\n",
    "new_train_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)\n",
    "new_val_loader = DataLoader(Dataset(x,y,0), shuffle=True, batch_size=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 100  # change me\n",
    "logdir = \"./logs/b99\"\n",
    "loaders = {\n",
    "    \"train\": new_train_loader,\n",
    "    \"valid\": new_val_loader\n",
    "}\n",
    "\n",
    "criterion = {\n",
    "   \"mse\": nn.MSELoss()\n",
    "}\n",
    "\n",
    "optimizer = torch.optim.Adam([\n",
    "    {'params': model_bin.u, 'lr': 3e-3}])\n",
    "\n",
    "\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, factor=0.9, patience=4)\n",
    "\n",
    "runner = SupervisedRunner(input_target_key=None,output_key=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.save_checkpoint();"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:00:32,519] \n",
      "1/100 * Epoch 1 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.8344 | _timers/batch_time=0.3453 | _timers/data_time=0.0003 | _timers/model_time=0.3449 | f1_score=1.611e-06 | loss=0.4885 | loss_dist=6.300e-06 | loss_mean=0.4885\n",
      "1/100 * Epoch 1 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.2343 | _timers/batch_time=0.1994 | _timers/data_time=0.0003 | _timers/model_time=0.1991 | f1_score=1.611e-06 | loss=0.4885 | loss_dist=6.300e-06 | loss_mean=0.4885\n",
      "[2019-12-06 20:00:37,207] \n",
      "2/100 * Epoch 2 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.7884 | _timers/batch_time=0.3467 | _timers/data_time=0.0004 | _timers/model_time=0.3463 | f1_score=1.635e-06 | loss=0.4897 | loss_dist=4.855e-06 | loss_mean=0.4897\n",
      "2/100 * Epoch 2 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=8.6677 | _timers/batch_time=0.2389 | _timers/data_time=0.0005 | _timers/model_time=0.2384 | f1_score=1.635e-06 | loss=0.4897 | loss_dist=4.855e-06 | loss_mean=0.4897\n",
      "[2019-12-06 20:00:41,904] \n",
      "3/100 * Epoch 3 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.1653 | _timers/batch_time=0.3916 | _timers/data_time=0.0003 | _timers/model_time=0.3913 | f1_score=1.607e-06 | loss=0.4912 | loss_dist=3.063e-08 | loss_mean=0.4912\n",
      "3/100 * Epoch 3 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.7964 | _timers/batch_time=0.2070 | _timers/data_time=0.0003 | _timers/model_time=0.2067 | f1_score=1.607e-06 | loss=0.4912 | loss_dist=3.063e-08 | loss_mean=0.4912\n",
      "[2019-12-06 20:00:46,333] \n",
      "4/100 * Epoch 4 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.6836 | _timers/batch_time=0.3543 | _timers/data_time=0.0004 | _timers/model_time=0.3538 | f1_score=1.609e-06 | loss=0.4886 | loss_dist=1.218e-08 | loss_mean=0.4886\n",
      "4/100 * Epoch 4 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=11.0414 | _timers/batch_time=0.1864 | _timers/data_time=0.0003 | _timers/model_time=0.1860 | f1_score=1.609e-06 | loss=0.4886 | loss_dist=1.218e-08 | loss_mean=0.4886\n",
      "[2019-12-06 20:00:50,480] \n",
      "5/100 * Epoch 5 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=6.4313 | _timers/batch_time=0.3133 | _timers/data_time=0.0003 | _timers/model_time=0.3129 | f1_score=1.605e-06 | loss=0.4895 | loss_dist=2.684e-07 | loss_mean=0.4895\n",
      "5/100 * Epoch 5 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=9.5899 | _timers/batch_time=0.2127 | _timers/data_time=0.0003 | _timers/model_time=0.2124 | f1_score=1.605e-06 | loss=0.4895 | loss_dist=2.684e-07 | loss_mean=0.4895\n",
      "[2019-12-06 20:00:55,067] \n",
      "6/100 * Epoch 6 (train): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=5.2638 | _timers/batch_time=0.3834 | _timers/data_time=0.0002 | _timers/model_time=0.3832 | f1_score=1.630e-06 | loss=0.4893 | loss_dist=4.184e-06 | loss_mean=0.4892\n",
      "6/100 * Epoch 6 (valid): _base/lr=0.0030 | _base/momentum=0.9000 | _timers/_fps=10.3438 | _timers/batch_time=0.1955 | _timers/data_time=0.0003 | _timers/model_time=0.1952 | f1_score=1.630e-06 | loss=0.4893 | loss_dist=4.184e-06 | loss_mean=0.4892\n",
      "[2019-12-06 20:00:59,351] \n",
      "7/100 * Epoch 7 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.8884 | _timers/batch_time=0.3404 | _timers/data_time=0.0003 | _timers/model_time=0.3400 | f1_score=1.612e-06 | loss=0.4915 | loss_dist=2.423e-08 | loss_mean=0.4915\n",
      "7/100 * Epoch 7 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.5422 | _timers/batch_time=0.1940 | _timers/data_time=0.0002 | _timers/model_time=0.1937 | f1_score=1.612e-06 | loss=0.4915 | loss_dist=2.423e-08 | loss_mean=0.4915\n",
      "[2019-12-06 20:01:03,780] \n",
      "8/100 * Epoch 8 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.8503 | _timers/batch_time=0.3423 | _timers/data_time=0.0003 | _timers/model_time=0.3420 | f1_score=1.640e-06 | loss=0.4898 | loss_dist=2.015e-05 | loss_mean=0.4898\n",
      "8/100 * Epoch 8 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.2296 | _timers/batch_time=0.1993 | _timers/data_time=0.0003 | _timers/model_time=0.1990 | f1_score=1.640e-06 | loss=0.4898 | loss_dist=2.015e-05 | loss_mean=0.4898\n",
      "[2019-12-06 20:01:08,077] \n",
      "9/100 * Epoch 9 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=5.9533 | _timers/batch_time=0.3378 | _timers/data_time=0.0003 | _timers/model_time=0.3374 | f1_score=1.609e-06 | loss=0.4889 | loss_dist=1.788e-08 | loss_mean=0.4889\n",
      "9/100 * Epoch 9 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=10.2412 | _timers/batch_time=0.1995 | _timers/data_time=0.0003 | _timers/model_time=0.1992 | f1_score=1.609e-06 | loss=0.4889 | loss_dist=1.788e-08 | loss_mean=0.4889\n",
      "[2019-12-06 20:01:12,371] \n",
      "10/100 * Epoch 10 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0321 | _timers/batch_time=0.3318 | _timers/data_time=0.0004 | _timers/model_time=0.3313 | f1_score=1.605e-06 | loss=0.4897 | loss_dist=5.824e-09 | loss_mean=0.4897\n",
      "10/100 * Epoch 10 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=9.8587 | _timers/batch_time=0.2052 | _timers/data_time=0.0003 | _timers/model_time=0.2049 | f1_score=1.605e-06 | loss=0.4897 | loss_dist=5.824e-09 | loss_mean=0.4897\n",
      "[2019-12-06 20:01:16,404] \n",
      "11/100 * Epoch 11 (train): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=6.0175 | _timers/batch_time=0.3327 | _timers/data_time=0.0003 | _timers/model_time=0.3324 | f1_score=1.611e-06 | loss=0.4894 | loss_dist=1.008e-10 | loss_mean=0.4894\n",
      "11/100 * Epoch 11 (valid): _base/lr=0.0027 | _base/momentum=0.9000 | _timers/_fps=11.3756 | _timers/batch_time=0.1790 | _timers/data_time=0.0003 | _timers/model_time=0.1787 | f1_score=1.611e-06 | loss=0.4894 | loss_dist=1.008e-10 | loss_mean=0.4894\n",
      "[2019-12-06 20:01:20,395] \n",
      "12/100 * Epoch 12 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.3217 | _timers/batch_time=0.3173 | _timers/data_time=0.0003 | _timers/model_time=0.3169 | f1_score=1.665e-06 | loss=0.4900 | loss_dist=0.0003 | loss_mean=0.4897\n",
      "12/100 * Epoch 12 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.0097 | _timers/batch_time=0.1858 | _timers/data_time=0.0003 | _timers/model_time=0.1855 | f1_score=1.665e-06 | loss=0.4900 | loss_dist=0.0003 | loss_mean=0.4897\n",
      "[2019-12-06 20:01:24,612] \n",
      "13/100 * Epoch 13 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.0778 | _timers/batch_time=0.3317 | _timers/data_time=0.0002 | _timers/model_time=0.3314 | f1_score=1.595e-06 | loss=0.4894 | loss_dist=9.146e-07 | loss_mean=0.4894\n",
      "13/100 * Epoch 13 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.0686 | _timers/batch_time=0.1835 | _timers/data_time=0.0004 | _timers/model_time=0.1831 | f1_score=1.595e-06 | loss=0.4894 | loss_dist=9.146e-07 | loss_mean=0.4894\n",
      "[2019-12-06 20:01:28,954] \n",
      "14/100 * Epoch 14 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.4897 | _timers/batch_time=0.3649 | _timers/data_time=0.0003 | _timers/model_time=0.3646 | f1_score=1.629e-06 | loss=0.4905 | loss_dist=2.328e-07 | loss_mean=0.4905\n",
      "14/100 * Epoch 14 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.2500 | _timers/batch_time=0.1990 | _timers/data_time=0.0002 | _timers/model_time=0.1988 | f1_score=1.629e-06 | loss=0.4905 | loss_dist=2.328e-07 | loss_mean=0.4905\n",
      "[2019-12-06 20:01:33,192] \n",
      "15/100 * Epoch 15 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=5.9311 | _timers/batch_time=0.3381 | _timers/data_time=0.0004 | _timers/model_time=0.3376 | f1_score=1.622e-06 | loss=0.4894 | loss_dist=1.908e-07 | loss_mean=0.4894\n",
      "15/100 * Epoch 15 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=10.5992 | _timers/batch_time=0.1917 | _timers/data_time=0.0005 | _timers/model_time=0.1912 | f1_score=1.622e-06 | loss=0.4894 | loss_dist=1.908e-07 | loss_mean=0.4894\n",
      "[2019-12-06 20:01:37,186] \n",
      "16/100 * Epoch 16 (train): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=6.1180 | _timers/batch_time=0.3271 | _timers/data_time=0.0005 | _timers/model_time=0.3266 | f1_score=1.609e-06 | loss=0.4918 | loss_dist=5.990e-06 | loss_mean=0.4918\n",
      "16/100 * Epoch 16 (valid): _base/lr=0.0024 | _base/momentum=0.9000 | _timers/_fps=11.5802 | _timers/batch_time=0.1771 | _timers/data_time=0.0002 | _timers/model_time=0.1769 | f1_score=1.609e-06 | loss=0.4918 | loss_dist=5.990e-06 | loss_mean=0.4918\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:01:41,301] \n",
      "17/100 * Epoch 17 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.2698 | _timers/batch_time=0.3212 | _timers/data_time=0.0004 | _timers/model_time=0.3208 | f1_score=1.572e-06 | loss=0.4902 | loss_dist=0.0002 | loss_mean=0.4900\n",
      "17/100 * Epoch 17 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=11.2537 | _timers/batch_time=0.1822 | _timers/data_time=0.0002 | _timers/model_time=0.1819 | f1_score=1.572e-06 | loss=0.4902 | loss_dist=0.0002 | loss_mean=0.4900\n",
      "[2019-12-06 20:01:45,276] \n",
      "18/100 * Epoch 18 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.5044 | _timers/batch_time=0.3086 | _timers/data_time=0.0004 | _timers/model_time=0.3081 | f1_score=1.616e-06 | loss=0.4896 | loss_dist=6.700e-06 | loss_mean=0.4896\n",
      "18/100 * Epoch 18 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.1222 | _timers/batch_time=0.1993 | _timers/data_time=0.0002 | _timers/model_time=0.1991 | f1_score=1.616e-06 | loss=0.4896 | loss_dist=6.700e-06 | loss_mean=0.4896\n",
      "[2019-12-06 20:01:49,559] \n",
      "19/100 * Epoch 19 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0528 | _timers/batch_time=0.3309 | _timers/data_time=0.0002 | _timers/model_time=0.3307 | f1_score=1.618e-06 | loss=0.4909 | loss_dist=0.0013 | loss_mean=0.4896\n",
      "19/100 * Epoch 19 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.0451 | _timers/batch_time=0.2035 | _timers/data_time=0.0003 | _timers/model_time=0.2032 | f1_score=1.618e-06 | loss=0.4909 | loss_dist=0.0013 | loss_mean=0.4896\n",
      "[2019-12-06 20:01:53,714] \n",
      "20/100 * Epoch 20 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0080 | _timers/batch_time=0.3341 | _timers/data_time=0.0004 | _timers/model_time=0.3336 | f1_score=1.621e-06 | loss=0.4918 | loss_dist=2.136e-07 | loss_mean=0.4918\n",
      "20/100 * Epoch 20 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.9352 | _timers/batch_time=0.1873 | _timers/data_time=0.0003 | _timers/model_time=0.1870 | f1_score=1.621e-06 | loss=0.4918 | loss_dist=2.136e-07 | loss_mean=0.4918\n",
      "[2019-12-06 20:01:57,936] \n",
      "21/100 * Epoch 21 (train): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=6.0068 | _timers/batch_time=0.3336 | _timers/data_time=0.0003 | _timers/model_time=0.3333 | f1_score=1.606e-06 | loss=0.4912 | loss_dist=5.439e-07 | loss_mean=0.4912\n",
      "21/100 * Epoch 21 (valid): _base/lr=0.0022 | _base/momentum=0.9000 | _timers/_fps=10.6432 | _timers/batch_time=0.1927 | _timers/data_time=0.0003 | _timers/model_time=0.1923 | f1_score=1.606e-06 | loss=0.4912 | loss_dist=5.439e-07 | loss_mean=0.4912\n",
      "[2019-12-06 20:02:01,952] \n",
      "22/100 * Epoch 22 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1422 | _timers/batch_time=0.3260 | _timers/data_time=0.0004 | _timers/model_time=0.3256 | f1_score=1.609e-06 | loss=0.4906 | loss_dist=9.921e-08 | loss_mean=0.4906\n",
      "22/100 * Epoch 22 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.4081 | _timers/batch_time=0.1783 | _timers/data_time=0.0003 | _timers/model_time=0.1780 | f1_score=1.609e-06 | loss=0.4906 | loss_dist=9.921e-08 | loss_mean=0.4906\n",
      "[2019-12-06 20:02:06,134] \n",
      "23/100 * Epoch 23 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1690 | _timers/batch_time=0.3262 | _timers/data_time=0.0003 | _timers/model_time=0.3258 | f1_score=1.603e-06 | loss=0.4894 | loss_dist=2.848e-07 | loss_mean=0.4894\n",
      "23/100 * Epoch 23 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.1059 | _timers/batch_time=0.2016 | _timers/data_time=0.0002 | _timers/model_time=0.2014 | f1_score=1.603e-06 | loss=0.4894 | loss_dist=2.848e-07 | loss_mean=0.4894\n",
      "[2019-12-06 20:02:10,195] \n",
      "24/100 * Epoch 24 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1934 | _timers/batch_time=0.3232 | _timers/data_time=0.0003 | _timers/model_time=0.3228 | f1_score=1.563e-06 | loss=0.4894 | loss_dist=0.0001 | loss_mean=0.4892\n",
      "24/100 * Epoch 24 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.9246 | _timers/batch_time=0.1856 | _timers/data_time=0.0003 | _timers/model_time=0.1853 | f1_score=1.563e-06 | loss=0.4894 | loss_dist=0.0001 | loss_mean=0.4892\n",
      "[2019-12-06 20:02:14,295] \n",
      "25/100 * Epoch 25 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.0897 | _timers/batch_time=0.3290 | _timers/data_time=0.0003 | _timers/model_time=0.3287 | f1_score=1.612e-06 | loss=0.4870 | loss_dist=2.245e-05 | loss_mean=0.4870\n",
      "25/100 * Epoch 25 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.3622 | _timers/batch_time=0.1794 | _timers/data_time=0.0003 | _timers/model_time=0.1791 | f1_score=1.612e-06 | loss=0.4870 | loss_dist=2.245e-05 | loss_mean=0.4870\n",
      "[2019-12-06 20:02:18,233] \n",
      "26/100 * Epoch 26 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.4361 | _timers/batch_time=0.3120 | _timers/data_time=0.0003 | _timers/model_time=0.3116 | f1_score=1.497e-06 | loss=0.4943 | loss_dist=0.0043 | loss_mean=0.4900\n",
      "26/100 * Epoch 26 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.3643 | _timers/batch_time=0.1792 | _timers/data_time=0.0003 | _timers/model_time=0.1790 | f1_score=1.497e-06 | loss=0.4943 | loss_dist=0.0043 | loss_mean=0.4900\n",
      "[2019-12-06 20:02:22,547] \n",
      "27/100 * Epoch 27 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.1107 | _timers/batch_time=0.3306 | _timers/data_time=0.0004 | _timers/model_time=0.3302 | f1_score=1.592e-06 | loss=0.4899 | loss_dist=6.242e-06 | loss_mean=0.4899\n",
      "27/100 * Epoch 27 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.0961 | _timers/batch_time=0.2025 | _timers/data_time=0.0002 | _timers/model_time=0.2023 | f1_score=1.592e-06 | loss=0.4899 | loss_dist=6.242e-06 | loss_mean=0.4899\n",
      "[2019-12-06 20:02:26,975] \n",
      "28/100 * Epoch 28 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=5.7515 | _timers/batch_time=0.3496 | _timers/data_time=0.0003 | _timers/model_time=0.3493 | f1_score=1.610e-06 | loss=0.4894 | loss_dist=9.391e-08 | loss_mean=0.4894\n",
      "28/100 * Epoch 28 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.0863 | _timers/batch_time=0.2019 | _timers/data_time=0.0002 | _timers/model_time=0.2017 | f1_score=1.610e-06 | loss=0.4894 | loss_dist=9.391e-08 | loss_mean=0.4894\n",
      "[2019-12-06 20:02:31,094] \n",
      "29/100 * Epoch 29 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.3516 | _timers/batch_time=0.3165 | _timers/data_time=0.0002 | _timers/model_time=0.3162 | f1_score=1.603e-06 | loss=0.4927 | loss_dist=0.0013 | loss_mean=0.4913\n",
      "29/100 * Epoch 29 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=10.9498 | _timers/batch_time=0.1877 | _timers/data_time=0.0002 | _timers/model_time=0.1874 | f1_score=1.603e-06 | loss=0.4927 | loss_dist=0.0013 | loss_mean=0.4913\n",
      "[2019-12-06 20:02:35,197] \n",
      "30/100 * Epoch 30 (train): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=6.2054 | _timers/batch_time=0.3263 | _timers/data_time=0.0004 | _timers/model_time=0.3259 | f1_score=1.617e-06 | loss=0.4908 | loss_dist=1.280e-07 | loss_mean=0.4908\n",
      "30/100 * Epoch 30 (valid): _base/lr=0.0020 | _base/momentum=0.9000 | _timers/_fps=11.5726 | _timers/batch_time=0.1771 | _timers/data_time=0.0002 | _timers/model_time=0.1768 | f1_score=1.617e-06 | loss=0.4908 | loss_dist=1.280e-07 | loss_mean=0.4908\n",
      "[2019-12-06 20:02:39,332] \n",
      "31/100 * Epoch 31 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.1176 | _timers/batch_time=0.3319 | _timers/data_time=0.0004 | _timers/model_time=0.3315 | f1_score=1.626e-06 | loss=0.4902 | loss_dist=3.216e-06 | loss_mean=0.4902\n",
      "31/100 * Epoch 31 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.8032 | _timers/batch_time=0.1920 | _timers/data_time=0.0002 | _timers/model_time=0.1918 | f1_score=1.626e-06 | loss=0.4902 | loss_dist=3.216e-06 | loss_mean=0.4902\n",
      "[2019-12-06 20:02:43,471] \n",
      "32/100 * Epoch 32 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0830 | _timers/batch_time=0.3331 | _timers/data_time=0.0004 | _timers/model_time=0.3327 | f1_score=1.635e-06 | loss=0.4917 | loss_dist=2.779e-06 | loss_mean=0.4917\n",
      "32/100 * Epoch 32 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.0212 | _timers/batch_time=0.1842 | _timers/data_time=0.0004 | _timers/model_time=0.1838 | f1_score=1.635e-06 | loss=0.4917 | loss_dist=2.779e-06 | loss_mean=0.4917\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:02:47,586] \n",
      "33/100 * Epoch 33 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.1492 | _timers/batch_time=0.3272 | _timers/data_time=0.0003 | _timers/model_time=0.3269 | f1_score=1.604e-06 | loss=0.4900 | loss_dist=2.742e-06 | loss_mean=0.4900\n",
      "33/100 * Epoch 33 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=10.6862 | _timers/batch_time=0.1911 | _timers/data_time=0.0003 | _timers/model_time=0.1908 | f1_score=1.604e-06 | loss=0.4900 | loss_dist=2.742e-06 | loss_mean=0.4900\n",
      "[2019-12-06 20:02:51,705] \n",
      "34/100 * Epoch 34 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.0575 | _timers/batch_time=0.3312 | _timers/data_time=0.0003 | _timers/model_time=0.3308 | f1_score=1.613e-06 | loss=0.4892 | loss_dist=3.705e-09 | loss_mean=0.4892\n",
      "34/100 * Epoch 34 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.1244 | _timers/batch_time=0.1849 | _timers/data_time=0.0004 | _timers/model_time=0.1845 | f1_score=1.613e-06 | loss=0.4892 | loss_dist=3.705e-09 | loss_mean=0.4892\n",
      "[2019-12-06 20:02:55,805] \n",
      "35/100 * Epoch 35 (train): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=6.2524 | _timers/batch_time=0.3220 | _timers/data_time=0.0003 | _timers/model_time=0.3216 | f1_score=1.609e-06 | loss=0.4903 | loss_dist=1.085e-07 | loss_mean=0.4903\n",
      "35/100 * Epoch 35 (valid): _base/lr=0.0018 | _base/momentum=0.9000 | _timers/_fps=11.0168 | _timers/batch_time=0.1851 | _timers/data_time=0.0004 | _timers/model_time=0.1847 | f1_score=1.609e-06 | loss=0.4903 | loss_dist=1.085e-07 | loss_mean=0.4903\n",
      "[2019-12-06 20:03:00,045] \n",
      "36/100 * Epoch 36 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.1255 | _timers/batch_time=0.3273 | _timers/data_time=0.0004 | _timers/model_time=0.3269 | f1_score=1.595e-06 | loss=0.4954 | loss_dist=0.0070 | loss_mean=0.4884\n",
      "36/100 * Epoch 36 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.5693 | _timers/batch_time=0.1941 | _timers/data_time=0.0003 | _timers/model_time=0.1938 | f1_score=1.595e-06 | loss=0.4954 | loss_dist=0.0070 | loss_mean=0.4884\n",
      "[2019-12-06 20:03:04,205] \n",
      "37/100 * Epoch 37 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.8688 | _timers/batch_time=0.3415 | _timers/data_time=0.0004 | _timers/model_time=0.3411 | f1_score=1.599e-06 | loss=0.4888 | loss_dist=3.005e-06 | loss_mean=0.4888\n",
      "37/100 * Epoch 37 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.0698 | _timers/batch_time=0.1857 | _timers/data_time=0.0003 | _timers/model_time=0.1854 | f1_score=1.599e-06 | loss=0.4888 | loss_dist=3.005e-06 | loss_mean=0.4888\n",
      "[2019-12-06 20:03:08,535] \n",
      "38/100 * Epoch 38 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=6.1374 | _timers/batch_time=0.3297 | _timers/data_time=0.0004 | _timers/model_time=0.3293 | f1_score=1.660e-06 | loss=0.4890 | loss_dist=0.0003 | loss_mean=0.4887\n",
      "38/100 * Epoch 38 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.5298 | _timers/batch_time=0.1949 | _timers/data_time=0.0003 | _timers/model_time=0.1945 | f1_score=1.660e-06 | loss=0.4890 | loss_dist=0.0003 | loss_mean=0.4887\n",
      "[2019-12-06 20:03:12,757] \n",
      "39/100 * Epoch 39 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.9286 | _timers/batch_time=0.3393 | _timers/data_time=0.0002 | _timers/model_time=0.3391 | f1_score=1.532e-06 | loss=0.4960 | loss_dist=0.0043 | loss_mean=0.4918\n",
      "39/100 * Epoch 39 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=11.0060 | _timers/batch_time=0.1852 | _timers/data_time=0.0004 | _timers/model_time=0.1848 | f1_score=1.532e-06 | loss=0.4960 | loss_dist=0.0043 | loss_mean=0.4918\n",
      "[2019-12-06 20:03:17,039] \n",
      "40/100 * Epoch 40 (train): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=5.9364 | _timers/batch_time=0.3404 | _timers/data_time=0.0004 | _timers/model_time=0.3400 | f1_score=1.542e-06 | loss=0.4939 | loss_dist=0.0042 | loss_mean=0.4897\n",
      "40/100 * Epoch 40 (valid): _base/lr=0.0016 | _base/momentum=0.9000 | _timers/_fps=10.7455 | _timers/batch_time=0.1909 | _timers/data_time=0.0003 | _timers/model_time=0.1906 | f1_score=1.542e-06 | loss=0.4939 | loss_dist=0.0042 | loss_mean=0.4897\n",
      "[2019-12-06 20:03:21,202] \n",
      "41/100 * Epoch 41 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=5.9498 | _timers/batch_time=0.3368 | _timers/data_time=0.0002 | _timers/model_time=0.3365 | f1_score=1.533e-06 | loss=0.4902 | loss_dist=0.0002 | loss_mean=0.4900\n",
      "41/100 * Epoch 41 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.1945 | _timers/batch_time=0.1846 | _timers/data_time=0.0003 | _timers/model_time=0.1843 | f1_score=1.533e-06 | loss=0.4902 | loss_dist=0.0002 | loss_mean=0.4900\n",
      "[2019-12-06 20:03:25,619] \n",
      "42/100 * Epoch 42 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=5.8270 | _timers/batch_time=0.3460 | _timers/data_time=0.0003 | _timers/model_time=0.3457 | f1_score=1.600e-06 | loss=0.4925 | loss_dist=3.054e-09 | loss_mean=0.4925\n",
      "42/100 * Epoch 42 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.2156 | _timers/batch_time=0.2007 | _timers/data_time=0.0003 | _timers/model_time=0.2003 | f1_score=1.600e-06 | loss=0.4925 | loss_dist=3.054e-09 | loss_mean=0.4925\n",
      "[2019-12-06 20:03:29,899] \n",
      "43/100 * Epoch 43 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=5.8819 | _timers/batch_time=0.3407 | _timers/data_time=0.0004 | _timers/model_time=0.3403 | f1_score=1.530e-06 | loss=0.4904 | loss_dist=0.0002 | loss_mean=0.4902\n",
      "43/100 * Epoch 43 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=11.0000 | _timers/batch_time=0.1878 | _timers/data_time=0.0003 | _timers/model_time=0.1874 | f1_score=1.530e-06 | loss=0.4904 | loss_dist=0.0002 | loss_mean=0.4902\n",
      "[2019-12-06 20:03:33,963] \n",
      "44/100 * Epoch 44 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=6.3513 | _timers/batch_time=0.3175 | _timers/data_time=0.0003 | _timers/model_time=0.3172 | f1_score=1.641e-06 | loss=0.4969 | loss_dist=0.0070 | loss_mean=0.4899\n",
      "44/100 * Epoch 44 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.8119 | _timers/batch_time=0.1917 | _timers/data_time=0.0003 | _timers/model_time=0.1913 | f1_score=1.641e-06 | loss=0.4969 | loss_dist=0.0070 | loss_mean=0.4899\n",
      "[2019-12-06 20:03:38,290] \n",
      "45/100 * Epoch 45 (train): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=5.8382 | _timers/batch_time=0.3435 | _timers/data_time=0.0003 | _timers/model_time=0.3431 | f1_score=1.611e-06 | loss=0.4902 | loss_dist=1.443e-07 | loss_mean=0.4902\n",
      "45/100 * Epoch 45 (valid): _base/lr=0.0014 | _base/momentum=0.9000 | _timers/_fps=10.1784 | _timers/batch_time=0.1999 | _timers/data_time=0.0002 | _timers/model_time=0.1997 | f1_score=1.611e-06 | loss=0.4902 | loss_dist=1.443e-07 | loss_mean=0.4902\n",
      "[2019-12-06 20:03:42,480] \n",
      "46/100 * Epoch 46 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1300 | _timers/batch_time=0.3267 | _timers/data_time=0.0003 | _timers/model_time=0.3264 | f1_score=1.546e-06 | loss=0.4904 | loss_dist=0.0002 | loss_mean=0.4902\n",
      "46/100 * Epoch 46 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.2350 | _timers/batch_time=0.1973 | _timers/data_time=0.0003 | _timers/model_time=0.1969 | f1_score=1.546e-06 | loss=0.4904 | loss_dist=0.0002 | loss_mean=0.4902\n",
      "[2019-12-06 20:03:46,683] \n",
      "47/100 * Epoch 47 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1487 | _timers/batch_time=0.3282 | _timers/data_time=0.0004 | _timers/model_time=0.3278 | f1_score=1.612e-06 | loss=0.4901 | loss_dist=9.378e-08 | loss_mean=0.4901\n",
      "47/100 * Epoch 47 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.2677 | _timers/batch_time=0.1804 | _timers/data_time=0.0003 | _timers/model_time=0.1801 | f1_score=1.612e-06 | loss=0.4901 | loss_dist=9.378e-08 | loss_mean=0.4901\n",
      "[2019-12-06 20:03:50,800] \n",
      "48/100 * Epoch 48 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.2831 | _timers/batch_time=0.3229 | _timers/data_time=0.0003 | _timers/model_time=0.3225 | f1_score=1.617e-06 | loss=0.4886 | loss_dist=1.689e-06 | loss_mean=0.4886\n",
      "48/100 * Epoch 48 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=11.0556 | _timers/batch_time=0.1865 | _timers/data_time=0.0002 | _timers/model_time=0.1862 | f1_score=1.617e-06 | loss=0.4886 | loss_dist=1.689e-06 | loss_mean=0.4886\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:03:55,187] \n",
      "49/100 * Epoch 49 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=5.8413 | _timers/batch_time=0.3467 | _timers/data_time=0.0004 | _timers/model_time=0.3462 | f1_score=1.520e-06 | loss=0.4911 | loss_dist=0.0004 | loss_mean=0.4907\n",
      "49/100 * Epoch 49 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.6046 | _timers/batch_time=0.1954 | _timers/data_time=0.0005 | _timers/model_time=0.1948 | f1_score=1.520e-06 | loss=0.4911 | loss_dist=0.0004 | loss_mean=0.4907\n",
      "[2019-12-06 20:03:59,294] \n",
      "50/100 * Epoch 50 (train): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=6.1822 | _timers/batch_time=0.3266 | _timers/data_time=0.0003 | _timers/model_time=0.3262 | f1_score=1.609e-06 | loss=0.4905 | loss_dist=1.240e-08 | loss_mean=0.4905\n",
      "50/100 * Epoch 50 (valid): _base/lr=0.0013 | _base/momentum=0.9000 | _timers/_fps=10.7855 | _timers/batch_time=0.1890 | _timers/data_time=0.0003 | _timers/model_time=0.1887 | f1_score=1.609e-06 | loss=0.4905 | loss_dist=1.240e-08 | loss_mean=0.4905\n",
      "[2019-12-06 20:04:03,520] \n",
      "51/100 * Epoch 51 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1603 | _timers/batch_time=0.3247 | _timers/data_time=0.0003 | _timers/model_time=0.3243 | f1_score=1.604e-06 | loss=0.4908 | loss_dist=9.439e-06 | loss_mean=0.4908\n",
      "51/100 * Epoch 51 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.0744 | _timers/batch_time=0.1861 | _timers/data_time=0.0004 | _timers/model_time=0.1857 | f1_score=1.604e-06 | loss=0.4908 | loss_dist=9.439e-06 | loss_mean=0.4908\n",
      "[2019-12-06 20:04:07,482] \n",
      "52/100 * Epoch 52 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.2402 | _timers/batch_time=0.3210 | _timers/data_time=0.0003 | _timers/model_time=0.3206 | f1_score=1.599e-06 | loss=0.4882 | loss_dist=2.028e-08 | loss_mean=0.4882\n",
      "52/100 * Epoch 52 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.5655 | _timers/batch_time=0.1764 | _timers/data_time=0.0004 | _timers/model_time=0.1759 | f1_score=1.599e-06 | loss=0.4882 | loss_dist=2.028e-08 | loss_mean=0.4882\n",
      "[2019-12-06 20:04:11,606] \n",
      "53/100 * Epoch 53 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.3388 | _timers/batch_time=0.3196 | _timers/data_time=0.0003 | _timers/model_time=0.3193 | f1_score=1.615e-06 | loss=0.4921 | loss_dist=0.0013 | loss_mean=0.4908\n",
      "53/100 * Epoch 53 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.5714 | _timers/batch_time=0.1947 | _timers/data_time=0.0003 | _timers/model_time=0.1943 | f1_score=1.615e-06 | loss=0.4921 | loss_dist=0.0013 | loss_mean=0.4908\n",
      "[2019-12-06 20:04:15,672] \n",
      "54/100 * Epoch 54 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.1745 | _timers/batch_time=0.3246 | _timers/data_time=0.0003 | _timers/model_time=0.3243 | f1_score=1.648e-06 | loss=0.4894 | loss_dist=5.087e-06 | loss_mean=0.4894\n",
      "54/100 * Epoch 54 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=11.2186 | _timers/batch_time=0.1849 | _timers/data_time=0.0003 | _timers/model_time=0.1845 | f1_score=1.648e-06 | loss=0.4894 | loss_dist=5.087e-06 | loss_mean=0.4894\n",
      "[2019-12-06 20:04:19,906] \n",
      "55/100 * Epoch 55 (train): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=6.0357 | _timers/batch_time=0.3331 | _timers/data_time=0.0004 | _timers/model_time=0.3327 | f1_score=1.643e-06 | loss=0.4898 | loss_dist=0.0002 | loss_mean=0.4896\n",
      "55/100 * Epoch 55 (valid): _base/lr=0.0012 | _base/momentum=0.9000 | _timers/_fps=10.6279 | _timers/batch_time=0.1946 | _timers/data_time=0.0003 | _timers/model_time=0.1943 | f1_score=1.643e-06 | loss=0.4898 | loss_dist=0.0002 | loss_mean=0.4896\n",
      "[2019-12-06 20:04:24,313] \n",
      "56/100 * Epoch 56 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.9385 | _timers/batch_time=0.3415 | _timers/data_time=0.0003 | _timers/model_time=0.3412 | f1_score=1.601e-06 | loss=0.4911 | loss_dist=9.091e-06 | loss_mean=0.4911\n",
      "56/100 * Epoch 56 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.0802 | _timers/batch_time=0.2054 | _timers/data_time=0.0002 | _timers/model_time=0.2052 | f1_score=1.601e-06 | loss=0.4911 | loss_dist=9.091e-06 | loss_mean=0.4911\n",
      "[2019-12-06 20:04:28,594] \n",
      "57/100 * Epoch 57 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.9202 | _timers/batch_time=0.3417 | _timers/data_time=0.0003 | _timers/model_time=0.3414 | f1_score=1.683e-06 | loss=0.4894 | loss_dist=0.0005 | loss_mean=0.4889\n",
      "57/100 * Epoch 57 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=11.1521 | _timers/batch_time=0.1841 | _timers/data_time=0.0003 | _timers/model_time=0.1837 | f1_score=1.683e-06 | loss=0.4894 | loss_dist=0.0005 | loss_mean=0.4889\n",
      "[2019-12-06 20:04:32,919] \n",
      "58/100 * Epoch 58 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=5.8281 | _timers/batch_time=0.3444 | _timers/data_time=0.0002 | _timers/model_time=0.3442 | f1_score=1.623e-06 | loss=0.4913 | loss_dist=1.562e-07 | loss_mean=0.4913\n",
      "58/100 * Epoch 58 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.2651 | _timers/batch_time=0.2018 | _timers/data_time=0.0003 | _timers/model_time=0.2014 | f1_score=1.623e-06 | loss=0.4913 | loss_dist=1.562e-07 | loss_mean=0.4913\n",
      "[2019-12-06 20:04:37,253] \n",
      "59/100 * Epoch 59 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.1785 | _timers/batch_time=0.3262 | _timers/data_time=0.0004 | _timers/model_time=0.3258 | f1_score=1.610e-06 | loss=0.4895 | loss_dist=2.479e-07 | loss_mean=0.4895\n",
      "59/100 * Epoch 59 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.2803 | _timers/batch_time=0.1971 | _timers/data_time=0.0003 | _timers/model_time=0.1968 | f1_score=1.610e-06 | loss=0.4895 | loss_dist=2.479e-07 | loss_mean=0.4895\n",
      "[2019-12-06 20:04:41,462] \n",
      "60/100 * Epoch 60 (train): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=6.0452 | _timers/batch_time=0.3312 | _timers/data_time=0.0003 | _timers/model_time=0.3309 | f1_score=1.619e-06 | loss=0.4896 | loss_dist=3.510e-06 | loss_mean=0.4896\n",
      "60/100 * Epoch 60 (valid): _base/lr=0.0010 | _base/momentum=0.9000 | _timers/_fps=10.9075 | _timers/batch_time=0.1863 | _timers/data_time=0.0003 | _timers/model_time=0.1860 | f1_score=1.619e-06 | loss=0.4896 | loss_dist=3.510e-06 | loss_mean=0.4896\n",
      "[2019-12-06 20:04:45,551] \n",
      "61/100 * Epoch 61 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.0839 | _timers/batch_time=0.3301 | _timers/data_time=0.0003 | _timers/model_time=0.3297 | f1_score=1.613e-06 | loss=0.4908 | loss_dist=4.467e-06 | loss_mean=0.4908\n",
      "61/100 * Epoch 61 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=11.3141 | _timers/batch_time=0.1803 | _timers/data_time=0.0003 | _timers/model_time=0.1799 | f1_score=1.613e-06 | loss=0.4908 | loss_dist=4.467e-06 | loss_mean=0.4908\n",
      "[2019-12-06 20:04:49,510] \n",
      "62/100 * Epoch 62 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4358 | _timers/batch_time=0.3115 | _timers/data_time=0.0003 | _timers/model_time=0.3112 | f1_score=1.616e-06 | loss=0.4891 | loss_dist=4.403e-07 | loss_mean=0.4891\n",
      "62/100 * Epoch 62 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.9367 | _timers/batch_time=0.1867 | _timers/data_time=0.0002 | _timers/model_time=0.1865 | f1_score=1.616e-06 | loss=0.4891 | loss_dist=4.403e-07 | loss_mean=0.4891\n",
      "[2019-12-06 20:04:53,480] \n",
      "63/100 * Epoch 63 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.4082 | _timers/batch_time=0.3133 | _timers/data_time=0.0003 | _timers/model_time=0.3130 | f1_score=1.640e-06 | loss=0.5047 | loss_dist=0.0153 | loss_mean=0.4895\n",
      "63/100 * Epoch 63 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.8700 | _timers/batch_time=0.1872 | _timers/data_time=0.0004 | _timers/model_time=0.1867 | f1_score=1.640e-06 | loss=0.5047 | loss_dist=0.0153 | loss_mean=0.4895\n",
      "[2019-12-06 20:04:57,698] \n",
      "64/100 * Epoch 64 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=6.0854 | _timers/batch_time=0.3303 | _timers/data_time=0.0004 | _timers/model_time=0.3299 | f1_score=1.595e-06 | loss=0.4891 | loss_dist=4.185e-07 | loss_mean=0.4891\n",
      "64/100 * Epoch 64 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.2292 | _timers/batch_time=0.1992 | _timers/data_time=0.0004 | _timers/model_time=0.1988 | f1_score=1.595e-06 | loss=0.4891 | loss_dist=4.185e-07 | loss_mean=0.4891\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:05:01,876] \n",
      "65/100 * Epoch 65 (train): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=5.9937 | _timers/batch_time=0.3342 | _timers/data_time=0.0004 | _timers/model_time=0.3337 | f1_score=1.554e-06 | loss=0.4895 | loss_dist=0.0002 | loss_mean=0.4894\n",
      "65/100 * Epoch 65 (valid): _base/lr=0.0009 | _base/momentum=0.9000 | _timers/_fps=10.9030 | _timers/batch_time=0.1870 | _timers/data_time=0.0004 | _timers/model_time=0.1866 | f1_score=1.554e-06 | loss=0.4895 | loss_dist=0.0002 | loss_mean=0.4894\n",
      "[2019-12-06 20:05:06,182] \n",
      "66/100 * Epoch 66 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1061 | _timers/batch_time=0.3283 | _timers/data_time=0.0004 | _timers/model_time=0.3279 | f1_score=1.623e-06 | loss=0.4889 | loss_dist=0.0003 | loss_mean=0.4886\n",
      "66/100 * Epoch 66 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.0226 | _timers/batch_time=0.2035 | _timers/data_time=0.0002 | _timers/model_time=0.2032 | f1_score=1.623e-06 | loss=0.4889 | loss_dist=0.0003 | loss_mean=0.4886\n",
      "[2019-12-06 20:05:10,377] \n",
      "67/100 * Epoch 67 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2085 | _timers/batch_time=0.3236 | _timers/data_time=0.0004 | _timers/model_time=0.3232 | f1_score=1.620e-06 | loss=0.4889 | loss_dist=8.468e-07 | loss_mean=0.4889\n",
      "67/100 * Epoch 67 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.1887 | _timers/batch_time=0.2001 | _timers/data_time=0.0004 | _timers/model_time=0.1997 | f1_score=1.620e-06 | loss=0.4889 | loss_dist=8.468e-07 | loss_mean=0.4889\n",
      "[2019-12-06 20:05:14,344] \n",
      "68/100 * Epoch 68 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2804 | _timers/batch_time=0.3186 | _timers/data_time=0.0003 | _timers/model_time=0.3183 | f1_score=1.589e-06 | loss=0.4900 | loss_dist=4.096e-08 | loss_mean=0.4900\n",
      "68/100 * Epoch 68 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.7199 | _timers/batch_time=0.1742 | _timers/data_time=0.0002 | _timers/model_time=0.1740 | f1_score=1.589e-06 | loss=0.4900 | loss_dist=4.096e-08 | loss_mean=0.4900\n",
      "[2019-12-06 20:05:18,609] \n",
      "69/100 * Epoch 69 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.0340 | _timers/batch_time=0.3358 | _timers/data_time=0.0003 | _timers/model_time=0.3354 | f1_score=1.603e-06 | loss=0.4895 | loss_dist=2.873e-08 | loss_mean=0.4895\n",
      "69/100 * Epoch 69 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.3767 | _timers/batch_time=0.1956 | _timers/data_time=0.0002 | _timers/model_time=0.1954 | f1_score=1.603e-06 | loss=0.4895 | loss_dist=2.873e-08 | loss_mean=0.4895\n",
      "[2019-12-06 20:05:22,671] \n",
      "70/100 * Epoch 70 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.2216 | _timers/batch_time=0.3218 | _timers/data_time=0.0004 | _timers/model_time=0.3215 | f1_score=1.606e-06 | loss=0.4898 | loss_dist=5.640e-10 | loss_mean=0.4898\n",
      "70/100 * Epoch 70 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.6320 | _timers/batch_time=0.1919 | _timers/data_time=0.0004 | _timers/model_time=0.1915 | f1_score=1.606e-06 | loss=0.4898 | loss_dist=5.640e-10 | loss_mean=0.4898\n",
      "[2019-12-06 20:05:26,839] \n",
      "71/100 * Epoch 71 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.1491 | _timers/batch_time=0.3264 | _timers/data_time=0.0003 | _timers/model_time=0.3260 | f1_score=1.625e-06 | loss=0.4891 | loss_dist=1.452e-05 | loss_mean=0.4890\n",
      "71/100 * Epoch 71 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.8114 | _timers/batch_time=0.1886 | _timers/data_time=0.0004 | _timers/model_time=0.1882 | f1_score=1.625e-06 | loss=0.4891 | loss_dist=1.452e-05 | loss_mean=0.4890\n",
      "[2019-12-06 20:05:30,833] \n",
      "72/100 * Epoch 72 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.3846 | _timers/batch_time=0.3143 | _timers/data_time=0.0004 | _timers/model_time=0.3139 | f1_score=1.634e-06 | loss=0.4922 | loss_dist=0.0003 | loss_mean=0.4920\n",
      "72/100 * Epoch 72 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.9588 | _timers/batch_time=0.1882 | _timers/data_time=0.0003 | _timers/model_time=0.1879 | f1_score=1.634e-06 | loss=0.4922 | loss_dist=0.0003 | loss_mean=0.4920\n",
      "[2019-12-06 20:05:34,667] \n",
      "73/100 * Epoch 73 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5989 | _timers/batch_time=0.3036 | _timers/data_time=0.0003 | _timers/model_time=0.3032 | f1_score=1.632e-06 | loss=0.4903 | loss_dist=2.877e-07 | loss_mean=0.4903\n",
      "73/100 * Epoch 73 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.8409 | _timers/batch_time=0.1725 | _timers/data_time=0.0002 | _timers/model_time=0.1723 | f1_score=1.632e-06 | loss=0.4903 | loss_dist=2.877e-07 | loss_mean=0.4903\n",
      "[2019-12-06 20:05:38,488] \n",
      "74/100 * Epoch 74 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.6075 | _timers/batch_time=0.3038 | _timers/data_time=0.0004 | _timers/model_time=0.3034 | f1_score=1.553e-06 | loss=0.4904 | loss_dist=0.0018 | loss_mean=0.4886\n",
      "74/100 * Epoch 74 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=11.6608 | _timers/batch_time=0.1749 | _timers/data_time=0.0004 | _timers/model_time=0.1744 | f1_score=1.553e-06 | loss=0.4904 | loss_dist=0.0018 | loss_mean=0.4886\n",
      "[2019-12-06 20:05:42,369] \n",
      "75/100 * Epoch 75 (train): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=6.5401 | _timers/batch_time=0.3065 | _timers/data_time=0.0004 | _timers/model_time=0.3060 | f1_score=1.595e-06 | loss=0.4900 | loss_dist=8.083e-06 | loss_mean=0.4900\n",
      "75/100 * Epoch 75 (valid): _base/lr=0.0008 | _base/momentum=0.9000 | _timers/_fps=10.8608 | _timers/batch_time=0.1885 | _timers/data_time=0.0003 | _timers/model_time=0.1882 | f1_score=1.595e-06 | loss=0.4900 | loss_dist=8.083e-06 | loss_mean=0.4900\n",
      "[2019-12-06 20:05:46,732] \n",
      "76/100 * Epoch 76 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.7839 | _timers/batch_time=0.3476 | _timers/data_time=0.0004 | _timers/model_time=0.3471 | f1_score=1.621e-06 | loss=0.4886 | loss_dist=1.368e-07 | loss_mean=0.4886\n",
      "76/100 * Epoch 76 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.4561 | _timers/batch_time=0.1969 | _timers/data_time=0.0002 | _timers/model_time=0.1966 | f1_score=1.621e-06 | loss=0.4886 | loss_dist=1.368e-07 | loss_mean=0.4886\n",
      "[2019-12-06 20:05:50,989] \n",
      "77/100 * Epoch 77 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.1148 | _timers/batch_time=0.3277 | _timers/data_time=0.0003 | _timers/model_time=0.3273 | f1_score=1.609e-06 | loss=0.4908 | loss_dist=4.795e-09 | loss_mean=0.4908\n",
      "77/100 * Epoch 77 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.3462 | _timers/batch_time=0.2022 | _timers/data_time=0.0003 | _timers/model_time=0.2019 | f1_score=1.609e-06 | loss=0.4908 | loss_dist=4.795e-09 | loss_mean=0.4908\n",
      "[2019-12-06 20:05:55,293] \n",
      "78/100 * Epoch 78 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.0285 | _timers/batch_time=0.3321 | _timers/data_time=0.0002 | _timers/model_time=0.3318 | f1_score=1.455e-06 | loss=0.4965 | loss_dist=0.0054 | loss_mean=0.4911\n",
      "78/100 * Epoch 78 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=9.8809 | _timers/batch_time=0.2037 | _timers/data_time=0.0003 | _timers/model_time=0.2033 | f1_score=1.455e-06 | loss=0.4965 | loss_dist=0.0054 | loss_mean=0.4911\n",
      "[2019-12-06 20:05:59,397] \n",
      "79/100 * Epoch 79 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=6.1650 | _timers/batch_time=0.3246 | _timers/data_time=0.0003 | _timers/model_time=0.3243 | f1_score=1.536e-06 | loss=0.4912 | loss_dist=0.0002 | loss_mean=0.4910\n",
      "79/100 * Epoch 79 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=10.9993 | _timers/batch_time=0.1852 | _timers/data_time=0.0003 | _timers/model_time=0.1849 | f1_score=1.536e-06 | loss=0.4912 | loss_dist=0.0002 | loss_mean=0.4910\n",
      "[2019-12-06 20:06:03,623] \n",
      "80/100 * Epoch 80 (train): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=5.9091 | _timers/batch_time=0.3434 | _timers/data_time=0.0003 | _timers/model_time=0.3430 | f1_score=1.614e-06 | loss=0.4955 | loss_dist=0.0070 | loss_mean=0.4885\n",
      "80/100 * Epoch 80 (valid): _base/lr=0.0007 | _base/momentum=0.9000 | _timers/_fps=11.1651 | _timers/batch_time=0.1831 | _timers/data_time=0.0003 | _timers/model_time=0.1828 | f1_score=1.614e-06 | loss=0.4955 | loss_dist=0.0070 | loss_mean=0.4885\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:06:07,636] \n",
      "81/100 * Epoch 81 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2121 | _timers/batch_time=0.3220 | _timers/data_time=0.0004 | _timers/model_time=0.3216 | f1_score=1.607e-06 | loss=0.4888 | loss_dist=5.937e-08 | loss_mean=0.4888\n",
      "81/100 * Epoch 81 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.8772 | _timers/batch_time=0.1873 | _timers/data_time=0.0004 | _timers/model_time=0.1869 | f1_score=1.607e-06 | loss=0.4888 | loss_dist=5.937e-08 | loss_mean=0.4888\n",
      "[2019-12-06 20:06:11,826] \n",
      "82/100 * Epoch 82 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.7567 | _timers/batch_time=0.3480 | _timers/data_time=0.0004 | _timers/model_time=0.3475 | f1_score=1.641e-06 | loss=0.4908 | loss_dist=0.0002 | loss_mean=0.4906\n",
      "82/100 * Epoch 82 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.4310 | _timers/batch_time=0.1797 | _timers/data_time=0.0003 | _timers/model_time=0.1793 | f1_score=1.641e-06 | loss=0.4908 | loss_dist=0.0002 | loss_mean=0.4906\n",
      "[2019-12-06 20:06:15,886] \n",
      "83/100 * Epoch 83 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2087 | _timers/batch_time=0.3241 | _timers/data_time=0.0003 | _timers/model_time=0.3237 | f1_score=1.622e-06 | loss=0.4894 | loss_dist=1.405e-07 | loss_mean=0.4894\n",
      "83/100 * Epoch 83 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.8406 | _timers/batch_time=0.1878 | _timers/data_time=0.0003 | _timers/model_time=0.1875 | f1_score=1.622e-06 | loss=0.4894 | loss_dist=1.405e-07 | loss_mean=0.4894\n",
      "[2019-12-06 20:06:20,079] \n",
      "84/100 * Epoch 84 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.4628 | _timers/batch_time=0.3115 | _timers/data_time=0.0004 | _timers/model_time=0.3111 | f1_score=1.572e-06 | loss=0.4922 | loss_dist=0.0002 | loss_mean=0.4920\n",
      "84/100 * Epoch 84 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=9.9551 | _timers/batch_time=0.2053 | _timers/data_time=0.0003 | _timers/model_time=0.2051 | f1_score=1.572e-06 | loss=0.4922 | loss_dist=0.0002 | loss_mean=0.4920\n",
      "[2019-12-06 20:06:24,183] \n",
      "85/100 * Epoch 85 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.2413 | _timers/batch_time=0.3218 | _timers/data_time=0.0002 | _timers/model_time=0.3215 | f1_score=1.619e-06 | loss=0.4906 | loss_dist=2.368e-08 | loss_mean=0.4906\n",
      "85/100 * Epoch 85 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.8884 | _timers/batch_time=0.1867 | _timers/data_time=0.0002 | _timers/model_time=0.1865 | f1_score=1.619e-06 | loss=0.4906 | loss_dist=2.368e-08 | loss_mean=0.4906\n",
      "[2019-12-06 20:06:28,475] \n",
      "86/100 * Epoch 86 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.8788 | _timers/batch_time=0.3406 | _timers/data_time=0.0004 | _timers/model_time=0.3402 | f1_score=1.641e-06 | loss=0.4894 | loss_dist=4.629e-06 | loss_mean=0.4894\n",
      "86/100 * Epoch 86 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.6197 | _timers/batch_time=0.1931 | _timers/data_time=0.0003 | _timers/model_time=0.1928 | f1_score=1.641e-06 | loss=0.4894 | loss_dist=4.629e-06 | loss_mean=0.4894\n",
      "[2019-12-06 20:06:32,450] \n",
      "87/100 * Epoch 87 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.3153 | _timers/batch_time=0.3172 | _timers/data_time=0.0003 | _timers/model_time=0.3169 | f1_score=1.620e-06 | loss=0.4899 | loss_dist=2.400e-07 | loss_mean=0.4899\n",
      "87/100 * Epoch 87 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.2066 | _timers/batch_time=0.1817 | _timers/data_time=0.0003 | _timers/model_time=0.1814 | f1_score=1.620e-06 | loss=0.4899 | loss_dist=2.400e-07 | loss_mean=0.4899\n",
      "[2019-12-06 20:06:36,443] \n",
      "88/100 * Epoch 88 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.3119 | _timers/batch_time=0.3183 | _timers/data_time=0.0005 | _timers/model_time=0.3177 | f1_score=1.613e-06 | loss=0.4912 | loss_dist=3.956e-06 | loss_mean=0.4912\n",
      "88/100 * Epoch 88 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=11.5429 | _timers/batch_time=0.1775 | _timers/data_time=0.0003 | _timers/model_time=0.1771 | f1_score=1.613e-06 | loss=0.4912 | loss_dist=3.956e-06 | loss_mean=0.4912\n",
      "[2019-12-06 20:06:40,743] \n",
      "89/100 * Epoch 89 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=5.9088 | _timers/batch_time=0.3401 | _timers/data_time=0.0003 | _timers/model_time=0.3398 | f1_score=1.529e-06 | loss=0.4942 | loss_dist=0.0039 | loss_mean=0.4903\n",
      "89/100 * Epoch 89 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.7496 | _timers/batch_time=0.1897 | _timers/data_time=0.0003 | _timers/model_time=0.1894 | f1_score=1.529e-06 | loss=0.4942 | loss_dist=0.0039 | loss_mean=0.4903\n",
      "[2019-12-06 20:06:44,879] \n",
      "90/100 * Epoch 90 (train): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=6.0761 | _timers/batch_time=0.3306 | _timers/data_time=0.0004 | _timers/model_time=0.3302 | f1_score=1.473e-06 | loss=0.4946 | loss_dist=0.0045 | loss_mean=0.4901\n",
      "90/100 * Epoch 90 (valid): _base/lr=0.0006 | _base/momentum=0.9000 | _timers/_fps=10.9047 | _timers/batch_time=0.1887 | _timers/data_time=0.0002 | _timers/model_time=0.1885 | f1_score=1.473e-06 | loss=0.4946 | loss_dist=0.0045 | loss_mean=0.4901\n",
      "[2019-12-06 20:06:48,979] \n",
      "91/100 * Epoch 91 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.1540 | _timers/batch_time=0.3260 | _timers/data_time=0.0003 | _timers/model_time=0.3256 | f1_score=1.610e-06 | loss=0.4900 | loss_dist=2.824e-09 | loss_mean=0.4900\n",
      "91/100 * Epoch 91 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.4413 | _timers/batch_time=0.1782 | _timers/data_time=0.0004 | _timers/model_time=0.1778 | f1_score=1.610e-06 | loss=0.4900 | loss_dist=2.824e-09 | loss_mean=0.4900\n",
      "[2019-12-06 20:06:53,105] \n",
      "92/100 * Epoch 92 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.3102 | _timers/batch_time=0.3187 | _timers/data_time=0.0003 | _timers/model_time=0.3184 | f1_score=1.633e-06 | loss=0.4903 | loss_dist=0.0003 | loss_mean=0.4900\n",
      "92/100 * Epoch 92 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.8773 | _timers/batch_time=0.1863 | _timers/data_time=0.0003 | _timers/model_time=0.1859 | f1_score=1.633e-06 | loss=0.4903 | loss_dist=0.0003 | loss_mean=0.4900\n",
      "[2019-12-06 20:06:57,076] \n",
      "93/100 * Epoch 93 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.5897 | _timers/batch_time=0.3045 | _timers/data_time=0.0003 | _timers/model_time=0.3042 | f1_score=1.609e-06 | loss=0.4895 | loss_dist=3.612e-10 | loss_mean=0.4895\n",
      "93/100 * Epoch 93 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.1543 | _timers/batch_time=0.1829 | _timers/data_time=0.0004 | _timers/model_time=0.1825 | f1_score=1.609e-06 | loss=0.4895 | loss_dist=3.612e-10 | loss_mean=0.4895\n",
      "[2019-12-06 20:07:01,050] \n",
      "94/100 * Epoch 94 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.2310 | _timers/batch_time=0.3248 | _timers/data_time=0.0003 | _timers/model_time=0.3244 | f1_score=1.625e-06 | loss=0.4907 | loss_dist=1.376e-05 | loss_mean=0.4907\n",
      "94/100 * Epoch 94 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.5021 | _timers/batch_time=0.1800 | _timers/data_time=0.0003 | _timers/model_time=0.1796 | f1_score=1.625e-06 | loss=0.4907 | loss_dist=1.376e-05 | loss_mean=0.4907\n",
      "[2019-12-06 20:07:05,242] \n",
      "95/100 * Epoch 95 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.2954 | _timers/batch_time=0.3198 | _timers/data_time=0.0004 | _timers/model_time=0.3194 | f1_score=1.603e-06 | loss=0.4896 | loss_dist=2.170e-08 | loss_mean=0.4896\n",
      "95/100 * Epoch 95 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.3389 | _timers/batch_time=0.1985 | _timers/data_time=0.0004 | _timers/model_time=0.1981 | f1_score=1.603e-06 | loss=0.4896 | loss_dist=2.170e-08 | loss_mean=0.4896\n",
      "[2019-12-06 20:07:09,542] \n",
      "96/100 * Epoch 96 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=5.9693 | _timers/batch_time=0.3386 | _timers/data_time=0.0005 | _timers/model_time=0.3381 | f1_score=1.626e-06 | loss=0.4890 | loss_dist=1.632e-06 | loss_mean=0.4890\n",
      "96/100 * Epoch 96 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.9025 | _timers/batch_time=0.1868 | _timers/data_time=0.0004 | _timers/model_time=0.1864 | f1_score=1.626e-06 | loss=0.4890 | loss_dist=1.632e-06 | loss_mean=0.4890\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019-12-06 20:07:13,699] \n",
      "97/100 * Epoch 97 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.0985 | _timers/batch_time=0.3297 | _timers/data_time=0.0003 | _timers/model_time=0.3294 | f1_score=1.496e-06 | loss=0.4941 | loss_dist=0.0039 | loss_mean=0.4902\n",
      "97/100 * Epoch 97 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.3443 | _timers/batch_time=0.1801 | _timers/data_time=0.0003 | _timers/model_time=0.1798 | f1_score=1.496e-06 | loss=0.4941 | loss_dist=0.0039 | loss_mean=0.4902\n",
      "[2019-12-06 20:07:17,876] \n",
      "98/100 * Epoch 98 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.2656 | _timers/batch_time=0.3222 | _timers/data_time=0.0003 | _timers/model_time=0.3218 | f1_score=1.630e-06 | loss=0.4928 | loss_dist=0.0027 | loss_mean=0.4901\n",
      "98/100 * Epoch 98 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.2134 | _timers/batch_time=0.1975 | _timers/data_time=0.0002 | _timers/model_time=0.1973 | f1_score=1.630e-06 | loss=0.4928 | loss_dist=0.0027 | loss_mean=0.4901\n",
      "[2019-12-06 20:07:22,035] \n",
      "99/100 * Epoch 99 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=5.9170 | _timers/batch_time=0.3386 | _timers/data_time=0.0004 | _timers/model_time=0.3381 | f1_score=1.626e-06 | loss=0.4887 | loss_dist=2.159e-06 | loss_mean=0.4887\n",
      "99/100 * Epoch 99 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=11.3908 | _timers/batch_time=0.1796 | _timers/data_time=0.0003 | _timers/model_time=0.1793 | f1_score=1.626e-06 | loss=0.4887 | loss_dist=2.159e-06 | loss_mean=0.4887\n",
      "[2019-12-06 20:07:26,274] \n",
      "100/100 * Epoch 100 (train): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=6.1046 | _timers/batch_time=0.3283 | _timers/data_time=0.0003 | _timers/model_time=0.3280 | f1_score=1.502e-06 | loss=0.4952 | loss_dist=0.0042 | loss_mean=0.4910\n",
      "100/100 * Epoch 100 (valid): _base/lr=0.0005 | _base/momentum=0.9000 | _timers/_fps=10.2165 | _timers/batch_time=0.1975 | _timers/data_time=0.0002 | _timers/model_time=0.1972 | f1_score=1.502e-06 | loss=0.4952 | loss_dist=0.0042 | loss_mean=0.4910\n",
      "Top best models:\n",
      "logs/b99/checkpoints/train.25.pth\t0.4870\n",
      "logs/b99/checkpoints/train.52.pth\t0.4882\n",
      "logs/b99/checkpoints/train.1.pth\t0.4885\n"
     ]
    }
   ],
   "source": [
    "runner.train(\n",
    "    model=model_bin,\n",
    "    criterion=criterion,\n",
    "    scheduler=scheduler,\n",
    "    callbacks=[   \n",
    "               CriterionCallback(prefix=\"loss_dist\", input_key='target_dist', output_key='dist',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "               CriterionCallback(prefix=\"loss_mean\", input_key='target_mean', output_key='mean',\n",
    "                     criterion_key='mse', multiplier=0.5),\n",
    "        \n",
    "               CriterionAggregatorCallback(prefix=\"loss\",loss_keys=['loss_dist','loss_mean']),\n",
    "#                AUCCallback(num_classes=1, input_key='target_cls', output_key='cls'), \n",
    "                F1ScoreCallback(input_key='target_cls', output_key='cls'),\n",
    "               CheckpointCallback(save_n_best=3)\n",
    "                ],\n",
    "\n",
    "    optimizer=optimizer,\n",
    "    main_metric='loss',\n",
    "    minimize_metric=True,\n",
    "    loaders=loaders,\n",
    "    logdir=logdir,\n",
    "    num_epochs=num_epochs,\n",
    "#     fp16={\"opt_level\": \"O1\"},\n",
    "    verbose=False\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}