20200424_NLG_Model_evaluation_with_BLEU.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "20200424_NLG Model evaluation with BLEU.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO7MO08OlZqu1rv6il5uOzx",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/luou-wen/NLP-learning/blob/main/20200424_NLG_Model_evaluation_with_BLEU.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FAFSIKLlHcIi"
      },
      "source": [
        "ML pipeline for undergraduate dissertation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qVKNztGHeoEp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "outputId": "f02b022b-e400-4351-bfa2-57c839300a3e"
      },
      "source": [
        "#mount google drive / My Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0U-2N0nLVNnO"
      },
      "source": [
        "!curl -L https://cpanmin.us | perl - App::cpanminus  # install cpanm\n",
        "!cpanm XML::Twig\n",
        "\n",
        "!gdown https://raw.githubusercontent.com/tuetschek/e2e-metrics/master/measure_scores.py\n",
        "!gdown https://github.com/tuetschek/e2e-metrics/archive/master.zip\n",
        "\n",
        "!unzip master.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Et7xix2ren8I",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "889f90bb-c78b-445e-9fa4-bd0c6fff90ac"
      },
      "source": [
        "!./e2e-metrics-master/measure_scores.py ./e2e-metrics-master/example-inputs/devel-conc.txt ./e2e-metrics-master/example-inputs/baseline-output.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running MS-COCO evaluator...\n",
            "creating index...\n",
            "index created!\n",
            "Loading and preparing results...     \n",
            "DONE (t=0.00s)\n",
            "creating index...\n",
            "index created!\n",
            "tokenization...\n",
            "PTBTokenizer tokenized 2198 tokens at 17069.01 tokens per second.\n",
            "PTBTokenizer tokenized 162 tokens at 1864.15 tokens per second.\n",
            "setting up scorers...\n",
            "computing METEOR score...\n",
            "METEOR: 0.480\n",
            "computing Rouge score...\n",
            "ROUGE_L: 0.791\n",
            "computing CIDEr score...\n",
            "CIDEr: 2.304\n",
            "Creating temp directory  /tmp/e2e-eval-81afan6g\n",
            "Running MTEval to compute BLEU & NIST...\n",
            "Use of 'Hyphen' in \\p{} or \\P{} is deprecated because: Supplanted by Line_Break property values; see www.unicode.org/reports/tr14; at /content/e2e-metrics-master/mteval/mteval-v13a-sig.pl line 993.\n",
            "MT evaluation scorer began on 2020 May 1 at 19:58:53\n",
            "command line:  /content/e2e-metrics-master/mteval/mteval-v13a-sig.pl -r /tmp/e2e-eval-81afan6g/mteval_ref.sgm -s /tmp/e2e-eval-81afan6g/mteval_src.sgm -t /tmp/e2e-eval-81afan6g/mteval_sys.sgm -f /tmp/e2e-eval-81afan6g/mteval_log.txt\n",
            "  Evaluation of any-to-en translation using:\n",
            "    src set \"e2e\" (1 docs, 10 segs)\n",
            "    ref set \"e2e\" (39 refs)\n",
            "    tst set \"e2e\" (1 systems)\n",
            "\n",
            "NIST score = 7.8212  BLEU score = 0.7203 for system \"tst\"\n",
            "\n",
            "# ------------------------------------------------------------------------\n",
            "\n",
            "Individual N-gram scoring\n",
            "        1-gram   2-gram   3-gram   4-gram   5-gram   6-gram   7-gram   8-gram   9-gram\n",
            "        ------   ------   ------   ------   ------   ------   ------   ------   ------\n",
            " NIST:  5.0188   1.2290   0.7479   0.5009   0.3246   0.0936   0.0385   0.0191   0.0080  \"tst\"\n",
            "\n",
            " BLEU:  0.9346   0.8112   0.6617   0.5366   0.3982   0.2816   0.1720   0.0843   0.0274  \"tst\"\n",
            "\n",
            "# ------------------------------------------------------------------------\n",
            "Cumulative N-gram scoring\n",
            "        1-gram   2-gram   3-gram   4-gram   5-gram   6-gram   7-gram   8-gram   9-gram\n",
            "        ------   ------   ------   ------   ------   ------   ------   ------   ------\n",
            " NIST:  5.0188   6.2478   6.9957   7.4965   7.8212   7.9148   7.9533   7.9724   7.9804  \"tst\"\n",
            "\n",
            " BLEU:  0.9346   0.8707   0.7946   0.7203   0.6398   0.5580   0.4717   0.3803   0.2839  \"tst\"\n",
            "MT evaluation scorer ended on 2020 May 1 at 19:58:53\n",
            "\n",
            "Removing temp directory\n",
            "SCORES:\n",
            "==============\n",
            "BLEU: 0.7203\n",
            "NIST: 7.8212\n",
            "METEOR: 0.4801\n",
            "ROUGE_L: 0.7910\n",
            "CIDEr: 2.3039\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IjNNHszyNiTb"
      },
      "source": [
        "#utils\n",
        "punc_tokens = {'!': ' <EXCLAIM> ',\n",
        " '.': ' <PERIOD> ',\n",
        " '?': ' <QMARK> ',\n",
        " ',': ' <COMMA> ',\n",
        " '(': ' <LPAREN> ',\n",
        " ')': ' <RPAREN> ',\n",
        " '\"': ' <QUOTE> ',\n",
        " ';': ' <SEMICOLON> ',\n",
        " '\\n': ' <RETURN> ',\n",
        " '\\t': ' <TAB> ',\n",
        " '~': ' <TILDE> ',\n",
        " '-': ' <HYPHEN> ',\n",
        " '\\'': ' <APOST> ',\n",
        " ':': ' <COLON> '\n",
        "}\n",
        "\n",
        "\n",
        "def replace_punctuation(dataset):\n",
        "    return [''.join([punc_tokens.get(char, char) for char in seq]) for seq in dataset]\n",
        "\n",
        "\n",
        "def extract_ngrams(sequence, n=2):\n",
        "    \"\"\" Extract n-grams from a sequence \"\"\"\n",
        "    ngrams = list(zip(*[sequence[ii:] for ii in range(n)]))\n",
        "\n",
        "    return ngrams\n",
        "\n",
        "\n",
        "def corrupt(dataset, p_drop=0.6):\n",
        "    \"\"\" Corrupt sequences in a dataset by randomly dropping words \"\"\"\n",
        "    values, counts = np.unique(np.concatenate(dataset), return_counts=True)\n",
        "    to_drop = set(values[counts > 100])\n",
        "\n",
        "    out_seq = [[each for each in seq if np.random.rand() > p_drop*int(each in to_drop)] for seq in dataset]\n",
        "\n",
        "    return out_seq\n",
        "\n",
        "\n",
        "def shuffle(original_seq, corrupted):\n",
        "    \"\"\" Shuffle elements in a corrupted sequence while keeping bigrams\n",
        "        appearing in original sequence.\n",
        "    \"\"\"\n",
        "\n",
        "    if not corrupted:\n",
        "        return corrupted\n",
        "\n",
        "    # Need to swap words around now but keep bigrams\n",
        "    # Get bigrams for original sequence\n",
        "    seq_grams = extract_ngrams(original_seq)\n",
        "    # Copy this\n",
        "    cor = corrupted.copy()\n",
        "\n",
        "    # Here I need to collect the tokens into n-grams that show up in the\n",
        "    # original sequence. That way when I shuffle, 2-grams, 3-grams, etc\n",
        "    # will stay together during the randomization.\n",
        "    to_shuffle = [[cor.pop(0)]]\n",
        "    while cor:\n",
        "        if len(cor) == 1:\n",
        "            to_shuffle.append([cor.pop()])\n",
        "        elif (to_shuffle[-1][-1], cor[0]) not in seq_grams:\n",
        "            to_shuffle.append([cor.pop(0)])\n",
        "        else:\n",
        "            to_shuffle[-1].append(cor.pop(0))\n",
        "\n",
        "    random.shuffle(to_shuffle)\n",
        "    flattened = [elem for lst in to_shuffle for elem in lst]\n",
        "    return flattened\n",
        "\n",
        "\n",
        "def get_tokens(dataset):\n",
        "    # Tokenize our dataset\n",
        "    corpus = \" \".join(dataset)\n",
        "    vocab_counter = Counter(corpus.split())\n",
        "    vocab = vocab_counter.keys()\n",
        "    total_words = sum(vocab_counter.values())\n",
        "\n",
        "    vocab_freqs = {word: count/total_words for word, count in vocab_counter.items()}\n",
        "    vocab_sorted = sorted(vocab, key=vocab_freqs.get, reverse=True)\n",
        "\n",
        "    # Starting at 3 here to reserve special tokens\n",
        "    vocab_to_int = dict(zip(vocab_sorted, range(3, len(vocab)+3)))\n",
        "\n",
        "    vocab_to_int[\"<SOS>\"] = 0 # Start of sentence\n",
        "    vocab_to_int[\"<EOS>\"] = 1 # End of sentence\n",
        "    vocab_to_int[\"<UNK>\"] = 2 # Unknown word\n",
        "\n",
        "    int_to_vocab = {val: key for key, val in vocab_to_int.items()}\n",
        "\n",
        "    return vocab_to_int, int_to_vocab"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QBxk_KVeoCa"
      },
      "source": [
        "!cp \"/content/gdrive/My Drive/nlg_2020-04-24_08:43:49.989893.pth\" /content/\n",
        "\n",
        "!cp -r \"/content/gdrive/My Drive/_Dissertation/e2e-data\" /content/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xQbefiwedhN"
      },
      "source": [
        "from collections import Counter\n",
        "import random\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "from datetime import datetime"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hJUuD3G2HRJA"
      },
      "source": [
        "class Encoder(nn.Module):\n",
        "    \n",
        "    def __init__(self, vocab_size, embedding_size=300, hidden_size=256, num_layers=2, drop_p=0.5):\n",
        "        super().__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.num_layers = num_layers\n",
        "        \n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.lstm = nn.LSTM(embedding_size, hidden_size, num_layers=num_layers, \n",
        "                            dropout=drop_p, bidirectional=True)\n",
        "        \n",
        "    def forward(self, input, hidden):\n",
        "        embedded = self.embedding(input)\n",
        "        output, hidden = self.lstm(embedded, hidden)\n",
        "        return output, hidden\n",
        "    \n",
        "    def init_hidden(self, device='cpu'):\n",
        "        \"\"\" Create two tensors with shape (num_layers * num_directions, batch, hidden_size)\n",
        "            for the hidden state and cell state\n",
        "        \"\"\"\n",
        "        h_0, c_0 = torch.zeros(2, 2*self.num_layers, 1, self.hidden_size, device=device)\n",
        "        \n",
        "        return h_0, c_0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SXXKom_7HRjI"
      },
      "source": [
        "# Attention network from http://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html\n",
        "class Decoder(nn.Module):\n",
        "    \n",
        "    def __init__(self, vocab_size, embedding_size=300, hidden_size=256, \n",
        "                       num_layers=2, drop_p=0.1, max_length=50):\n",
        "        \n",
        "        super().__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.num_layers = num_layers\n",
        "        self.max_length = max_length\n",
        "\n",
        "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
        "        self.attn = nn.Linear(self.hidden_size + embedding_size, self.max_length)\n",
        "        self.attn_combine = nn.Linear(self.hidden_size * 2 + embedding_size, self.hidden_size)\n",
        "        self.dropout = nn.Dropout(drop_p)\n",
        "        self.lstm = nn.LSTM(hidden_size, hidden_size, num_layers=num_layers, \n",
        "                            dropout=drop_p, bidirectional=True)\n",
        "        \n",
        "        self.out = nn.Linear(2 * hidden_size, vocab_size)\n",
        "        self.log_softmax = nn.LogSoftmax(dim=1)\n",
        "        \n",
        "    def forward(self, input, hidden, encoder_outputs):\n",
        "        embedded = self.embedding(input)\n",
        "        embedded = self.dropout(embedded)\n",
        "        \n",
        "        # Learns the attention vector (a probability distribution) here for weighting\n",
        "        # encoder outputs based on the decoder input and encoder hidden vector\n",
        "        attn_weights = F.softmax(self.attn(torch.cat((embedded[0], hidden[0][0]), 1)), dim=1)\n",
        "        \n",
        "        # Applies the attention vector (again, a probability distribution) to the encoder\n",
        "        # outputs which weight the encoder_outputs\n",
        "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
        "                                 encoder_outputs.unsqueeze(0))\n",
        "        \n",
        "        # Now the decoder input is combined with the weighted encoder_outputs and\n",
        "        # passed through a linear transformation as input to the LSTM layer\n",
        "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
        "        output = self.attn_combine(output).unsqueeze(0)\n",
        "        output = F.relu(output)\n",
        "        \n",
        "        output, hidden = self.lstm(output, hidden)\n",
        "        output = self.out(output).view(1, -1)\n",
        "        output = self.log_softmax(output)\n",
        "    \n",
        "        return output, hidden, attn_weights\n",
        "        \n",
        "    def init_hidden(self, device='cpu'):\n",
        "        \"\"\" Create two tensors with shape (num_layers * num_directions, batch, hidden_size)\n",
        "            for the hidden state and cell state\n",
        "        \"\"\"\n",
        "        h_0, c_0 = torch.zeros(2, 2*self.num_layers, 1, self.hidden_size, device=device)\n",
        "        return h_0, c_0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMLL9vUq5DR8"
      },
      "source": [
        "def predict(input_tensor, encoder, decoder):\n",
        "  max_length = 50\n",
        "  with torch.no_grad():\n",
        "    encoder.eval()\n",
        "    decoder.eval()\n",
        "\n",
        "    h, c = encoder.init_hidden(device=device)\n",
        "    encoder_outputs = torch.zeros(max_length, 2*encoder.hidden_size).to(device)\n",
        "    enc_outputs, enc_hidden = encoder.forward(input_tensor, (h, c))\n",
        "    encoder_outputs[:min(enc_outputs.shape[0], max_length)] = enc_outputs[:max_length,0,:]\n",
        "    \n",
        "     # First decoder input is the <SOS> token\n",
        "    dec_input = torch.Tensor([[0]]).type(torch.LongTensor).to(device)\n",
        "    dec_hidden = enc_hidden\n",
        "\n",
        "    EOS_TOKEN = vocab_to_int['<EOS>']\n",
        "    dec_outputs = []\n",
        "\n",
        "    for i in range(50):\n",
        "      dec_out, dec_hidden, dec_attn = decoder.forward(dec_input, dec_hidden, encoder_outputs)\n",
        "      _, out_token = dec_out.topk(1)\n",
        "\n",
        "      dec_input = out_token.detach().to(device)\n",
        "\n",
        "      dec_outputs.append(out_token)\n",
        "\n",
        "      if out_token == EOS_TOKEN:\n",
        "        break\n",
        "\n",
        "    return dec_outputs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CMWmNcXF5RkZ"
      },
      "source": [
        "def get_test_data(dataset,trainset_size=1000,testset_size=100):\n",
        "  input_tensors = []\n",
        "  target_tensors = []\n",
        "  for input_tensor, target_tensor in dataloader(dataset[trainset_size:trainset_size+testset_size]):\n",
        "    input_tensor = input_tensor.to(device)\n",
        "    target_tensor = target_tensor.to(device)\n",
        "    input_tensors.append(input_tensor)\n",
        "    target_tensors.append(target_tensor)\n",
        "  return input_tensors, target_tensors"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ji622C_8eDdm"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UvagVxbyMTbU"
      },
      "source": [
        "!cp \"/content/gdrive/My Drive/_Dissertation/dissertation/e2e-submission-papers/challenge_submissions/corpus.csv\"  /content/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jS100lWjH4Bv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "outputId": "6106acc6-84cf-4b0e-c0c1-87d203d8cbda"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "encoder = Encoder(2943, hidden_size=512, drop_p=0.1).to(device)\n",
        "decoder = Decoder(2943, hidden_size=512, drop_p=0.1, max_length=50).to(device)\n",
        "\n",
        "checkpoint = torch.load(\"/content/nlg_2020-04-24_08:43:49.989893.pth\")\n",
        "encoder.load_state_dict(checkpoint['encoder_sd'])\n",
        "decoder.load_state_dict(checkpoint['decoder_sd'])\n",
        "\n",
        "encoder.eval()\n",
        "decoder.eval()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Decoder(\n",
              "  (embedding): Embedding(2943, 300)\n",
              "  (attn): Linear(in_features=812, out_features=50, bias=True)\n",
              "  (attn_combine): Linear(in_features=1324, out_features=512, bias=True)\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (lstm): LSTM(512, 512, num_layers=2, dropout=0.1, bidirectional=True)\n",
              "  (out): Linear(in_features=1024, out_features=2943, bias=True)\n",
              "  (log_softmax): LogSoftmax()\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z9Wwbz_fepZw"
      },
      "source": [
        "trainset = pd.read_csv('/content/e2e-data/e2e-dataset/trainset.csv')\n",
        "trainset = trainset.assign(clean=replace_punctuation(trainset['ref']))\n",
        "vocab_to_int, int_to_vocab = get_tokens(trainset['clean'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eAmuoGnTg1Zb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "41a9e166-306d-44db-9cd1-cfaa11f9edea"
      },
      "source": [
        "temp_input = \"Blue Spice coffee shop city centre\"\n",
        "temp_tokens = []\n",
        "for item in temp_input.split(\" \"):\n",
        "  temp_tokens.append(vocab_to_int[item])\n",
        "temp_tokens"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[82, 118, 26, 27, 31, 32]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOahfOu7aooP",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "56e41ed1-4403-4bf3-c846-7c66249f0b8e"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "test_tensor = torch.tensor(temp_tokens).view(-1,1).type(torch.LongTensor)\n",
        "dec_outputs = predict(test_tensor.to(device), encoder, decoder)\n",
        "print([int_to_vocab[each.item()] for each in dec_outputs])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['Blue', 'Spice', 'is', 'a', 'low', '<HYPHEN>', 'priced', 'coffee', 'shop', 'in', 'the', 'city', 'centre', 'centre', 'near', '<EOS>']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IzbpN1Fu8d2d"
      },
      "source": [
        "testset = pd.read_csv(\"/content/e2e-data/e2e-dataset/testset.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gPoIdpJh-kM9"
      },
      "source": [
        "import re\n",
        "# mr = \"name[The Vaults], eatType[pub], food[Italian], priceRange[more than £30], customer rating[high], area[city centre], familyFriendly[yes], near[Rainbow Vegetarian Café]\"\n",
        "def mr_to_tokens(mr):\n",
        "  pattern = \"\\[.*\\]\"\n",
        "  input_string = []\n",
        "  input_tokens = []\n",
        "  for string in mr.split(\", \"):\n",
        "    match = re.search(pattern, string)\n",
        "    tmp = match.group(0).replace(\"[\", \"\").replace(\"]\", \"\")\n",
        "    input_string.append(tmp)\n",
        "  x = \" \".join(input_string)\n",
        "  for word in x.split(\" \"):\n",
        "    if (word in vocab_to_int):\n",
        "      input_tokens.append(vocab_to_int[word])\n",
        "  return input_tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dHv8_6aw-2q5"
      },
      "source": [
        "output_words = []\n",
        "for mr in testset[\"MR\"]:\n",
        "  test_tokens = mr_to_tokens(mr)\n",
        "  test_tensor = torch.tensor(test_tokens).view(-1,1).type(torch.LongTensor)\n",
        "  dec_outputs = predict(test_tensor.to(device), encoder, decoder)\n",
        "  output_words.append(\" \".join([int_to_vocab[each.item()] for each in dec_outputs]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yCrCpCBPQ6st",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        },
        "outputId": "15ba25a2-e298-4c8d-b8bc-6cccf860822d"
      },
      "source": [
        "output_words[-10:]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['The Wrestlers is a low <HYPHEN> priced coffee shop near near the riverside Cuisine near the riverside <EOS>',\n",
              " 'The coffee shop near The Wrestlers is a friendly coffee shop near Raja Indian Cuisine <PERIOD> <EOS>',\n",
              " 'Zizzi is a coffee shop shop shop the riverside area <PERIOD> <EOS>',\n",
              " 'The coffee shop shop The The near The near The Portland <PERIOD> The average customer rating and is family friendly <EOS>',\n",
              " 'The coffee shop shop The The near The Portland near The Portland Arms <PERIOD> <EOS>',\n",
              " 'The coffee shop shop The The Sorrento is The The is <PERIOD> The Sorrento <PERIOD> <EOS>',\n",
              " 'Zizzi is a family friendly pub pub the riverside area <PERIOD> <EOS>',\n",
              " 'The a average shop shop near near near The near The riverside <PERIOD> <EOS>',\n",
              " 'The near The Portland Arms <COMMA> a high friendly coffee shop shop high customer rating <PERIOD> <EOS>',\n",
              " 'The a pub friendly pub The The The The The The The <PERIOD> <EOS>']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qU_VuAj3tza0"
      },
      "source": [
        "punc_tokens = {'<EXCLAIM>': '!',\n",
        " '<PERIOD>': '.',\n",
        " '<QMARK>': '?',\n",
        " '<COMMA>': ',',\n",
        " '<LPAREN>': '(',\n",
        " '<RPAREN>': ')',\n",
        " '<QUOTE> ': '\"',\n",
        " '<SEMICOLON>': ';',\n",
        " '<RETURN>': '\\n',\n",
        " '<TAB>': '\\t',\n",
        " '<TILDE>': '~',\n",
        " '<HYPHEN>': '-',\n",
        " '<APOST>': '\\'',\n",
        " '<COLON>': ':'\n",
        "}\n",
        "\n",
        "\n",
        "def return_punctuation(dataset):\n",
        "    ret_punc = []\n",
        "    for seq in dataset:\n",
        "      for word in seq.split(\" \"):\n",
        "        punctuation = punc_tokens.get(word)\n",
        "        if punctuation:\n",
        "          seq = seq.replace(\" \"+word+\" \", punctuation)\n",
        "      seq = seq.replace('<EOS>', '')\n",
        "      ret_punc.append(seq)\n",
        "    return ret_punc\n",
        "\n",
        "test_outputw = output_words\n",
        "testso = return_punctuation(test_outputw)\n",
        "test_df = pd.DataFrame(testso)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CD1zkxPRuphz"
      },
      "source": [
        "test_out = pd.read_csv(\"/content/e2e-data/e2e-dataset/testset_w_refs.csv\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kue2Ayjyup0L"
      },
      "source": [
        "with open(\"test_out.txt\", \"w\") as f:\n",
        "  for name, group in test_out.groupby(\"mr\"):  \n",
        "    for item in group[\"ref\"]:\n",
        "      f.write(item+\"\\n\")\n",
        "    f.write(\"\\n\")\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GW4ualXGFZP7"
      },
      "source": [
        "with open(\"test_model_out.txt\", \"w\") as f:\n",
        "  for line in testso:\n",
        "    f.write(line+\"\\n\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kJOMj-fgFZJc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "fd27f4b6-eabd-4785-ad29-6746abc274e2"
      },
      "source": [
        "!./e2e-metrics-master/measure_scores.py ./test_out.txt ./test_model_out.txt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running MS-COCO evaluator...\n",
            "creating index...\n",
            "index created!\n",
            "Loading and preparing results...     \n",
            "DONE (t=0.00s)\n",
            "creating index...\n",
            "index created!\n",
            "tokenization...\n",
            "PTBTokenizer tokenized 129948 tokens at 309448.71 tokens per second.\n",
            "PTBTokenizer tokenized 14221 tokens at 65655.75 tokens per second.\n",
            "setting up scorers...\n",
            "computing METEOR score...\n",
            "METEOR: 0.210\n",
            "computing Rouge score...\n",
            "ROUGE_L: 0.409\n",
            "computing CIDEr score...\n",
            "CIDEr: 0.489\n",
            "Creating temp directory  /tmp/e2e-eval-vusvbtt7\n",
            "Running MTEval to compute BLEU & NIST...\n",
            "Use of 'Hyphen' in \\p{} or \\P{} is deprecated because: Supplanted by Line_Break property values; see www.unicode.org/reports/tr14; at /content/e2e-metrics-master/mteval/mteval-v13a-sig.pl line 993.\n",
            "MT evaluation scorer began on 2020 May 1 at 20:00:29\n",
            "command line:  /content/e2e-metrics-master/mteval/mteval-v13a-sig.pl -r /tmp/e2e-eval-vusvbtt7/mteval_ref.sgm -s /tmp/e2e-eval-vusvbtt7/mteval_src.sgm -t /tmp/e2e-eval-vusvbtt7/mteval_sys.sgm -f /tmp/e2e-eval-vusvbtt7/mteval_log.txt\n",
            "  Evaluation of any-to-en translation using:\n",
            "    src set \"e2e\" (1 docs, 630 segs)\n",
            "    ref set \"e2e\" (45 refs)\n",
            "    tst set \"e2e\" (1 systems)\n",
            "\n",
            "NIST score = 3.7764  BLEU score = 0.2335 for system \"tst\"\n",
            "\n",
            "# ------------------------------------------------------------------------\n",
            "\n",
            "Individual N-gram scoring\n",
            "        1-gram   2-gram   3-gram   4-gram   5-gram   6-gram   7-gram   8-gram   9-gram\n",
            "        ------   ------   ------   ------   ------   ------   ------   ------   ------\n",
            " NIST:  2.8695   0.5565   0.2047   0.0927   0.0530   0.0174   0.0052   0.0023   0.0010  \"tst\"\n",
            "\n",
            " BLEU:  0.6145   0.3451   0.1865   0.1066   0.0514   0.0248   0.0114   0.0059   0.0033  \"tst\"\n",
            "\n",
            "# ------------------------------------------------------------------------\n",
            "Cumulative N-gram scoring\n",
            "        1-gram   2-gram   3-gram   4-gram   5-gram   6-gram   7-gram   8-gram   9-gram\n",
            "        ------   ------   ------   ------   ------   ------   ------   ------   ------\n",
            " NIST:  2.8695   3.4260   3.6307   3.7234   3.7764   3.7937   3.7989   3.8012   3.8022  \"tst\"\n",
            "\n",
            " BLEU:  0.5630   0.4219   0.3121   0.2335   0.1695   0.1213   0.0854   0.0605   0.0434  \"tst\"\n",
            "MT evaluation scorer ended on 2020 May 1 at 20:00:41\n",
            "\n",
            "Removing temp directory\n",
            "SCORES:\n",
            "==============\n",
            "BLEU: 0.2335\n",
            "NIST: 3.7764\n",
            "METEOR: 0.2102\n",
            "ROUGE_L: 0.4091\n",
            "CIDEr: 0.4894\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QVhLtd12E1Ti"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}