model_n_gram.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "model_n_gram.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPevbAg+AVWojI295HLE3us",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/JimmyXiaodong/pytorch-study/blob/master/model_n_gram.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sbr8M_V-hv_R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "CONTEXT_SIZE = 2\n",
        "EMBEDDING_DIM = 10\n",
        "# 我们用莎士比亚的十四行诗 Sonnet 2\n",
        "test_sentence = \"\"\"When forty winters shall besiege thy brow,\n",
        "And dig deep trenches in thy beauty's field,\n",
        "Thy youth's proud livery so gazed on now,\n",
        "Will be a totter'd weed of small worth held:\n",
        "Then being asked, where all thy beauty lies,\n",
        "Where all the treasure of thy lusty days;\n",
        "To say, within thine own deep sunken eyes,\n",
        "Were an all-eating shame, and thriftless praise.\n",
        "How much more praise deserv'd thy beauty's use,\n",
        "If thou couldst answer 'This fair child of mine\n",
        "Shall sum my count, and make my old excuse,'\n",
        "Proving his beauty by succession thine!\n",
        "This were to be new made when thou art old,\n",
        "And see thy blood warm when thou feel'st it cold.\"\"\".split()"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qXWgkQdkikZn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 应该对输入变量进行标记，但暂时忽略。\n",
        "# 创建一系列的元组，每个元组都是([ word_i-2, word_i-1 ], target word)的形式。\n",
        "trigrams = [([test_sentence[i], test_sentence[i + 1]], test_sentence[i + 2])\n",
        "            for i in range(len(test_sentence) - 2)]\n",
        "\n",
        "vocab = set(test_sentence)\n",
        "word_to_ix = {word: i for i, word in enumerate(vocab)}"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LQh9IS4l4sHS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class NGramLanguageModeler(nn.Module):\n",
        "\n",
        "    def __init__(self, vocab_size, embedding_dim, context_size):\n",
        "        super(NGramLanguageModeler, self).__init__()\n",
        "        self.embeddings = nn.Embedding(vocab_size, embedding_dim)\n",
        "        self.linear1 = nn.Linear(context_size * embedding_dim, 128)\n",
        "        self.linear2 = nn.Linear(128, vocab_size)\n",
        "\n",
        "    def forward(self, inputs):\n",
        "        embeds = self.embeddings(inputs).view((1, -1))\n",
        "        out = F.relu(self.linear1(embeds))\n",
        "        out = self.linear2(out)\n",
        "        log_probs = F.log_softmax(out, dim=1)\n",
        "        return log_probs\n",
        "\n",
        "\n",
        "losses = []\n",
        "loss_function = nn.NLLLoss()\n",
        "model = NGramLanguageModeler(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.001)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aJlP4iWZ4wD5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "92e3e5b9-a1a1-4916-8232-601f9c313e85"
      },
      "source": [
        "for epoch in range(10):\n",
        "    total_loss = 0\n",
        "    for context, target in trigrams:\n",
        "\n",
        "        # 步骤 1\\. 准备好进入模型的数据 (例如将单词转换成整数索引,并将其封装在变量中)\n",
        "        context_idxs = torch.tensor([word_to_ix[w] for w in context], dtype=torch.long)\n",
        "\n",
        "        # 步骤 2\\. 回调torch累乘梯度\n",
        "        # 在传入一个新实例之前，需要把旧实例的梯度置零。\n",
        "        model.zero_grad()\n",
        "\n",
        "        # 步骤 3\\. 继续运行代码，得到单词的log概率值。\n",
        "        log_probs = model(context_idxs)\n",
        "\n",
        "        # 步骤 4\\. 计算损失函数（再次注意，Torch需要将目标单词封装在变量里）。\n",
        "        loss = loss_function(log_probs, torch.tensor([word_to_ix[target]], dtype=torch.long))\n",
        "\n",
        "        # 步骤 5\\. 反向传播更新梯度\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # 通过调tensor.item()得到单个Python数值。\n",
        "        total_loss += loss.item()\n",
        "    losses.append(total_loss)\n",
        "print(losses)  # 用训练数据每次迭代，损失函数都会下降。"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[522.4252758026123, 520.0589151382446, 517.7059609889984, 515.366669178009, 513.0399055480957, 510.7254102230072, 508.42188239097595, 506.129191160202, 503.8461592197418, 501.5722041130066]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iO_X10vW40jZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}