tasks/demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip3 install tkseem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frequency Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkseem as tk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read, preprocess then train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training WordTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.WordTokenizer()\n",
    "tokenizer.train('samples/data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WordTokenizer\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['السلام', 'عليكم']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"السلام عليكم\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Encode as ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[557, 798]\n"
     ]
    }
   ],
   "source": [
    "encoded = tokenizer.encode(\"السلام عليكم\")\n",
    "print(encoded)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decode back to tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['السلام', 'عليكم']\n"
     ]
    }
   ],
   "source": [
    "decoded = tokenizer.decode(encoded)\n",
    "print(decoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "السلام عليكم\n"
     ]
    }
   ],
   "source": [
    "detokenized = tokenizer.detokenize(decoded)\n",
    "print(detokenized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SentencePiece Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read, preprocess then train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training SentencePiece ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.SentencePieceTokenizer()\n",
    "tokenizer.train('samples/data.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['▁صباح', '▁الخير', '▁يا', '▁أص', 'د', 'قاء']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"صباح الخير يا أصدقاء\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Encode as ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1799, 2741]\n"
     ]
    }
   ],
   "source": [
    "encoded = tokenizer.encode(\"السلام عليكم\")\n",
    "print(encoded)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decode back to tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['▁السلام', '▁عليكم']\n"
     ]
    }
   ],
   "source": [
    "decoded = tokenizer.decode(encoded)\n",
    "print(decoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " السلام عليكم\n"
     ]
    }
   ],
   "source": [
    "detokenized = tokenizer.detokenize(decoded)\n",
    "print(detokenized)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morphological Tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read, preprocess then train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MorphologicalTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.MorphologicalTokenizer()\n",
    "tokenizer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ال', '##سلام', 'علي', '##كم']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"السلام عليكم\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Encode as ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 367, 764, 184]\n"
     ]
    }
   ],
   "source": [
    "encoded = tokenizer.encode(\"السلام عليكم\")\n",
    "print(encoded)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decode back to tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ال', '##سلام', 'علي', '##كم']\n"
     ]
    }
   ],
   "source": [
    "decoded = tokenizer.decode(encoded)\n",
    "print(decoded)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training RandomTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.B\n",
    "tokenizer.train('samples/data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['السل', '##ام', 'علي', '##كم', 'أي', '##ها', 'الأص', '##دقا', '##ء']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"السلام عليكم أيها الأصدقاء\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Disjoint Letter Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training DisjointLetterTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.DisjointLetterTokenizer()\n",
    "tokenizer.train('samples/data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['ا', '##لسلا', '##م', 'عليكم', 'أ', '##يها', 'ا', '##لأ', '##صد', '##قا', '##ء']\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.tokenize(\"السلام عليكم أيها الأصدقاء\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Character Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training CharacterTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.CharacterTokenizer()\n",
    "tokenizer.train('samples/data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ا', '##ل', '##س', '##ل', '##ا', '##م', 'ع', '##ل', '##ي', '##ك', '##م']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"السلام عليكم\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BruteForce Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training RandomTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.BruteForceTokenizer()\n",
    "tokenizer.train('samples/data.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ال', '##سل', '##ام', 'عل', '##يك', '##م']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize(\"السلام عليكم\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compression Factor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tkseem as tk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training WordTokenizer ...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.15384615384615385"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = tk.WordTokenizer()\n",
    "tokenizer.train('samples/data.txt')\n",
    "tokenizer.tokenize('السلام عليكم')\n",
    "tokenizer.calculate_compression_factor('السلام عليكم')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training WordTokenizer ...\n",
      "Training SentencePiece ...\n",
      "Training RandomTokenizer ...\n",
      "Training DisjointLetterTokenizer ...\n",
      "Training CharacterTokenizer ...\n",
      "Training MorphologicalTokenizer ...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVzUlEQVR4nO3dfbRddX3n8ffHYIRVVFRuOw4BQ2nQRmGw3MFRHMEKawU7QjtiV7LUiqPNdE3jw8g4g6MLKc6silbtjIZq7FCpDw1IO86lphOpiO3woLmUEAgIptFKsDNc8Km2Cga/88feVzaHm3vPJefeS3bfr7Wysvdv/+4+330ePud39j57n1QVkqQD3+OWugBJ0mgY6JLUEwa6JPWEgS5JPWGgS1JPHLRUN3z44YfXypUrl+rmJemAdOONN95bVWMzLVuyQF+5ciWTk5NLdfOSdEBK8jf7WuYuF0nqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6YqhAT7ImyR1JdiU5b4blRyX5QpKbkuxI8tLRlypJms2cgZ5kGbAROANYDaxLsnqg2zuAy6vqucBa4OJRFypJmt0wI/STgF1VtbuqHgA2A2cN9CngSe30k4Fvjq5ESdIwhjlT9Ajgrs78HuB5A30uAD6X5A3ATwGnjaQ6SYvmQ+deudQlzMuG971sqUt4zBnVQdF1wMeqagXwUuDjSR6x7iTrk0wmmZyamhrRTUuSYLhAvxs4sjO/om3reh1wOUBVXQ8cDBw+uKKq2lRV41U1PjY247VlJEmP0jCBvg1YleToJMtpDnpODPT5BvASgCQ/TxPoDsElaRHNGehVtRfYAGwFbqf5NsvOJBcmObPtdi7w60luBv4IOKf89WlJWlRDXT63qrYAWwbazu9M3wacPNrSJEnz4ZmiktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUE0MFepI1Se5IsivJeTMs/0CS7e2/O5N8Z/SlSpJmM+dP0CVZBmwETgf2ANuSTLQ/OwdAVf37Tv83AM9dgFolSbMYZoR+ErCrqnZX1QPAZuCsWfqvo/mhaEnSIhom0I8A7urM72nbHiHJM4Cjgav3sXx9kskkk1NTU/OtVZI0i1EfFF0LXFFVD860sKo2VdV4VY2PjY2N+KYl6R+3YQL9buDIzvyKtm0ma3F3iyQtiWECfRuwKsnRSZbThPbEYKckzwKeAlw/2hIlScOYM9Crai+wAdgK3A5cXlU7k1yY5MxO17XA5qqqhSlVkjSbOb+2CFBVW4AtA23nD8xfMLqyJEnz5ZmiktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPXEUF9blKQD3X991dlLXcK8vP0TV8z7bxyhS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUE0MFepI1Se5IsivJefvo86tJbkuyM8mnRlumJGkuc17LJckyYCNwOrAH2JZkoqpu6/RZBbwNOLmqvp3kpxeqYEnSzIYZoZ8E7Kqq3VX1ALAZOGugz68DG6vq2wBVdc9oy5QkzWWYQD8CuKszv6dt6zoWODbJtUluSLJmphUlWZ9kMsnk1NTUo6tYkjSjUR0UPQhYBZwKrAM+muSwwU5VtamqxqtqfGxsbEQ3LUmC4QL9buDIzvyKtq1rDzBRVT+qqq8Bd9IEvCRpkQwT6NuAVUmOTrIcWAtMDPT5DM3onCSH0+yC2T3COiVJc5gz0KtqL7AB2ArcDlxeVTuTXJjkzLbbVuC+JLcBXwDeWlX3LVTRkqRHGuon6KpqC7BloO38znQBb2n/SZKWgGeKSlJPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9cRQ30NfTCe+9Q+XuoR5ufG9v7bUJUgS4AhdknrDQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4w0CWpJwx0SeoJA12SemKoQE+yJskdSXYlOW+G5eckmUqyvf33+tGXKkmazZyn/idZBmwETgf2ANuSTFTVbQNdL6uqDQtQoyRpCMOM0E8CdlXV7qp6ANgMnLWwZUmS5muYQD8CuKszv6dtG/TyJDuSXJHkyJlWlGR9kskkk1NTU4+iXEnSvozqoOiVwMqqOh64Crh0pk5VtamqxqtqfGxsbEQ3LUmC4QL9bqA74l7Rtv1EVd1XVfe3s78PnDia8iRJwxom0LcBq5IcnWQ5sBaY6HZI8vTO7JnA7aMrUZI0jDm/5VJVe5NsALYCy4BLqmpnkguByaqaAN6Y5ExgL/At4JwFrFmSNIOhfrGoqrYAWwbazu9Mvw1422hLkyTNh2eKSlJPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPDHW1RUnwxRedstQlzNspf/HFpS5Bi8gRuiT1hIEuST1hoEtSTwwV6EnWJLkjya4k583S7+VJKsn46EqUJA1jzkBPsgzYCJwBrAbWJVk9Q78nAm8CvjTqIiVJcxtmhH4SsKuqdlfVA8Bm4KwZ+r0LuAj44QjrkyQNaZhAPwK4qzO/p237iSS/ABxZVZ+dbUVJ1ieZTDI5NTU172IlSfu23wdFkzwOeD9w7lx9q2pTVY1X1fjY2Nj+3rQkqWOYQL8bOLIzv6Jtm/ZE4DnANUm+DvwLYMIDo5K0uIYJ9G3AqiRHJ1kOrAUmphdW1Xer6vCqWllVK4EbgDOranJBKpYkzWjOQK+qvcAGYCtwO3B5Ve1McmGSMxe6QEnScIa6lktVbQG2DLSdv4++p+5/WZKk+fJMUUnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wp+g08ic/MGTl7qEebv2DdcudQnSyDhCl6SeMNAlqScMdEnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4YKtCTrElyR5JdSc6bYflvJLklyfYk/yfJ6tGXKkmazZyBnmQZsBE4A1gNrJshsD9VVcdV1QnAe4D3j7xSSdKshhmhnwTsqqrdVfUAsBk4q9uhqr7Xmf0poEZXoiRpGMNcnOsI4K7O/B7geYOdkvwm8BZgOfCLM60oyXpgPcBRRx0131olSbMY2UHRqtpYVccA/wl4xz76bKqq8aoaHxsbG9VNS5IYboR+N3BkZ35F27Yvm4Hf25+i+uobFx631CXM21Hn37LUJUga0jAj9G3AqiRHJ1kOrAUmuh2SrOrM/hLw1dGVKEkaxpwj9Kram2QDsBVYBlxSVTuTXAhMVtUEsCHJacCPgG8Dr1nIoiVJjzTULxZV1RZgy0Db+Z3pN424LknSPHmmqCT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9cRQgZ5kTZI7kuxKct4My9+S5LYkO5J8PskzRl+qJGk2cwZ6kmXARuAMYDWwLsnqgW43AeNVdTxwBfCeURcqSZrdMCP0k4BdVbW7qh4ANgNndTtU1Req6h/a2RuAFaMtU5I0l2EC/Qjgrs78nrZtX14H/NlMC5KsTzKZZHJqamr4KiVJcxrpQdEkrwLGgffOtLyqNlXVeFWNj42NjfKmJekfvYOG6HM3cGRnfkXb9jBJTgPeDpxSVfePpjxJ0rCGGaFvA1YlOTrJcmAtMNHtkOS5wEeAM6vqntGXKUmay5yBXlV7gQ3AVuB24PKq2pnkwiRntt3eCxwKfDrJ9iQT+1idJGmBDLPLharaAmwZaDu/M33aiOuSJM2TZ4pKUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPDBXoSdYkuSPJriTnzbD8RUn+KsneJGePvkxJ0lzmDPQky4CNwBnAamBdktUD3b4BnAN8atQFSpKGM8xvip4E7Kqq3QBJNgNnAbdNd6iqr7fLfrwANUqShjDMLpcjgLs683vatnlLsj7JZJLJqampR7MKSdI+LOpB0araVFXjVTU+Nja2mDctSb03TKDfDRzZmV/RtkmSHkOGCfRtwKokRydZDqwFJha2LEnSfM0Z6FW1F9gAbAVuBy6vqp1JLkxyJkCSf55kD/AK4CNJdi5k0ZKkRxrmWy5U1RZgy0Db+Z3pbTS7YiRJS8QzRSWpJwx0SeoJA12SesJAl6SeMNAlqScMdEnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4w0CWpJwx0SeoJA12SesJAl6SeMNAlqSeGCvQka5LckWRXkvNmWP6EJJe1y7+UZOWoC5UkzW7OQE+yDNgInAGsBtYlWT3Q7XXAt6vq54APABeNulBJ0uyGGaGfBOyqqt1V9QCwGThroM9ZwKXt9BXAS5JkdGVKkuaSqpq9Q3I2sKaqXt/Ovxp4XlVt6PS5te2zp53/67bPvQPrWg+sb2efCdwxqg0ZwuHAvXP2OnC5fQeuPm8buH2j9oyqGptpwUGLWARVtQnYtJi3OS3JZFWNL8VtLwa378DV520Dt28xDbPL5W7gyM78irZtxj5JDgKeDNw3igIlScMZJtC3AauSHJ1kObAWmBjoMwG8pp0+G7i65tqXI0kaqTl3uVTV3iQbgK3AMuCSqtqZ5EJgsqomgP8BfDzJLuBbNKH/WLMku3oWkdt34OrztoHbt2jmPCgqSToweKaoJPWEgS5JPXFABnqSDyR5c2d+a5Lf78y/L8lbHsV6T03yp6Oqc6EkeXuSnUl2JNme5HlJrmkvz3BzkmuTPHOJa3ywre3WJFcmOWxE6z0nyYdGsa791dnGne39fm6Sx7XLxpP891n+9p8muWKO9c+6jrbPYUn+3aPbgoet558k2Zzkr5PcmGRLkvWL/XpI8p8X8/Y6t1tJPtGZPyjJ1Ki2P8nK9nydBXVABjpwLfACgPYFdDjw7M7yFwDXzbWS9rIGB5Qkzwf+FfALVXU8cBpwV7v4lVX1z2jO2n3vEpU47QdVdUJVPYfmQPlvLnE9C2F6G58NnE5zeYx3AlTVZFW9cV9/WFXfrKqzZ1v5XOtoHQbsV6C3Z3X/T+Caqjqmqk4E3gb8zH6u99Gc5zLvQB/R6/jvgeckOaSdP51Hfj17rjoW9byemRyogX4d8Px2+tnArcDfJXlKkicAPw88OclNSW5JcknbTpKvJ7koyV8Br2gvPPaVdv5fL8XGzNPTgXur6n6Aqrq3qr450OcvgJ9b9Mr27XrgCIAkJyW5vn1srpv+JNGOvP8kyf9O8tUk75n+4ySvTXJnki8DJ3faVya5uv2k8vkkR7XtH0vye0luSLK7/eR1SZLbk3xsITawqu6hOQt6Qxo/+bSX5JR2JL+93e4ndkdsSQ5O8gftc/WmJC9u27vruKDdhmvabZoO+ncDx7TrfrRv4i8GflRVH+5sz83AXwKHJrmifY18sg1/kpyfZFv7CWxTp/2aJL+bZBJ4U5KXpblg301J/jzJz7T9Du1s844kL0/ybuCQdls+2fZ7VZIvt20fmQ7vJN9P80n8Zh7Kgv21Bfildnod8EfTC5I8Ncln2lpvSHJ8235Bko8nuZbmm37nJPlf7f3w1STv7Kx/WZKPpvlE97nOm8foVNUB+Q/4GnAU8G+B3wDeBbyU5gW/jWbUemzb9w+BN7fTXwf+Yzt9cNtvFRDgcuBPl3rb5tjuQ4HtwJ3AxcApbfs1wHg7/VbgsiWu8/vt/8uAT9NcGgLgScBB7fRpwB+30+cAu2lOSjsY+Buak9WeDnwDGAOW03w6+1D7N1cCr2mn/w3wmXb6YzTXHArNdYa+BxxHM4C5EThhlNs40PYdmpHtqdPPpbbOkzuP30HASuDWtu1cmq8DAzyr3d6DB9ZxAc1A5gk0n0jvAx7fXc9+bMcbgQ/M0H4q8F2akwkfR/PG/MJ22VM7/T4OvKzzPLy4s+wpPPRtutcD72unLwJ+t9tv8D6lGZhdCTy+nb8Y+LV2uoBfHeXzFTie5lpUB9O8xrr3/weBd7bTvwhs7zwuNwKHdJ7Hfws8DTiEZrA53j5Oe6efezRZ86pRv+4O1BE6NE/uF7T/rm//Tc/vAb5WVXe2fS8FXtT528va/5/V9vtqNffyJ3iMq6rvAyfSjAangMuSnNMu/mSS7TRvav9haSr8iUPaWv4vTcBd1bY/Gfh0Ozr9AA/fVfb5qvpuVf0QuA14BvA8ml0BU9VcHO6yTv/nA59qpz8OvLCz7Mr2Mb0F+H9VdUtV/RjYSfPiWkzXAu9vR9WHVdXegeUvpH3uVdVXaN7Mjp1hPZ+tqvuruUbSPeznLpEhfbmq9rT33XYeuu9e3I68b6EJuO7j2H2MVgBb235v7fQ7jeYqrgBU1bdnuO2X0DzXt7XPpZcAP9suexD44/3ZsEFVtYNm+9bRjNa7XkjzHKOqrgaeluRJ7bKJqvpBp+9VVXVf2/YnPPS8/FpVbW+nb2QBnocHcqBP70c/juZd8AaaF/gLaEYJs/n7Ba1sgVXVg1V1TVW9E9gAvLxd9Mpq9un+clXdNcsqFsMPquoEmlAOD+1DfxfwhWr2rb+MZjQ07f7O9IPs37WGptf144H1/ng/17tPSX6Wpu57uu1V9W6a0ekhwLVJnvUob2KU90/XTprgHOo2kxxMM1o+u6qOAz7Kwx/H7uvrgzSfqI6j+TTd7TeXAJe2z+kTquqZVXVBu+yHVfXgPNY1rAngd+jsbhnCYJ4MntwzPb9Qj99PHMiBfh3NwcFvtQH3LZoDRM+needemWR6P/KrgS/OsI6vtP2OaefXLXDN+y3JM5Os6jSdQDOie0yqqn+g+Uh/bh66zs/0waZzhljFl4BTkjwtyeOBV3SWXcdDZyW/kmaf75JIMgZ8mCa8amDZMe0nhItodgcOBvpf0tRPkmNpdiUOeyXSvwOeuD+1A1cDT0hzNdTpmo8H/uU++k+H8r1JDqW53Me+dB/v13Tar6JzoDzJU9rJH7WPM8DngbOT/HTb56lJnjHE9uyPS4DfqqpbBtq7j9GpNMexvrePdZze1noI8Ms0g89FcSAH+i00+xJvGGj7bjWX8X0tzUf7W2hGZR8eXEH70X498Nk0B0XvGezzGHQocGmS25LsoPnRkQuWtqTZVdVNwA6aN8z3AL+d5CaGu/TE39Js3/U0L4zbO4vfALy2vR9eDbxptJXPafoA3k7gz4HPAb81Q783twcPdwA/Av6sbZ8O/ouBx7XP1cuAc6o96D2XqrqPZtR/66M9KNq+Af0KcFqary3uBH6bZnfZTP2/QzMqv5XmkiDbZln9BTSvwxt5+CVm/wvwlLbum2kOzEJzGv2OJJ+sqtuAdwCfa++7q2iOqSyYdvfSTF8VvQA4sa3j3Tz8zWnQl2kGlTtojhFNjrzQffDUf2kJJDkReH9VnbLUtWh02uNZ49X5vYjFdCCP0KUDUpJxmn20/22pa1G/OEKXpJ5whC5JPWGgS1JPGOiS1BMGuiT1hIEuST3x/wHNObM9FdbcdQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import time \n",
    "\n",
    "def calc_comp(fun):\n",
    "    tokenizer = fun()\n",
    "    # morph tokenizer doesn't take arguments\n",
    "    if str(tokenizer) == 'MorphologicalTokenizer':\n",
    "        tokenizer.train()\n",
    "    else:\n",
    "        tokenizer.train('samples/data.txt')\n",
    "    text = open('samples/data.txt', 'r').read()\n",
    "    return tokenizer.calculate_compression_factor(text)\n",
    "\n",
    "factors = {}\n",
    "\n",
    "factors['Word'] = calc_comp(tk.WordTokenizer)\n",
    "factors['SP'] = calc_comp(tk.SentencePieceTokenizer)\n",
    "factors['Random'] = calc_comp(tk.RandomTokenizer)\n",
    "factors['Disjoint'] = calc_comp(tk.DisjointLetterTokenizer)\n",
    "factors['Character'] = calc_comp(tk.CharacterTokenizer)\n",
    "factors['Morph'] = calc_comp(tk.MorphologicalTokenizer)\n",
    "plt = sns.barplot(data = pd.DataFrame.from_dict([factors]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Models can be saved for deployment and reloading."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training WordTokenizer ...\n",
      "Saving as pickle file ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.WordTokenizer()\n",
    "tokenizer.train('samples/data.txt')\n",
    "tokenizer.save_model('freq.pl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "load model without pretraining"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading as pickle file ...\n"
     ]
    }
   ],
   "source": [
    "tokenizer = tk.WordTokenizer()\n",
    "tokenizer.load_model('freq.pl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['السلام', 'عليكم']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize('السلام عليكم')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benchmarking"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing tokenizers in terms of training time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training WordTokenizer ...\n",
      "Training SentencePiece ...\n",
      "Training RandomTokenizer ...\n",
      "Training DisjointLetterTokenizer ...\n",
      "Training CharacterTokenizer ...\n",
      "Training MorphologicalTokenizer ...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASP0lEQVR4nO3df5BlZX3n8fdHBoUKKiCdWVaCYxQ1GpQsXRiEDSi6xcYYSUKoUGoGgzXZ2pBohTVrkq2Am6QCSRRdXddFJU6URAhqAPNDySiagAIzYZhhwIhBSSDoDCr+SIwR/OaP87RzaXqmb3ff7uaZer+qpvqc5zz33O9zf3zuc8+5906qCklSfx612gVIkhbHAJekThngktQpA1ySOmWAS1Kn1qzklR122GG1bt26lbxKSereli1b7quqqdntKxrg69atY/PmzSt5lZLUvSR3zdU+VoAn+TzwdeBB4IGqmk5yKHAZsA74PHBGVX1lEsVKkua3kGPgz6+qY6pquq2/DthUVUcBm9q6JGmFLOUk5kuBjW15I3Da0suRJI1r3AAv4CNJtiTZ0NrWVtW9bfkLwNqJVydJ2qNxT2KeWFX3JPle4Joknx7dWFWVZM4fVWmBvwHgyCOPXFKxkqTdxpqBV9U97e9O4IPAccAXkxwO0P7u3MNlL66q6aqanpp62KdgJEmLNG+AJ/meJI+dWQb+C3ArcBWwvnVbD1y5XEVKkh5unEMoa4EPJpnp/0dV9ZdJbgIuT3I2cBdwxvKVKUmabd4Ar6o7gefM0f4l4JTlKEqSNL8V/Sam9j0nvOWE1S5hwa77xetWuwRpIvwxK0nqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6tTYAZ5kvyQ3J/lQW39ykhuSfDbJZUkevXxlSpJmW8gM/NXA7SPrFwIXVdVTga8AZ0+yMEnS3o0V4EmOAF4MvLOtB3gBcEXrshE4bTkKlCTNbdwZ+JuAXwG+09afANxfVQ+09buBJ851wSQbkmxOsnnXrl1LKlaStNu8AZ7kx4CdVbVlMVdQVRdX1XRVTU9NTS1mF5KkOawZo88JwI8n+VHgAOBxwJuBg5OsabPwI4B7lq9MSdJs887Aq+pXq+qIqloH/Azw0ap6GfAx4PTWbT1w5bJVKUl6mKV8Dvx/Ar+c5LMMx8TfNZmSJEnjGOcQyndV1bXAtW35TuC4yZckSRqH38SUpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROzRvgSQ5IcmOSW5LsSPL61v7kJDck+WySy5I8evnLlSTNGGcG/i3gBVX1HOAY4NQkPwxcCFxUVU8FvgKcvXxlSpJmmzfAa/CNtrp/+1fAC4ArWvtG4LRlqVCSNKexjoEn2S/JVmAncA3w98D9VfVA63I38MQ9XHZDks1JNu/atWsSNUuSGDPAq+rBqjoGOAI4DnjGuFdQVRdX1XRVTU9NTS2yTEnSbAv6FEpV3Q98DDgeODjJmrbpCOCeCdcmSdqLcT6FMpXk4LZ8IPAi4HaGID+9dVsPXLlcRUqSHm7N/F04HNiYZD+GwL+8qj6U5DbgfUl+C7gZeNcy1ilJmmXeAK+qbcAPzdF+J8PxcEnSKvCbmJLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6NW+AJ/m+JB9LcluSHUle3doPTXJNkjva30OWv1xJ0oxxZuAPAOdW1TOBHwZ+IckzgdcBm6rqKGBTW5ckrZB5A7yq7q2qv23LXwduB54IvBTY2LptBE5briIlSQ+3oGPgSdYBPwTcAKytqnvbpi8Aa/dwmQ1JNifZvGvXriWUKkkaNXaAJzkIeD/wmqr62ui2qiqg5rpcVV1cVdNVNT01NbWkYiVJu40V4En2ZwjvS6vqA635i0kOb9sPB3YuT4mSpLmM8ymUAO8Cbq+qN45sugpY35bXA1dOvjxJ0p6sGaPPCcArgO1Jtra2XwMuAC5PcjZwF3DG8pQoSZrLvAFeVX8DZA+bT5lsOZKkcflNTEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6tS8AZ7kkiQ7k9w60nZokmuS3NH+HrK8ZUqSZhtnBv5u4NRZba8DNlXVUcCmti5JWkHzBnhVfQL48qzmlwIb2/JG4LQJ1yVJmsdij4Gvrap72/IXgLV76phkQ5LNSTbv2rVrkVcnSZptyScxq6qA2sv2i6tquqqmp6amlnp1kqRmsQH+xSSHA7S/OydXkiRpHIsN8KuA9W15PXDlZMqRJI1rzXwdkvwxcDJwWJK7gfOAC4DLk5wN3AWcsZxF9uwf/vfRq13Cgh35G9tXuwRJY5g3wKvqzD1sOmXCtUiSFsBvYkpSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqfm/SampH3XW8+9erVLWJBz3vCS1S7hEcUZuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnfJz4JL2Sb/98tNXu4QF+/X3XrGg/s7AJalTBrgkdcoAl6ROeQxc2ouP/8hJq13Cgpz0iY+vdglaQc7AJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktSpJf2YVZJTgTcD+wHvrKoLFrqPY1/7h0spYVVs+b2fXe0SJGnxM/Ak+wH/F/ivwDOBM5M8c1KFSZL2bimHUI4DPltVd1bVvwHvA146mbIkSfNJVS3ugsnpwKlV9aq2/grguVV1zqx+G4ANbfXpwN8tvtwFOwy4bwWvb6Xty+Pbl8cGjq93Kz2+J1XV1OzGZf8PHarqYuDi5b6euSTZXFXTq3HdK2FfHt++PDZwfL17pIxvKYdQ7gG+b2T9iNYmSVoBSwnwm4Cjkjw5yaOBnwGumkxZkqT5LPoQSlU9kOQc4MMMHyO8pKp2TKyyyViVQzcraF8e3748NnB8vXtEjG/RJzElSavLb2JKUqcMcEnqVDcBnuSiJK8ZWf9wkneOrL8hyS8vYr8nJ/nQpOpcLkl+PcmOJNuSbE3y3CTXJvm7JLckuS7J01e5xgdbbbcmuTrJwRPa71lJ3jqJfS3VyBh3tNv93CSPatumk/yfvVz2Pya5Yp7973Ufrc/BSf774kbwkP38hyTvS/L3SbYk+fMkG1b6+ZDk11by+kaut5K8d2R9TZJdkxp/knVJbp3EvvakmwAHrgOeB9CeMIcBzxrZ/jzg+vl20n4CoCtJjgd+DPhPVfVs4IXAP7bNL6uq5wAbgd9bpRJnfLOqjqmqHwS+DPzCKtezHGbG+CzgRQw/JXEeQFVtrqpf2tMFq+qfqur0ve18vn00BwNLCvAkAT4IXFtVT6mqY4FfBdYucb+L+WDEggN8Qs/jfwZ+MMmBbf1FLPCj0Isc78T0FODXA8e35WcBtwJfT3JIkscAPwA8PsnNSbYnuaS1k+TzSS5M8rfATyc5Ncmn2/pPrsZgFuhw4L6q+hZAVd1XVf80q88ngKeueGV79kngiQBJjkvyyXbfXD/zTqHNrD+Q5C+T3JHkd2cunOSVST6T5EbghJH2dUk+2t6JbEpyZGt/d5L/l+RTSe5s76wuSXJ7kncvxwCraifDt4zPyeC77+aSnNRm6lvbuB87OiNLckCSP2iP1ZuTPL+1j+7j/DaGa9uYZoL9AuApbd+LfdF+PvDtqnr7yHhuAf4aOCjJFe05cmkLe5L8RpKb2jusi0far03ypiSbgVcneUmSG9q4/irJ2tbvoJExb0vyU0kuAA5sY7m09Xt5khtb2/+fCesk38jwTvsWdmfBUv058OK2fCbwxzMbkhya5E9brZ9K8uzWfn6S9yS5DnhPexxf2W6HO5KcN7L//ZK8I8M7to+MvFhMRlV18w/4HHAk8PPAfwN+E/hRhif4TQyz0qe1vn8IvKYtfx74lbZ8QOt3FBDgcuBDqz22ecZ9ELAV+AzwNuCk1n4tMN2WXwtctsp1fqP93Q/4E4afWgB4HLCmLb8QeH9bPgu4E3h8u1/uYvhy2OHAPwBTwKMZ3n29tV3mamB9W/454E/b8rsZfo8nDL/J8zXgaIZJyhbgmEmOcVbb/Qwz15NnHkutzhNG7r81wDrg1tZ2LsNHbwGe0cZ7wKx9nM8wcXkMwzvOLwH7j+5nCeP4JeCiOdpPBr7K8MW8RzG8EJ/Yth060u89wEtGHodvG9l2CLs/4fYq4A1t+ULgTaP9Zt+mDBOxq4H92/rbgJ9tywWcMcnHK/Bs4Ip222+ddfu/BTivLb8A2Dpyv2wBDhx5HN8LPAE4kGFyOd3upwdmHnsMWfPyST7nepqBw/Bgfl7798n2b2b9buBzVfWZ1ncj8CMjl72s/X1G63dHDbfqe3mEq6pvAMcyzPZ2AZclOattvjTJVoYXsf+xOhV+14Gtli8wBNo1rf3xwJ+02edFPPTQ16aq+mpV/StwG/Ak4LkMb+131fBDaZeN9D8e+KO2/B7gxJFtV7f7dDvwxaraXlXfAXYwPJlW0nXAG9us+eCqemDW9hNpj72q+jTDi9fT5tjPn1XVt6rqPmAnSzzEMaYbq+rudtttZfdt9/w2s97OEGij9+PofXQE8OHW77Uj/V7I8AumAFTVV+a47lMYHus3tcfSKcD3t20PAu9fysBmq6ptDOM7k2E2PupEhscYVfVR4AlJHte2XVVV3xzpe01Vfam1fYDdj8vPVdXWtryFCT8OewvwmePgRzO8yn2K4Qn9PIZZwN7887JWtsyq6sGquraqzgPOAX6qbXpZDcdkT6uqf9zLLlbCN6vqGIYQDruPgf8m8LEajo2/hGG2M+NbI8sPsrTf55nZ13dm7fc7S9zvHiX5foa6d4621/Db+K9imJFdl+QZi7yKSd4+o3YwBOVY15nkAIbZ8OlVdTTwDh56P44+v97C8I7paIZ3y6P95hNgY3tMH1NVT6+q89u2f62qBxewr3FdBfw+I4dPxjA7T2Z/oWZmfbnuP6C/AL+e4WTel1ugfZnhhM7xDK/M65LMHAd+BfDxOfbx6dbvKW39zGWuecmSPD3JUSNNxzDM2B6RqupfGN6in5vhJM/j2X1y6KwxdnEDcFKSJyTZH/jpkW3XM/xsA8DLGI7ZrookU8DbGcKqZm17SnsHcCHD4b3ZAf7XDPWT5GkMhwbH/aXOrwOPXUrtwEeBx2T4tdCZmp8N/Oc99J8J4fuSHATs7WTs6P29fqT9GkZObCc5pC1+u93PAJuA05N8b+tzaJInjTGepbgEeH1VbZ/VPnofncxwHupre9jHi1qtBwKnMUw2l11vAb6d4Vjgp2a1fbWq7gZeyfBWfTvDrOvts3fQ3qpvAP4sw0nMnbP7PAIdBGxMcluSbQz/gcb5q1vS3lXVzcA2hhfI3wV+J8nNjDEDqap7Gcb3SYYnwu0jm38ReGW7HV4BvHqylc9r5oTbDuCvgI8Ar5+j32vayb5twLeBv2jtM0H/NuBR7bF6GXBWtZPU86mqLzHM6m9d7EnM9oLzE8ALM3yMcAfwOwyHv+bqfz/DrPtWhp/PuGkvuz+f4Xm4hYf+5OpvAYe0um9hOJEKw9fStyW5tKpuA/4X8JF2213DcE5k2bTDRXN9dPN84NhWxwU89MVothsZJpHbGM7xbJ54oXPwq/TSCklyLPDGqjpptWvR5LTzUdM16/9CWAm9zcClLiWZZjjG+ubVrkX7DmfgktQpZ+CS1CkDXJI6ZYBLUqcMcEnqlAEuSZ36d//uSCP0EDBrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import time \n",
    "\n",
    "def calc_time(fun):\n",
    "    tokenizer = fun()\n",
    "    start_time = time.time()\n",
    "    # morph tokenizer doesn't take arguments\n",
    "    if str(tokenizer) == 'MorphologicalTokenizer':\n",
    "        tokenizer.train()\n",
    "    else:\n",
    "        tokenizer.train('samples/data.txt')\n",
    "    return time.time() - start_time\n",
    "\n",
    "running_times = {}\n",
    "\n",
    "running_times['Word'] = calc_time(tk.WordTokenizer)\n",
    "running_times['SP'] = calc_time(tk.SentencePieceTokenizer)\n",
    "running_times['Random'] = calc_time(tk.RandomTokenizer)\n",
    "running_times['Disjoint'] = calc_time(tk.DisjointLetterTokenizer)\n",
    "running_times['Character'] = calc_time(tk.CharacterTokenizer)\n",
    "running_times['Morph'] = calc_time(tk.MorphologicalTokenizer)\n",
    "plt = sns.barplot(data = pd.DataFrame.from_dict([running_times]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "comparing tokenizers in tokenization time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training WordTokenizer ...\n",
      "Training SentencePiece ...\n",
      "Training RandomTokenizer ...\n",
      "Training DisjointLetterTokenizer ...\n",
      "Training CharacterTokenizer ...\n",
      "Training MorphologicalTokenizer ...\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD5CAYAAADLL+UrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYAUlEQVR4nO3de5gcVZ3G8e9rIgFFSSBjxFyciAGWmwgjoKCAQQx4CbuiDzwIAeMz6xpUFlYE3cegro/gLYIIboRIQJaLiBI0CjEQcYFAJhByAYTZIGRiIAOBeEEugd/+UWdM0cxkZrp7ehLO+3meeabqnNNVp6Z73j59qrpbEYGZmeXhVYPdATMzaxyHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRob21kDSLOCDwNqI2KNU/hlgGvAC8KuIOD2VnwlMTeWfjYgbUvkk4FxgCHBRRJzd275HjhwZzc3N/T0mM7OsLV68+PGIaOqurtfQBy4Bzgcu7SqQdCgwGXhbRDwr6Q2pfDfgGGB34E3AbyXtnG72A+B9QAewSNKciLh3Uztubm6mra2tD100M7Mukh7uqa7X0I+IWyQ1VxT/G3B2RDyb2qxN5ZOBK1P5Q5Lagf1SXXtErEwdujK13WTom5lZfVU7p78z8G5Jd0j6naR3pPLRwKpSu45U1lO5mZk1UF+md3q63fbAAcA7gKslvaUeHZLUCrQCjBs3rh6bNDOzpNqRfgdwbRTuBF4ERgKrgbGldmNSWU/lLxMRMyOiJSJampq6PQ9hZmZVqjb0fwEcCpBO1G4FPA7MAY6RNEzSeGACcCewCJggabykrShO9s6ptfNmZtY/fblk8wrgEGCkpA5gOjALmCVpOfAcMCWKj+tcIelqihO0G4BpEfFC2s7JwA0Ul2zOiogVA3A8Zma2CdqcP1q5paUlfMmmmVn/SFocES3d1fkduWZmGXHom5llpNpLNm0APfLVPQe7C/0y7svLBrsLZtZHHumbmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRnoNfUmzJK1N34dbWXeapJA0Mq1L0nmS2iUtlbRPqe0USQ+mnyn1PQwzM+uLvoz0LwEmVRZKGgscDjxSKj4CmJB+WoELU9vtKb5QfX9gP2C6pBG1dNzMzPqv19CPiFuAdd1UzQBOB8rfrD4ZuDQKC4HhknYE3g/Mi4h1EfEkMI9unkjMzGxgVTWnL2kysDoi7qmoGg2sKq13pLKeys3MrIH6/R25kl4DfJFiaqfuJLVSTA0xbty4gdiFmVm2qhnp7wSMB+6R9EdgDHCXpDcCq4GxpbZjUllP5S8TETMjoiUiWpqamqronpmZ9aTfoR8RyyLiDRHRHBHNFFM1+0TEo8Ac4IR0Fc8BwPqIWAPcABwuaUQ6gXt4KjMzswbqyyWbVwC3A7tI6pA0dRPN5wIrgXbgR8CnASJiHfA1YFH6+WoqMzOzBup1Tj8iju2lvrm0HMC0HtrNAmb1s39mZlZHfkeumVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhnpy3fkzpK0VtLyUtm3JN0vaamkn0saXqo7U1K7pD9Ien+pfFIqa5d0Rv0PxczMetOXkf4lwKSKsnnAHhGxF/AAcCaApN2AY4Dd020ukDRE0hDgB8ARwG7AsamtmZk1UK+hHxG3AOsqym6MiA1pdSEwJi1PBq6MiGcj4iGgHdgv/bRHxMqIeA64MrU1M7MGqsec/ieAX6fl0cCqUl1HKuup3MzMGqim0Jf0JWADcHl9ugOSWiW1SWrr7Oys12bNzIwaQl/SicAHgeMiIlLxamBsqdmYVNZT+ctExMyIaImIlqampmq7Z2Zm3agq9CVNAk4HPhwRT5eq5gDHSBomaTwwAbgTWARMkDRe0lYUJ3vn1NZ1MzPrr6G9NZB0BXAIMFJSBzCd4mqdYcA8SQALI+JTEbFC0tXAvRTTPtMi4oW0nZOBG4AhwKyIWDEAx2NmZpvQa+hHxLHdFF+8ifZfB77eTflcYG6/emdmZnXld+SamWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWWk19CXNEvSWknLS2XbS5on6cH0e0Qql6TzJLVLWippn9JtpqT2D0qaMjCHY2Zmm9KXkf4lwKSKsjOA+RExAZif1gGOoPgy9AlAK3AhFE8SFN+tuz+wHzC964nCzMwap9fQj4hbgHUVxZOB2Wl5NnBUqfzSKCwEhkvaEXg/MC8i1kXEk8A8Xv5EYmZmA6zaOf1REbEmLT8KjErLo4FVpXYdqayncjMza6CaT+RGRABRh74AIKlVUpukts7Oznpt1szMqD70H0vTNqTfa1P5amBsqd2YVNZT+ctExMyIaImIlqampiq7Z2Zm3ak29OcAXVfgTAGuK5WfkK7iOQBYn6aBbgAOlzQincA9PJWZmVkDDe2tgaQrgEOAkZI6KK7CORu4WtJU4GHgY6n5XOBIoB14GjgJICLWSfoasCi1+2pEVJ4cNjOzAdZr6EfEsT1UTeymbQDTetjOLGBWv3pnZmZ15XfkmpllxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpaRmkJf0r9LWiFpuaQrJG0tabykOyS1S7pK0lap7bC03p7qm+txAGZm1ndVh76k0cBngZaI2AMYAhwDnAPMiIi3Ak8CU9NNpgJPpvIZqZ2ZmTVQrdM7Q4FtJA0FXgOsAd4LXJPqZwNHpeXJaZ1UP1GSaty/mZn1Q9WhHxGrgW8Dj1CE/XpgMfBURGxIzTqA0Wl5NLAq3XZDar9Dtfs3M7P+q2V6ZwTF6H088CbgtcCkWjskqVVSm6S2zs7OWjdnZmYltUzvHAY8FBGdEfE8cC1wIDA8TfcAjAFWp+XVwFiAVL8d8ETlRiNiZkS0RERLU1NTDd0zM7NKtYT+I8ABkl6T5uYnAvcCNwNHpzZTgOvS8py0Tqq/KSKihv2bmVk/1TKnfwfFCdm7gGVpWzOBLwCnSmqnmLO/ON3kYmCHVH4qcEYN/TYzsyoM7b1JzyJiOjC9onglsF83bZ8BPlrL/szMrDZ+R66ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGakp9CUNl3SNpPsl3SfpnZK2lzRP0oPp94jUVpLOk9QuaamkfepzCGZm1le1jvTPBX4TEbsCbwPuo/jC8/kRMQGYz8YvQD8CmJB+WoELa9y3mZn1U9WhL2k74D3AxQAR8VxEPAVMBmanZrOBo9LyZODSKCwEhkvaseqem5lZv9Uy0h8PdAI/lnS3pIskvRYYFRFrUptHgVFpeTSwqnT7jlRmZmYNUkvoDwX2AS6MiLcDf2PjVA4AERFA9GejkloltUlq6+zsrKF7ZmZWqZbQ7wA6IuKOtH4NxZPAY13TNun32lS/Ghhbuv2YVPYSETEzIloioqWpqamG7pmZWaWqQz8iHgVWSdolFU0E7gXmAFNS2RTgurQ8BzghXcVzALC+NA1kZmYNMLTG238GuFzSVsBK4CSKJ5KrJU0FHgY+ltrOBY4E2oGnU1szM2ugmkI/IpYALd1UTeymbQDTatmfmZnVxu/INTPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLSM2hL2mIpLsl/TKtj5d0h6R2SVelr1JE0rC03p7qm2vdt5mZ9U89RvqfA+4rrZ8DzIiItwJPAlNT+VTgyVQ+I7UzM7MGqin0JY0BPgBclNYFvBe4JjWZDRyVliendVL9xNTezMwapNaR/veA04EX0/oOwFMRsSGtdwCj0/JoYBVAql+f2puZWYNUHfqSPgisjYjFdewPkloltUlq6+zsrOemzcyyV8tI/0Dgw5L+CFxJMa1zLjBc0tDUZgywOi2vBsYCpPrtgCcqNxoRMyOiJSJampqaauiemZlVqjr0I+LMiBgTEc3AMcBNEXEccDNwdGo2BbguLc9J66T6myIiqt2/mZn130Bcp/8F4FRJ7RRz9hen8ouBHVL5qcAZA7BvMzPbhKG9N+ldRCwAFqTllcB+3bR5BvhoPfZnZmbV8Ttyzcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8tI1aEvaaykmyXdK2mFpM+l8u0lzZP0YPo9IpVL0nmS2iUtlbRPvQ7CzMz6ppaR/gbgtIjYDTgAmCZpN4ovPJ8fEROA+Wz8AvQjgAnppxW4sIZ9m5lZFaoO/YhYExF3peW/APcBo4HJwOzUbDZwVFqeDFwahYXAcEk7Vt1zMzPrt7rM6UtqBt4O3AGMiog1qepRYFRaHg2sKt2sI5WZmVmD1Bz6krYFfgacEhF/LtdFRADRz+21SmqT1NbZ2Vlr98zMrKSm0Jf0aorAvzwirk3Fj3VN26Tfa1P5amBs6eZjUtlLRMTMiGiJiJampqZaumdmZhVquXpHwMXAfRHx3VLVHGBKWp4CXFcqPyFdxXMAsL40DWRmZg0wtIbbHggcDyyTtCSVfRE4G7ha0lTgYeBjqW4ucCTQDjwNnFTDvs3MrApVh35E/C+gHqondtM+gGnV7s/MzGrnd+SamWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlpGGh76kSZL+IKld0hmN3r+ZWc4aGvqShgA/AI4AdgOOlbRbI/tgZpazqr8YvUr7Ae0RsRJA0pXAZODeBvfDzKpw/mnXD3YX+u3k73xosLuwWWl06I8GVpXWO4D9G9wHG2QHfv/Awe5Cv9z6mVv73PZ37zl4AHsyMA6+5XeD3YXNxtc/fvRgd6HfvvSTa/rVXhExQF3pZmfS0cCkiPhkWj8e2D8iTi61aQVa0+ouwB8a1kEYCTzewP01mo9vy+bj23I1+tjeHBFN3VU0eqS/GhhbWh+Tyv4hImYCMxvZqS6S2iKiZTD23Qg+vi2bj2/LtTkdW6Ov3lkETJA0XtJWwDHAnAb3wcwsWw0d6UfEBkknAzcAQ4BZEbGikX0wM8tZo6d3iIi5wNxG77ePBmVaqYF8fFs2H9+Wa7M5toaeyDUzs8Hlj2EwM8vIKzr0Jc2QdEpp/QZJF5XWvyPp1Cq2e4ikX9arnwNF0pckrZC0VNISSftLWpA+BuMeSbdK2mUQ+/dC6tdySddLGl6n7Z4o6fx6bKseSse5Iv3dT5P0qlTXIum8Tdz2TZI2eSF2b9tIbYZL+nR1R/CPbbxR0pWS/k/SYklzJbU2+n9B0hcbub+KfYekn5TWh0rqrNffQFKzpOX12FZPXtGhD9wKvAsg/ZONBHYv1b8LuK23jaSPj9iiSHon8EFgn4jYCziMjW+MOy4i3gbMBr41SF0E+HtE7B0RewDrgGmD2JeB1HWcuwPvo/gYkukAEdEWEZ/t6YYR8aeI2OQ7hnrbRjIcqDr0JQn4ObAgInaKiH2BM4FR1W4zbbea84r9Dv06/g//DdhD0jZp/X1UXHbeh740/Fxq2Ss99G8D3pmWdweWA3+RNELSMOCfgO0k3S1pmaRZqRxJf5R0jqS7gI+mD4q7P63/y2AcTD/tCDweEc8CRMTjEfGnija3AG9teM+6dzvFO7aRtJ+k29P9clvXq5E0gr9W0m8kPSjpm103lnSSpAck3QkcWCpvlnRTerUzX9K4VH6JpAslLZS0Mr16myXpPkmXDNRBRsRaijcfnqzCP141Sjo4vSJYko79deWRn6StJf04PVbvlnRoKi9v46x0HAvScXU9GZwN7JS2Xc0T/aHA8xHxw9Kx3AP8HthW0jXp/+Py9ASBpC9LWpReyc0slS+Q9D1JbcDnJH1I0h3pmH4raVRqt23peJdK+oiks4Ft0nFcntp9XNKdqey/uwJe0l9VvJq/h405UA9zgQ+k5WOBK7oqJG0v6Repvwsl7ZXKz5J0maRbgcvSY/m69Ld4UNL00vaHSPqRileGN5aeYOojIl7RP8BDwDjgX4FPAV8DjqQIhkUUo9+dU9tLgVPS8h+B09Py1qndBEDA1cAvB/vYejnubYElwAPABcDBqXwB0JKWPw9cNYh9/Gv6PQT4KcW7tQFeDwxNy4cBP0vLJwIrge3SffIwxZv9dgQeAZqArShe4Z2fbnM9MCUtfwL4RVq+BLgy3Z+TgT8De1IMhBYDe9f7OCvKnqIYJR/S9VhKfT2wdP8NBZqB5ansNIrLnAF2Tce8dcU2zqIY7AyjeGX7BPDq8naqPIbPAjO6KT8EWE/xRstXUTx5H5Tqti+1uwz4UOkxeEGpbgQbLyr5JPCdtHwO8L1yu8q/J8XA7Xrg1Wn9AuCEtBzAx+r9mAX2Aq5Jf/slFX//7wPT0/J7gSWl+2UxsE3psbwG2AHYhmJA2pLupw1djz+KrPl4PY/hlT7Sh+If4F3p5/b007XeATwUEQ+ktrOB95Rue1X6vWtq92AU98RP2MxFxF+BfSlGlZ3AVZJOTNWXS1pC8cT3H4PTQyCN2IBHKQJwXirfDvhpGuHO4KVTcvMjYn1EPEPxQX1vpvj8pgUR0RkRz7HxfoNihPc/afky4KBS3fXp/lwGPBYRyyLiRWAFxT9fo90KfDeNzodHxIaK+oNIj72IuJ/iSW/nbrbzq4h4NiIeB9ZS4xRMH9wZER3pb7eEjX+7Q9MIfhlFAJbvx/J9NAa4IbX7fKndYRSfygtARDzZzb4nUjzOF6XH0kTgLanuBeBntRxYdyJiKcUxHsvLLz8/iOJxRkTcBOwg6fWpbk5E/L3Udl5EPJHKrmXjY/OhiFiSlhdT58diDqHfNa+/J8Wz6UKKIHgXxYhjU/42oD0bYBHxQkQsiIjpwMnAR1LVcVHMMR8VEas2sYmB9veI2JsiuMXGOf2vATdHMdf/IYoRVZdnS8svUNt7Tbq29WLFdl+scbubJOktFH1fWy6PiLMpRrrbALdK2rXKXdTzb9RlBUW49ml/kramGHUfHRF7Aj/ipfdj+X/r+xSvzPakeEVebtcbAbPT43nviNglIs5Kdc9ExAv92FZ/zAG+TWlqpw8q86Tyevmu9YG4//4hh9C/jeKE5roUgusoTmq9k2IU0Cypa177eKC7jxy8P7XbKa0fO8B9rpmkXSRNKBXtTTEy3OxExNMU0wenqTjJtR0bT46d2IdN3AEcLGkHSa8GPlqqu43i4z4AjqOYgx40kpqAH1KEXFTU7ZRebZxDMfVYGfq/pzgGJO1MMW3Z1w8k/Avwuhq6fhMwTMUHInb1dy/g3T207wruxyVtC2zqZHT5/p5SKp9H6eS+pBFp8fl0PwPMB46W9IbUZntJb+7D8dRqFvCViFhWUV6+jw6hOK/25x628b7U322AoygGqAMuh9BfRjG3ubCibH1EdAAnUUwlLKMY4f2wcgNpKqEV+JWKE7lrK9tshrYFZku6V9JSii+tOWtwu9SziLgbWErxhPpN4BuS7qYPo5yIWENxbLdT/OPcV6r+DHBS+hscD3yuvj3vk64TjyuA3wI3Al/ppt0p6aTnUuB54NepvOvJ4QLgVemxehVwYqQT9b2JiCcoXj0sr+ZEbnqC+mfgMBWXbK4AvkExNddd+6coRvfLKT52ZdEmNn8Wxf/gYl76SZT/BYxIfb6H4mQyFO9uXSrp8oi4F/hP4Mb0d5tHcY5nQKXprO4ukz0L2Df15Wxe+iRW6U6KgedSivNWbXXvaDf8jlyzzZikfYHvRsSW90H91qN0fq0lSh8r3yg5jPTNtkiSWijmjM8d7L7YK4dH+mZmGfFI38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OM/D93X6YmJiS8QgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import time \n",
    "\n",
    "def calc_time(fun):\n",
    "    tokenizer = fun()\n",
    "    # morph tokenizer doesn't take arguments\n",
    "    if str(tokenizer) == 'MorphologicalTokenizer':\n",
    "        tokenizer.train()\n",
    "    else:\n",
    "        tokenizer.train('samples/data.txt')\n",
    "    start_time = time.time()\n",
    "    tokenizer.tokenize(open('samples/data.txt', 'r').read())\n",
    "    return time.time() - start_time\n",
    "\n",
    "running_times = {}\n",
    "\n",
    "running_times['Word'] = calc_time(tk.WordTokenizer)\n",
    "running_times['SP'] = calc_time(tk.SentencePieceTokenizer)\n",
    "running_times['Random'] = calc_time(tk.RandomTokenizer)\n",
    "running_times['Disjoint'] = calc_time(tk.DisjointLetterTokenizer)\n",
    "running_times['Character'] = calc_time(tk.CharacterTokenizer)\n",
    "running_times['Morph'] = calc_time(tk.MorphologicalTokenizer)\n",
    "plt = sns.barplot(data = pd.DataFrame.from_dict([running_times]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Caching"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Caching is used for speeding up the tokenization process. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MorphologicalTokenizer ...\n"
     ]
    }
   ],
   "source": [
    "import tkseem as tk\n",
    "tokenizer = tk.MorphologicalTokenizer()\n",
    "tokenizer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.82 s ± 277 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "out = tokenizer.tokenize(open('samples/data.txt', 'r').read(), use_cache = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.14 s ± 296 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "out = tokenizer.tokenize(open('samples/data.txt', 'r').read(), use_cache = True, max_cache_size = 10000)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}