notebooks/208_bi_encoder_margin_mse_loss.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "139bee68",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "028823c5",
   "metadata": {},
   "source": [
    "# Train a bi-encoder to learn name-to-vec encodings\n",
    "\n",
    "**Deprecated by 224_train_bi_encoder**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da4d9cdc",
   "metadata": {},
   "source": [
    "### Remove all non-negatives from anchor-pos-neg triplets and test data\n",
    "\n",
    "| phon/sub | easy/common negs | notes  | uni/bi | size | loss | errors | negs |\n",
    "| -------- | ---- | ------ | ------ | ---- | ---- | ------ | ---- |\n",
    "| Subwords | 100+1000 | noback10mask 64dim 5epochs | unigrams | 2000 | 0.036 | 62/23/44 | 3/1/523/136 |\n",
    "| Subwords | 100+1000 | noback10mask 64dim 10epochs | unigrams | 2000 | 0.036 | 56/17/42 | 4/1/532/135 |\n",
    "| Subwords | 100+2000 | noback10mask 64dim 5epochs | unigrams | 2000 | 0.028 | 64/16/51 | 6/1/577/146 |\n",
    "| Subwords | 100+1000x2 | noback10mask 64dim 5epochs | unigrams | 2000 | 0.032 | 69/28/49 | 3/1/508/129 |\n",
    "| Subwords | 100+1000 | noback10mask 128dim 5epochs | unigrams | 2000 | 0.031 | 66/28/41 | 3/1/376/117 |\n",
    "| Subwords | 100+1000x2 | noback10mask 128dim 5epochs | unigrams | 2000 | 0.026 | 65/28/38 | 2/1/356/93 |\n",
    "| Subwords | 100+1000x2 | noback10mask 256dim 5epochs | unigrams | 2000 | 0.023 | 57/22/39 | 1/0/267/67 |\n",
    "| Subwords | 100+1000x2 | trainall noback10mask 256dim 5epochs | unigrams | 2000 | 0.023 | 54/18/42 | 2/0/272/83 |\n",
    "| Subwords | 100+1000x2 | noback10mask 256dim 10epochs | unigrams | 2000 | 0.023 | 57/26/33 | 0/0/268/92 |\n",
    "| Subwords | 100+2000x2 | noback10mask 256dim 5epochs | unigrams | 2000 | 0.013 | 66/23/44 | 1/0/279/87 |\n",
    "| Subwords | 100+1000x2 | pos-anchor noback10mask 256dim 5epochs | unigrams | 2000 | 0.031 | 53/21/34 | 2/1/325/113 |\n",
    "| Subwords | 200+1000x2 | pos-anchor noback10mask 256dim 5epochs | unigrams | 2000 | 0.028 | 50/17/35 | 2/0/311/105 |\n",
    "| Subwords | 100+1000x2 | noback10mask 128dim 5epochs | bigrams | 2000 | 0.026 | 58/27/33 | 2/0/341/104 |\n",
    "| Subwords | 100+1000x2 | noback10mask 256dim 5epochs | bigrams | 2000 | 0.024 | 52/25/27 | 0/0/256/83  |\n",
    "| Subwords | 0+1000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.031 | 109/79/38 | 0/0/287/89 |\n",
    "| Subwords | 20+1000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.31 | 75/39/37 | 0/0/258/64 |\n",
    "| Subwords | 20+2000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.016 | 86/53/39 | 1/0/269/59 |\n",
    "| Subwords | 50+1000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.030 | 58/27/33 | 1/0/256/54 |\n",
    "| Subwords | 50+2000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.017 | 63/30/38 | 0/0/237/52 |\n",
    "| Subwords | 100+1000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.028 | 56/23/35 | 1/0/281/67 |\n",
    "| Subwords | 100+2000 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.017 | 67/24/47 | 2/0/261/56 |\n",
    "| Subwords | 100+1000x2 | trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.023 | 56/23/34 | 2/0/237/52 |\n",
    "| Subwords | 100+1000x2 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.015 | **44/6/38** | **2/0/227/33** |\n",
    "| Subwords | 100+1000x2 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.015 | 49/2/47 | 1/0/219/37 |\n",
    "| Subwords | 100+1000x2 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000 | 0.015 | **43/3/40** | 1/0/216/39 |\n",
    "| Subwords | 100+1000x2 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000f | 0.015 | **40/2/38** | **1/1/169/38** |\n",
    "| Subwords | 100+1000x2 | dotprod score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000f | 0.014 | **45/4/41** | **2/0/96/29** |\n",
    "| Subwords | 100+1000x2 | dotprod score^20% trainall noback10mask 256dim 10epochs | unigrams | 2000f | 0.013 | **47/3/44** | **1/0/93/28** |\n",
    "| Subwords | 100+1000x2 | rmsgrad dotprod score^20% trainall noback10mask 256dim 10epochs | unigrams | 2000f | 0.033 | 76/12/65 | 13/3/1297/363 |\n",
    "| Subwords | 100+2000 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000p | 0.013 | 53/9/44 | 3/1/203/56 |\n",
    "| Subwords | 100+2000 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000p | 0.013 | 51/5/47 | 3/1/182/54 |\n",
    "| Subwords | 100+1000x2 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 2000p | 0.16 | 52/5/48 | 3/1/170/50 |\n",
    "| Subwords | 100+2000 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 1800p | 0.013  | 54/7/47 | 1/1/219/54 |\n",
    "| Subwords | 100+2000 | score^20% trainall noback10mask 256dim 6epochs | unigrams | 1500p | 0.014 | 59/11/50 | 2/1/272/78 |\n",
    "\n",
    "### Clean Negatives, new way to measure errors\n",
    "All tests are score^20% trainall noback10mask 256dim 6epochs\n",
    "\n",
    "| phon/sub | easy pos/neg+common negs | notes  | uni/bi | size | loss | errors | negs |\n",
    "| -------- | ---- | ------ | ------ | ---- | ---- | ------ | ---- |\n",
    "| Subwords | 100+1000x2 | base | unigrams | 2000f | 0.015 | **26/5/22** | 1/1/42/6 |\n",
    "| Subwords | 100/100+1000x2 | anchor-neg-easyneg | unigrams | 2000f | 0.02 | 109/96/18 | **0/0/25/1** |\n",
    "| Subwords | 100/10+1000x2 | anchor-neg-easyneg | unigrams | 2000f |  | 82/74/18 | **0/0/20/3** |\n",
    "| Subwords | 100/10+1000x2 | anchor-pos-negx2 anchor-neg-easyneg | unigrams | 2000f | 0.025 | 44/24/21 | **1/1/29/6** |\n",
    "| Subwords | 100/5+1000x2 | anchor-neg-easyneg | unigrams | 2000f | 0.02 | **34/12/24** | **1/0/28/3** |\n",
    "| Subwords | 100/5+1000x2 | init anchor-neg-easyneg | unigrams | 2000f | 0.02 | 45/22/25 | **1/0/33/3** |\n",
    "| Subwords | 100/5+1000x2 | 12epochs push-more-apart anchor-neg-easyneg | unigrams | 2000f | 0.02 | **34/14/22** | **0/0/19/2** |\n",
    "| Subwords | 100/5+1000x2 | 128dims 12epochs push-more-apart anchor-neg-easyneg | unigrams | 2000f | 0.022 | **37/17/25** | **1/0/26/2** |"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c022354",
   "metadata": {},
   "source": [
    "### Results before we removed non-negatives from anchor-pos-neg triples and test data\n",
    "\n",
    "Try phonemes, subwords, and/or n-grams using anchor-pos-neg triplets with MarginMSELoss\n",
    "\n",
    "| phon/sub | easy/common negs | notes  | uni/bi | size | loss | errors | negs |\n",
    "| -------- | ---- | ------ | ------ | ---- | ---- | ------ | ---- |\n",
    "| Phonemes | 100  |        | unigrams |      | 0.102 | ?, 155 | ?, ?, 146 |\n",
    "| Phonemes | 100  |        | bigrams  |      | 0.073 | 8.5, 94 | 12, 43, 56 |\n",
    "| Subwords | 100  |        | unigrams | 500  | 0.74 | ?, 89 | 12, 54, 60, |\n",
    "| Subwords | 100  |        | unigrams | 1000 | 0.063 | 7.6, 69 | 7, 22, 30 |\n",
    "| Subwords | 100  |        | unigrams | 2000 | 0.052 | 5.5, 61 | 7, 25, 32 |\n",
    "| Subwords | 200  |        | unigrams | 1000 | 0.055 | 9.6, 80 | 4, 27, 37 |\n",
    "| Subwords | 200  |        | unigrams | 2000 | 0.047 | 6.2, 56 | 5, 24, 32 |\n",
    "| Subwords | 200  | noback | unigrams | 2000 | 0.049 | 7.4, 46 | 6, 28, 39 |\n",
    "| Subwords | 200  | noback9 | unigrams | 2000 | 0.049 | 3.7, 62 | 7, 31, 46 |\n",
    "| Subwords | 200  | noback9mask | unigrams | 2000 | 0.049 | 7.6, 76 | 6, 30, 46 |\n",
    "| Subwords | 100+1000 | noback9mask | unigrams | 2000 | 0.050 | 2.6, 66 | 4, 21, 33 |\n",
    "| Subwords | 100+1000 | noback10mask | unigrams | 2000 | 0.049 | 11.6, 77 | 6, 24, 34 |\n",
    "| Subwords | 100+1000 | noback10mask 64dim 40epochs | unigrams | 2000 | 0.036 | 17.8, 75 | 2, 12, 21 |\n",
    "| Subwords | 100+1000 | noback10mask 64dim 10epochs | unigrams | 2000 | 0.037 | 11.2, 80 | 2, 10, 18 |\n",
    "| Subwords | 100+1000 | noback10mask 64dim 5epochs | unigrams | 2000 | 0.037 | 6, 75 | 1, 11, 22 |\n",
    "| Subwords | 150+1000 | noback10mask 64dim 5epochs | unigrams | 2000 | 0.035 | 11.2, 80 | 3, 15, 22 |\n",
    "| Subwords | 100+1500 | noback9mask | unigrams | 2000 | 0.046 | 6.4. 80 | 4, 22, 34 |\n",
    "| Subwords | 200 | nofront | unigrams | 2000 | 0.048 | 6.7, 56 | 6, 28, 37 |\n",
    "| Subwords | 200 | noback  | unigrams | 1500 | 0.052 | 6.3, 57 | 8, 31, 47 |\n",
    "| Subwords | 100  |        | bigrams  | 500  | 0.066 | 19.5, 85 | 13, 23, 31 |\n",
    "| Subwords | 100  |        | bigrams  | 1000 | 0.56 | 12, 63 | 7, 30, 40 |\n",
    "| Subwords | 100  |        | bigrams  | 2000 | 0.040 | 9.6, 64 | 16, 56, 66 |\n",
    "| Subwords | 100  | edit   | unigrams | 500  | | | |\n",
    "| Subwords | 100  | edit   | unigrams | 1000 | 0.065 | 12.2, 102, | 7, 34, 36 |\n",
    "| Subwords | 100  | edit   | unigrams | 2000 | 0.055 | 6.9, 77 | 11, 42, 46 |\n",
    "| Subwords | 100  | edit   | bigrams  | 500  | | | |\n",
    "| Subwords | 100  | edit   | bigrams  | 1000 | | | |\n",
    "| Subwords | 100  | edit   | bigrams  | 2000 | | | |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0364890",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import random\n",
    "import re\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "import torch\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "from tqdm.auto import tqdm\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from src.data.utils import read_csv\n",
    "from src.models.biencoder import BiEncoder\n",
    "from src.models.tokenizer import get_tokenize_function_and_vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "774de195",
   "metadata": {},
   "outputs": [],
   "source": [
    "given_surname = \"given\"\n",
    "num_common_names = 10000\n",
    "report_size = 10000\n",
    "max_tokens = 10\n",
    "\n",
    "use_phonemes = False\n",
    "use_edit_subwords = False\n",
    "vocab_type = 'f'  # tokenizer based upon training name frequency\n",
    "use_bigrams = False\n",
    "use_pretrained_embeddings = False\n",
    "subword_vocab_size = 2000  # 500, 1000, 1500, 2000\n",
    "num_easy_pos_negs = 100\n",
    "num_easy_neg_negs = 5\n",
    "num_common_negs = 1000\n",
    "num_common_neg_copies = 2\n",
    "num_anchor_pos_neg_copies = 1\n",
    "under_represented_threshold = 500\n",
    "\n",
    "# hyperparameters\n",
    "embedding_dim = 128\n",
    "learning_rate = 0.001\n",
    "batch_size = 64\n",
    "num_epochs = 12\n",
    "use_amsgrad = False\n",
    "\n",
    "pref_path = f\"s3://familysearch-names/processed/tree-preferred-{given_surname}-aggr.csv.gz\"\n",
    "triplets_path=f\"../data/processed/tree-hr-{given_surname}-triplets-v2-1000.csv.gz\"\n",
    "tfidf_path=f\"s3://nama-data/data/models/fs-{given_surname}-tfidf-v2.joblib\"\n",
    "phoneme_vocab_path = None  # f\"../data/models/fs-{given_surname}-espeak_phoneme_vocab.json\"\n",
    "phoneme_bigrams_vocab_path = None  # f\"../data/models/fs-{given_surname}-espeak_phoneme_vocab_bigrams.json\"\n",
    "\n",
    "nama_bucket = 'nama-data'\n",
    "subwords_path = f\"../data/models/fs-{given_surname}-subword-tokenizer-{subword_vocab_size}{vocab_type}.json\"\n",
    "edit_subwords_path = None  # f\"../data/models/fs-{given_surname}-edit-subword-tokenizer-{subword_vocab_size}.json\"\n",
    "subwords_bigrams_vocab_path = None  # f\"../data/models/fs-{given_surname}-tokenizer_vocab_bigrams-{subword_vocab_size}{vocab_type}.json\"\n",
    "edit_subwords_bigrams_vocab_path = None  # f\"../data/models/fs-{given_surname}-edit_tokenizer_vocab_bigrams-{subword_vocab_size}.json\"\n",
    "common_non_negatives_path = f\"../data/processed/common_{given_surname}_non_negatives.csv\"\n",
    "\n",
    "model_path = f\"../data/models/bi_encoder-{given_surname}-orig.pth\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34474d97",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()\n",
    "print(torch.cuda.is_available())\n",
    "print(\"cuda total\", torch.cuda.get_device_properties(0).total_memory)\n",
    "print(\"cuda reserved\", torch.cuda.memory_reserved(0))\n",
    "print(\"cuda allocated\", torch.cuda.memory_allocated(0))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7c030b7",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a60fb5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read triplets\n",
    "triplets_df = read_csv(triplets_path)\n",
    "print(len(triplets_df))\n",
    "triplets_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c20e14fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# count the number of unique anchor-pos pairs and anchor-neg pairs\n",
    "anchor_pos = set()\n",
    "anchor_neg = set()\n",
    "for tup in tqdm(triplets_df.itertuples()):\n",
    "    anchor = tup.anchor\n",
    "    pos = tup.positive\n",
    "    neg = tup.negative\n",
    "    anchor_pos.add(f\"{anchor}:{pos}\")\n",
    "    anchor_neg.add(f\"{anchor}:{neg}\")\n",
    "print(len(anchor_pos))\n",
    "print(len(anchor_neg))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d0d0d78",
   "metadata": {},
   "source": [
    "### read common names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ee56867",
   "metadata": {},
   "outputs": [],
   "source": [
    "pref_df = read_csv(pref_path)\n",
    "common_names = [name for name in pref_df['name'][:num_common_names].tolist() \\\n",
    "                if len(name) > 1 and re.fullmatch(r'[a-z]+', name)]\n",
    "pref_df = None\n",
    "len(common_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "401ad99c",
   "metadata": {},
   "source": [
    "### read common non-negatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3606383a",
   "metadata": {},
   "outputs": [],
   "source": [
    "common_non_negatives = set()\n",
    "\n",
    "common_non_negatives_df = read_csv(common_non_negatives_path)\n",
    "for name1, name2 in common_non_negatives_df.values.tolist():\n",
    "    common_non_negatives.add((name1, name2))\n",
    "len(common_non_negatives)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b6a6eec",
   "metadata": {},
   "source": [
    "## Create training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b51952a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# don't add anchor-pos pairs that teach the model bad habits\n",
    "bad_anchor_pos_pairs = [('maria', 'annamaria'), \n",
    "                        ('marie', 'annamarie'),\n",
    "                       ]\n",
    "\n",
    "def is_bad_anchor_pos_pair(name1, name2):\n",
    "    for bad_name1, bad_name2 in bad_anchor_pos_pairs:\n",
    "        if (name1 == bad_name1 and name2 == bad_name2) or \\\n",
    "           (name2 == bad_name1 and name1 == bad_name2):\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47871b51",
   "metadata": {},
   "source": [
    "### Get tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c28eaab",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize, tokenizer_vocab = get_tokenize_function_and_vocab(\n",
    "    use_phonemes=use_phonemes, \n",
    "    use_bigrams=use_bigrams, \n",
    "    use_edit_subwords=use_edit_subwords,\n",
    "    max_tokens=max_tokens,\n",
    "\n",
    "    subwords_path=subwords_path,\n",
    "    edit_subwords_path=edit_subwords_path,\n",
    "    subwords_bigrams_vocab_path=subwords_bigrams_vocab_path,\n",
    "    edit_subwords_bigrams_vocab_path=edit_subwords_bigrams_vocab_path,\n",
    "    phoneme_vocab_path=phoneme_vocab_path,\n",
    "    phoneme_bigrams_vocab_path=phoneme_bigrams_vocab_path,\n",
    "    nama_bucket=nama_bucket,\n",
    ")\n",
    "len(tokenizer_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9447cfa9",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize('dallan')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "181fba23",
   "metadata": {},
   "source": [
    "### Add anchor-pos-neg triplets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daa47c61",
   "metadata": {},
   "outputs": [],
   "source": [
    "neg = '123'\n",
    "type(neg) == int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdba9648",
   "metadata": {},
   "outputs": [],
   "source": [
    "# array of (anchor_tokens, pos_tokens, neg_tokens, target_margin)\n",
    "all_data = []\n",
    "seen_anchor_pos = set()\n",
    "seen_anchor_neg = set()\n",
    "for tup in tqdm(triplets_df.itertuples()):\n",
    "    anchor = tup.anchor\n",
    "    pos = tup.positive\n",
    "    neg = tup.negative\n",
    "    if is_bad_anchor_pos_pair(anchor, pos):\n",
    "        continue\n",
    "    if (anchor, neg) not in common_non_negatives:\n",
    "        # only add anchor-pos-neg if the neg isn't really a non-negative\n",
    "        anchor_tokens = tokenize(anchor)\n",
    "        pos_tokens = tokenize(pos)\n",
    "        neg_tokens = tokenize(neg)\n",
    "        target_margin = tup.positive_score - tup.negative_score\n",
    "        # anchor, positive, hard-negative\n",
    "        for _ in range(num_anchor_pos_neg_copies):\n",
    "            all_data.append({\n",
    "                'anchor': torch.tensor(anchor_tokens),\n",
    "                'pos': torch.tensor(pos_tokens),\n",
    "                'neg': torch.tensor(neg_tokens),\n",
    "                'target': torch.tensor(target_margin, dtype=torch.float),\n",
    "            })\n",
    "\n",
    "    # add anchor-pos-easy-negatives\n",
    "    # only add easy negatives the first time we see this anchor,pos pair\n",
    "    anchor_pos = f\"{anchor},{pos}\"\n",
    "    if anchor_pos not in seen_anchor_pos:\n",
    "        seen_anchor_pos.add(anchor_pos)\n",
    "        ix = 0\n",
    "        while ix < num_easy_pos_negs:\n",
    "            # anchor, positive, easy-negative\n",
    "            easy_neg = random.choice(common_names)\n",
    "            # only add anchor-pos-easy-neg if easy-neg isn't really a non-negative\n",
    "            if (anchor, easy_neg) in common_non_negatives:\n",
    "                continue\n",
    "            easy_neg_tokens = tokenize(easy_neg)\n",
    "            if anchor_tokens == easy_neg_tokens:\n",
    "                continue\n",
    "            all_data.append({\n",
    "                'anchor': torch.tensor(anchor_tokens),\n",
    "                'pos': torch.tensor(pos_tokens),\n",
    "                'neg': torch.tensor(easy_neg_tokens),\n",
    "                'target': torch.tensor(tup.positive_score, dtype=torch.float)\n",
    "            })\n",
    "            ix += 1\n",
    "\n",
    "    # add anchor-neg-easy-negatives\n",
    "    # only add easy negatives the first time we see this anchor,neg pair\n",
    "    neg_anchor = f\"{neg},{anchor}\"\n",
    "    if neg_anchor not in seen_anchor_neg:\n",
    "        seen_anchor_neg.add(neg_anchor)\n",
    "        ix = 0\n",
    "        while ix < num_easy_neg_negs:\n",
    "            easy_neg = random.choice(common_names)\n",
    "            # only add anchor-neg-easy-neg if easy-neg isn't really a non-negative\n",
    "            if (anchor, easy_neg) in common_non_negatives:\n",
    "                continue\n",
    "            easy_neg_tokens = tokenize(easy_neg)\n",
    "            if anchor_tokens == easy_neg_tokens:\n",
    "                continue\n",
    "            all_data.append({\n",
    "                'anchor': torch.tensor(anchor_tokens),\n",
    "                'pos': torch.tensor(neg_tokens),\n",
    "                'neg': torch.tensor(easy_neg_tokens),\n",
    "                'target': torch.tensor(tup.negative_score, dtype=torch.float)\n",
    "            })\n",
    "            ix += 1\n",
    "            \n",
    "len(all_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7e9f680",
   "metadata": {},
   "source": [
    "### Add pos-pos-easyneg triplets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48a15d3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = 0\n",
    "for pos in tqdm(common_names[:num_common_negs]):\n",
    "    pos_tokens = tokenize(pos)\n",
    "    for neg in common_names[:num_common_negs]:\n",
    "        if pos == neg or (pos, neg) in common_non_negatives:\n",
    "            continue\n",
    "#         pos_neg = f\"{pos},{neg}\"\n",
    "#         if pos_neg in seen_anchor_pos or pos_neg in seen_anchor_neg:\n",
    "#             continue\n",
    "#         neg_pos = f\"{neg},{pos}\"\n",
    "#         if neg_pos in seen_anchor_pos or neg_pos in seen_anchor_neg:\n",
    "#             continue\n",
    "        neg_tokens = tokenize(neg)\n",
    "        for _ in range(num_common_neg_copies):\n",
    "            all_data.append({\n",
    "                'anchor': torch.tensor(pos_tokens),\n",
    "                'pos': torch.tensor(pos_tokens),\n",
    "                'neg': torch.tensor(neg_tokens),\n",
    "                'target': torch.tensor(1.0, dtype=torch.float)\n",
    "            })\n",
    "            cnt += 1\n",
    "print(cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "706ca1ed",
   "metadata": {},
   "source": [
    "### Add triplets for names we want to push apart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c32997ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "push_apart_pairs = [('charles', 'frances'),\n",
    "                    ('marie', 'annie'),\n",
    "                    ('james', 'jane'),\n",
    "                    ('jane', 'janos'),\n",
    "                    ('hannah', 'hans'),\n",
    "                    ('frank', 'frederick'),\n",
    "                    ('anne', 'anders'),\n",
    "                    ('maria', 'manuel'),\n",
    "                    ('maria', 'manuela'),\n",
    "                    ('juan','julia'),\n",
    "                    ('margaret','violet'),\n",
    "                    ('antonio','emilio'),\n",
    "                    ('edward','edwin'),\n",
    "                    ('samuel','smith'),\n",
    "                    ('martin','augustin'),\n",
    "                    ('eva','evan'),\n",
    "                    ('dejesus','de'),\n",
    "                    ('dejesus','dean'),\n",
    "                    ('eliza','luiza'),\n",
    "                    ('frank','mark'),\n",
    "                    ('benjamin','benita'),\n",
    "                    ('andrew','matthew'),\n",
    "                    ('andrew','mathew'),\n",
    "                    ('guadalupe','guy'),\n",
    "                    ('jeanne','susanne'),\n",
    "                    ('delacruz','delaconcepcion'),\n",
    "                    ('rebecca','veronica'),\n",
    "                    ('rebecca','francesca'),\n",
    "                    ('karl','karen'),\n",
    "                    ('karl','karin'),\n",
    "                    ('adam','ada'),\n",
    "                    ('adam','addie'),\n",
    "                    ('bertha','bruce'),\n",
    "                    ('edith','edmond'),\n",
    "                    ('mathias','elias'),\n",
    "                    ('anton','anta'),\n",
    "                    ('ethel','effie'),\n",
    "                    ('delcarmen','oscar'),\n",
    "                    ('santiago','santos'),\n",
    "                    ('vicente','clemente'),\n",
    "                    ('ysabel','ysidro'),\n",
    "                    ('karen','karolina'),\n",
    "                    ('ralph','christoph'),\n",
    "                    ('raymond','reyes'),\n",
    "                    ('maren','christen'),\n",
    "                    ('christoph','jph'),\n",
    "                    ('erzsebet','jozsef'),\n",
    "                    ('carlos','marcos'),\n",
    "                    ('ada','adamus'),\n",
    "                    ('delaluz','dela'),\n",
    "                    ('jennie','jemima'),\n",
    "                    ('lorenzo','vincenzo'),\n",
    "                    ('stina','stella'),\n",
    "                    ('pearl','per'),\n",
    "                    ('pearl','pehr'),\n",
    "                    ('oscar','encarnacion'),\n",
    "                    ('veronica','francesca'),\n",
    "                    ('sebastiana','victoriana'),\n",
    "                    ('elias','matias'),\n",
    "                    ('myrtle','estelle'),\n",
    "                    ('bernardo','leonardo'),\n",
    "                    ('amy','amos'),\n",
    "                    ('leslie','lester'),\n",
    "                    ('rosario','hilario'),\n",
    "                    ('karin','karolina'),\n",
    "                    ('nora','norma'),\n",
    "                    ('michaela','mc'),\n",
    "                    ('christiana','luciana'),\n",
    "                    ('chen','chester'),\n",
    "                    ('angelina','augustina'),\n",
    "                    ('sam','smith'),\n",
    "                    ('soledad','solomon'),\n",
    "                    ('mari','jacobi'),\n",
    "                    ('mari','eli'),\n",
    "                    ('mari','josephi'),\n",
    "                    ('mari','li'),\n",
    "                    ('delacrus','delaconcepcion'),\n",
    "                    ('etta','etienne'),\n",
    "                    ('imre','ines'),\n",
    "                    ('florentina','valentina'),\n",
    "                    ('jacobi','josephi'),\n",
    "                    ('joanne','susanne'),\n",
    "                    ('bernardino','florentino'),\n",
    "                    ('josefina','rufina'),\n",
    "                    ('eli','josephi'),\n",
    "                    ('dean','delia'),\n",
    "                    ('emilio','mario'),\n",
    "                    ('jenny','jemima'),\n",
    "                    ('paulino','antonino'),\n",
    "                    # 13 Oct 2023\n",
    "                    ('anne','anders'),\n",
    "                    ('ann','amy'),\n",
    "                    ('dejesus','de'),\n",
    "                    ('dejesus','dedios'),\n",
    "                    ('anders','an'),\n",
    "                    ('ana','anastacia'),\n",
    "                    ('de','dedios'),\n",
    "                    ('mae','mette'),\n",
    "                    ('betty','bell'),\n",
    "                    ('jesus','julius'),\n",
    "                    ('joao','joel'),\n",
    "                    ('clarence','claire'),\n",
    "                    ('martina','marta'),\n",
    "                    ('roy','ray'),\n",
    "                    ('pearl','per'),\n",
    "                    ('veronica','domenica'),\n",
    "                    ('elias','elisa'),\n",
    "                    ('lidia','li'),\n",
    "                    ('sven','sue'),\n",
    "                    ('bernardino','paulino'),\n",
    "                    ('bernardino','antonino'),\n",
    "                    ('eli','elin'),\n",
    "                    ('emilio','mario'),\n",
    "                    ('antal','an'),\n",
    "                    ('anta','an'),\n",
    "                    ('paulino','antonino'),\n",
    "                   ]\n",
    "push_apart_copies = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9eb430d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "for anchor, neg in push_apart_pairs:\n",
    "    anchor_tokens = tokenize(anchor)\n",
    "    neg_tokens = tokenize(neg)\n",
    "    for _ in range(push_apart_copies):\n",
    "        all_data.append({\n",
    "            'anchor': torch.tensor(anchor_tokens),\n",
    "            'pos': torch.tensor(anchor_tokens),\n",
    "            'neg': torch.tensor(neg_tokens),\n",
    "            'target': torch.tensor(1.0, dtype=torch.float)\n",
    "        })"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99431d39",
   "metadata": {},
   "source": [
    "## Analyze training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5c83ad9",
   "metadata": {},
   "outputs": [],
   "source": [
    "token_id2text = {}\n",
    "for text, id_ in tokenizer_vocab.items():\n",
    "    token_id2text[id_] = text\n",
    "\n",
    "counter = Counter()\n",
    "for row in tqdm(all_data):\n",
    "    for key in ['anchor', 'pos', 'neg']:\n",
    "        for token in row[key].tolist():\n",
    "            if token == 1:\n",
    "                break\n",
    "            counter[token] += 1\n",
    "for ix, (token, cnt) in enumerate(counter.most_common()):\n",
    "    print(ix, token, token_id2text[token], cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da51621c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize('jewel')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d3c6b49",
   "metadata": {},
   "source": [
    "### Find names that contain under-represented tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1905c8ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "under_represented_token_ids = set([id_ for id_ in tokenizer_vocab.values() if counter[id_] < under_represented_threshold])\n",
    "print(len(under_represented_token_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a60b125",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_names = set()\n",
    "for tup in tqdm(triplets_df.itertuples()):\n",
    "    anchor = tup.anchor\n",
    "    pos = tup.positive\n",
    "    neg = tup.negative\n",
    "    all_names.add(anchor)\n",
    "    all_names.add(pos)\n",
    "    all_names.add(neg)\n",
    "all_names.update(common_names)\n",
    "len(all_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4e66b77",
   "metadata": {},
   "outputs": [],
   "source": [
    "under_represented_names = set()\n",
    "for name in all_names:\n",
    "    found_token = False\n",
    "    for token in tokenize(name):\n",
    "        if token == 1:\n",
    "            break\n",
    "        if token in under_represented_token_ids:\n",
    "            found_token = True\n",
    "            break\n",
    "    if found_token:\n",
    "        under_represented_names.add(name)\n",
    "print(len(under_represented_names))\n",
    "for name in under_represented_names:\n",
    "    token_counts = []\n",
    "    for token in tokenize(name):\n",
    "        if token == 1:\n",
    "            break\n",
    "        token_counts.append((token, token_id2text[token], counter[token]))\n",
    "    print(name, token_counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dfe8e3c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# add any under-represented tokens that don't start with ## as under-represented names\n",
    "for token, id_ in tokenizer_vocab.items():\n",
    "    if counter[id_] >= under_represented_threshold:\n",
    "        continue\n",
    "    if '[' in token or '#' in token:\n",
    "        continue\n",
    "    print(token)\n",
    "    under_represented_names.add(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3ba9b03",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(under_represented_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58f0b3de",
   "metadata": {},
   "source": [
    "### Add names that contain under-represented tokens to all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce437155",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = 0\n",
    "for pos in tqdm(under_represented_names):\n",
    "    pos_tokens = tokenize(pos)\n",
    "    for neg in common_names[:under_represented_threshold]:\n",
    "        if pos == neg or (pos, neg) in common_non_negatives:\n",
    "            continue\n",
    "        pos_neg = f\"{pos},{neg}\"\n",
    "        if pos_neg in seen_anchor_pos:\n",
    "            continue\n",
    "        neg_pos = f\"{neg},{pos}\"\n",
    "        if neg_pos in seen_anchor_pos:\n",
    "            continue\n",
    "        neg_tokens = tokenize(neg)\n",
    "        all_data.append({\n",
    "            'anchor': torch.tensor(pos_tokens),\n",
    "            'pos': torch.tensor(pos_tokens),\n",
    "            'neg': torch.tensor(neg_tokens),\n",
    "            'target': torch.tensor(1.0, dtype=torch.float)\n",
    "        })\n",
    "        cnt += 1\n",
    "print(cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55f127ae",
   "metadata": {},
   "source": [
    "## Split training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f021d010",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data, val_data = train_test_split(all_data, test_size=0.10, random_seed=42)\n",
    "print(len(train_data), len(val_data))        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2dbb696",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5debcca1",
   "metadata": {},
   "source": [
    "## Re-Analyze training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec2e9ae3",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = Counter()\n",
    "for row in tqdm(train_data):\n",
    "    for key in ['anchor', 'pos', 'neg']:\n",
    "        for token in row[key].tolist():\n",
    "            if token == 1:\n",
    "                break\n",
    "            counter[token] += 1\n",
    "for ix, (token, cnt) in enumerate(counter.most_common()):\n",
    "    print(ix, token, token_id2text[token], cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "205c4769",
   "metadata": {},
   "outputs": [],
   "source": [
    "for token, id_ in tokenizer_vocab.items():\n",
    "    if counter[id_] == 0:\n",
    "        print(id_, token, counter[id_])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a766a635",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenize('zetty')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dba77e31",
   "metadata": {},
   "source": [
    "## Train model and Review predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49b444fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pretrained_embeddings(tokenizer_vocab, embedding_dim):\n",
    "    embeddings = [[0.0 for _ in range(embedding_dim)] for _ in range(len(tokenizer_vocab))]\n",
    "    dims_per_char = embedding_dim // 26\n",
    "    ones = [1.0] * dims_per_char\n",
    "    for token, id_ in tokenizer_vocab.items():\n",
    "        embedding = embeddings[id_]\n",
    "        if token[0] == '[':\n",
    "            continue\n",
    "        for ch in token.strip('#'):\n",
    "            # set dimensions for ch in embedding to 1.0\n",
    "            start = (ord(ch) - ord('a')) * dims_per_char\n",
    "            embedding[start:start+dims_per_char] = ones\n",
    "        embeddings[id_] = embedding\n",
    "    return torch.FloatTensor(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b50b883",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "# Construct pre-trained embeddings based upon bag-of-characters\n",
    "pretrained_embeddings = None\n",
    "if use_pretrained_embeddings:\n",
    "    pretrained_embeddings = get_pretrained_embeddings(tokenizer_vocab, embedding_dim)\n",
    "    \n",
    "# Create an instance of the bi-encoder model\n",
    "model = BiEncoder(embedding_dim, len(tokenizer_vocab), max_tokens, tokenizer_vocab['[PAD]'], pretrained_embeddings)\n",
    "\n",
    "# Define the optimizer\n",
    "optimizer = optim.AdamW(model.parameters(), lr=learning_rate, amsgrad=use_amsgrad)\n",
    "\n",
    "# Create data loader\n",
    "### NOTE: train on all_data in place of train_data\n",
    "train_loader = DataLoader(all_data, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_data, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "train(model, train_loader, val_loader, loss_fn, optimizer, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09002d90",
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_threshold = 0.5\n",
    "num_error = 0\n",
    "num_low_pos = 0\n",
    "num_incorrect = 0\n",
    "for ix, row in enumerate(triplets_df.head(1000).itertuples()):\n",
    "    anchor = row.anchor\n",
    "    pos = row.positive\n",
    "    neg = row.negative\n",
    "    anchor_tokens = tokenize(anchor)\n",
    "    pos_tokens = tokenize(pos)\n",
    "    neg_tokens = tokenize(neg)\n",
    "    pos_predict = model.predict(anchor_tokens, pos_tokens)\n",
    "    neg_predict = model.predict(anchor_tokens, neg_tokens)\n",
    "    if pos_predict < pos_threshold:\n",
    "        num_low_pos += 1\n",
    "    # difference between predicted and true score ratios must be small\n",
    "    correct = abs(neg_predict / pos_predict - row.negative_score / row.positive_score) < 0.67\n",
    "    if not correct:\n",
    "        num_incorrect += 1\n",
    "    if pos_predict < pos_threshold or not correct:\n",
    "        num_error += 1\n",
    "        print(num_error, anchor, pos, pos_predict, row.positive_score, '' if correct else 'ERROR')\n",
    "        print('  ', anchor, neg, neg_predict, row.negative_score)\n",
    "print(num_error, num_low_pos, num_incorrect)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7cb2cf9",
   "metadata": {},
   "source": [
    "### count common name pairs that score above thresholds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf0c628c",
   "metadata": {},
   "outputs": [],
   "source": [
    "cnt = 0\n",
    "bad_cnt = 0\n",
    "n_names = 100\n",
    "pos_threshold = 0.5\n",
    "for ix, pos in enumerate(common_names[:n_names]):\n",
    "    pos_tokens = tokenize(pos)\n",
    "    for neg in common_names[ix+1:n_names]:\n",
    "        if (pos, neg) in common_non_negatives:\n",
    "            continue\n",
    "        neg_tokens = tokenize(neg)\n",
    "        sim = model.predict(pos_tokens, neg_tokens)\n",
    "        if sim > pos_threshold:\n",
    "            bad = sim >= 0.6\n",
    "            print(pos, neg, sim, '***' if bad else '', pos_tokens, neg_tokens)\n",
    "            cnt += 1\n",
    "            if bad:\n",
    "                bad_cnt += 1\n",
    "print(cnt, bad_cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e9e72dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "cnt = 0\n",
    "bad_cnt = 0\n",
    "n_names = 1000\n",
    "pos_threshold = 0.6\n",
    "bad_threshold = 0.7\n",
    "common_negative_scores = []\n",
    "for ix, pos in enumerate(common_names[:n_names]):\n",
    "    pos_tokens = tokenize(pos)\n",
    "    for neg in common_names[ix+1:n_names]:\n",
    "        if (pos, neg) in common_non_negatives:\n",
    "            continue\n",
    "        neg_tokens = tokenize(neg)\n",
    "        sim = model.predict(pos_tokens, neg_tokens)\n",
    "        common_negative_scores.append(sim)\n",
    "        if sim > pos_threshold:\n",
    "            print(pos, neg, sim, '***' if sim > bad_threshold else '')\n",
    "            cnt += 1\n",
    "            bad_cnt += 1 if sim > bad_threshold else 0\n",
    "print(cnt, bad_cnt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0df5a95",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efe649b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model, model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11cff598",
   "metadata": {},
   "source": [
    "## Compare negative to positive scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ebd811f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_threshold = 0.3\n",
    "upper_threshold = 0.5\n",
    "non_negative_scores = []\n",
    "cnt = 0\n",
    "for pair in tqdm(common_non_negatives):\n",
    "    name1, name2 = pair.split(':')\n",
    "    sim = model.predict(tokenize(name1), tokenize(name2))\n",
    "    non_negative_scores.append(sim)\n",
    "    if lower_threshold < sim < upper_threshold:\n",
    "        if cnt < 100:\n",
    "            print(name1, name2, sim)\n",
    "        cnt += 1\n",
    "print(cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91b60349",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(non_negative_scores), len(common_negative_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4de91c8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "below_threshold = [score for score in non_negative_scores if score < 0.3]\n",
    "len(below_threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96f71503",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(non_negative_scores, bins=30, alpha=0.5, label=\"Current Clusters/Variants/Nicknames\", color='green')\n",
    "plt.hist(common_negative_scores, bins=30, alpha=0.5, label=\"Common negatives\", color='red')\n",
    "plt.title('Overlapping Histogram')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.legend(loc='upper right')\n",
    "\n",
    "# Show the plot\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78ddfc8b",
   "metadata": {},
   "source": [
    "### Compare results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cbc6219",
   "metadata": {},
   "outputs": [],
   "source": [
    "lower_threshold = 0.1\n",
    "non_negative_scores = []\n",
    "cnt = 0\n",
    "for pair in tqdm(common_non_negatives):\n",
    "    name1, name2 = pair.split(':')\n",
    "    sim = model.predict(tokenize(name1), tokenize(name2))\n",
    "    non_negative_scores.append(sim)\n",
    "    if sim < lower_threshold:\n",
    "        print(name1, name2, sim)\n",
    "        cnt += 1\n",
    "print(cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "322c50bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(non_negative_scores), len(common_negative_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "982797b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(non_negative_scores, bins=30, alpha=0.5, label=\"Current Clusters\", color='green')\n",
    "plt.hist(common_negative_scores, bins=30, alpha=0.5, label=\"Common negatives\", color='red')\n",
    "plt.title('Overlapping Histogram')\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.legend(loc='upper right')\n",
    "\n",
    "# Show the plot\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd31da69",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nama",
   "language": "python",
   "name": "nama"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}