notebooks/102_baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8837d91e",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8622d4a1",
   "metadata": {},
   "source": [
    "# Compute Frequency-based Precision & Recall for various codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f94d97f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "\n",
    "import boto3\n",
    "import jellyfish\n",
    "import pandas as pd\n",
    "from py4j.java_gateway import JavaGateway\n",
    "from tqdm.autonotebook import tqdm\n",
    "\n",
    "from src.eval.freq_metrics import calc_avg_precision_recall\n",
    "from src.models.cluster import read_cluster_scores\n",
    "from src.models.utils import remove_padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c63a44a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "given_surname = \"surname\"\n",
    "n_to_cluster = 250000\n",
    "cluster_threshold = 0.15\n",
    "# cluster_scores_threshold = 0.95\n",
    "# cluster_scores_limit = 20\n",
    "\n",
    "train_path = f\"s3://familysearch-names/processed/tree-hr-{given_surname}-train-v2.csv.gz\"\n",
    "test_path = f\"s3://familysearch-names/processed/tree-hr-{given_surname}-test-v2.csv.gz\"\n",
    "query_path = f\"s3://familysearch-names/processed/query-names-{given_surname}-v2.csv.gz\"\n",
    "pref_path = f\"s3://familysearch-names/processed/tree-preferred-{given_surname}-aggr.csv.gz\"\n",
    "\n",
    "# upper = 0.95\n",
    "# lower = 0.6\n",
    "# m = 0.059133018459459175\n",
    "# b = 0.10088609567188966\n",
    "# cluster_path = f\"../data/models/fs-{given_surname}-cluster-greedy-{n_to_cluster}-upper_{upper}-lower_{lower}-m_{m}-b_{b}.csv\"\n",
    "cluster_path = f\"s3://nama-data/data/models/fs-{given_surname}-cluster-names-{n_to_cluster}-{cluster_threshold}.csv\"\n",
    "# cluster_path=f\"s3://nama-data/data/models/fs-{given_surname}-cluster-names.csv\"\n",
    "\n",
    "# vocab_size = 610000 if given_surname == \"given\" else 2100000\n",
    "# embed_dim = 100\n",
    "# cluster_scores_path=f\"s3://nama-data/data/processed/fs-{given_surname}-cluster-scores-{vocab_size}-{embed_dim}-precomputed.jsonl.gz\"\n",
    "\n",
    "nickname_bucket = \"familysearch-names\"\n",
    "nickname_path = \"processed/givenname_nicknames.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1e362a2",
   "metadata": {},
   "source": [
    "## Read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb546c0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(train_path, na_filter=False)\n",
    "print(train_df.shape)\n",
    "train_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89c537b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = pd.read_csv(test_path, na_filter=False)\n",
    "print(test_df.shape)\n",
    "test_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97e2110c",
   "metadata": {},
   "outputs": [],
   "source": [
    "query_names = pd.read_csv(query_path, na_filter=False)[\"name\"].tolist()\n",
    "print(len(query_names))\n",
    "query_names[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9abd093",
   "metadata": {},
   "outputs": [],
   "source": [
    "pref_df = pd.read_csv(pref_path, na_filter=False)\n",
    "print(pref_df.shape)\n",
    "pref_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2912f63d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the most-common names\n",
    "common_names = set(pref_df.nlargest(n_to_cluster, 'frequency')['name'].tolist())\n",
    "len(common_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52d989fe",
   "metadata": {},
   "source": [
    "### Read Nama Cluster Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e8e0fbe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_cluster_scores = read_cluster_scores(cluster_scores_path)\n",
    "# print(len(all_cluster_scores))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6bc0a8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cluster_scores = {}\n",
    "# clustered_names = set(remove_padding(name) for name in all_cluster_scores.keys() \\\n",
    "#                       if remove_padding(name) in common_names)\n",
    "# total_cluster_scores = 0\n",
    "# for name, all_scores in all_cluster_scores.items():\n",
    "#     name = remove_padding(name)\n",
    "#     if name not in clustered_names:\n",
    "#         continue\n",
    "#     scores = [(name, score) for name, score in all_scores if name in clustered_names and score >= cluster_scores_threshold]\n",
    "#     if len(scores) == 0:\n",
    "#         continue\n",
    "#     cluster_scores[name] = scores\n",
    "#     total_cluster_scores += len(scores)\n",
    "# print(len(cluster_scores))\n",
    "# print(total_cluster_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce6940f1",
   "metadata": {},
   "source": [
    "### Read Nama Clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26b740e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(cluster_path, na_filter=False)\n",
    "print(len(df))\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "755d21af",
   "metadata": {},
   "outputs": [],
   "source": [
    "nama_name2clusters = defaultdict(set)\n",
    "clusters = set()\n",
    "for name, cluster in zip(df['name'], df['cluster']):\n",
    "    nama_name2clusters[name].add(cluster)\n",
    "    clusters.add(cluster)\n",
    "    \n",
    "print(len(nama_name2clusters))\n",
    "print(len(clusters))\n",
    "nama_name2clusters['ronald']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c99bdc2f",
   "metadata": {},
   "source": [
    "### Read Nicknames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe78c4be",
   "metadata": {},
   "outputs": [],
   "source": [
    "s3 = boto3.resource('s3')\n",
    "\n",
    "nama_nicknames = defaultdict(set)\n",
    "if given_surname == \"given\":\n",
    "    obj = s3.Object(nickname_bucket, nickname_path)\n",
    "    contents = obj.get()['Body'].read().decode('utf-8')\n",
    "    for ix, line in enumerate(contents.split('\\n')):\n",
    "        line = line.strip()\n",
    "        names = line.split(',')\n",
    "        headname = names[0]\n",
    "        for name in names[1:]:\n",
    "            if name != headname:\n",
    "                nama_nicknames[name].add(headname)\n",
    "print(len(nama_nicknames))\n",
    "nama_nicknames['zachery']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37c1ec13",
   "metadata": {},
   "source": [
    "## Compute codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52724407",
   "metadata": {},
   "outputs": [],
   "source": [
    "gateway = JavaGateway()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "369f7671",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_identity(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in names:\n",
    "        code = name\n",
    "        name2codes[name].add(code)\n",
    "        code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "def get_nysiis(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in names:\n",
    "        code = jellyfish.nysiis(name)\n",
    "        name2codes[name].add(code)\n",
    "        code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "def get_soundex(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in names:\n",
    "        code = jellyfish.soundex(name)\n",
    "        name2codes[name].add(code)\n",
    "        code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "def get_fs(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in tqdm(names):\n",
    "        result = gateway.getClusters(name, given_surname == \"surname\")\n",
    "        for code in result.split(','):\n",
    "            name2codes[name].add(code)\n",
    "            code2names[code].add(name)\n",
    "    return name2codes, code2names   \n",
    "\n",
    "def get_fs_soundex(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in tqdm(names):\n",
    "        result = gateway.getClustersUsingSoundex(name, given_surname == \"surname\")\n",
    "        codes = result.split(',')\n",
    "        for code in codes:\n",
    "            name2codes[name].add(code)\n",
    "            if len(codes) > 1 and code.startswith(\"_\"):\n",
    "                # if this name is associated with multiple clusters, \n",
    "                # and this cluster is a soundex code, \n",
    "                # then don't associate this name with this soundex cluster,\n",
    "                # because it is already associated (indexed) under another cluster\n",
    "                continue\n",
    "            code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "def get_fs_nysiis(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in tqdm(names):\n",
    "        result = gateway.getClustersUsingNysiis(name, given_surname == \"surname\")\n",
    "        codes = result.split(',')\n",
    "        for code in codes:\n",
    "            name2codes[name].add(code)\n",
    "            if len(codes) > 1 and code.startswith(\"_\"):\n",
    "                # if this name is associated with multiple clusters, \n",
    "                # and this cluster is a nysiis code, \n",
    "                # then don't associate this name with this nysiis cluster,\n",
    "                # because it is already associated (indexed) under another cluster\n",
    "                continue\n",
    "            code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "def _get_nama_standards(name):\n",
    "    standards = set()\n",
    "    lookups = set([name])\n",
    "    if given_surname == \"given\" and name in nama_nicknames:\n",
    "        lookups.update(nama_nicknames[name])\n",
    "    for lookup in lookups:\n",
    "        if lookup in nama_name2clusters:\n",
    "            standards.update(nama_name2clusters[lookup])\n",
    "    return standards\n",
    "\n",
    "def get_nama_soundex(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in tqdm(names):\n",
    "        codes = _get_nama_standards(name)\n",
    "        for code in codes:\n",
    "            name2codes[name].add(code)\n",
    "            code2names[code].add(name)\n",
    "        code = jellyfish.soundex(name)\n",
    "        # always query soundex code\n",
    "        # name2codes simulates query: given a name, what codes to lookup\n",
    "        name2codes[name].add(code)\n",
    "        # add name to soundex bucket only if it isn't in another bucket\n",
    "        # code2names simulates an index lookup: given code, what names are indexed with that code\n",
    "        if len(codes) == 0:\n",
    "            code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "def get_nama_nysiis(names):\n",
    "    name2codes = defaultdict(set)\n",
    "    code2names = defaultdict(set)\n",
    "    for name in tqdm(names):\n",
    "        codes = _get_nama_standards(name)\n",
    "        for code in codes:\n",
    "            name2codes[name].add(code)\n",
    "            code2names[code].add(name)\n",
    "        code = jellyfish.nysiis(name)\n",
    "        # always query nysiis code\n",
    "        # name2codes simulates query: given a name, what codes to lookup\n",
    "        name2codes[name].add(code)\n",
    "        # add name to nysiis bucket only if it isn't in another bucket\n",
    "        # code2names simulates an index lookup: given code, what names are indexed with that code\n",
    "        if len(codes) == 0:\n",
    "            code2names[code].add(name)\n",
    "    return name2codes, code2names\n",
    "\n",
    "# def _get_multi_nama_standards(name, limit):\n",
    "#     standards = set()\n",
    "#     lookups = set([name])\n",
    "#     if given_surname == \"given\" and name in nama_nicknames:\n",
    "#         lookups.update(nama_nicknames[name])\n",
    "#     for lookup in lookups:\n",
    "#         if lookup in cluster_scores:\n",
    "#             clusters = [cluster for cluster, _ in cluster_scores[lookup]]\n",
    "#             lim = limit if lookup == name else 1\n",
    "#             standards.update(clusters[0:limit])\n",
    "#     return standards\n",
    "\n",
    "# def get_multi_nama_nysiis(names):\n",
    "#     name2codes = defaultdict(set)\n",
    "#     code2names = defaultdict(set)\n",
    "#     for name in tqdm(names):\n",
    "#         nysiis_code = jellyfish.nysiis(name)\n",
    "#         # get index codes\n",
    "#         index_codes = _get_multi_nama_standards(name, 1)\n",
    "#         if len(index_codes) == 0:\n",
    "#             code2names[nysiis_code].add(name)\n",
    "#         else:\n",
    "#             for code in index_codes:\n",
    "#                 code2names[code].add(name)\n",
    "#         # get query codes\n",
    "#         query_codes = _get_multi_nama_standards(name, cluster_scores_limit)\n",
    "#         for code in query_codes:\n",
    "#             name2codes[name].add(code)\n",
    "#         name2codes[name].add(nysiis_code)\n",
    "#     return name2codes, code2names\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34a9abc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_nama_nysiis(['ann', 'anna', 'anne'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6775f19d",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_fs_nysiis(['ann', 'anna', 'anne'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c480f435",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_nama_soundex(['john', 'dallan', 'johnny', 'elizabeth', 'quass'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8a29810",
   "metadata": {},
   "source": [
    "# Compute Precision and Recall on Train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0af34be9",
   "metadata": {},
   "source": [
    "## Identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd34e706",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_identity(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19e59ddd",
   "metadata": {},
   "source": [
    "## NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f45be30b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2252f507",
   "metadata": {},
   "source": [
    "## Soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c292fce8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_soundex(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ff696ca",
   "metadata": {},
   "source": [
    "## Nama-Soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "560f5628",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_nama_soundex(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "print(\"total names\", len(name2codes))\n",
    "print(\"total index entries\", sum(len(names) for names in code2names.values()))\n",
    "print(\"total queries\", len(query_names))\n",
    "print(\"total lookups\", sum(len(name2codes[query]) for query in query_names))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "071494c4",
   "metadata": {},
   "source": [
    "## Nama-NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "743444a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_nama_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "print(\"total names\", len(name2codes))\n",
    "print(\"total index entries\", sum(len(names) for names in code2names.values()))\n",
    "print(\"total codes\", len(code2names))\n",
    "print(\"total queries\", len(query_names))\n",
    "print(\"total lookups\", sum(len(name2codes[query]) for query in query_names))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3cc4ef2d",
   "metadata": {},
   "source": [
    "## Multi-Nama-NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d6fe0f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df = train_df\n",
    "# name2codes, code2names = get_multi_nama_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "# print(\"total names\", len(name2codes))\n",
    "# print(\"total index entries\", sum(len(names) for names in code2names.values()))\n",
    "# print(\"total queries\", len(query_names))\n",
    "# print(\"total lookups\", sum(len(name2codes[query]) for query in query_names))\n",
    "# precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "# print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a61470f",
   "metadata": {},
   "source": [
    "## FS-Soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc5cda62",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_fs_soundex(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5828bdb2",
   "metadata": {},
   "source": [
    "## FS-NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c77b5299",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_fs_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "print(\"total names\", len(name2codes))\n",
    "print(\"total index entries\", sum(len(names) for names in code2names.values()))\n",
    "print(\"total codes\", len(code2names))\n",
    "print(\"total queries\", len(query_names))\n",
    "print(\"total lookups\", sum(len(name2codes[query]) for query in query_names))\n",
    "# precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "# print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "addbd5a9",
   "metadata": {},
   "source": [
    "## FS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1c9efc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = train_df\n",
    "name2codes, code2names = get_fs(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c525f32",
   "metadata": {},
   "source": [
    "# Compute Precision and Recall on Train and Test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a316e54",
   "metadata": {},
   "source": [
    "## Identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58e292c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_identity(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d002996",
   "metadata": {},
   "source": [
    "## NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7463c63",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3be55e0e",
   "metadata": {},
   "source": [
    "## Soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bde5656",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_soundex(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85dbf9f6",
   "metadata": {},
   "source": [
    "## Nama-Soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bc9b403",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_nama_soundex(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "828bfbc6",
   "metadata": {},
   "source": [
    "## Nama-NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9147674",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_nama_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f02740c5",
   "metadata": {},
   "source": [
    "## FS-Soundex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4c687c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_fs_soundex(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "388b16dd",
   "metadata": {},
   "source": [
    "## FS-NYSIIS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd735011",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_fs_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "73f5a957",
   "metadata": {},
   "source": [
    "## FS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66b82141",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.concat([train_df, test_df], ignore_index=True)\n",
    "name2codes, code2names = get_fs(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "precision, recall, f1, f2 = calc_avg_precision_recall(query_names, name2codes, code2names, df)\n",
    "print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f1f1fb19",
   "metadata": {},
   "source": [
    "# Remove"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "733a0948",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_to_cluster_values = [50000]\n",
    "upper_values = [0.9]\n",
    "lower_values = [0.6]\n",
    "high_freq_ix_values = [100]  # 100, 500, 2000\n",
    "low_freq_ix_values = [25000, 100000]  # 10000, 25000, 100000\n",
    "\n",
    "query_names_sample = query_names[:2000]\n",
    "\n",
    "for n_to_cluster in n_to_cluster_values:\n",
    "    for high_freq_ix in high_freq_ix_values:\n",
    "        for low_freq_ix in low_freq_ix_values:\n",
    "            for upper in upper_values:\n",
    "                for lower in lower_values:\n",
    "                    print(n_to_cluster, high_freq_ix, low_freq_ix, upper, lower)\n",
    "                    path = f\"../data/models/fs-{given_surname}-cluster-greedy-{n_to_cluster}-upper_{upper}-lower_{lower}-high_freq_ix_{high_freq_ix}-low_freq_ix_{low_freq_ix}.csv\"\n",
    "                    df = pd.read_csv(path, na_filter=False)\n",
    "                    nama_name2clusters = defaultdict(set)\n",
    "                    clusters = set()\n",
    "                    for name, cluster in zip(df['name'], df['cluster']):\n",
    "                        nama_name2clusters[name].add(cluster)\n",
    "                        clusters.add(cluster)\n",
    "                    df = train_df\n",
    "                    name2codes, code2names = get_nama_nysiis(set(df[\"tree_name\"]) | set(df[\"record_name\"]))\n",
    "                    print(\"total names\", len(name2codes))\n",
    "                    print(\"total index entries\", sum(len(names) for names in code2names.values()))\n",
    "                    print(\"total codes\", len(code2names))\n",
    "                    print(\"total queries\", len(query_names_sample))\n",
    "                    print(\"total lookups\", sum(len(name2codes[query]) for query in query_names_sample))\n",
    "                    precision, recall, f1, f2 = calc_avg_precision_recall(query_names_sample, name2codes, code2names, df)\n",
    "                    print(f\"precision={precision}, recall={recall} f1={f1} f2={f2}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "021c90fc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e17a30c9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "396c8928",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f510bc4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nama",
   "language": "python",
   "name": "nama"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}