notebooks/360_augment_train_for_swivel.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-28T18:55:35.399248Z",
     "start_time": "2024-10-28T18:55:35.371233Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Augment the training dataset with other matching pairs based upon names having the same code or levenshtein similarity\n",
    "\n",
    "## DEPRECATED\n",
    "S\n",
    "We found that even though we had millions of pairs of names, it wasn't nearly enough.\n",
    "Many good name pairs were not represented in the data.\n",
    "So we augmented the data with additional pairs, and gave them frequencies based upon how many coding algorithms assigned them the same code, and how similar they were in levenshtein.\n",
    "This turns out to be key to the success of the algorithm.\n",
    "Otherwise these pairs are represented as negatives, which is a significant issue.\n",
    "\n",
    "This notebook takes about 24 hours to run for given names, and probably about 72 hours for surnames.\n",
    "\n",
    "TODO Currently, we don't add scores for having the same soundex code, high levenshtein similarity, etc. to the name associated with itself.\n",
    "But maybe we should?\n",
    "\n",
    "TODO Currently we don't add scores for names in the same bucket. But we almost certainly should! What score should we add?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-28T18:55:37.446596Z",
     "start_time": "2024-10-28T18:55:35.402249Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from collections import namedtuple\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "from nama.data.augment import generate_augmented_name_pairs, augment_dataset\n",
    "from nama.data.filesystem import download_file_from_s3, save_file\n",
    "from nama.data.utils import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-28T18:55:37.503169Z",
     "start_time": "2024-10-28T18:55:37.471174Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Config\n",
    "\n",
    "# run on 1536Gb machine high-memory instance\n",
    "\n",
    "# TODO do for given and surname\n",
    "given_surname = \"given\"\n",
    "# given_surname = \"surname\"\n",
    "\n",
    "threshold = 3\n",
    "augment_discount = 1\n",
    "original_data_multiplier = 8\n",
    "batch_size = 5000\n",
    "n_jobs=64\n",
    "Config = namedtuple(\"Config\", \"in_path threshold augment_discount original_data_multiplier augments_path combined_path\")\n",
    "config = Config(\n",
    "    in_path=f\"s3://fs-nama-data/2024/familysearch-names/processed/tree-hr-{given_surname}-train.csv.gz\",\n",
    "    threshold=threshold,\n",
    "    augment_discount=augment_discount,\n",
    "    original_data_multiplier=original_data_multiplier,\n",
    "    augments_path=f\"s3://fs-nama-data/2024/familysearch-names/processed/tree-hr-{given_surname}-train-augments.csv.gz\",\n",
    "    combined_path=f\"s3://fs-nama-data/2024/familysearch-names/processed/tree-hr-{given_surname}-train-augmented.csv.gz\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-28T18:56:07.985165Z",
     "start_time": "2024-10-28T18:55:37.505463Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# load data\n",
    "in_path = download_file_from_s3(config.in_path) if config.in_path.startswith(\"s3://\") else config.in_path\n",
    "input_names, record_name_frequencies, candidate_names = load_dataset(in_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-28T18:56:08.008562Z",
     "start_time": "2024-10-28T18:56:07.986559Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# sample to make things go faster\n",
    "# sample_size = 10000\n",
    "# batch_size = 1000\n",
    "# input_names, record_name_frequencies, candidate_names = select_frequent_k(input_names, record_name_frequencies, candidate_names, k=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-28T18:56:08.296770Z",
     "start_time": "2024-10-28T18:56:08.009859Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "836285\n"
     ]
    }
   ],
   "source": [
    "all_names = set(input_names).union(set(candidate_names))\n",
    "print(len(all_names))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2024-10-28T18:56:08.298217Z"
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|                                                                                         | 0/168 [2:01:50<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "# get augmented name-pair data\n",
    "name_pairs_df = generate_augmented_name_pairs(\n",
    "    input_names, \n",
    "    record_name_frequencies, \n",
    "    candidate_names, \n",
    "    threshold=config.threshold, \n",
    "    discount=config.augment_discount, \n",
    "    batch_size=batch_size,\n",
    "    n_jobs=n_jobs,\n",
    ")"
   ]
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": [
    "# rename columns\n",
    "name_pairs_df = name_pairs_df.rename(\n",
    "    columns={'name1': 'tree_name', \n",
    "             'name2': 'record_name',\n",
    "             'co_occurrence': 'frequency',\n",
    "            }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(len(name_pairs_df))\n",
    "name_pairs_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# free memory\n",
    "input_names = record_name_frequencies = candidate_names = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# write augments\n",
    "save_file(config.augments_path,\n",
    "          lambda local_out_path : name_pairs_df.to_csv(local_out_path, index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# read raw input dataset\n",
    "in_path = download_file_from_s3(config.in_path) if config.in_path.startswith(\"s3://\") else config.in_path\n",
    "raw_input_df = pd.read_csv(in_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(len(raw_input_df))\n",
    "print(len(set(raw_input_df[\"tree_name\"])))\n",
    "print(len(set(raw_input_df[\"record_name\"])))\n",
    "print(len(set(raw_input_df[\"tree_name\"]).union(set(raw_input_df[\"record_name\"]))))\n",
    "raw_input_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# augment raw input dataset with augmented name pairs\n",
    "augmented_df = augment_dataset(raw_input_df, name_pairs_df, multiplier=config.original_data_multiplier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# free memory\n",
    "raw_input_df = name_pairs_df = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(len(augmented_df))\n",
    "print(len(set(augmented_df[\"tree_name\"])))\n",
    "print(len(set(augmented_df[\"record_name\"])))\n",
    "print(len(set(augmented_df[\"tree_name\"]).union(set(augmented_df[\"record_name\"]))))\n",
    "augmented_df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# write merged dataset\n",
    "save_file(config.combined_path,\n",
    "          lambda local_out_path : augmented_df.to_csv(local_out_path, index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Analyze results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "print(len(augmented_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "print(augmented_df[\"frequency\"].max())"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "print(len(augmented_df[augmented_df[\"frequency\"] == 2]))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "print(len(augmented_df[augmented_df[\"frequency\"] == 3]))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "print(len(augmented_df[augmented_df[\"frequency\"] == 4]))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "print(len(augmented_df[augmented_df[\"frequency\"] > 1000]))"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "augmented_df[augmented_df[\"frequency\"] <= 100]['frequency'].plot(kind=\"hist\", bins=100, figsize=(20, 6), logy=True)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "augmented_df[augmented_df[\"frequency\"] == 4].sample(100)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "augmented_df[augmented_df[\"frequency\"] == 3].sample(100)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": "augmented_df[augmented_df[\"frequency\"] == 2].sample(100)"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nama",
   "language": "python",
   "name": "nama"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}