notebooks/393_compress_cluster_files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "2882991e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-16T18:04:34.474970Z",
     "start_time": "2024-12-16T18:04:34.445382Z"
    }
   },
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "markdown",
   "id": "d671df72",
   "metadata": {},
   "source": [
    "# Compress cluster and super-cluster files\n",
    "\n",
    "Read the existing cluster and super-cluster files, reduce the number of decimals in the centroids and write them in gzip format in order to keep them below 100Mb."
   ]
  },
  {
   "cell_type": "code",
   "id": "3ab5d9f3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-16T18:05:12.226421Z",
     "start_time": "2024-12-16T18:05:12.066013Z"
    }
   },
   "source": [
    "import gzip\n",
    "import json\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from nama.data.filesystem import download_file_from_s3, save_file"
   ],
   "outputs": [],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "id": "1ff83f53",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-16T18:04:36.320252Z",
     "start_time": "2024-12-16T18:04:36.292714Z"
    }
   },
   "source": [
    "# config\n",
    "# TODO run both given and surname\n",
    "given_surname = \"given\"\n",
    "# given_surname = \"surname\"\n",
    "\n",
    "n_decimals = 8\n",
    "\n",
    "linkage = \"average\"\n",
    "similarity_threshold = 0.10 if given_surname == \"given\" else 0.25\n",
    "cluster_freq_normalizer = \"none\"\n",
    "\n",
    "augmented_clusters_path = f\"s3://fs-nama-data/2024/nama-data/data/processed/clusters_{given_surname}-{linkage}-{similarity_threshold}-{cluster_freq_normalizer}-augmented.json\"\n",
    "super_clusters_path = f\"s3://fs-nama-data/2024/nama-data/data/processed/super_clusters_{given_surname}-{linkage}-{similarity_threshold}-{cluster_freq_normalizer}.json\""
   ],
   "outputs": [],
   "execution_count": 3
  },
  {
   "cell_type": "markdown",
   "id": "ad87f02c",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "id": "deaa1fe1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-16T18:06:46.639334Z",
     "start_time": "2024-12-16T18:06:05.158478Z"
    }
   },
   "source": [
    "path = download_file_from_s3(augmented_clusters_path) if augmented_clusters_path.startswith(\"s3://\") else augmented_clusters_path\n",
    "with open(path, 'r') as f:\n",
    "    clusters = json.load(f)  # cluster label -> names, centroid\n",
    "\n",
    "path = download_file_from_s3(super_clusters_path) if super_clusters_path.startswith(\"s3://\") else super_clusters_path\n",
    "with open(path, 'r') as f:\n",
    "    super_clusters = json.load(f)  # cluster label -> names, centroid"
   ],
   "outputs": [],
   "execution_count": 5
  },
  {
   "cell_type": "markdown",
   "id": "713d5ecc-08cd-4825-a025-cf2387e09c12",
   "metadata": {},
   "source": [
    "## Reduce centroid decimal places"
   ]
  },
  {
   "cell_type": "code",
   "id": "f57a1850-d154-493c-a391-2baf01c0d541",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-16T18:06:47.172705Z",
     "start_time": "2024-12-16T18:06:46.641274Z"
    }
   },
   "source": [
    "for cluster in clusters.values():\n",
    "    cluster['centroid'] = np.around(np.array(cluster['centroid']), decimals=n_decimals).tolist()"
   ],
   "outputs": [],
   "execution_count": 6
  },
  {
   "cell_type": "markdown",
   "id": "4d21aadc-fd95-4780-9198-db66511c85be",
   "metadata": {},
   "source": [
    "## Save gzipped"
   ]
  },
  {
   "cell_type": "code",
   "id": "b855c628",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-12-16T18:09:25.607213Z",
     "start_time": "2024-12-16T18:08:42.997133Z"
    }
   },
   "source": [
    "save_file(f\"{augmented_clusters_path}.gz\",\n",
    "          lambda local_out_path : json.dump(clusters, gzip.open(local_out_path, 'wt', encoding='UTF-8')))\n",
    "\n",
    "save_file(f\"{super_clusters_path}.gz\",\n",
    "          lambda local_out_path : json.dump(super_clusters, gzip.open(local_out_path, 'wt', encoding='UTF-8')))"
   ],
   "outputs": [],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5146991",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}