test_interpretation_correlation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# My library\n",
    "from molgraph.graphmodel import *\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.stats\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "from tqdm import tqdm\n",
    "import os\n",
    "from matplotlib.offsetbox import AnchoredText"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SINGLE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model\n",
    "file = 'bbbp'\n",
    "model = 'GAT'\n",
    "schema = 'A'\n",
    "reduced = ['']\n",
    "\n",
    "ts = \"2023-Jan-09-23:29:08\"\n",
    "reduced_list = '_'.join(reduced)\n",
    "log_folder_name = os.path.join(*[file, model+'_'+schema+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "path = './dataset/'+log_folder_name+'/attention0.pickle'\n",
    "with open(path, 'rb') as handle:\n",
    "    attention1 = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model\n",
    "file = 'bbbp'\n",
    "model = 'GIN'\n",
    "schema = 'R'\n",
    "reduced = ['functional']\n",
    "\n",
    "ts = \"2022-Nov-19-05:42:56\"\n",
    "reduced_list = '_'.join(reduced)\n",
    "log_folder_name = os.path.join(*[file, model+'_'+schema+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "path = './dataset/'+log_folder_name+'/attention.pickle'\n",
    "with open(path, 'rb') as handle:\n",
    "    attention2 = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_pearson = []\n",
    "scatter_all = [list(), list()]\n",
    "for a in tqdm(attention1):\n",
    "    compare = []\n",
    "    for g in attention1[a]:\n",
    "        compare.append(list(attention1[a][g].values()))\n",
    "        scatter_all[0].extend(list(attention1[a][g].values()))\n",
    "    for g in attention2[a]:\n",
    "        compare.append(list(attention2[a][g].values()))\n",
    "        scatter_all[1].extend(list(attention2[a][g].values()))\n",
    "    attention_pearson.append(scipy.stats.pearsonr(compare[0], compare[1])[0])\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.histplot(data=attention_pearson, bins=np.arange(-1.1, 1.1, 0.05))\n",
    "fig.axvline(x=np.mean(attention_pearson), linewidth=1, color='black')\n",
    "fig.set_xlim([-1,1])\n",
    "plt.xlabel('spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.scatterplot(x=scatter_all[0], y=scatter_all[1])\n",
    "fig.set_xlim([0,1])\n",
    "fig.set_ylim([0,1])\n",
    "plt.xlabel('atom')\n",
    "plt.ylabel('reduced')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# COMBINE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load model\n",
    "file = 'bbbp'\n",
    "model = 'GIN'\n",
    "schema = 'AR_0'\n",
    "reduced = ['pharmacophore']\n",
    "\n",
    "ts = \"2023-Apr-29-17:22:22\"\n",
    "reduced_list = '_'.join(reduced)\n",
    "log_folder_name = os.path.join(*[file, model+'_'+schema+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "path = './dataset/'+log_folder_name+'/attention1.pickle'\n",
    "with open(path, 'rb') as handle:\n",
    "    attention = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_pearson = []\n",
    "scatter_all = dict()\n",
    "for a in tqdm(attention):\n",
    "    compare = []\n",
    "    if len(attention[a]) != 2:\n",
    "        print(a, attention[a])\n",
    "    for g in attention[a]:\n",
    "        if g not in scatter_all:\n",
    "            scatter_all[g] = list()\n",
    "        compare.append(list(attention[a][g].values()))\n",
    "        scatter_all[g].extend(list(attention[a][g].values()))\n",
    "    # attention_pearson.append(scipy.stats.pearsonr(compare[0], compare[1])[0])\n",
    "    r = scipy.stats.spearmanr(compare[0], compare[1])[0]\n",
    "    attention_pearson.append(r)\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=150)\n",
    "fig = sns.histplot(data=attention_pearson, bins=np.arange(-1.1, 1.1, 0.1))\n",
    "fig.axvline(x=np.nanmean(attention_pearson), linewidth=1, color='black')\n",
    "fig.set_xlim([-1,1])\n",
    "plt.xlabel('spearmanr')\n",
    "plt.title('Attention Correlation - '+'A+'+reduced[0][0].upper())\n",
    "print(len(attention_pearson))\n",
    "print(np.nanmean(attention_pearson))\n",
    "anchored_text = AnchoredText(\"AVG = {:.4f}\".format(np.nanmean(attention_pearson)), loc='upper right', prop=dict(size='small'))\n",
    "fig.add_artist(anchored_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.scatterplot(x=scatter_all[list(scatter_all.keys())[0]], y=scatter_all[list(scatter_all.keys())[1]])\n",
    "fig.set_xlim([0,1])\n",
    "fig.set_ylim([0,1])\n",
    "plt.xlabel('atom')\n",
    "plt.ylabel('reduced')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CROSS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_pearson = []\n",
    "scatter_all = [list(), list()]\n",
    "for a in tqdm(attention):\n",
    "    compare = []\n",
    "    compare.append(list(attention1[a]['atom'].values()))\n",
    "    scatter_all[0].extend(list(attention1[a]['atom'].values()))\n",
    "    compare.append(list(attention[a]['substructure'].values()))\n",
    "    scatter_all[1].extend(list(attention[a]['substructure'].values()))\n",
    "    # attention_pearson.append(scipy.stats.pearsonr(compare[0], compare[1])[0])\n",
    "    attention_pearson.append(scipy.stats.spearmanr(compare[0], compare[1])[0])\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.histplot(data=attention_pearson, bins=np.arange(-1.1, 1.1, 0.1))\n",
    "fig.axvline(x=np.mean(attention_pearson), linewidth=1, color='black')\n",
    "fig.set_xlim([-1,1])\n",
    "plt.xlabel('spearmanr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.scatterplot(x=scatter_all[0], y=scatter_all[1])\n",
    "fig.set_xlim([0,1])\n",
    "fig.set_ylim([0,1])\n",
    "plt.xlabel('atom')\n",
    "plt.ylabel('reduced')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_pearson = []\n",
    "scatter_all = [list(), list()]\n",
    "for a in tqdm(attention):\n",
    "    compare = []\n",
    "    compare.append(list(attention2[a]['functional'].values()))\n",
    "    scatter_all[0].extend(list(attention2[a]['functional'].values()))\n",
    "    compare.append(list(attention[a]['functional'].values()))\n",
    "    scatter_all[1].extend(list(attention[a]['functional'].values()))\n",
    "    attention_pearson.append(scipy.stats.pearsonr(compare[0], compare[1])[0])\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.histplot(data=attention_pearson, bins=np.arange(-1.1, 1.1, 0.1))\n",
    "fig.axvline(x=np.mean(attention_pearson), linewidth=1, color='black')\n",
    "fig.set_xlim([-1,1])\n",
    "plt.xlabel('spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "figure(figsize=(8, 5), dpi=80)\n",
    "fig = sns.scatterplot(x=scatter_all[0], y=scatter_all[1])\n",
    "fig.set_xlim([0,1])\n",
    "fig.set_ylim([0,1])\n",
    "plt.xlabel('atom')\n",
    "plt.ylabel('reduced')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SINGLE ALL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_all = list()\n",
    "# Load model\n",
    "file = 't04_CYP2C8_533'\n",
    "model = 'GIN'\n",
    "schema = 'A'\n",
    "reduced = []\n",
    "fold = 5\n",
    "\n",
    "for f in range(fold):\n",
    "    s = schema\n",
    "    if s == 'A':\n",
    "        directory = './dataset/'+os.path.join(*[file, model+'_'+s+'_'])\n",
    "        ts = next(os.walk(directory))[1][0]\n",
    "        reduced_list = ''\n",
    "        log_folder_name = os.path.join(*[file, model+'_'+s+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "        path = './dataset/'+log_folder_name+'/attention'+str(f)+'.pickle'\n",
    "        with open(path, 'rb') as handle:\n",
    "            attention_all.append(pickle.load(handle))\n",
    "    else:\n",
    "        for r in reduced:\n",
    "            directory = './dataset/'+os.path.join(*[file, model+'_'+s+'_'+r])\n",
    "            ts = sorted(next(os.walk(directory))[1])[0]\n",
    "            reduced_list = r\n",
    "            log_folder_name = os.path.join(*[file, model+'_'+s+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "            path = './dataset/'+log_folder_name+'/attention'+str(f)+'.pickle'\n",
    "            with open(path, 'rb') as handle:\n",
    "                attention_all.append(pickle.load(handle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(attention_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for a in attention_all:\n",
    "    for aa in a:\n",
    "        print(a[aa])\n",
    "        break\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit.Chem import AllChem\n",
    "fingerprint = list()\n",
    "test_y = list()\n",
    "attention_result = [[] for _ in range(len(attention_all))]\n",
    "\n",
    "for a in attention_all[0]: # loop smilse from atom graph\n",
    "    for i in range(len(attention_all)): # loop graph\n",
    "        for g in attention_all[i][a]: # loop node of smiles a in graph i\n",
    "            # if len(list(attention_all[i][a][g])) != 0 and g != 'atom':\n",
    "            if len(list(attention_all[i][a][g])) != 0:\n",
    "                attention_result[i].append(list(attention_all[i][a][g].values()))\n",
    "\n",
    "attention_result_t = []\n",
    "for r in attention_result:\n",
    "    attention_result_t.append(np.transpose(r))\n",
    "\n",
    "attention_result = attention_result_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(attention_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in attention_result:\n",
    "    print(len(i))\n",
    "    for ii in i:\n",
    "        print(ii)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(len(attention_result), len(attention_result), sharex=True, sharey=True, figsize=(15,15))\n",
    "label = range(fold)\n",
    "\n",
    "for r_i in tqdm(range(len(attention_result))): # loop graph as index r_i\n",
    "    for r_j in range(len(attention_result)): # loop graph as index r_j\n",
    "        if r_i > r_j:\n",
    "            attention_pearson = list()\n",
    "            for ii, aa in enumerate(attention_result[r_i]): # loop smiles as index ii\n",
    "                corr = scipy.stats.spearmanr(attention_result[r_i][ii], attention_result[r_j][ii])[0]\n",
    "                attention_pearson.append(corr)\n",
    "\n",
    "            sns.histplot(data=attention_pearson, bins=np.arange(-1.1, 1.1, 0.05), ax=ax[r_i, r_j], label=str(label[r_i])+'-'+str(label[r_j]))\n",
    "            ax[r_i, r_j].axvline(x=np.mean(attention_pearson), linewidth=1, color='black')\n",
    "            ax[r_i, r_j].set_xlim([-1,1])\n",
    "            plt.xlabel('spearman')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SINGLE COMPARE ALL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "attention_all = list()\n",
    "# Load model\n",
    "file = 'bbbp'\n",
    "model = 'GAT'\n",
    "schema = ['A', 'AR_0']\n",
    "reduced = ['junctiontree', 'cluster', 'functional', 'pharmacophore', 'substructure']\n",
    "\n",
    "for s in schema:\n",
    "    if s == 'A':\n",
    "        directory = './dataset/'+os.path.join(*[file, model+'_'+s+'_'])\n",
    "        ts = next(os.walk(directory))[1][0]\n",
    "        reduced_list = ''\n",
    "        log_folder_name = os.path.join(*[file, model+'_'+s+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "        path = './dataset/'+log_folder_name+'/attention.pickle'\n",
    "        with open(path, 'rb') as handle:\n",
    "            attention_all.append(pickle.load(handle))\n",
    "    else:\n",
    "        for r in reduced:\n",
    "            directory = './dataset/'+os.path.join(*[file, model+'_'+s+'_'+r])\n",
    "            ts = sorted(next(os.walk(directory))[1])[0]\n",
    "            reduced_list = r\n",
    "            log_folder_name = os.path.join(*[file, model+'_'+s+'_'+reduced_list, f\"{ts}\"])\n",
    "\n",
    "            path = './dataset/'+log_folder_name+'/attention.pickle'\n",
    "            with open(path, 'rb') as handle:\n",
    "                attention_all.append(pickle.load(handle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(attention_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for a in attention_all:\n",
    "    for aa in a:\n",
    "        print(a[aa])\n",
    "        break\n",
    "    # break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from rdkit.Chem import AllChem\n",
    "# import functools\n",
    "fingerprint = list()\n",
    "test_y = list()\n",
    "attention_result = [[] for _ in range(len(attention_all))]\n",
    "\n",
    "for a in attention_all[0]: # loop smilse from atom graph\n",
    "    for i in range(len(attention_all)): # loop graph\n",
    "        # for g in attention_all[i][a]: # loop node of smiles a in graph i\n",
    "        if len(attention_all[i][a]) !=1:\n",
    "            dict_list = []\n",
    "            for g in attention_all[i][a]:\n",
    "                dict_list.append(attention_all[i][a][g])\n",
    "            # print(dict_list)\n",
    "            dictf = {k: dict_list[0].get(k, 0) + dict_list[1].get(k, 0) for k in set(dict_list[0]) & set(dict_list[1])}\n",
    "            attention_result[i].append(list(dictf.values()))\n",
    "        elif len(attention_all[i][a]) ==1:\n",
    "            for g in attention_all[i][a]: # loop node of smiles a in graph i\n",
    "                attention_result[i].append(list(attention_all[i][a][g].values()))\n",
    "\n",
    "attention_result_t = []\n",
    "for r in attention_result:\n",
    "    attention_result_t.append(np.transpose(r))\n",
    "\n",
    "attention_result = attention_result_t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(attention_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in attention_result:\n",
    "    print(len(i))\n",
    "    for ii in i:\n",
    "        print(ii)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# attention_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(len(attention_result), len(attention_result), sharex=True, sharey=True, figsize=(15,15))\n",
    "# label = ['atom'] + reduced + ['ECFP6']\n",
    "label = ['atom'] + reduced\n",
    "\n",
    "for r_i in tqdm(range(len(attention_result))): # loop graph as index r_i\n",
    "    for r_j in range(len(attention_result)): # loop graph as index r_j\n",
    "        if r_i > r_j:\n",
    "            attention_pearson = list()\n",
    "            for ii, aa in enumerate(attention_result[r_i]): # loop smiles as index ii\n",
    "                corr = scipy.stats.spearmanr(attention_result[r_i][ii], attention_result[r_j][ii])[0]\n",
    "                attention_pearson.append(corr)\n",
    "\n",
    "            sns.histplot(data=attention_pearson, bins=np.arange(-1.1, 1.1, 0.05), ax=ax[r_i, r_j], label=label[r_i]+'-'+label[r_j])\n",
    "            ax[r_i, r_j].axvline(x=np.mean(attention_pearson), linewidth=1, color='black')\n",
    "            ax[r_i, r_j].set_xlim([-1,1])\n",
    "            plt.xlabel('spearman')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('myenv38')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "39c542c5c02b7a511e1ae79f0bb478cef0c12733a4ae7993722600208abb2b65"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}