building and evaluating SVM-based models.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建、测试并调试基于特征工程和机器学习的实体属性预测模型\n",
    "# 使用SVM多分类模型\n",
    "# 特征工程使用 1. 触发词是否出现；2. 触发词相对位置； 3. 是否有语义分割 三类特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 改进方向: 小样本复制一下上采样 (表征基本上是一样的) \n",
    "# 完善触发词规则，将数量型触发信号加入进来看看"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import json\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize.punkt import PunktSentenceTokenizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import WhitespaceTokenizer\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from nltk import pos_tag\n",
    "from nltk.util import ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flashtext import KeywordProcessor\n",
    "from interval import Interval\n",
    "from collections import Counter\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.multiclass import OneVsRestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 词形还原器\n",
    "wordnet_lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 相关数据文件路径\n",
    "data_dir = \"/home/denglizong/SSUMiner/corpus/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 疾病描述文本文件路径\n",
    "corpus_path = '/home/denglizong/SSUMiner/corpus/WikiPediaR5'\n",
    "# os.listdir( corpus_path )[0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "['Acinetobacter infections', 'Kawasaki disease', 'Paragonimiasis']"
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "# 疾病名称列表\n",
    "list_of_diseases = [ filename.replace('.txt',\"\") \n",
    "                    for filename in os.listdir( corpus_path ) if filename.endswith(\".txt\")]\n",
    "#\n",
    "list_of_diseases[0:3]                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(133, 60)"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "# 读入训练集和测试集的疾病名称索引\n",
    "diseases_used_for_training = []\n",
    "diseases_used_for_test = []\n",
    "\n",
    "# 用于训练模型或调优规则的疾病文档\n",
    "file_of_diseases_for_training = os.path.join( data_dir+\"TrainTestSplit\", \"diseases_for_training.txt\" )\n",
    "with open( file_of_diseases_for_training, 'r', encoding='utf-8' ) as f:\n",
    "    for line in f.readlines():\n",
    "        if line.strip() != \"\":\n",
    "            diseases_used_for_training.append( line.strip()  )\n",
    "\n",
    "# 用于测试模型或测试规则的疾病文档\n",
    "file_of_diseases_for_test = os.path.join( data_dir+\"TrainTestSplit\", \"diseases_for_test.txt\" )\n",
    "with open( file_of_diseases_for_test, 'r', encoding='utf-8' ) as f:\n",
    "    for line in f.readlines():\n",
    "        if line.strip() != \"\":\n",
    "            diseases_used_for_test.append( line.strip()  )\n",
    "\n",
    "#\n",
    "len( diseases_used_for_training ), len( diseases_used_for_test )            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取brat标注文件的函数\n",
    "# 获取brat对应的标注文件中的实体span,名称及类型\n",
    "# dict_of_brat_annotations.setdefault( ent_id, ('FindingSite', ent_name, pos_info) )    \n",
    "# 将ent_id 和 pos_info的位置置换一下   \n",
    "def read_brat_annotation_file( file_of_brat_annotation ):\n",
    "    # 求解目标\n",
    "    dict_of_brat_annotations = {}\n",
    "    # 原文字符串\n",
    "    text_string = \"\"\n",
    "    with open(file_of_brat_annotation,'r',encoding='utf-8') as f:\n",
    "        text_string = f.read()    \n",
    "    # 注释文件内容\n",
    "    ann_lines = []\n",
    "    with open(file_of_brat_annotation,'r',encoding='utf-8') as f:\n",
    "        ann_lines = f.readlines()\n",
    "    # 解析注释文件中的内容\n",
    "    for line in ann_lines:\n",
    "        # \n",
    "        if line.startswith('T'): \n",
    "            # 表型实体\n",
    "            if re.search('Phenotype',line,re.I):\n",
    "                # T1\tPhenotype 69 86\tpainful abscesses\n",
    "                ent_id, ent_info, ent_name = line.strip().split('\\t')\n",
    "                # 位置信息 153 157;180 194\n",
    "                pos_info = ent_info.replace('Phenotype ','')\n",
    "                # \n",
    "                # dict_of_brat_annotations.setdefault( ent_id, ('Phenotype', ent_name, pos_info) )   \n",
    "                dict_of_brat_annotations.setdefault( pos_info, ('Phenotype', ent_name, ent_id) )\n",
    "            # 部位实体\n",
    "            elif re.search('FindingSite',line,re.I):\n",
    "                # T4\tFindingSite 114 120\tbreast\n",
    "                ent_id, ent_info, ent_name = line.strip().split('\\t')\n",
    "                # 位置信息 114 120\n",
    "                pos_info = ent_info.replace('FindingSite ','')   \n",
    "                #\n",
    "                # dict_of_brat_annotations.setdefault( ent_id, ('FindingSite', ent_name, pos_info) )   \n",
    "                dict_of_brat_annotations.setdefault( pos_info, ('FindingSite', ent_name, ent_id) )     \n",
    "    #\n",
    "    return dict_of_brat_annotations   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取通过brat标注产生的表型术语和部位术语 (需要移除的对象)(避免干扰)\n",
    "# dict_of_brat_annotations.setdefault( pos_info, ('Phenotype', ent_name, ent_id) )\n",
    "# dict_of_brat_annotations.setdefault( pos_info, ('FindingSite', ent_name, ent_id) )\n",
    "\n",
    "# expert_annotated_terms = set()\n",
    "expert_annotated_terms = {}\n",
    "\n",
    "for disease_name in diseases_used_for_training:\n",
    "    # 疾病百科原文\n",
    "    file_of_original_text  = os.path.join( corpus_path, disease_name + '.txt' )\n",
    "    raw_text = open( file_of_original_text, 'r', encoding='utf-8').read() \n",
    "    # 莫非是 raw_text 没有替换'\\n'的问题？(因为标注的时候有替换)\n",
    "    # 还真的是这个问题\n",
    "    raw_text = raw_text.replace('\\n',',')\n",
    "\n",
    "    # 获取该疾病文本的人工标注\n",
    "    # 原来是这里出错了，将 disease_name 设置为了 \"Acinetobacter infections\"\n",
    "    file_of_brat_annotation  = os.path.join( corpus_path, disease_name + '.ann' )\n",
    "    dict_of_brat_annotations = read_brat_annotation_file(file_of_brat_annotation) \n",
    "\n",
    "    # \n",
    "    for span_key in dict_of_brat_annotations:\n",
    "        # expert_annotated_terms.add( dict_of_brat_annotations[span_key][1] )\n",
    "        annotated_term = dict_of_brat_annotations[span_key][1]\n",
    "        annotated_type = dict_of_brat_annotations[span_key][0]\n",
    "        #\n",
    "        expert_annotated_terms.setdefault( annotated_term, annotated_type )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 属性名称列表\n",
    "# list_of_attributes = ['Assertion','Severity','Temporal','Sensation','Color','Magnitudes','Shape','Age','Gender',\n",
    "#                      'Frequency','Relevance','Laterality','Quadrant','Distribution',\n",
    "#                       'Stage','Phase','Type','SOI','Complication']\n",
    "# 待预测的属性名称\n",
    "# list_of_attributes = ['Assertion','Severity','Temporal','Sensation','Color','Age','Gender',\n",
    "#                      'Frequency','Laterality','Quadrant','Distribution','SOI']       \n",
    "# 不预测 'Laterality','Quadrant'                                 \n",
    "list_of_attributes = ['Assertion','Severity','Temporal','Sensation','Color','Age','Gender',\n",
    "                     'Frequency','Distribution','SOI']  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 观察 某一具体属性(wanted_attr_name)在疾病文档(disease_names)中取值分布的函数\n",
    "# {value 1: count 1, value 2: count 2}\n",
    "# occurences_of_attribute_values = {}\n",
    "def stat_occurences_of_attribute_values( wanted_attr_name, disease_names ):\n",
    "    # \n",
    "    dict_of_attribute_value_distribution = {}\n",
    "\n",
    "    #\n",
    "    for disease_name in disease_names:\n",
    "        # 这一疾病对应的文本文件和标注文件\n",
    "        # textfilepath = os.path.join( filepath, disease_names+'.txt')\n",
    "        annfilepath  = os.path.join( corpus_path, disease_name+'.ann')\n",
    "        \n",
    "        # 读入标注文件\n",
    "        with open(annfilepath,'r',encoding='utf-8') as f:\n",
    "            # 载入标注文件的行\n",
    "            lines = f.readlines()     \n",
    "            # 找到属性行\n",
    "            for line in lines:\n",
    "                # 解析属性行，\n",
    "                if line.startswith('A'):\n",
    "                    # 解析属性行中的属性名称\n",
    "                    # A2\tAssertion T6 Possible\n",
    "                    info_a, info_b = line.strip().split('\\t')\n",
    "                    att_name, ent_id, att_value = info_b.split(' ')                     \n",
    "                    # 如果是需要统计的属性\n",
    "                    if att_name == wanted_attr_name:\n",
    "                        # 统计属性值\n",
    "                        if att_value not in dict_of_attribute_value_distribution:\n",
    "                            dict_of_attribute_value_distribution.setdefault(att_value, 1)\n",
    "                        else:\n",
    "                            dict_of_attribute_value_distribution[att_value] +=1\n",
    "    #\n",
    "    return dict_of_attribute_value_distribution                                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "# 目标变量: annotated_phenotypes_info_of_diseases.setdefault(disease_name, info_of_annotated_phenotypes )\n",
    "# list of info_of_annotated_phenotypes 以表型的id为key,记录表型的标注信息\n",
    "# info_of_annotated_phenotypes.setdefault(pheno_id, tmpdict)\n",
    "# 以表型为单位，包含以下信息记录到tmpdict\n",
    "# tmpdict.setdefault(\"pheno_name\",pheno_name)\n",
    "# tmpdict.setdefault(\"pheno_pos\",pheno_pos)\n",
    "# tmpdict.setdefault(\"associated_sites\", ';'.join(associated_finding_sites) )\n",
    "# tmpdict.setdefault(\"associated_attributes\", ';'.join(associated_attributes) )      \n",
    "annotated_phenotypes_info_of_diseases = {}\n",
    "\n",
    "for disease_name in list_of_diseases:\n",
    "    # 疾病百科原文\n",
    "    file_of_original_text  = os.path.join( corpus_path, disease_name + '.txt' )\n",
    "    raw_text = open( file_of_original_text, 'r', encoding='utf-8').read() \n",
    "    # 莫非是 raw_text 没有替换'\\n'的问题？(因为标注的时候有替换)\n",
    "    # 还真的是这个问题\n",
    "    raw_text = raw_text.replace('\\n',' ')\n",
    "\n",
    "    # 获取该疾病文本的人工标注\n",
    "    # 原来是这里出错了，将 disease_name 设置为了 \"Acinetobacter infections\"\n",
    "    file_of_brat_annotation  = os.path.join( corpus_path, disease_name + '.ann' )\n",
    "\n",
    "    # 解析其中标注的表型信息\n",
    "    info_of_annotated_phenotypes = {}  \n",
    "\n",
    "    with open(file_of_brat_annotation,'r',encoding='utf-8') as f:\n",
    "\n",
    "        # 载入标注文件的行\n",
    "        lines = f.readlines()\n",
    "        # 搜索标注的表型实体\n",
    "        # 获取该文本中出现过的表型实体编号及其名称   ent_id, ent_name \n",
    "        dict_of_phenotypic_id_and_name = {}\n",
    "        # 获取该文本中出现过的部位实体编号及其名称   ent_id, ent_name \n",
    "        dict_of_findingsite_id_and_name = {}\n",
    "        # 记录表型实体与部位实体之间的关联          ent1_id, [ent2_id] 可能会关联多个部位\n",
    "        dict_of_phenotypes_related_sites = {}   \n",
    "        # 记录表型实体具有的属性信息                ent_id, [(att_name,att_value)] 可能会关联多个属性\n",
    "        dict_of_phenotypes_related_attributes = {}         \n",
    "        #\n",
    "        for line in lines:\n",
    "            # 表型实体，不包括部位实体\n",
    "            if line.startswith('T'): \n",
    "                if re.search('Phenotype',line,re.I):\n",
    "                    # T1\tPhenotype 69 86\tpainful abscesses\n",
    "                    ent_id, ent_info, ent_name = line.strip().split('\\t')\n",
    "                    # 位置信息 153 157;180 194\n",
    "                    ent_pos = ent_info.replace('Phenotype ','')\n",
    "                    # \n",
    "                    dict_of_phenotypic_id_and_name.setdefault( ent_id, (ent_name,ent_pos) )   \n",
    "                elif re.search('FindingSite',line,re.I):\n",
    "                    # T4\tFindingSite 114 120\tbreast\n",
    "                    ent_id, ent_info, ent_name = line.strip().split('\\t')\n",
    "                    dict_of_findingsite_id_and_name.setdefault( ent_id, ent_name )                \n",
    "            # 找到关系 ，记录标注有部位的表型\n",
    "            elif line.startswith('R'):\n",
    "                if re.search('locate',line,re.I):\n",
    "                    # R1\tlocate Arg1:T1 Arg2:T2\t\n",
    "                    rel_id, rel_info, tmp_str = line.split('\\t')\n",
    "                    # locate Arg1:T1 Arg2:T2\n",
    "                    part_a, part_b, part_c = rel_info.split(' ')\n",
    "                    # \n",
    "                    ent1_id = part_b.split(':')[1]\n",
    "                    ent2_id = part_c.split(':')[1]\n",
    "                    #\n",
    "                    if ent1_id not in dict_of_phenotypes_related_sites:\n",
    "                        dict_of_phenotypes_related_sites.setdefault( ent1_id, [ent2_id] )     \n",
    "                    else:\n",
    "                        dict_of_phenotypes_related_sites[ent1_id].append( ent2_id )\n",
    "            # 属性行，关联到对应的表型实体\n",
    "            elif line.startswith('A'):\n",
    "                # A2\tAssertion T6 Possible\n",
    "                info_a, info_b = line.strip().split('\\t')\n",
    "                att_name, ent_id, att_value = info_b.split(' ') \n",
    "                # 除外'AgreeAnn','EqualRaw'属性\n",
    "                if att_name not in ['AgreeAnn','EqualRaw']:\n",
    "                    # 表型id, 属性名称，属性值\n",
    "                    if ent_id not in dict_of_phenotypes_related_attributes:\n",
    "                        dict_of_phenotypes_related_attributes.setdefault( ent_id, [(att_name,att_value)]  )\n",
    "                    else:\n",
    "                        dict_of_phenotypes_related_attributes[ent_id].append( (att_name,att_value) ) \n",
    "        #\n",
    "        for pheno_id in dict_of_phenotypic_id_and_name:\n",
    "            # 文档名称\n",
    "            doc_name = disease_name\n",
    "            # 表型名称  (ent_name,end_start,end_end)\n",
    "            pheno_name = dict_of_phenotypic_id_and_name[pheno_id][0]\n",
    "            # 表型位置\n",
    "            pheno_pos = dict_of_phenotypic_id_and_name[pheno_id][1]\n",
    "            # 关联部位名称 (分号隔开)\n",
    "            associated_finding_sites = []\n",
    "            if pheno_id in dict_of_phenotypes_related_sites:\n",
    "                site_ids = dict_of_phenotypes_related_sites[pheno_id]\n",
    "                for site_id in site_ids:\n",
    "                    if site_id in dict_of_findingsite_id_and_name:\n",
    "                        associated_finding_sites.append( dict_of_findingsite_id_and_name[site_id] )\n",
    "            # 关联属性\n",
    "            associated_attributes = []\n",
    "            if pheno_id in dict_of_phenotypes_related_attributes:\n",
    "                list_of_attrs = dict_of_phenotypes_related_attributes[pheno_id]\n",
    "                for (_attr, _value) in list_of_attrs:\n",
    "                    associated_attributes.append( _attr+\":\"+_value) \n",
    "            # 记录\n",
    "            tmpdict = {}\n",
    "            tmpdict.setdefault(\"pheno_name\",pheno_name)\n",
    "            tmpdict.setdefault(\"pheno_pos\",pheno_pos)\n",
    "            tmpdict.setdefault(\"associated_sites\", ';'.join(associated_finding_sites) )\n",
    "            tmpdict.setdefault(\"associated_attributes\", ';'.join(associated_attributes) )                       \n",
    "            # \n",
    "            info_of_annotated_phenotypes.setdefault(pheno_id, tmpdict)\n",
    "            annotated_phenotypes_info_of_diseases.setdefault(disease_name, info_of_annotated_phenotypes )  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annotated_phenotypes_info_of_diseases = {}\n",
    "# annotated_phenotypes_info_of_diseases.setdefault(disease_name, info_of_annotated_phenotypes )  \n",
    "# 类似S5A_基于规则预测实体属性，在其内增加一个算法预测的属性模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "{'T1': {'pheno_name': 'painful abscesses',\n  'pheno_pos': '69 86',\n  'associated_sites': 'mouth;lungs;breast;gastrointestinal tract',\n  'associated_attributes': 'Assertion:Present;Relevance:Distinctive_finding'},\n 'T6': {'pheno_name': 'pus',\n  'pheno_pos': '366 369',\n  'associated_sites': 'skin',\n  'associated_attributes': 'Assertion:Possible;SOI:Severe_problem'},\n 'T8': {'pheno_name': 'sulfur granules',\n  'pheno_pos': '417 432',\n  'associated_sites': '',\n  'associated_attributes': 'Assertion:Possible;Frequency:Frequent;Relevance:Distinctive_finding;SOI:Severe_problem'}}"
     },
     "metadata": {},
     "execution_count": 20
    }
   ],
   "source": [
    "# 给定一份疾病文档，输出专家标注的表型信息\n",
    "annotated_phenotypes_info_of_diseases['Actinomycosis']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 进一步的，对于每一个标注的表型，记录表型所在的句子文本信息\n",
    "# 将之增添到 annotated_phenotypes_info_of_diseases 变量中\n",
    "for disease_name in list_of_diseases:\n",
    "    # for the disease_name \n",
    "\n",
    "    # 疾病百科原文\n",
    "    file_of_original_text  = os.path.join( corpus_path, disease_name + '.txt' )\n",
    "    raw_text = open( file_of_original_text, 'r', encoding='utf-8').read() \n",
    "    # 莫非是 raw_text 没有替换'\\n'的问题？(因为标注的时候有替换)\n",
    "    # 还真的是这个问题 \n",
    "    # 在S1A 使用MetaMap注释语料的过程中, 替换其中的'\\n'为','\n",
    "    raw_text = raw_text.replace('\\n',',')\n",
    "\n",
    "    # 读入文本文件，获取句子信息\n",
    "    # text_sents = []\n",
    "    for start, end in PunktSentenceTokenizer().span_tokenize(raw_text):\n",
    "        # text_sents.append( (start,end, raw_text[start:end]) ) \n",
    "        sent_text = raw_text[start:end]\n",
    "        \n",
    "        # 遍历标注的表型\n",
    "        for pheno_id in annotated_phenotypes_info_of_diseases[disease_name]:\n",
    "            # 解析表型的起止位置\n",
    "            pheno_info = annotated_phenotypes_info_of_diseases[disease_name][pheno_id]\n",
    "            pheno_spos = int( pheno_info['pheno_pos'].split(' ')[0] )\n",
    "            pheno_epos = int( pheno_info['pheno_pos'].split(' ')[-1] )\n",
    "\n",
    "            # 进一步定位表型所在的句子,\n",
    "            if pheno_spos >= start and pheno_epos<= end:  \n",
    "                # 记录表型所在的句子\n",
    "                if 'pheno_context' not in annotated_phenotypes_info_of_diseases[disease_name][pheno_id]:\n",
    "                    annotated_phenotypes_info_of_diseases[disease_name][pheno_id].setdefault( 'pheno_context', sent_text )\n",
    "                else:\n",
    "                    annotated_phenotypes_info_of_diseases[disease_name][pheno_id]['pheno_context'] = sent_text\n",
    "                # 记录表型在句子中的起始位置\n",
    "                pheno_spos_in_sent = pheno_spos - start\n",
    "                pheno_epos_in_sent = pheno_epos - start\n",
    "                pos_info = (pheno_spos_in_sent, pheno_epos_in_sent)\n",
    "                if pos_info not in annotated_phenotypes_info_of_diseases[disease_name][pheno_id]:\n",
    "                    annotated_phenotypes_info_of_diseases[disease_name][pheno_id].setdefault( 'pheno_pos_in_sent', pos_info )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "{'T1': {'pheno_name': 'painful abscesses',\n  'pheno_pos': '69 86',\n  'associated_sites': 'mouth;lungs;breast;gastrointestinal tract',\n  'associated_attributes': 'Assertion:Present;Relevance:Distinctive_finding',\n  'pheno_context': 'Signs and symptoms,,The disease is characterised by the formation of painful abscesses in the mouth, lungs,[3][4] breast,[5] or gastrointestinal tract.',\n  'pheno_pos_in_sent': (69, 86)},\n 'T6': {'pheno_name': 'pus',\n  'pheno_pos': '366 369',\n  'associated_sites': 'skin',\n  'associated_attributes': 'Assertion:Possible;SOI:Severe_problem',\n  'pheno_context': 'In severe cases, they may penetrate the surrounding bone and muscle to the skin, where they break open and leak large amounts of pus, which often contains characteristic granules (sulfur granules) filled with progeny bacteria.',\n  'pheno_pos_in_sent': (129, 132)},\n 'T8': {'pheno_name': 'sulfur granules',\n  'pheno_pos': '417 432',\n  'associated_sites': '',\n  'associated_attributes': 'Assertion:Possible;Frequency:Frequent;Relevance:Distinctive_finding;SOI:Severe_problem',\n  'pheno_context': 'In severe cases, they may penetrate the surrounding bone and muscle to the skin, where they break open and leak large amounts of pus, which often contains characteristic granules (sulfur granules) filled with progeny bacteria.',\n  'pheno_pos_in_sent': (180, 195)}}"
     },
     "metadata": {},
     "execution_count": 22
    }
   ],
   "source": [
    "# 再看看现在记录到变量中的信息\n",
    "annotated_phenotypes_info_of_diseases['Actinomycosis']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给定 属性:取值(wanted_attribute), 并指定疾病文档(wanted_diseases), 输出这些疾病文档中包含有指定属性取值的表型信息\n",
    "def get_phenotypes_with_wanted_attribute_value( annotated_phenotypes_info_of_diseases, wanted_diseases, wanted_attribute_value ):\n",
    "    #\n",
    "    list_of_phenotypes_with_wanted_attribute_value = []\n",
    "    list_of_phenotypes_without_wanted_attribute_value = []\n",
    "\n",
    "    # 遍历标注的表型\n",
    "    for disease_name in wanted_diseases:\n",
    "        for pheno_id in annotated_phenotypes_info_of_diseases[disease_name]:\n",
    "            # infodict of annotated phenotypes\n",
    "            pheno_info = annotated_phenotypes_info_of_diseases[disease_name][pheno_id]  \n",
    "\n",
    "            if \"pheno_context\" not in pheno_info:\n",
    "                continue\n",
    "\n",
    "            # 观察是否该表型是否具有指定的属性值\n",
    "            pheno_values = pheno_info[\"associated_attributes\"].split(';')\n",
    "            # Severity:Severe\n",
    "            if wanted_attribute_value in pheno_values:\n",
    "                list_of_phenotypes_with_wanted_attribute_value.append( pheno_info )\n",
    "            else:\n",
    "                list_of_phenotypes_without_wanted_attribute_value.append( pheno_info )\n",
    "    \n",
    "    #\n",
    "    return list_of_phenotypes_with_wanted_attribute_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给定 属性 (wanted_attribute), 并指定疾病文档(wanted_diseases), 输出这些疾病文档中没有选择该属性取值的表型信息\n",
    "def get_phenotypes_without_wanted_attribute( annotated_phenotypes_info_of_diseases, wanted_diseases, wanted_attribute ):\n",
    "    #\n",
    "    list_of_phenotypes_with_wanted_attribute = []\n",
    "    list_of_phenotypes_without_wanted_attribute = []\n",
    "\n",
    "    # 遍历标注的表型\n",
    "    for disease_name in wanted_diseases:\n",
    "        for pheno_id in annotated_phenotypes_info_of_diseases[disease_name]:\n",
    "            # infodict of annotated phenotypes\n",
    "            pheno_info = annotated_phenotypes_info_of_diseases[disease_name][pheno_id]  \n",
    "\n",
    "            if 'pheno_context' not in pheno_info:\n",
    "                continue\n",
    "\n",
    "            # 观察是否该表型是否具有指定的属性值\n",
    "            # \"Assertion:Present;Severity:Severe\" \n",
    "            pheno_values = pheno_info[\"associated_attributes\"]\n",
    "            # \"Severity:\" in \"Assertion:Present;Severity:Severe\" \n",
    "            if wanted_attribute+':' in pheno_values:\n",
    "                list_of_phenotypes_with_wanted_attribute.append( pheno_info )\n",
    "            else:\n",
    "                list_of_phenotypes_without_wanted_attribute.append( pheno_info )\n",
    "    \n",
    "    #\n",
    "    return list_of_phenotypes_without_wanted_attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读入实体属性标准取值的触发词表知识库\n",
    "trigger_kb_file = \"/home/denglizong/SSUMiner/corpus/TriggerWords/attribute_triggers.xlsx\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   属性中文名称     属性英文名称          标准取值   样本数   训练集  测试集  \\\n0  表型存在情况  Assertion       Present  1229   862  367   \n1  表型存在情况  Assertion      Possible  2692  1976  716   \n2  表型存在情况  Assertion   Conditional    76    62   14   \n3  表型存在情况  Assertion  Hypothetical     4     4    0   \n4  表型存在情况  Assertion        Absent    20    13    7   \n\n                                                 触发词   触发范围  \n0                                                  0   long  \n1  may;might;can;could;possible;possibly;likely;i...   long  \n2         if;while;when;in case of;without treatment   long  \n3                                                  0   long  \n4  no;without;never developed;never had;no compla...  short  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>属性中文名称</th>\n      <th>属性英文名称</th>\n      <th>标准取值</th>\n      <th>样本数</th>\n      <th>训练集</th>\n      <th>测试集</th>\n      <th>触发词</th>\n      <th>触发范围</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>表型存在情况</td>\n      <td>Assertion</td>\n      <td>Present</td>\n      <td>1229</td>\n      <td>862</td>\n      <td>367</td>\n      <td>0</td>\n      <td>long</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>表型存在情况</td>\n      <td>Assertion</td>\n      <td>Possible</td>\n      <td>2692</td>\n      <td>1976</td>\n      <td>716</td>\n      <td>may;might;can;could;possible;possibly;likely;i...</td>\n      <td>long</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>表型存在情况</td>\n      <td>Assertion</td>\n      <td>Conditional</td>\n      <td>76</td>\n      <td>62</td>\n      <td>14</td>\n      <td>if;while;when;in case of;without treatment</td>\n      <td>long</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>表型存在情况</td>\n      <td>Assertion</td>\n      <td>Hypothetical</td>\n      <td>4</td>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>long</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>表型存在情况</td>\n      <td>Assertion</td>\n      <td>Absent</td>\n      <td>20</td>\n      <td>13</td>\n      <td>7</td>\n      <td>no;without;never developed;never had;no compla...</td>\n      <td>short</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 26
    }
   ],
   "source": [
    "df_of_attribute_triggers = pd.read_excel(trigger_kb_file, dtype=str).fillna('0')\n",
    "df_of_attribute_triggers.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "   属性中文名称    属性英文名称                       标准取值  样本数 训练集 测试集  \\\n5  表型严重程度  Severity                       Mild   52  31  21   \n6  表型严重程度  Severity           Mild_to_Moderate    2   2   0   \n7  表型严重程度  Severity         Moderate_to_Severe    9   8   1   \n8  表型严重程度  Severity                     Severe  128  89  39   \n9  表型严重程度  Severity  Life_threatening_severity    6   5   1   \n\n                                                 触发词   触发范围  \n5  mild;low-grade;low grade;subtle;mildly;a littl...  short  \n6                  mild to moderate;mild or moderate  short  \n7           moderate or very high;moderate to severe  short  \n8  severe;high;extreme;intense;profound;strong;si...  short  \n9             life-threatening;dangerously;dangerous  short  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>属性中文名称</th>\n      <th>属性英文名称</th>\n      <th>标准取值</th>\n      <th>样本数</th>\n      <th>训练集</th>\n      <th>测试集</th>\n      <th>触发词</th>\n      <th>触发范围</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>5</th>\n      <td>表型严重程度</td>\n      <td>Severity</td>\n      <td>Mild</td>\n      <td>52</td>\n      <td>31</td>\n      <td>21</td>\n      <td>mild;low-grade;low grade;subtle;mildly;a littl...</td>\n      <td>short</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>表型严重程度</td>\n      <td>Severity</td>\n      <td>Mild_to_Moderate</td>\n      <td>2</td>\n      <td>2</td>\n      <td>0</td>\n      <td>mild to moderate;mild or moderate</td>\n      <td>short</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>表型严重程度</td>\n      <td>Severity</td>\n      <td>Moderate_to_Severe</td>\n      <td>9</td>\n      <td>8</td>\n      <td>1</td>\n      <td>moderate or very high;moderate to severe</td>\n      <td>short</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>表型严重程度</td>\n      <td>Severity</td>\n      <td>Severe</td>\n      <td>128</td>\n      <td>89</td>\n      <td>39</td>\n      <td>severe;high;extreme;intense;profound;strong;si...</td>\n      <td>short</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>表型严重程度</td>\n      <td>Severity</td>\n      <td>Life_threatening_severity</td>\n      <td>6</td>\n      <td>5</td>\n      <td>1</td>\n      <td>life-threatening;dangerously;dangerous</td>\n      <td>short</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 27
    }
   ],
   "source": [
    "df_of_attribute_triggers[ df_of_attribute_triggers['属性英文名称'] == 'Severity' ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# svm多分类示例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([\n",
    "                [10, 10],\n",
    "                [8, 10],\n",
    "                [-5, 5.5],\n",
    "                [-5.4, 5.5],\n",
    "                [-20, -20],\n",
    "                [-15, -20]\n",
    "            ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.array([0, 0, 1, 1, 2, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = OneVsRestClassifier(SVC()).fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([2, 0, 1])"
     },
     "metadata": {},
     "execution_count": 32
    }
   ],
   "source": [
    "clf.predict([[-19, -20], [9, 9], [-5, 5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X [ [表征]     ]\n",
    "# y [ [标签]     ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建实体属性值预测的多分类模型demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 第一阶段 求解目标: \n",
    "# trained classifier for each of attributes\n",
    "# dict_of_trained_classifiers = {}\n",
    "# dict_of_trained_classifiers.sefdefault( attribute_name, clf )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dict_of_trained_classifiers = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "{}"
     },
     "metadata": {},
     "execution_count": 37
    }
   ],
   "source": [
    "dict_of_trained_classifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "['Assertion', 'Severity', 'Temporal', 'Sensation', 'Color', 'Age', 'Gender', 'Frequency', 'Distribution', 'SOI']\n"
    }
   ],
   "source": [
    "# 数据量不够 Laterality; Quadrant;\n",
    "print(list_of_attributes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 299,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 回到这里\n",
    "# 目标属性名称\n",
    "target_attribute = 'SOI'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'None': 0, 'Severe_problem': 1, 'Mild_problem': 2}\n"
    }
   ],
   "source": [
    "# 对目标属性的取值进行数字编码\n",
    "codes_of_attribute_values = {}\n",
    "\n",
    "# default的属性取值设置为0\n",
    "if target_attribute == 'Assertion':\n",
    "    codes_of_attribute_values.setdefault( \"Present\", 0 )\n",
    "else:\n",
    "    codes_of_attribute_values.setdefault( \"None\", 0 )  \n",
    "\n",
    "\n",
    "# 观察目标属性在语料集中的出现情况\n",
    "# 观察目标属性在训练集中的出现情况\n",
    "present_of_attribute_values = stat_occurences_of_attribute_values(target_attribute, diseases_used_for_training)\n",
    "# 该属性在训练集中出现的类别数 len( present_of_attribute_values )\n",
    "\n",
    "# 对出现过的目标属性取值进行数字编码\n",
    "for attribute_value in present_of_attribute_values:\n",
    "    if attribute_value != 'Present':\n",
    "        # 赋予编号\n",
    "        # 只有当该属性值具有的样本数大于类别数时才考虑\n",
    "        count_of_attribute_value = present_of_attribute_values[ attribute_value ] \n",
    "        # 有样本即可\n",
    "        if count_of_attribute_value >= 0 :\n",
    "            code_of_attribute_value = len( codes_of_attribute_values )\n",
    "            codes_of_attribute_values.setdefault( attribute_value , code_of_attribute_value )  \n",
    "\n",
    "# 符合样本数条件的属性类\n",
    "print(codes_of_attribute_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 301,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 提取训练集中具有目标属性取值的表型信息\n",
    "# {\"Severity:Severe\":[], }\n",
    "# phenotypes_with_target_attribute.setdefault( attribute_value, list_of_phenoinfo )\n",
    "phenotypes_with_target_attribute = {}\n",
    "\n",
    "# 对于在语料中出现过的属性取值 [只需要考虑符合样本数条件的属性类]\n",
    "for attribute_value in codes_of_attribute_values:\n",
    "    if attribute_value == 'None':\n",
    "        continue\n",
    "    # 获取它们所依附的表型信息\n",
    "    list_of_phenoinfo = get_phenotypes_with_wanted_attribute_value(annotated_phenotypes_info_of_diseases, \n",
    "                        diseases_used_for_training, target_attribute+ ':'+ attribute_value)\n",
    "    # 保存\n",
    "    phenotypes_with_target_attribute.setdefault( attribute_value, list_of_phenoinfo )\n",
    "    \n",
    "\n",
    "# 如果是可选属性(Assertion之外的属性)，还要考虑这些属性有'None'的取值\n",
    "if target_attribute != 'Assertion':\n",
    "    # 获取不具有该属性的表型标注\n",
    "    list_of_phenoinfo = get_phenotypes_without_wanted_attribute(annotated_phenotypes_info_of_diseases, \n",
    "                                                                    diseases_used_for_training, target_attribute)\n",
    "    # 保存\n",
    "    phenotypes_with_target_attribute.setdefault( \"None\", list_of_phenoinfo )  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Severe_problem 52\nMild_problem 9\nNone 2846\n"
    }
   ],
   "source": [
    "for attribute_value in phenotypes_with_target_attribute:\n",
    "    print( attribute_value, len( phenotypes_with_target_attribute[attribute_value] ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 阴性数据和阳性数据的确非常不平衡，可以考虑对阴性数据进行下采样 [使得阴阳数据平衡]\n",
    "# 阳性数据数目\n",
    "count_of_positive_samples = 0\n",
    "\n",
    "for attribute_value in phenotypes_with_target_attribute:\n",
    "    if attribute_value != 'None':\n",
    "        count_of_positive_samples += len( phenotypes_with_target_attribute[attribute_value] )\n",
    "\n",
    "# print( count_of_positive_samples, len(phenotypes_with_target_attribute['None']) )\n",
    "\n",
    "# 如果阴性数据的数目大于了阳性数据\n",
    "# 对None阴性数据进行下采样 (随机下采样即可，因为它们的表征在理论上，都应该是0 0 0)\n",
    "if 'None' in phenotypes_with_target_attribute:\n",
    "    resampled_negative_samples = []\n",
    "    resampled_negative_samples = random.sample( phenotypes_with_target_attribute['None'], count_of_positive_samples )\n",
    "    # 更新 phenotypes_with_target_attribute 中的 None 样本列表\n",
    "    phenotypes_with_target_attribute['None'] = resampled_negative_samples    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Severe_problem 52\nMild_problem 9\nNone 61\n"
    }
   ],
   "source": [
    "for attribute_value in phenotypes_with_target_attribute:\n",
    "    print( attribute_value, len( phenotypes_with_target_attribute[attribute_value] ) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'Mild_problem': ['mild illness', 'mild case', 'mild infection', 'mild symtom', 'mild one'], 'Severe_problem': ['severe form', 'severe case', 'severe infection', 'severe disease', 'severe illness', 'severe case', 'severe infection', 'severe one', 'when severe', 'severe acute disease', 'serious one', 'serious form', 'severe symptoms'], 'PseudoTrigger': ['no increase', 'no significant change', 'not only', 'not drain', 'not certain whether', 'when urinating', 'when symptomatic', 'urinate frequently', 'most notably', 'most important', 'a few days', 'common cold', 'most common cause', 'most severe', 'most serious', 'common name', 'half of the body', 'a few months', 'difficulty becoming pregnant', 'complications of pregnancy', 'ectopic (tubal) pregnancy', 'adult worms', 'common cold', 'female worms', 'men and women', 'other than pregnant women', 'high numbers', 'many other', 'general ill', 'as many as', 'general abdominal pain', 'often referred to', 'without significant', 'heavy breathing', 'high blood', 'less severe than', 'fear of night', 'in hot', 'some elevation', 'generalized malaise', 'distal ileum', 'initial infection', 'the first disease', 'secondary bacterial pneumonia', 'mild to severe disease', 'less severe illnesses', 'many of the symptom', 'red blood cell', 'white blood cell', 'many months']}\n"
    }
   ],
   "source": [
    "# 获取该属性不同属性取值的触发词\n",
    "# 不考虑default的属性取值\n",
    "triggers_of_attribute_values = {}\n",
    "\n",
    "for index, row in df_of_attribute_triggers[ df_of_attribute_triggers['属性英文名称'] == target_attribute ].iterrows():\n",
    "    attribute_name = row['属性英文名称']\n",
    "    attribute_value = row['标准取值']\n",
    "    attribute_triggers = row['触发词']\n",
    "\n",
    "    # 跳过必选属性的默认取值Present\n",
    "    if attribute_value == 'Present':\n",
    "        continue\n",
    "\n",
    "    # 写入属性取值-触发词变量\n",
    "    if attribute_triggers != '0':\n",
    "        triggers_of_attribute_values.setdefault( attribute_value, attribute_triggers.split(';') )\n",
    "\n",
    "# add\n",
    "# 需要考虑 PseudoTrigger\n",
    "for index, row in df_of_attribute_triggers[ df_of_attribute_triggers['属性英文名称'] == 'PT' ].iterrows():\n",
    "    attribute_name = row['属性英文名称']\n",
    "    attribute_value = row['标准取值']\n",
    "    attribute_triggers = row['触发词']\n",
    "\n",
    "    # 写入属性取值-触发词变量\n",
    "    if attribute_triggers != '0':\n",
    "        triggers_of_attribute_values.setdefault( attribute_value, attribute_triggers.split(';') )\n",
    "\n",
    "\n",
    "# 如果是Assertion:Possible属性，需要考虑Frequency的触发词并入到Possible里\n",
    "if target_attribute == 'Assertion':\n",
    "    # \n",
    "    for index, row in df_of_attribute_triggers[ df_of_attribute_triggers['属性英文名称'] == 'Frequency' ].iterrows():\n",
    "        attribute_name = row['属性英文名称']\n",
    "        attribute_value = row['标准取值']\n",
    "        attribute_triggers = row['触发词']\n",
    "\n",
    "        # 写入属性取值-触发词变量\n",
    "        if attribute_triggers != '0' and attribute_value != \"Obligate\":\n",
    "            triggers_of_attribute_values['Possible'] += attribute_triggers.split(';')\n",
    "\n",
    "# 如果是Severity的属性，需要把Severe Case作为虚假触发信号\n",
    "if target_attribute == 'Severity':       \n",
    "    # \n",
    "    for index, row in df_of_attribute_triggers[ df_of_attribute_triggers['属性英文名称'] == 'SOI' ].iterrows():\n",
    "        attribute_name = row['属性英文名称']\n",
    "        attribute_value = row['标准取值']\n",
    "        attribute_triggers = row['触发词']\n",
    "\n",
    "        # 写入属性取值-触发词变量\n",
    "        if attribute_triggers != '0':\n",
    "            triggers_of_attribute_values['PseudoTrigger'] += attribute_triggers.split(';')    \n",
    "\n",
    "print(triggers_of_attribute_values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将 triggers_of_attribute_values 转换为 trigger 到 标准取值的形式，便于处理\n",
    "mapping_btw_trigger_and_std_value = {}\n",
    "\n",
    "for attribute_value in triggers_of_attribute_values:\n",
    "    for trigger_word in triggers_of_attribute_values[attribute_value]:\n",
    "        mapping_btw_trigger_and_std_value.setdefault( trigger_word, attribute_value )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'None': 0, 'Severe_problem': 1, 'Mild_problem': 2}\n"
    }
   ],
   "source": [
    "print(codes_of_attribute_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mapping_btw_trigger_and_std_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# 训练集表征\n",
    "# 尝试以表型为单位，生成该表型的特征矢量(X), 并记录对应的分类标签(y)\n",
    "# 由于是有监督学习，用于训练的样本量还是要有一定保障，\n",
    "# 要求在训练集中一个类别至少要有五个以上的样本 (5个样本以下就是小样本学习了)\n",
    "# 对于小样本，尝试使用基于规则的方法来预测其属性，而非用机器学习\n",
    "# 然后合并生成 y_pred 与 y_true进行比较\n",
    "# \n",
    "# 对于Severity而言, 当表型对象是Fever的时候，摄氏度也要算作触发信号\n",
    "# 如果某一个标准取值有多个触发信号，选择(保留)离核心词最近的那个\n",
    "# 如果触发词超过了50个token, 忽略\n",
    "features_of_phenotypes = []\n",
    "labels_of_phenotypes   = []\n",
    "\n",
    "# 对于Assertion属性的每一个具有足够样本量的属性取值\n",
    "# {'Present': 0, 'Possible': 1, 'Conditional': 2, 'Absent': 3}\n",
    "for attribute_value in codes_of_attribute_values:\n",
    "    # 自带了限制，该属性取值的样本数不能少于该属性具有的取值分类数目\n",
    "    # if attribute_value != 'None':\n",
    "    #     # 该属性取值的样本数 < 该属性具有的分类数\n",
    "    #     if present_of_attribute_values[ attribute_value ] < len(present_of_attribute_values):\n",
    "    #         continue\n",
    "\n",
    "    # 观测具有该属性取值的表型信息\n",
    "    for pheno_info in phenotypes_with_target_attribute[attribute_value]:\n",
    "        # 表型名称\n",
    "        pheno_name = pheno_info['pheno_name']\n",
    "        # 表型所在的句子\n",
    "        pheno_text = pheno_info['pheno_context']\n",
    "        # 表型在句子中的位置\n",
    "        pheno_spos, pheno_epos = pheno_info['pheno_pos_in_sent']\n",
    "        pheno_epos = pheno_epos-1\n",
    "\n",
    "        # 表型属性取值的类别标签\n",
    "        # value_code = codes_of_attribute_values[attribute_value]\n",
    "\n",
    "        # 逗号替换为空格，避免它依附在单词上带来干扰 (有TreebankWordTokenizer就不必了)\n",
    "        # pheno_text = pheno_text.replace(',',' ')\n",
    "\n",
    "        # 对句子进行带位置信息的tokenize操作\n",
    "        span_generator = TreebankWordTokenizer().span_tokenize(pheno_text)  \n",
    "\n",
    "        # 对句子的token进行小写化处理和词形还原处理\n",
    "        text_words_cleaned_with_pos = []\n",
    "        text_words_cleaned = []\n",
    "\n",
    "        for span in span_generator:\n",
    "            # token原始形式\n",
    "            raw_word = pheno_text[ span[0]:span[1] ]\n",
    "            # token处理后形式\n",
    "            cleaned_word = wordnet_lemmatizer.lemmatize( raw_word.lower() )\n",
    "            # 记录token原始形式的位置和token处理后形式\n",
    "            text_words_cleaned_with_pos.append( (span[0], span[1], cleaned_word) ) \n",
    "            # 记录token处理后形式\n",
    "            text_words_cleaned.append( cleaned_word )\n",
    "\n",
    "\n",
    "        # 特征矢量构建\n",
    "        # 1. 不同属性取值的触发词的出现情况  [] 维度等于属性取值的数目，排序按code排列\n",
    "        # 2. 如果出现，相对于中心词的位置 (token数)    \n",
    "        # 3. 如果出现，与中心词之间是否存在语义分割?\n",
    "        # 句中的触发信号 出现 1/未出现 0\n",
    "        vector_of_trigger_signal    = [0]*len( codes_of_attribute_values )\n",
    "        # 触发信号到核心词的距离 未出现 对应 -1; 出现 距离的绝对值/100；超过100个token设置为1\n",
    "        # 不能设置为0，因为0是有特殊意义的\n",
    "        vector_of_relative_distance = [-1]*len( codes_of_attribute_values )\n",
    "        # 触发信号与核心词之间有无标点符号  (-)1 触发词未出现  (0)(无标点符号)  (1)(有标点符号)\n",
    "        vector_of_semantic_split    = [0]*len( codes_of_attribute_values )\n",
    "\n",
    "\n",
    "        # 扫描表型上下文中的触发词 (主要是看是哪一个属性取值的触发词)\n",
    "        # 只要找到一个即可？还是所有的触发词都要找到？根据双向扫描的经验，应该是都要找到\n",
    "        present_of_triggers = {}\n",
    "\n",
    "        for raw_trigger_word in mapping_btw_trigger_and_std_value:\n",
    "            # trigger_word小写处理\n",
    "            trigger_word = raw_trigger_word.lower()\n",
    "\n",
    "            # 避免错写的那种异常情况\n",
    "            if trigger_word in ['',' ']:\n",
    "                continue     \n",
    "\n",
    "            # 如果触发词是unigram，那么在 text_words_cleaned 寻找其存在\n",
    "            if ' ' not in trigger_word:\n",
    "                # 观察该triger_word是否在text_words_cleaned(小写化和词形还原处理后)的列表中存在\n",
    "                # 因为可能有多次出现 用 trigger word in/index只能找到1个\n",
    "                for text_word_idx, text_word in enumerate( text_words_cleaned ):                    \n",
    "                    # 避免一种异常情况\n",
    "                    if ',' in text_word:\n",
    "                        text_word = text_word.replace(',','')\n",
    "                    #\n",
    "                    if trigger_word == text_word:\n",
    "                        # 提取 trigger_word 在句子中出现的位置\n",
    "                        trigger_spos = text_words_cleaned_with_pos[ text_word_idx ][0]\n",
    "                        trigger_epos = text_words_cleaned_with_pos[ text_word_idx ][1] -1\n",
    "                        # 将之区间化，并记录到 present_of_triggers 中\n",
    "                        trigger_interval = Interval( trigger_spos, trigger_epos )       \n",
    "                        # \n",
    "                        if trigger_interval not in present_of_triggers:\n",
    "                            # 该触发词触发的标准值\n",
    "                            trigged_value = mapping_btw_trigger_and_std_value[trigger_word]\n",
    "                            # 记录该区间存在的触发词及其触发的标准值\n",
    "                            present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) )                  \n",
    "            # 如果触发词不是unigram, 直接在句子层面匹配(因为这种情况下产生冗余的可能性很小)\n",
    "            else:\n",
    "                # ' ' in trigger_word\n",
    "                # 为了进一步排除干扰，还可以加一个空格 (由多个token组成的，就没有必要加空格了)\n",
    "                # 不然 severe case 遇到severe cases就歇菜了\n",
    "                if trigger_word in pheno_text:\n",
    "                    # 提取 trigger_word 在句子中出现的位置\n",
    "                    trigger_spos = pheno_text.index( trigger_word )\n",
    "                    trigger_epos = trigger_spos + len(trigger_word) - 1  \n",
    "                    # 将之区间化，并记录到 present_of_triggers 中\n",
    "                    trigger_interval = Interval( trigger_spos, trigger_epos ) \n",
    "                    # \n",
    "                    if trigger_interval not in present_of_triggers:\n",
    "                        # 该触发词触发的标准值\n",
    "                        trigged_value = mapping_btw_trigger_and_std_value[trigger_word]\n",
    "                        # 记录该区间存在的触发词及其触发的标准值\n",
    "                        present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) )                                               \n",
    "\n",
    "        # 数值型触发信号\n",
    "        # 对于Frequency属性，数量型触发词的判断\n",
    "        # 数量型触发词 , 用 of, people, case的存在减少假阳性\n",
    "        # 频率数字生效的条件同 present_of_trigger_words (的确可以考虑合并进去)\n",
    "        # 一个问题是 XXX in 26%, XXX ... 这种情况 (还好，长程触发信号无此问题)(没有进行逗号的替换)\n",
    "        # 26% of case has XXX, XXX \n",
    "        # present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) )        \n",
    "        if re.search('\\sof\\s|\\sin\\s|\\speople|\\scase|\\spatient|\\sperson', pheno_text):\n",
    "            # 搜索数值型触发信号\n",
    "            for match_pt in re.finditer( '\\d+(\\.\\d+)?%', pheno_text ):\n",
    "                # 观察该 match_pt去除%后的match_num 能否转化为数字\n",
    "                match_num = match_pt.group().replace('%','')\n",
    "                if match_num.replace( '.','',1 ).isdigit():\n",
    "                    # 拿到频率数字\n",
    "                    freq_num = float( match_num )\n",
    "                    #\n",
    "                    # 频率数字的位置\n",
    "                    trigger_spos = pheno_text.index( match_pt.group() )\n",
    "                    trigger_epos = trigger_spos + len( match_pt.group() ) -1    \n",
    "                    # 将之区间化，并记录到 present_of_triggers 中\n",
    "                    trigger_interval = Interval( trigger_spos, trigger_epos ) \n",
    "                    # 触发词是百分比\n",
    "                    trigger_word = match_pt.group()                     \n",
    "                    #                                      \n",
    "                    # 根据频率数字判断频率取值\n",
    "                    trigged_value = \"\"\n",
    "                    # \n",
    "                    if freq_num == 100:\n",
    "                        trigged_value = \"Frequency:Obligate\"\n",
    "                    elif freq_num >=80 and freq_num <100:\n",
    "                        trigged_value = \"Frequency:Very_frequent\"\n",
    "                    elif freq_num >=30 and freq_num <80:\n",
    "                        trigged_value = \"Frequency:Frequent\"\n",
    "                    elif freq_num >=5 and freq_num <30:\n",
    "                        trigged_value = \"Frequency:Occasional\"\n",
    "                    elif freq_num >=1 and freq_num <5:\n",
    "                        trigged_value = \"Frequency:Very_rare\"\n",
    "                    # 记录该百分比触发信号及其对应触发的属性标准值\n",
    "                    if trigged_value != \"\":\n",
    "                        present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) ) \n",
    "\n",
    "\n",
    "        # 初始触发信号扫描结果 present_of_triggers = {}\n",
    "        # 对找到的触发信号进行去冗余\n",
    "        # present_of_triggers = {}    \n",
    "        # trigger_interval, (trigger_word, std_value)\n",
    "        excluded_trigger_keys = set()\n",
    "\n",
    "        for trigger_interval_A in present_of_triggers:\n",
    "            for trigger_interval_B in present_of_triggers:\n",
    "                if trigger_interval_A != trigger_interval_B:\n",
    "                    # 看区间A或区间B是否被包含?\n",
    "                    if trigger_interval_A in trigger_interval_B:\n",
    "                        excluded_trigger_keys.add( trigger_interval_A ) \n",
    "                    elif trigger_interval_B in trigger_interval_A:\n",
    "                        excluded_trigger_keys.add( trigger_interval_B ) \n",
    "        \n",
    "        # \n",
    "        for excluded_trigger_key in excluded_trigger_keys:\n",
    "            del present_of_triggers[excluded_trigger_key]\n",
    "\n",
    "\n",
    "        # 如果存在该属性的属性值的触发信号\n",
    "        if len( present_of_triggers ) != 0 :    \n",
    "            # 对触发信号根据位置进行排序\n",
    "            sorted_trigger_positions = []\n",
    "            for trigger_interval in present_of_triggers:\n",
    "                sorted_trigger_positions.append( (trigger_interval.lower_bound, trigger_interval.upper_bound) )\n",
    "            sorted_trigger_positions.sort()\n",
    "\n",
    "            # 还原成Interval\n",
    "            sorted_trigger_interval = [ Interval(trigger_pos[0], trigger_pos[1]) for trigger_pos in sorted_trigger_positions]\n",
    "                    \n",
    "\n",
    "            # 从左到右阅读trigger信号\n",
    "            for trigger_interval in sorted_trigger_interval:\n",
    "                # 触发词及其触发的属性标准值\n",
    "                trigger_word, trigged_value = present_of_triggers[trigger_interval]\n",
    "\n",
    "                # 如果是虚假触发信号，跳过    \n",
    "                if 'PseudoTrigger' in trigged_value:\n",
    "                    continue\n",
    "\n",
    "                # 触发信号的起止位置\n",
    "                trigger_spos = trigger_interval.lower_bound\n",
    "                trigger_epos = trigger_interval.upper_bound\n",
    "\n",
    "                # 计算触发信号与核心词之间的距离，以及与核心词之间是否存在语义分割\n",
    "                token_based_distance = -1     # 触发词未出现\n",
    "                with_semantic_split  = False  # 没有语义分割\n",
    "\n",
    "                # 若触发词在表型的左边\n",
    "                if trigger_epos < pheno_spos :\n",
    "                    # 相对距离计算\n",
    "                    # 提取两者之间的文本\n",
    "                    text_btw   = pheno_text[ trigger_epos+1:pheno_spos ]\n",
    "                    # 计数两者之间的token数目 (不包括标点符号在内)\n",
    "                    tokens_btw = word_tokenize(text_btw)\n",
    "                    tokens_btw = [token for token in tokens_btw if token.isalpha()]\n",
    "                    # left \n",
    "                    token_based_distance = len(tokens_btw)\n",
    "                    # 距离归一化\n",
    "                    if token_based_distance <=100:\n",
    "                        token_based_distance = token_based_distance/100\n",
    "                    else:\n",
    "                        token_based_distance = 1\n",
    "                    #\n",
    "                    # 两者之间是否存在语义分割？\n",
    "                    if re.search(',|;|:', text_btw):\n",
    "                        with_semantic_split = True\n",
    "                # 若触发词在表型的右边\n",
    "                elif trigger_spos > pheno_epos:\n",
    "                    # 相对距离计算\n",
    "                    # 提取两者之间的文本\n",
    "                    text_btw   = pheno_text[ pheno_epos+1:trigger_spos ]\n",
    "                    # 计数两者之间的token数目 (不包括标点符号在内)\n",
    "                    tokens_btw = word_tokenize(text_btw)\n",
    "                    tokens_btw = [token for token in tokens_btw if token.isalpha()]\n",
    "                    # right + \n",
    "                    token_based_distance = len(tokens_btw)\n",
    "                    # 距离归一化\n",
    "                    if token_based_distance <=100:\n",
    "                        token_based_distance = token_based_distance/100     \n",
    "                    else:\n",
    "                        token_based_distance = 1                                               \n",
    "                    #\n",
    "                    # 两者之间是否存在语义分割？\n",
    "                    if re.search(',|;|:', text_btw):\n",
    "                        with_semantic_split = True\n",
    "                # 若触发词就位于表型概念中\n",
    "                elif trigger_spos >= pheno_spos and trigger_epos <= pheno_epos:\n",
    "                    # 相对距离设置为0                                                \n",
    "                    token_based_distance = 0\n",
    "                    # 没有语义分割\n",
    "                    with_semantic_split  = False\n",
    "                #\n",
    "                # 触发信号存在，触发信号距离\n",
    "                # token_based_distance = -1     # 触发词未出现\n",
    "                # with_semantic_split  = False  # 没有语义分割                    \n",
    "                # 基于上述计算更新该属性触发信号的存在情况\n",
    "                # 更新特征矢量，同一标准取值有多个特征矢量仅更新最近的那一个信号\n",
    "                # \n",
    "                # 首先找到触发词对应的属性标准取值对应的数字编码\n",
    "                # 只考虑需要预测的属性类型的触发信号(Nocturnal不在其中)\n",
    "                if trigged_value in codes_of_attribute_values:\n",
    "                    # 触发词触发的属性标准值对应的数字编码\n",
    "                    trigged_value_idx = codes_of_attribute_values[ trigged_value ]                    \n",
    "                    # 如果现在还没有该属性值的触发信号\n",
    "                    if vector_of_trigger_signal[ trigged_value_idx ] == 0:\n",
    "                        # 触发信号出现\n",
    "                        vector_of_trigger_signal[ trigged_value_idx ] = 1                    \n",
    "                        # 相对距离\n",
    "                        vector_of_relative_distance[ trigged_value_idx ] = token_based_distance\n",
    "                    else:\n",
    "                        if token_based_distance < vector_of_relative_distance[ trigged_value_idx ]:\n",
    "                            # 触发信号出现\n",
    "                            vector_of_trigger_signal[ trigged_value_idx ] = 1                    \n",
    "                            # 相对距离\n",
    "                            vector_of_relative_distance[ trigged_value_idx ] = token_based_distance                                \n",
    "                \n",
    "\n",
    "        # 特征矢量构建\n",
    "        # vector_of_trigger_signal    = [0]*len( codes_of_attribute_values )\n",
    "        # vector_of_relative_distance = [0]*len( codes_of_attribute_values )\n",
    "        # vector_of_semantic_split    = [0]*len( codes_of_attribute_values )\n",
    "\n",
    "        # 如果在句子中没有找到触发信号，而属性的取值又不是None或Present，这样的数据不要写入\n",
    "        if vector_of_trigger_signal == [0]*len( codes_of_attribute_values ):\n",
    "            # 是阳性样本，而不是None，句子中没有找到触发信号，跳过 \n",
    "            if not re.search('None|Present', attribute_value) :\n",
    "                continue\n",
    "            else:\n",
    "                # 是None样本，写入\n",
    "                # 记录该表型的特征矢量\n",
    "                # features_of_phenotypes.append( vector_of_trigger_signal + vector_of_relative_distance  )\n",
    "                features_of_phenotypes.append( vector_of_trigger_signal + vector_of_relative_distance )\n",
    "                # 记录该表型属性取值的类别标签\n",
    "                labels_of_phenotypes.append( codes_of_attribute_values[attribute_value] )\n",
    "        else:\n",
    "            # 记录该表型的特征矢量\n",
    "            # features_of_phenotypes.append( vector_of_trigger_signal + vector_of_relative_distance  )\n",
    "            features_of_phenotypes.append( vector_of_trigger_signal + vector_of_relative_distance )\n",
    "            # 记录该表型属性取值的类别标签\n",
    "            labels_of_phenotypes.append( codes_of_attribute_values[attribute_value] )                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(111, 111)"
     },
     "metadata": {},
     "execution_count": 310
    }
   ],
   "source": [
    "len( features_of_phenotypes), len( labels_of_phenotypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Counter({0: 61, 1: 45, 2: 5})"
     },
     "metadata": {},
     "execution_count": 311
    }
   ],
   "source": [
    "Counter( labels_of_phenotypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'None': 0, 'Severe_problem': 1, 'Mild_problem': 2}\n"
    }
   ],
   "source": [
    "print(codes_of_attribute_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{0: 'None', 1: 'Severe_problem', 2: 'Mild_problem'}\n"
    }
   ],
   "source": [
    "mapping_btw_codes_and_value = {}\n",
    "\n",
    "for attribute_value in codes_of_attribute_values:\n",
    "    mapping_btw_codes_and_value.setdefault( codes_of_attribute_values[attribute_value], attribute_value )\n",
    "\n",
    "print(mapping_btw_codes_and_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 314,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "([0, 0, 0, -1, -1, -1], 0)"
     },
     "metadata": {},
     "execution_count": 314
    }
   ],
   "source": [
    "features_of_phenotypes[1], labels_of_phenotypes[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对类别中的小类进行上采样，直接复制features_of_phenotypes中的数据即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "added_features_of_phenotypes = []\n",
    "added_labels_of_phenotypes = []\n",
    "\n",
    "# Counter({0: 811, 1: 1844, 2: 48, 3: 13, 4: 4})\n",
    "for value_code in Counter( labels_of_phenotypes ):\n",
    "    value_count = Counter( labels_of_phenotypes )[value_code]\n",
    "    # print(value_code, value_count)\n",
    "    # 如果该属性取值的value_count过少，复制一些数据出来\n",
    "    if value_count <= 20:\n",
    "        # 找到 labels_of_phenotypes 对应的 value_code\n",
    "        if value_code in labels_of_phenotypes:\n",
    "            idx = labels_of_phenotypes.index(value_code)\n",
    "            # 提取特征\n",
    "            vec = features_of_phenotypes[idx]\n",
    "            # 复制20份加入进去\n",
    "            copy_num = 20 - value_count\n",
    "            for i in range(copy_num):\n",
    "                added_features_of_phenotypes.append( vec )\n",
    "                added_labels_of_phenotypes.append( value_code )\n",
    "\n",
    "# added_labels_of_phenotypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Counter({0: 61, 1: 45, 2: 20})"
     },
     "metadata": {},
     "execution_count": 317
    }
   ],
   "source": [
    "# 添加到训练集中\n",
    "features_of_phenotypes += added_features_of_phenotypes\n",
    "labels_of_phenotypes   += added_labels_of_phenotypes\n",
    "\n",
    "Counter( labels_of_phenotypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这种上采样方式应该问题不大，因为从特征层面的抽象角度讲，就是该都差不多的。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建机器学习模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 参数空间\n",
    "param_grid = {'C': [0.1,1, 10, 100], 'gamma': [1,0.1,0.01,0.001],'kernel': ['rbf']}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "CPU times: user 264 ms, sys: 5.34 ms, total: 270 ms\nWall time: 267 ms\n"
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "GridSearchCV(estimator=SVC(),\n             param_grid={'C': [0.1, 1, 10, 100], 'gamma': [1, 0.1, 0.01, 0.001],\n                         'kernel': ['rbf']})"
     },
     "metadata": {},
     "execution_count": 321
    }
   ],
   "source": [
    "%%time\n",
    "grid = GridSearchCV(SVC(),param_grid,refit=True)\n",
    "grid.fit(features_of_phenotypes, labels_of_phenotypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "{'C': 1, 'gamma': 1, 'kernel': 'rbf'}"
     },
     "metadata": {},
     "execution_count": 322
    }
   ],
   "source": [
    "grid.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 记录该模型\n",
    "# 注意 setsetdefault不会改变默认值\n",
    "if target_attribute not in dict_of_trained_classifiers:\n",
    "    dict_of_trained_classifiers.setdefault( target_attribute, grid )\n",
    "else:\n",
    "    dict_of_trained_classifiers[target_attribute] = grid "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 324,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Counter({0: 59, 1: 45, 2: 22})"
     },
     "metadata": {},
     "execution_count": 324
    }
   ],
   "source": [
    "# dict_of_trained_classifiers[target_attribute].best_params_\n",
    "# 训练集上的预测效果 [还是倾向于大类，不过没问题]\n",
    "Counter( list(dict_of_trained_classifiers[target_attribute].predict( features_of_phenotypes )) ) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "{'None': 0, 'Severe_problem': 1, 'Mild_problem': 2}\n"
    }
   ],
   "source": [
    "print(codes_of_attribute_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 326,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Assertion {'C': 0.1, 'gamma': 0.1, 'kernel': 'rbf'}\nSeverity {'C': 100, 'gamma': 1, 'kernel': 'rbf'}\nTemporal {'C': 10, 'gamma': 1, 'kernel': 'rbf'}\nSensation {'C': 0.1, 'gamma': 1, 'kernel': 'rbf'}\nColor {'C': 1, 'gamma': 0.1, 'kernel': 'rbf'}\nAge {'C': 100, 'gamma': 1, 'kernel': 'rbf'}\nGender {'C': 0.1, 'gamma': 1, 'kernel': 'rbf'}\nFrequency {'C': 100, 'gamma': 1, 'kernel': 'rbf'}\nDistribution {'C': 0.1, 'gamma': 1, 'kernel': 'rbf'}\nSOI {'C': 1, 'gamma': 1, 'kernel': 'rbf'}\n"
    }
   ],
   "source": [
    "for attribute_name in dict_of_trained_classifiers:\n",
    "    print( attribute_name, dict_of_trained_classifiers[attribute_name].best_params_ )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试集\n",
    "# 对以上属性进行预测, 将预测结果解码后记录下来\n",
    "# 这个流程是低效的，但作为评估流程尽快拿到数据，能用就行\n",
    "# 应效仿我在深度学习上的方法，那个方法快"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 327,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "'Poliomyelitis'"
     },
     "metadata": {},
     "execution_count": 327
    }
   ],
   "source": [
    "diseases_used_for_test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "CPU times: user 3min 30s, sys: 16.1 s, total: 3min 46s\nWall time: 3min 46s\n"
    }
   ],
   "source": [
    "%%time\n",
    "for disease_name in list( annotated_phenotypes_info_of_diseases.keys() ):\n",
    "    # 仅看训练集中的疾病\n",
    "    if disease_name not in diseases_used_for_test:\n",
    "        continue\n",
    "\n",
    "    # disease_name = 'Poliomyelitis'\n",
    "\n",
    "    # \n",
    "    # 该疾病文档中标注的表型及其关联属性\n",
    "    info_of_annotated_phenotypes = annotated_phenotypes_info_of_diseases[disease_name]\n",
    "\n",
    "    # 对于该疾病文档中标注的每一个表型\n",
    "    for pheno_id in info_of_annotated_phenotypes:\n",
    "        # 获取该变量中存储的表型相关信息\n",
    "        pheno_info = info_of_annotated_phenotypes[pheno_id]\n",
    "\n",
    "\n",
    "        # 设置该表型属性槽的默认值 (同时记录判断依据)\n",
    "        default_values_of_attributes = {}\n",
    "\n",
    "        # 这里其实没有触发信号，但还是这样保留着吧\n",
    "        for attribute_name in list_of_attributes:\n",
    "            if attribute_name == 'Assertion':\n",
    "                default_values_of_attributes.setdefault( attribute_name, ['Present',''] )\n",
    "            else:\n",
    "                default_values_of_attributes.setdefault( attribute_name, ['None',''] )   \n",
    "\n",
    "        #  \n",
    "        # 之前的一个问题 (这种表型的属性槽设置为全部默认即可) 数量不多，9个\n",
    "        if 'pheno_context' not in pheno_info:\n",
    "            # print(disease_name)\n",
    "            if 'pred_attribute_values' not in annotated_phenotypes_info_of_diseases[disease_name][pheno_id]:\n",
    "                annotated_phenotypes_info_of_diseases[disease_name][pheno_id].setdefault( 'pred_attribute_values', \n",
    "                                                                                        default_values_of_attributes )\n",
    "            else:\n",
    "                annotated_phenotypes_info_of_diseases[disease_name][pheno_id]['pred_attribute_values'] =        default_values_of_attributes               \n",
    "            # count += 1\n",
    "            continue     \n",
    "\n",
    "\n",
    "        # 表型的名称\n",
    "        pheno_name = pheno_info['pheno_name']\n",
    "        # 表型所在的句子 (有的表型没有\"pheno_context\") )\n",
    "        pheno_text = pheno_info['pheno_context']\n",
    "        # 表型在句子中的位置 (注意pheno_epos-1)才是表型最后一个字符串的位置\n",
    "        pheno_spos, pheno_epos = pheno_info['pheno_pos_in_sent']     \n",
    "        pheno_epos = pheno_epos-1\n",
    "        # 表型的关联属性\n",
    "        pheno_labels = pheno_info['associated_attributes']  \n",
    "\n",
    "        # 使用机器学习模型根据表型的上下文表征预测属性 (更新default_attribute_values字典)\n",
    "        # 生成该表型的用于机器学习的表征，以便调用模型进行预测\n",
    "        # 注意有两个属性没有机器学习模型\n",
    "        for target_attribute in dict_of_trained_classifiers:\n",
    "            # pheno_features = [] --> pred_label\n",
    "            # 1\n",
    "            # 对目标属性的取值进行数字编码\n",
    "            codes_of_attribute_values = {}\n",
    "\n",
    "            # default的属性取值设置为0\n",
    "            if target_attribute == 'Assertion':\n",
    "                codes_of_attribute_values.setdefault( \"Present\", 0 )\n",
    "            else:\n",
    "                codes_of_attribute_values.setdefault( \"None\", 0 )  \n",
    "\n",
    "\n",
    "            # 观察目标属性在语料集中的出现情况\n",
    "            # 观察目标属性在训练集中的出现情况\n",
    "            present_of_attribute_values = stat_occurences_of_attribute_values(target_attribute, diseases_used_for_training)\n",
    "            # 该属性在训练集中出现的类别数 len( present_of_attribute_values )\n",
    "\n",
    "            # 对出现过的目标属性取值进行数字编码\n",
    "            for attribute_value in present_of_attribute_values:\n",
    "                if attribute_value != 'Present':\n",
    "                    # 赋予编号\n",
    "                    # 只有当该属性值具有的样本数大于类别数时才考虑\n",
    "                    count_of_attribute_value = present_of_attribute_values[ attribute_value ] \n",
    "                    # 不设限制\n",
    "                    if count_of_attribute_value >= 0 :\n",
    "                        code_of_attribute_value = len( codes_of_attribute_values )\n",
    "                        codes_of_attribute_values.setdefault( attribute_value , code_of_attribute_value )  \n",
    "\n",
    "            # \n",
    "            # codes_of_attribute_values = {}\n",
    "            mapping_codes_to_values = {}\n",
    "            for attribute_value in codes_of_attribute_values:\n",
    "                mapping_codes_to_values.setdefault( codes_of_attribute_values[attribute_value], attribute_value )\n",
    "\n",
    "\n",
    "            # 2\n",
    "            # 获取该属性不同属性取值的触发词\n",
    "            # 不考虑default的属性取值\n",
    "            triggers_of_attribute_values = {}\n",
    "\n",
    "            for index, row in df_of_attribute_triggers[ df_of_attribute_triggers['属性英文名称'] == target_attribute ].iterrows():\n",
    "                attribute_name = row['属性英文名称']\n",
    "                attribute_value = row['标准取值']\n",
    "                attribute_triggers = row['触发词']\n",
    "\n",
    "                # 跳过必选属性的默认取值Present\n",
    "                if attribute_value == 'Present':\n",
    "                    continue\n",
    "\n",
    "                # 写入属性取值-触发词变量\n",
    "                if attribute_triggers != '0':\n",
    "                    triggers_of_attribute_values.setdefault( attribute_value, attribute_triggers.split(';') )    \n",
    "\n",
    "\n",
    "            # 3\n",
    "            # 将 triggers_of_attribute_values 转换为 trigger 到 标准取值的形式，便于处理\n",
    "            mapping_btw_trigger_and_std_value = {}\n",
    "\n",
    "            for attribute_value in triggers_of_attribute_values:\n",
    "                for trigger_word in triggers_of_attribute_values[attribute_value]:\n",
    "                    mapping_btw_trigger_and_std_value.setdefault( trigger_word, attribute_value )\n",
    "\n",
    "            # 4\n",
    "            # 对pheno_text进行处理，生成其表征\n",
    "            pheno_features = []\n",
    "\n",
    "            # 对句子进行带位置信息的tokenize操作\n",
    "            span_generator = TreebankWordTokenizer().span_tokenize(pheno_text)  \n",
    "\n",
    "            # 对句子的token进行小写化处理和词形还原处理\n",
    "            text_words_cleaned_with_pos = []\n",
    "            text_words_cleaned = []\n",
    "\n",
    "            for span in span_generator:\n",
    "                # token原始形式\n",
    "                raw_word = pheno_text[ span[0]:span[1] ]\n",
    "                # token处理后形式\n",
    "                cleaned_word = wordnet_lemmatizer.lemmatize( raw_word.lower() )\n",
    "                # 记录token原始形式的位置和token处理后形式\n",
    "                text_words_cleaned_with_pos.append( (span[0], span[1], cleaned_word) ) \n",
    "                # 记录token处理后形式\n",
    "                text_words_cleaned.append( cleaned_word )\n",
    "\n",
    "\n",
    "            # 各分量特征矢量构建\n",
    "            # 1. 不同属性取值的触发词的出现情况  [] 维度等于属性取值的数目，排序按code排列\n",
    "            # 2. 如果出现，相对于中心词的位置 (token数)    \n",
    "            # 3. 如果出现，与中心词之间是否存在语义分割?\n",
    "            # 句中的触发信号 出现 1/未出现 0\n",
    "            vector_of_trigger_signal    = [0]*len( codes_of_attribute_values )\n",
    "            # 触发信号到核心词的距离 未出现 对应 -1; 出现 距离的绝对值/100；超过100个token设置为1\n",
    "            # 不能设置为0，因为0是有特殊意义的\n",
    "            vector_of_relative_distance = [-1]*len( codes_of_attribute_values )\n",
    "            # 触发信号与核心词之间有无标点符号  (-)1 触发词未出现  (0)(无标点符号)  (1)(有标点符号)\n",
    "            vector_of_semantic_split    = [0]*len( codes_of_attribute_values )\n",
    "\n",
    "\n",
    "            # 扫描表型上下文中的触发词 (主要是看是哪一个属性取值的触发词)\n",
    "            # 只要找到一个即可？还是所有的触发词都要找到？根据双向扫描的经验，应该是都要找到\n",
    "            present_of_triggers = {}\n",
    "\n",
    "            for raw_trigger_word in mapping_btw_trigger_and_std_value:\n",
    "                # trigger_word小写处理\n",
    "                trigger_word = raw_trigger_word.lower()\n",
    "\n",
    "                # 避免错写的那种异常情况\n",
    "                if trigger_word in ['',' ']:\n",
    "                    continue     \n",
    "\n",
    "                # 如果触发词是unigram，那么在 text_words_cleaned 寻找其存在\n",
    "                if ' ' not in trigger_word:\n",
    "                    # 观察该triger_word是否在text_words_cleaned(小写化和词形还原处理后)的列表中存在\n",
    "                    # 因为可能有多次出现 用 trigger word in/index只能找到1个\n",
    "                    for text_word_idx, text_word in enumerate( text_words_cleaned ):                    \n",
    "                        # 避免一种异常情况\n",
    "                        if ',' in text_word:\n",
    "                            text_word = text_word.replace(',','')\n",
    "                        #\n",
    "                        if trigger_word == text_word:\n",
    "                            # 提取 trigger_word 在句子中出现的位置\n",
    "                            trigger_spos = text_words_cleaned_with_pos[ text_word_idx ][0]\n",
    "                            trigger_epos = text_words_cleaned_with_pos[ text_word_idx ][1] -1\n",
    "                            # 将之区间化，并记录到 present_of_triggers 中\n",
    "                            trigger_interval = Interval( trigger_spos, trigger_epos )       \n",
    "                            # \n",
    "                            if trigger_interval not in present_of_triggers:\n",
    "                                # 该触发词触发的标准值\n",
    "                                trigged_value = mapping_btw_trigger_and_std_value[trigger_word]\n",
    "                                # 记录该区间存在的触发词及其触发的标准值\n",
    "                                present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) )                  \n",
    "                # 如果触发词不是unigram, 直接在句子层面匹配(因为这种情况下产生冗余的可能性很小)\n",
    "                else:\n",
    "                    # ' ' in trigger_word\n",
    "                    # 为了进一步排除干扰，还可以加一个空格 (由多个token组成的，就没有必要加空格了)\n",
    "                    # 不然 severe case 遇到severe cases就歇菜了\n",
    "                    if trigger_word in pheno_text:\n",
    "                        # 提取 trigger_word 在句子中出现的位置\n",
    "                        trigger_spos = pheno_text.index( trigger_word )\n",
    "                        trigger_epos = trigger_spos + len(trigger_word) - 1  \n",
    "                        # 将之区间化，并记录到 present_of_triggers 中\n",
    "                        trigger_interval = Interval( trigger_spos, trigger_epos ) \n",
    "                        # \n",
    "                        if trigger_interval not in present_of_triggers:\n",
    "                            # 该触发词触发的标准值\n",
    "                            trigged_value = mapping_btw_trigger_and_std_value[trigger_word]\n",
    "                            # 记录该区间存在的触发词及其触发的标准值\n",
    "                            present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) )                                               \n",
    "\n",
    "            # 数量型触发词 , 用 of, people, case的存在减少假阳性\n",
    "            # 频率数字生效的条件同 present_of_trigger_words (的确可以考虑合并进去)\n",
    "            # 一个问题是 XXX in 26%, XXX ... 这种情况 (还好，长程触发信号无此问题)(没有进行逗号的替换)\n",
    "            # 26% of case has XXX, XXX \n",
    "            # present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) )        \n",
    "            if re.search('\\sof\\s|\\sin\\s|\\speople|\\scase|\\spatient|\\sperson', pheno_text):\n",
    "                # 搜索数值型触发信号\n",
    "                for match_pt in re.finditer( '\\d+(\\.\\d+)?%', pheno_text ):\n",
    "                    # 观察该 match_pt去除%后的match_num 能否转化为数字\n",
    "                    match_num = match_pt.group().replace('%','')\n",
    "                    if match_num.replace( '.','',1 ).isdigit():\n",
    "                        # 拿到频率数字\n",
    "                        freq_num = float( match_num )\n",
    "                        #\n",
    "                        # 频率数字的位置\n",
    "                        trigger_spos = pheno_text.index( match_pt.group() )\n",
    "                        trigger_epos = trigger_spos + len( match_pt.group() ) -1    \n",
    "                        # 将之区间化，并记录到 present_of_triggers 中\n",
    "                        trigger_interval = Interval( trigger_spos, trigger_epos ) \n",
    "                        # 触发词是百分比\n",
    "                        trigger_word = match_pt.group()                     \n",
    "                        #                                      \n",
    "                        # 根据频率数字判断频率取值\n",
    "                        trigged_value = \"\"\n",
    "                        # \n",
    "                        if freq_num == 100:\n",
    "                            trigged_value = \"Frequency:Obligate\"\n",
    "                        elif freq_num >=80 and freq_num <100:\n",
    "                            trigged_value = \"Frequency:Very_frequent\"\n",
    "                        elif freq_num >=30 and freq_num <80:\n",
    "                            trigged_value = \"Frequency:Frequent\"\n",
    "                        elif freq_num >=5 and freq_num <30:\n",
    "                            trigged_value = \"Frequency:Occasional\"\n",
    "                        elif freq_num >=1 and freq_num <5:\n",
    "                            trigged_value = \"Frequency:Very_rare\"\n",
    "                        # 记录该百分比触发信号及其对应触发的属性标准值\n",
    "                        if trigged_value != \"\":\n",
    "                            present_of_triggers.setdefault(  trigger_interval, (trigger_word, trigged_value) ) \n",
    "\n",
    "\n",
    "\n",
    "            # 初始触发信号扫描结果 present_of_triggers = {}\n",
    "            # 对找到的触发信号进行去冗余\n",
    "            # present_of_triggers = {}    \n",
    "            # trigger_interval, (trigger_word, std_value)\n",
    "            excluded_trigger_keys = set()\n",
    "\n",
    "            for trigger_interval_A in present_of_triggers:\n",
    "                for trigger_interval_B in present_of_triggers:\n",
    "                    if trigger_interval_A != trigger_interval_B:\n",
    "                        # 看区间A或区间B是否被包含?\n",
    "                        if trigger_interval_A in trigger_interval_B:\n",
    "                            excluded_trigger_keys.add( trigger_interval_A ) \n",
    "                        elif trigger_interval_B in trigger_interval_A:\n",
    "                            excluded_trigger_keys.add( trigger_interval_B ) \n",
    "            \n",
    "            # \n",
    "            for excluded_trigger_key in excluded_trigger_keys:\n",
    "                del present_of_triggers[excluded_trigger_key]\n",
    "\n",
    "\n",
    "            # 如果存在该属性的属性值的触发信号\n",
    "            if len( present_of_triggers ) != 0 :    \n",
    "                # 对触发信号根据位置进行排序\n",
    "                sorted_trigger_positions = []\n",
    "                for trigger_interval in present_of_triggers:\n",
    "                    sorted_trigger_positions.append( (trigger_interval.lower_bound, trigger_interval.upper_bound) )\n",
    "                sorted_trigger_positions.sort()\n",
    "\n",
    "                # 还原成Interval\n",
    "                sorted_trigger_interval = [ Interval(trigger_pos[0], trigger_pos[1]) for trigger_pos in sorted_trigger_positions]\n",
    "                        \n",
    "\n",
    "                # 从左到右阅读trigger信号\n",
    "                for trigger_interval in sorted_trigger_interval:\n",
    "                    # 触发词及其触发的属性标准值\n",
    "                    trigger_word, trigged_value = present_of_triggers[trigger_interval]\n",
    "\n",
    "                    # 如果是虚假触发信号，跳过    \n",
    "                    if 'PseudoTrigger' in trigged_value:\n",
    "                        continue\n",
    "\n",
    "                    # 触发信号的起止位置\n",
    "                    trigger_spos = trigger_interval.lower_bound\n",
    "                    trigger_epos = trigger_interval.upper_bound\n",
    "\n",
    "                    # 计算触发信号与核心词之间的距离，以及与核心词之间是否存在语义分割\n",
    "                    token_based_distance = -1     # 触发词未出现\n",
    "                    with_semantic_split  = False  # 没有语义分割\n",
    "\n",
    "                    # 若触发词在表型的左边\n",
    "                    if trigger_epos < pheno_spos :\n",
    "                        # 相对距离计算\n",
    "                        # 提取两者之间的文本\n",
    "                        text_btw   = pheno_text[ trigger_epos+1:pheno_spos ]\n",
    "                        # 计数两者之间的token数目 (不包括标点符号在内)\n",
    "                        tokens_btw = word_tokenize(text_btw)\n",
    "                        tokens_btw = [token for token in tokens_btw if token.isalpha()]\n",
    "                        # left \n",
    "                        token_based_distance = len(tokens_btw)\n",
    "                        # 距离归一化\n",
    "                        if token_based_distance <=100:\n",
    "                            token_based_distance = token_based_distance/100\n",
    "                        else:\n",
    "                            token_based_distance = 1\n",
    "                        #\n",
    "                        # 两者之间是否存在语义分割？\n",
    "                        if re.search(',|;|:', text_btw):\n",
    "                            with_semantic_split = True\n",
    "                    # 若触发词在表型的右边\n",
    "                    elif trigger_spos > pheno_epos:\n",
    "                        # 相对距离计算\n",
    "                        # 提取两者之间的文本\n",
    "                        text_btw   = pheno_text[ pheno_epos+1:trigger_spos ]\n",
    "                        # 计数两者之间的token数目 (不包括标点符号在内)\n",
    "                        tokens_btw = word_tokenize(text_btw)\n",
    "                        tokens_btw = [token for token in tokens_btw if token.isalpha()]\n",
    "                        # right + \n",
    "                        token_based_distance = len(tokens_btw)\n",
    "                        # 距离归一化\n",
    "                        if token_based_distance <=100:\n",
    "                            token_based_distance = token_based_distance/100     \n",
    "                        else:\n",
    "                            token_based_distance = 1                                               \n",
    "                        #\n",
    "                        # 两者之间是否存在语义分割？\n",
    "                        if re.search(',|;|:', text_btw):\n",
    "                            with_semantic_split = True\n",
    "                    # 若触发词就位于表型概念中\n",
    "                    elif trigger_spos >= pheno_spos and trigger_epos <= pheno_epos:\n",
    "                        # 相对距离设置为0                                                \n",
    "                        token_based_distance = 0\n",
    "                        # 没有语义分割\n",
    "                        with_semantic_split  = False\n",
    "                    #\n",
    "                    # 触发信号存在，触发信号距离\n",
    "                    # token_based_distance = -1     # 触发词未出现\n",
    "                    # with_semantic_split  = False  # 没有语义分割                    \n",
    "                    # 基于上述计算更新该属性触发信号的存在情况\n",
    "                    # 更新特征矢量，同一标准取值有多个特征矢量仅更新最近的那一个信号\n",
    "                    # \n",
    "                    # 首先找到触发词对应的属性标准取值对应的数字编码\n",
    "                    # 只考虑需要预测的属性类型的触发信号(Nocturnal不在其中)\n",
    "                    if trigged_value in codes_of_attribute_values:\n",
    "                        # 触发词触发的属性标准值对应的数字编码\n",
    "                        trigged_value_idx = codes_of_attribute_values[ trigged_value ]                    \n",
    "                        # 如果现在还没有该属性值的触发信号\n",
    "                        if vector_of_trigger_signal[ trigged_value_idx ] == 0:\n",
    "                            # 触发信号出现\n",
    "                            vector_of_trigger_signal[ trigged_value_idx ] = 1                    \n",
    "                            # 相对距离\n",
    "                            vector_of_relative_distance[ trigged_value_idx ] = token_based_distance\n",
    "                        else:\n",
    "                            if token_based_distance < vector_of_relative_distance[ trigged_value_idx ]:\n",
    "                                # 触发信号出现\n",
    "                                vector_of_trigger_signal[ trigged_value_idx ] = 1                    \n",
    "                                # 相对距离\n",
    "                                vector_of_relative_distance[ trigged_value_idx ] = token_based_distance\n",
    "\n",
    "            # 4\n",
    "            # 特征矢量拼接\n",
    "            # vector_of_trigger_signal    = [0]*len( codes_of_attribute_values )\n",
    "            # vector_of_relative_distance = [0]*len( codes_of_attribute_values )\n",
    "            # vector_of_semantic_split    = [0]*len( codes_of_attribute_values )\n",
    "            pheno_features = vector_of_trigger_signal + vector_of_relative_distance\n",
    "            \n",
    "\n",
    "            # 5\n",
    "            # 根据特征矢量进行预测\n",
    "            pred_value_code = dict_of_trained_classifiers[target_attribute].predict( [pheno_features] )[0]\n",
    "            # print( target_attribute, pheno_features, pred_value_code)\n",
    "\n",
    "            # \n",
    "            pred_value_name =  mapping_codes_to_values[pred_value_code]\n",
    "\n",
    "            # 更新 default 属性槽\n",
    "            default_values_of_attributes[target_attribute][0] = pred_value_name\n",
    "\n",
    "        # 默认属性槽全部更新完后\n",
    "        # 记录更改后的 default_values_of_attributes 到该表型的信息种\n",
    "        # 为 annotated_phenotypes_info_of_diseases 增加一个 pred_attribute_values key\n",
    "        if 'pred_attribute_values' not in annotated_phenotypes_info_of_diseases[disease_name][pheno_id]:\n",
    "            annotated_phenotypes_info_of_diseases[disease_name][pheno_id].setdefault( 'pred_attribute_values', \n",
    "                                                                                    default_values_of_attributes )\n",
    "        else:\n",
    "            annotated_phenotypes_info_of_diseases[disease_name][pheno_id]['pred_attribute_values'] = default_values_of_attributes                           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [],
   "source": [
    "# annotated_phenotypes_info_of_diseases[disease_name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试集评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 330,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 所有/单个属性的预测准确率 (accuracy)\n",
    "# 所有属性带权重的预测准确率 (weighted accuracy) -- 这里没有单个的说法了\n",
    "\n",
    "# 对于不同的属性\n",
    "# {\"Severity\":[表型的属性列表]}\n",
    "y_preds = {}\n",
    "y_trues = {}\n",
    "\n",
    "for attribute in list_of_attributes:\n",
    "    y_preds.setdefault(attribute,[])\n",
    "    y_trues.setdefault(attribute,[])\n",
    "\n",
    "# \n",
    "for disease_name in list( annotated_phenotypes_info_of_diseases.keys() ):\n",
    "    # 仅看测试集中的疾病\n",
    "    if disease_name not in diseases_used_for_test:\n",
    "        continue\n",
    "\n",
    "    # print(disease_name)\n",
    "\n",
    "    # 该疾病文档中标注的表型及其关联属性\n",
    "    info_of_annotated_phenotypes = annotated_phenotypes_info_of_diseases[disease_name]\n",
    "\n",
    "    # 对于该疾病文档中标注的每一个表型\n",
    "    for pheno_id in info_of_annotated_phenotypes:\n",
    "        # 获取该变量中存储的表型相关信息\n",
    "        pheno_info = info_of_annotated_phenotypes[pheno_id]\n",
    "\n",
    "        # 该表型人工标注的属性\n",
    "        string_of_true_attribute_values = pheno_info['associated_attributes']   \n",
    "        # 如果人工标注的属性中有Stage:Stage_1, 去除\n",
    "        true_attribute_values = []\n",
    "        \n",
    "        for attribute_value in string_of_true_attribute_values.split(';'):\n",
    "            if 'Stage:Stage_' not in attribute_value:\n",
    "                true_attribute_values.append( attribute_value )\n",
    "\n",
    "\n",
    "        # 写入到 y_trues \n",
    "        for attribute_name in y_trues:\n",
    "            # 这里只有可选属性的时候会触发\n",
    "            if attribute_name not in string_of_true_attribute_values:\n",
    "                # 设为 attribute_name +':' + 'None'\n",
    "                y_trues[attribute_name].append('None')\n",
    "            else:\n",
    "                # 如果 attribute_name 存在, 直接使用 true_attribute_values 中的值\n",
    "                for attribute_value in true_attribute_values:\n",
    "                    selected_attribute, selected_value = attribute_value.split(':')\n",
    "                    #\n",
    "                    if attribute_name == selected_attribute:\n",
    "                        y_trues[attribute_name].append( selected_value )\n",
    "\n",
    "\n",
    "        # 对应的，该表型算法预测的属性\n",
    "        pred_attribute_values = pheno_info['pred_attribute_values']     \n",
    "\n",
    "        # 写入到 y_preds \n",
    "        for attribute_name in y_preds:\n",
    "            # 这条规则是不会触发的\n",
    "            if attribute_name not in pred_attribute_values:\n",
    "                # 设为 attribute_name +':' + 'None'\n",
    "                y_preds[attribute_name].append('None')    \n",
    "            else:\n",
    "                # 如果 attribute_name 存在\n",
    "                for selected_attribute in pred_attribute_values:\n",
    "                    selected_value = pred_attribute_values[selected_attribute][0]\n",
    "                    #\n",
    "                    if attribute_name == selected_attribute:\n",
    "                        y_preds[attribute_name].append( selected_value )                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "\n*******\n属性名称: Assertion\n预测正确率: 1104 727 0.659\n0.659 (727/1104)\n\n*******\n属性名称: Severity\n预测正确率: 1104 930 0.842\n0.842 (930/1104)\n\n*******\n属性名称: Temporal\n预测正确率: 1104 957 0.867\n0.867 (957/1104)\n\n*******\n属性名称: Sensation\n预测正确率: 1104 1036 0.938\n0.938 (1036/1104)\n\n*******\n属性名称: Color\n预测正确率: 1104 985 0.892\n0.892 (985/1104)\n\n*******\n属性名称: Age\n预测正确率: 1104 1062 0.962\n0.962 (1062/1104)\n\n*******\n属性名称: Gender\n预测正确率: 1104 1090 0.987\n0.987 (1090/1104)\n\n*******\n属性名称: Frequency\n预测正确率: 1104 856 0.775\n0.775 (856/1104)\n\n*******\n属性名称: Distribution\n预测正确率: 1104 1049 0.95\n0.950 (1049/1104)\n\n*******\n属性名称: SOI\n预测正确率: 1104 1089 0.986\n0.986 (1089/1104)\n"
    }
   ],
   "source": [
    "# 根据 y_preds 和 y_trues 计算各个属性预测的准确率\n",
    "# 不同属性预测正确的数目\n",
    "\n",
    "for attribute_name in y_trues:\n",
    "    # 该属性预测正确的计数\n",
    "    count_correct_pred_all = 0\n",
    "    # 该属性各分类预测正确的计数\n",
    "    count_correct_pred_eachs = {}\n",
    "\n",
    "    true_labels = y_trues[attribute_name]\n",
    "    pred_labels = y_preds[attribute_name]\n",
    "\n",
    "    # 计算一致性\n",
    "    for true_label, pred_label in zip( true_labels, pred_labels ):\n",
    "        if true_label == pred_label:\n",
    "            count_correct_pred_all +=1\n",
    "            # \n",
    "            if true_label not in count_correct_pred_eachs:\n",
    "                count_correct_pred_eachs.setdefault( true_label, 1 )\n",
    "            else:\n",
    "                count_correct_pred_eachs[true_label] +=1\n",
    "    \n",
    "    # \n",
    "    print(\"\\n*******\")\n",
    "    print(\"属性名称:\", attribute_name )\n",
    "    print(\"预测正确率:\", len(true_labels), count_correct_pred_all, round( count_correct_pred_all/len(true_labels), 3) )\n",
    "    print( '%.3f'%( count_correct_pred_all/len(true_labels) ), \n",
    "            '(' + str(count_correct_pred_all) + '/' + str(len(true_labels) ) + ')' )    \n",
    "    # 需要知道各分类的数目, 用Counter来统计各分类\n",
    "    # print(\"各分类预测正确率:\")\n",
    "    # result = Counter(true_labels)\n",
    "    # for label_name in result:\n",
    "    #     label_count = result[label_name]\n",
    "    #     correct_pred = count_correct_pred_eachs[label_name]\n",
    "    #     print( label_count, correct_pred, round( correct_pred/label_count, 2) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算测试集上各属性预测的weighted accuracy (它比宏平均更好)\n",
    "# 核心指标\n",
    "# overall evaluation metrics for task 2a\n",
    "# accuracy\n",
    "# weighted accuracy\n",
    "# per-slot weighted accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "4019"
     },
     "metadata": {},
     "execution_count": 333
    }
   ],
   "source": [
    "# 维基语料集中总的标注的表型实体数量\n",
    "total_annotated_phenotypes = 0\n",
    "\n",
    "counts_for_assertion_values  = stat_occurences_of_attribute_values('Assertion', list_of_diseases)\n",
    "for assertion_value in counts_for_assertion_values:\n",
    "    total_annotated_phenotypes += counts_for_assertion_values[assertion_value]\n",
    "\n",
    "total_annotated_phenotypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每一个属性取值的prevalence\n",
    "prevalences_of_attribute_values = {}\n",
    "\n",
    "for attribute_name in list_of_attributes:\n",
    "    counts_for_attribute_values = stat_occurences_of_attribute_values(attribute_name, list_of_diseases)\n",
    "\n",
    "    tmpdict = {}\n",
    "\n",
    "    sum_of_values = 0 \n",
    "    for attribute_value in counts_for_attribute_values:\n",
    "        #\n",
    "        count_of_value = counts_for_attribute_values[attribute_value]\n",
    "        sum_of_values += count_of_value\n",
    "        #\n",
    "        value_prevalence = count_of_value/total_annotated_phenotypes\n",
    "        tmpdict.setdefault( attribute_value, value_prevalence )\n",
    "    \n",
    "    # 对于None这种情况 (可选属性都有None)\n",
    "    if attribute_name != 'Assertion':\n",
    "        count_of_none = total_annotated_phenotypes - sum_of_values\n",
    "        value_prevalence = count_of_none/total_annotated_phenotypes\n",
    "        tmpdict.setdefault( 'None', value_prevalence )\n",
    "    \n",
    "    #\n",
    "    prevalences_of_attribute_values.setdefault( attribute_name, tmpdict )\n",
    "\n",
    "# prevalences_of_attribute_values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "# overall_accuracy  = 0\n",
    "# overall_weighted_acc = 0\n",
    "\n",
    "list_of_unweighted_accuracy_for_phenotypes = []\n",
    "list_of_weighted_accuracy_for_phenotypes   = []\n",
    "\n",
    "# 每一个属性槽的weighted acc\n",
    "dict_of_weighted_accuracy_for_slots = {}\n",
    "\n",
    "# 对于4020个表型，每一个属性，其取值有一个权重，记录为[]; 这一取值是否正确，记录一个 weighted identify\n",
    "# 然后分别求和，然后可计算\n",
    "for attribute_name in list_of_attributes:\n",
    "    dict_of_weighted_accuracy_for_slots.setdefault( attribute_name, ([],[]) )\n",
    "\n",
    "\n",
    "for disease_name in list( annotated_phenotypes_info_of_diseases.keys() ):\n",
    "    # 仅看测试集中的疾病\n",
    "    if disease_name not in diseases_used_for_test:\n",
    "        continue\n",
    "\n",
    "    # print(disease_name)\n",
    "\n",
    "    # 该疾病文档中标注的表型及其关联属性\n",
    "    info_of_annotated_phenotypes = annotated_phenotypes_info_of_diseases[disease_name]\n",
    "\n",
    "    # 对于该疾病文档中标注的每一个表型\n",
    "    for pheno_id in info_of_annotated_phenotypes:\n",
    "        # 获取该变量中存储的表型相关信息\n",
    "        pheno_info = info_of_annotated_phenotypes[pheno_id]\n",
    "\n",
    "        # \n",
    "        # 该表型人工标注的属性\n",
    "        string_of_true_attribute_values = pheno_info['associated_attributes']   \n",
    "\n",
    "        # 仅考虑 list_of_attributes 中的属性 (12个)\n",
    "        # 'Assertion:Possible'\n",
    "        true_attribute_values = {}\n",
    "        \n",
    "        # 对于这12个属性\n",
    "        for attribute_name in list_of_attributes:\n",
    "            # 观察它们是否存在于人工标注中\n",
    "            # 如果存在，直接赋予人工标注的值\n",
    "            if attribute_name + ':' in string_of_true_attribute_values:\n",
    "                # 将这个值找出来\n",
    "                for selected_attribute_value in string_of_true_attribute_values.split(';'):\n",
    "                    selected_attribute, selected_value = selected_attribute_value.split(':')\n",
    "                    if selected_attribute == attribute_name:\n",
    "                        true_attribute_values.setdefault(attribute_name, selected_value)\n",
    "                        break\n",
    "            # 如果不存在，赋予默认值 (Present or None)\n",
    "            elif attribute_name + ':' not in string_of_true_attribute_values:\n",
    "                if attribute_name == 'Assertion':\n",
    "                    true_attribute_values.setdefault(attribute_name, 'Present')\n",
    "                else:\n",
    "                    true_attribute_values.setdefault(attribute_name, 'None')\n",
    "\n",
    "\n",
    "        #\n",
    "        # 该表型算法预测的属性\n",
    "        # 原始的pred_attribute_values包含了 detected cue word , 可以去掉\n",
    "        pred_attribute_values = {}\n",
    "\n",
    "        for attribute_name in pheno_info['pred_attribute_values']:\n",
    "            predicted_value = pheno_info['pred_attribute_values'][attribute_name][0]\n",
    "            pred_attribute_values.setdefault( attribute_name, predicted_value )\n",
    "\n",
    "        \n",
    "        # \n",
    "        # true_attribute_values = {}\n",
    "        # pred_attribute_values = {}\n",
    "\n",
    "        # \n",
    "        # 先考虑 per-phenotype unweighted accuracy\n",
    "        # per-phenotype\n",
    "        # K : number of slots\n",
    "        number_of_slots = len( list_of_attributes )\n",
    "\n",
    "        per_phenotype_unweighted_accuracy = 0\n",
    "        for attribute_name in list_of_attributes:\n",
    "            # identify function : I(true_for_slot_k, pred_for_slot_k)\n",
    "            if true_attribute_values[attribute_name] == pred_attribute_values[attribute_name]:\n",
    "                per_phenotype_unweighted_accuracy += 1\n",
    "            else:\n",
    "                per_phenotype_unweighted_accuracy += 0\n",
    "        \n",
    "        # 记录该表型的 identify score\n",
    "        list_of_unweighted_accuracy_for_phenotypes.append( per_phenotype_unweighted_accuracy/number_of_slots )\n",
    "\n",
    "        \n",
    "        #\n",
    "        # 再考虑 per-phenotype unweighted accuracy\n",
    "        # 就是 Identify 打分要带上该属性值的权重了\n",
    "        # 12个属性槽上取值的权重之和 \n",
    "        per_phenotype_weighted_identify = 0\n",
    "        sum_of_weights_on_slots = 0\n",
    "\n",
    "        for attribute_name in list_of_attributes:\n",
    "            # 该属性对应的取值\n",
    "            true_selected_value = true_attribute_values[attribute_name]\n",
    "            # 该属性取值的权重\n",
    "            weight_of_attribute_value = 1 - prevalences_of_attribute_values[attribute_name][true_selected_value]\n",
    "            # 这些属性取值对应的权重之和\n",
    "            sum_of_weights_on_slots += weight_of_attribute_value\n",
    "\n",
    "            # identify function : I(true_for_slot_k, pred_for_slot_k)\n",
    "            if true_attribute_values[attribute_name] == pred_attribute_values[attribute_name]:\n",
    "                # 引入该属性值的权重\n",
    "                per_phenotype_weighted_identify += 1*weight_of_attribute_value\n",
    "            else:\n",
    "                per_phenotype_weighted_identify += 0*weight_of_attribute_value    \n",
    "\n",
    "        # 记录该表型的 weighted identify score\n",
    "        list_of_weighted_accuracy_for_phenotypes.append( per_phenotype_weighted_identify/sum_of_weights_on_slots )\n",
    "\n",
    "\n",
    "        #\n",
    "        # for per-slot accuracy\n",
    "        # 对于该表型的每一个属性槽\n",
    "        for attribute_name in list_of_attributes:\n",
    "            # 该属性对应的取值\n",
    "            true_selected_value = true_attribute_values[attribute_name]\n",
    "            # 该属性取值的权重\n",
    "            weight_of_attribute_value = 1 - prevalences_of_attribute_values[attribute_name][true_selected_value]\n",
    "            # 该属性取值的identify score\n",
    "            identify_of_attribute_value = 0\n",
    "\n",
    "            # identify function : I(true_for_slot_k, pred_for_slot_k)\n",
    "            if true_attribute_values[attribute_name] == pred_attribute_values[attribute_name]:\n",
    "                # 引入该属性值的权重\n",
    "                identify_of_attribute_value = 1*weight_of_attribute_value\n",
    "            else:\n",
    "                identify_of_attribute_value = 0*weight_of_attribute_value  \n",
    "\n",
    "            # 记录该表型每一个属性槽上的加权得分\n",
    "            # identify_of_attribute_value += 1*weight_of_attribute_value 其实就是1和0\n",
    "            # 属性取值的权重\n",
    "            dict_of_weighted_accuracy_for_slots[attribute_name][1].append( weight_of_attribute_value )\n",
    "            # 取值正确或错误的加权打分\n",
    "            dict_of_weighted_accuracy_for_slots[attribute_name][0].append( identify_of_attribute_value )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(1104, 1104)"
     },
     "metadata": {},
     "execution_count": 336
    }
   ],
   "source": [
    "len( list_of_weighted_accuracy_for_phenotypes ), len( list_of_unweighted_accuracy_for_phenotypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 337,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0.8859601449275284"
     },
     "metadata": {},
     "execution_count": 337
    }
   ],
   "source": [
    "# unweighted acc\n",
    "sum( list_of_unweighted_accuracy_for_phenotypes )/len( list_of_unweighted_accuracy_for_phenotypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 338,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "0.7745842908604559"
     },
     "metadata": {},
     "execution_count": 338
    }
   ],
   "source": [
    "# weighted acc\n",
    "sum( list_of_weighted_accuracy_for_phenotypes )/len( list_of_weighted_accuracy_for_phenotypes )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "Assertion 0.676\nSeverity 0.891\nTemporal 0.759\nSensation 0.888\nColor 0.794\nAge 0.823\nGender 0.964\nFrequency 0.765\nDistribution 0.909\nSOI 0.788\n"
    }
   ],
   "source": [
    "# per-slot weighted accuracy\n",
    "for attribute_name in list_of_attributes:\n",
    "    # dict_of_weighted_accuracy_for_slots[attribute_name]\n",
    "    list_of_identity, list_of_weights = dict_of_weighted_accuracy_for_slots[attribute_name]\n",
    "    print( attribute_name, '%.3f'%(sum(list_of_identity)/sum(list_of_weights)) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}