create_COVIDx_binary.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import random \n",
    "from shutil import copyfile\n",
    "import pydicom as dicom\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set parameters here\n",
    "savepath = 'data'\n",
    "seed = 0\n",
    "np.random.seed(seed) # Reset the seed so all runs are the same.\n",
    "random.seed(seed)\n",
    "MAXVAL = 255  # Range [0 255]\n",
    "\n",
    "# path to covid-19 dataset from https://github.com/ieee8023/covid-chestxray-dataset\n",
    "cohen_imgpath = '../covid-chestxray-dataset/images' \n",
    "cohen_csvpath = '../covid-chestxray-dataset/metadata.csv'\n",
    "\n",
    "# path to covid-19 dataset from https://github.com/agchung/Figure1-COVID-chestxray-dataset\n",
    "fig1_imgpath = '../Figure1-COVID-chestxray-dataset/images'\n",
    "fig1_csvpath = '../Figure1-COVID-chestxray-dataset/metadata.csv'\n",
    "\n",
    "# path to covid-19 dataset from https://github.com/agchung/Actualmed-COVID-chestxray-dataset\n",
    "actmed_imgpath = '../Actualmed-COVID-chestxray-dataset/images'\n",
    "actmed_csvpath = '../Actualmed-COVID-chestxray-dataset/metadata.csv'\n",
    "\n",
    "# path to covid-19 dataset from https://www.kaggle.com/tawsifurrahman/covid19-radiography-database\n",
    "sirm_imgpath = '../COVID-19-Radiography-Database/COVID'\n",
    "sirm_csvpath = '../COVID-19-Radiography-Database/COVID.metadata.xlsx'\n",
    "\n",
    "# path to https://www.kaggle.com/c/rsna-pneumonia-detection-challenge\n",
    "rsna_datapath = '../rsna-pneumonia-detection-challenge'\n",
    "# get all the normal from here\n",
    "rsna_csvname = 'stage_2_detailed_class_info.csv' \n",
    "# get all the 1s from here since 1 indicate pneumonia\n",
    "# found that images that aren't pneunmonia and also not normal are classified as 0s\n",
    "rsna_csvname2 = 'stage_2_train_labels.csv' \n",
    "rsna_imgpath = 'stage_2_train_images'\n",
    "\n",
    "# parameters for COVIDx dataset\n",
    "train = []\n",
    "test = []\n",
    "test_count = {'negative': 0,'normal': 0, 'pneumonia': 0, 'COVID-19': 0}\n",
    "train_count = {'negative': 0,'normal': 0, 'pneumonia': 0, 'COVID-19': 0}\n",
    "\n",
    "mapping = dict()\n",
    "mapping['COVID-19'] = 'COVID-19'\n",
    "mapping['SARS'] = 'pneumonia'\n",
    "mapping['MERS'] = 'pneumonia'\n",
    "mapping['Streptococcus'] = 'pneumonia'\n",
    "mapping['Klebsiella'] = 'pneumonia'\n",
    "mapping['Chlamydophila'] = 'pneumonia'\n",
    "mapping['Legionella'] = 'pneumonia'\n",
    "mapping['E.Coli'] = 'pneumonia'\n",
    "mapping['Normal'] = 'normal'\n",
    "mapping['Lung Opacity'] = 'pneumonia'\n",
    "mapping['1'] = 'pneumonia'\n",
    "\n",
    "# train/test split\n",
    "split = 0.1\n",
    "\n",
    "# to avoid duplicates\n",
    "patient_imgpath = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adapted from https://github.com/mlmed/torchxrayvision/blob/master/torchxrayvision/datasets.py#L814\n",
    "cohen_csv = pd.read_csv(cohen_csvpath, nrows=None)\n",
    "#idx_pa = csv[\"view\"] == \"PA\"  # Keep only the PA view\n",
    "views = [\"PA\", \"AP\", \"AP Supine\", \"AP semi erect\", \"AP erect\"]\n",
    "cohen_idx_keep = cohen_csv.view.isin(views)\n",
    "cohen_csv = cohen_csv[cohen_idx_keep]\n",
    "\n",
    "fig1_csv = pd.read_csv(fig1_csvpath, encoding='ISO-8859-1', nrows=None)\n",
    "actmed_csv = pd.read_csv(actmed_csvpath, nrows=None)\n",
    "\n",
    "sirm_csv = pd.read_excel(sirm_csvpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data distribution from covid datasets:\n",
      "{'negative': 373, 'normal': 0, 'pneumonia': 57, 'COVID-19': 1770}\n"
     ]
    }
   ],
   "source": [
    "# get non-COVID19 viral, bacteria, and COVID-19 infections from covid-chestxray-dataset, figure1 and actualmed\n",
    "# stored as patient id, image filename and label\n",
    "filename_label = {'negative':[],'normal': [], 'pneumonia': [], 'COVID-19': []}\n",
    "count = {'negative':0,'normal': 0, 'pneumonia': 0, 'COVID-19': 0}\n",
    "covid_ds = {'cohen': [], 'fig1': [], 'actmed': [], 'sirm': []}\n",
    "\n",
    "for index, row in cohen_csv.iterrows():\n",
    "    f = row['finding'].split('/')[-1] # take final finding in hierarchy, for the case of COVID-19, ARDS\n",
    "    if f in mapping: # \n",
    "        count[mapping[f]] += 1\n",
    "        entry = [str(row['patientid']), row['filename'], mapping[f], 'cohen']\n",
    "        filename_label[mapping[f]].append(entry)\n",
    "        if mapping[f] == 'COVID-19':\n",
    "            covid_ds['cohen'].append(str(row['patientid']))\n",
    "    else:\n",
    "        count['negative'] += 1\n",
    "        entry = [str(row['patientid']), row['filename'], 'negative', 'cohen']\n",
    "        filename_label['negative'].append(entry)\n",
    "        \n",
    "for index, row in fig1_csv.iterrows():\n",
    "    if not str(row['finding']) == 'nan':\n",
    "        f = row['finding'].split(',')[0] # take the first finding\n",
    "        if f in mapping: # \n",
    "            count[mapping[f]] += 1\n",
    "            if os.path.exists(os.path.join(fig1_imgpath, row['patientid'] + '.jpg')):\n",
    "                entry = [row['patientid'], row['patientid'] + '.jpg', mapping[f], 'fig1']\n",
    "            elif os.path.exists(os.path.join(fig1_imgpath, row['patientid'] + '.png')):\n",
    "                entry = [row['patientid'], row['patientid'] + '.png', mapping[f], 'fig1']\n",
    "            filename_label[mapping[f]].append(entry)\n",
    "            if mapping[f] == 'COVID-19':\n",
    "                covid_ds['fig1'].append(row['patientid'])\n",
    "\n",
    "for index, row in actmed_csv.iterrows():\n",
    "    if not str(row['finding']) == 'nan':\n",
    "        f = row['finding'].split(',')[0]\n",
    "        if f in mapping:\n",
    "            count[mapping[f]] += 1\n",
    "            entry = [row['patientid'], row['imagename'], mapping[f], 'actmed']\n",
    "            filename_label[mapping[f]].append(entry)\n",
    "            if mapping[f] == 'COVID-19':\n",
    "                covid_ds['actmed'].append(row['patientid'])\n",
    "        else:\n",
    "            count['negative'] += 1\n",
    "            entry = [row['patientid'], row['imagename'], 'negative', 'actmed']\n",
    "            filename_label['negative'].append(entry)\n",
    "    \n",
    "sirm = set(sirm_csv['URL'])\n",
    "cohen = set(cohen_csv['url'])\n",
    "discard = ['100', '101', '102', '103', '104', '105', \n",
    "           '110', '111', '112', '113', '122', '123', \n",
    "           '124', '125', '126', '217']\n",
    "\n",
    "for idx, row in sirm_csv.iterrows():\n",
    "    patientid = row['FILE NAME']\n",
    "    if row['URL'] not in cohen and patientid[patientid.find('(')+1:patientid.find(')')] not in discard:\n",
    "        count[mapping['COVID-19']] += 1\n",
    "        imagename = patientid + '.' + row['FORMAT'].lower()\n",
    "        if not os.path.exists(os.path.join(sirm_imgpath, imagename)):\n",
    "            imagename = \"COVID ({}).png\".format(imagename.rsplit(\".png\")[0].split(\"COVID \")[1])\n",
    "        entry = [patientid, imagename, mapping['COVID-19'], 'sirm']\n",
    "        filename_label[mapping['COVID-19']].append(entry)\n",
    "        covid_ds['sirm'].append(patientid)\n",
    "    \n",
    "print('Data distribution from covid datasets:')\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a random sample of 20 'negative' actmed patients to be added to the test set,\n",
    "# the rest will be added to the train set\n",
    "actmed_patients = []\n",
    "duplicates = []\n",
    "\n",
    "for entry in filename_label['negative']:\n",
    "    if entry[3] == 'actmed':\n",
    "        if entry[0] not in actmed_patients:\n",
    "            actmed_patients.append(entry[0])\n",
    "        else:\n",
    "            duplicates.append(entry[0])\n",
    "\n",
    "for num in duplicates:\n",
    "    if num in actmed_patients:\n",
    "        actmed_patients.remove(num)\n",
    "    \n",
    "test_patients_actmed = random.sample(actmed_patients, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Key:  negative\n",
      "Test patients:  ['ANON148', 'ANON6', 'ANON152', 'ANON93', 'ANON2', 'ANON193', 'ANON156', 'ANON28', 'ANON143', 'ANON186', 'ANON15', 'ANON65', 'ANON128', 'ANON168', 'ANON120', 'ANON194', 'ANON216', 'ANON131', 'ANON175', 'ANON141']\n",
      "Key:  pneumonia\n",
      "Test patients:  ['8', '31']\n",
      "Key:  COVID-19\n",
      "Test patients:  ['19', '20', '36', '42', '86', '94', '97', '117', '132', '138', '144', '150', '163', '169', '174', '175', '179', '190', '191COVID-00024', 'COVID-00025', 'COVID-00026', 'COVID-00027', 'COVID-00029', 'COVID-00030', 'COVID-00032', 'COVID-00033', 'COVID-00035', 'COVID-00036', 'COVID-00037', 'COVID-00038', 'ANON24', 'ANON45', 'ANON126', 'ANON106', 'ANON67', 'ANON153', 'ANON135', 'ANON44', 'ANON29', 'ANON201', 'ANON191', 'ANON234', 'ANON110', 'ANON112', 'ANON73', 'ANON220', 'ANON189', 'ANON30', 'ANON53', 'ANON46', 'ANON218', 'ANON240', 'ANON100', 'ANON237', 'ANON158', 'ANON174', 'ANON19', 'ANON195', 'COVID 119', 'COVID 87', 'COVID 70', 'COVID 94', 'COVID 215', 'COVID 77', 'COVID 213', 'COVID 81', 'COVID 216', 'COVID 72', 'COVID 106', 'COVID 131', 'COVID 107', 'COVID 116', 'COVID 95', 'COVID 214', 'COVID 129']\n",
      "test count:  {'negative': 20, 'normal': 0, 'pneumonia': 5, 'COVID-19': 100}\n",
      "train count:  {'negative': 353, 'normal': 0, 'pneumonia': 52, 'COVID-19': 1670}\n"
     ]
    }
   ],
   "source": [
    "# add covid-chestxray-dataset, figure1 and actualmed into COVIDx dataset\n",
    "# since these datasets don't have test dataset, split into train/test by patientid\n",
    "# for covid-chestxray-dataset:\n",
    "# patient 8 is used as non-COVID19 viral test\n",
    "# patient 31 is used as bacterial test\n",
    "# patients 19, 20, 36, 42, 86 are used as COVID-19 viral test\n",
    "# for figure 1:\n",
    "# patients 24, 25, 27, 29, 30, 32, 33, 36, 37, 38\n",
    "\n",
    "ds_imgpath = {'cohen': cohen_imgpath, 'fig1': fig1_imgpath, 'actmed': actmed_imgpath, 'sirm': sirm_imgpath}\n",
    "\n",
    "for key in filename_label.keys():\n",
    "    arr = np.array(filename_label[key])\n",
    "    if arr.size == 0:\n",
    "        continue\n",
    "    # split by patients\n",
    "    num_diff_patients = len(np.unique(arr[:,0]))\n",
    "    # num_test = max(1, round(split*num_diff_patients))\n",
    "    # select num_test number of random patients\n",
    "    # random.sample(list(arr[:,0]), num_test)\n",
    "    if key == 'pneumonia':\n",
    "        test_patients = ['8', '31']\n",
    "    elif key == 'COVID-19':\n",
    "        test_patients = ['19', '20', '36', '42', '86', \n",
    "                         '94', '97', '117', '132', \n",
    "                         '138', '144', '150', '163', '169', '174', '175', '179', '190', '191'\n",
    "                         'COVID-00024', 'COVID-00025', 'COVID-00026', 'COVID-00027', 'COVID-00029',\n",
    "                         'COVID-00030', 'COVID-00032', 'COVID-00033', 'COVID-00035', 'COVID-00036',\n",
    "                         'COVID-00037', 'COVID-00038',\n",
    "                         'ANON24', 'ANON45', 'ANON126', 'ANON106', 'ANON67',\n",
    "                         'ANON153', 'ANON135', 'ANON44', 'ANON29', 'ANON201', \n",
    "                         'ANON191', 'ANON234', 'ANON110', 'ANON112', 'ANON73', \n",
    "                         'ANON220', 'ANON189', 'ANON30', 'ANON53', 'ANON46',\n",
    "                         'ANON218', 'ANON240', 'ANON100', 'ANON237', 'ANON158',\n",
    "                         'ANON174', 'ANON19', 'ANON195',\n",
    "                         'COVID 119', 'COVID 87', 'COVID 70', 'COVID 94', \n",
    "                         'COVID 215', 'COVID 77', 'COVID 213', 'COVID 81', \n",
    "                         'COVID 216', 'COVID 72', 'COVID 106', 'COVID 131', \n",
    "                         'COVID 107', 'COVID 116', 'COVID 95', 'COVID 214', \n",
    "                         'COVID 129']\n",
    "    elif key == 'negative':\n",
    "        test_patients = test_patients_actmed\n",
    "    else: \n",
    "        test_patients = []\n",
    "    print('Key: ', key)\n",
    "    print('Test patients: ', test_patients)\n",
    "    # go through all the patients\n",
    "    for patient in arr:\n",
    "        if patient[0] not in patient_imgpath:\n",
    "            patient_imgpath[patient[0]] = [patient[1]]\n",
    "        else:\n",
    "            if patient[1] not in patient_imgpath[patient[0]]:\n",
    "                patient_imgpath[patient[0]].append(patient[1])\n",
    "            else:\n",
    "                continue  # skip since image has already been written\n",
    "        if patient[0] in test_patients:\n",
    "            if patient[3] == 'sirm':\n",
    "                image = cv2.imread(os.path.join(ds_imgpath[patient[3]], patient[1]))\n",
    "                gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
    "                patient[1] = patient[1].replace(' ', '')\n",
    "                cv2.imwrite(os.path.join(savepath, 'test', patient[1]), gray)\n",
    "            else:\n",
    "                copyfile(os.path.join(ds_imgpath[patient[3]], patient[1]), os.path.join(savepath, 'test', patient[1]))\n",
    "            test.append(patient)\n",
    "            test_count[patient[2]] += 1\n",
    "        else:\n",
    "            if patient[3] == 'sirm':\n",
    "                image = cv2.imread(os.path.join(ds_imgpath[patient[3]], patient[1]))\n",
    "                gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n",
    "                patient[1] = patient[1].replace(' ', '')\n",
    "                cv2.imwrite(os.path.join(savepath, 'train', patient[1]), gray)\n",
    "            else:\n",
    "                copyfile(os.path.join(ds_imgpath[patient[3]], patient[1]), os.path.join(savepath, 'train', patient[1]))\n",
    "            train.append(patient)\n",
    "            train_count[patient[2]] += 1\n",
    "\n",
    "print('test count: ', test_count)\n",
    "print('train count: ', train_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test count:  {'negative': 20, 'normal': 885, 'pneumonia': 594, 'COVID-19': 100}\n",
      "train count:  {'negative': 353, 'normal': 7966, 'pneumonia': 5475, 'COVID-19': 1670}\n"
     ]
    }
   ],
   "source": [
    "# add normal and rest of pneumonia cases from https://www.kaggle.com/c/rsna-pneumonia-detection-challenge\n",
    "csv_normal = pd.read_csv(os.path.join(rsna_datapath, rsna_csvname), nrows=None)\n",
    "csv_pneu = pd.read_csv(os.path.join(rsna_datapath, rsna_csvname2), nrows=None)\n",
    "patients = {'normal': [], 'pneumonia': []}\n",
    "\n",
    "for index, row in csv_normal.iterrows():\n",
    "    if row['class'] == 'Normal':\n",
    "        patients['normal'].append(row['patientId'])\n",
    "\n",
    "for index, row in csv_pneu.iterrows():\n",
    "    if int(row['Target']) == 1:\n",
    "        patients['pneumonia'].append(row['patientId'])\n",
    "\n",
    "for key in patients.keys():\n",
    "    arr = np.array(patients[key])\n",
    "    if arr.size == 0:\n",
    "        continue\n",
    "    # split by patients \n",
    "    num_diff_patients = len(np.unique(arr))\n",
    "    # num_test = max(1, round(split*num_diff_patients))\n",
    "    test_patients = np.load('rsna_test_patients_{}.npy'.format(key)) # random.sample(list(arr), num_test), download the .npy files from the repo.\n",
    "    # np.save('rsna_test_patients_{}.npy'.format(key), np.array(test_patients))\n",
    "    for patient in arr:\n",
    "        if patient not in patient_imgpath:\n",
    "            patient_imgpath[patient] = [patient]\n",
    "        else:\n",
    "            continue  # skip since image has already been written\n",
    "                \n",
    "        ds = dicom.dcmread(os.path.join(rsna_datapath, rsna_imgpath, patient + '.dcm'))\n",
    "        pixel_array_numpy = ds.pixel_array\n",
    "        imgname = patient + '.png'\n",
    "        if patient in test_patients:\n",
    "            cv2.imwrite(os.path.join(savepath, 'test', imgname), pixel_array_numpy)\n",
    "            test.append([patient, imgname, key, 'rsna'])\n",
    "            test_count[key] += 1\n",
    "        else:\n",
    "            cv2.imwrite(os.path.join(savepath, 'train', imgname), pixel_array_numpy)\n",
    "            train.append([patient, imgname, key, 'rsna'])\n",
    "            train_count[key] += 1\n",
    "\n",
    "print('test count: ', test_count)\n",
    "print('train count: ', train_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final stats\n",
      "Train count:  {'negative': 353, 'normal': 7966, 'pneumonia': 5475, 'COVID-19': 1670}\n",
      "Test count:  {'negative': 20, 'normal': 885, 'pneumonia': 594, 'COVID-19': 100}\n",
      "Total length of train:  15464\n",
      "Total length of test:  1599\n"
     ]
    }
   ],
   "source": [
    "# final stats\n",
    "print('Final stats')\n",
    "print('Train count: ', train_count)\n",
    "print('Test count: ', test_count)\n",
    "print('Total length of train: ', len(train))\n",
    "print('Total length of test: ', len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of final test set :  200\n"
     ]
    }
   ],
   "source": [
    "# Create a test set containing the 100 COVID-19 test samples and a random selection of \n",
    "# 10 normal cases, 70 pneumonia cases, and 20 negative cases from Actuamed\n",
    "final_test = []\n",
    "\n",
    "normal_cases = []\n",
    "pneumonia_cases = []\n",
    "\n",
    "for entry in test:\n",
    "    if entry[3] == 'actmed' and entry[2] == 'negative':\n",
    "        final_test.append(entry)\n",
    "    if entry[2] == 'pneumonia':\n",
    "        pneumonia_cases.append(entry)\n",
    "    if entry[2] == 'normal':\n",
    "        normal_cases.append(entry)\n",
    "    if entry[2] == 'COVID-19':\n",
    "        final_test.append(entry)\n",
    "\n",
    "final_test = final_test + random.sample(normal_cases, 10) + random.sample(pneumonia_cases, 70)\n",
    "print('Length of final test set : ', len(final_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export to train and test csv\n",
    "# format as patientid, filename, label, separated by a space\n",
    "# where label is either \"positive\" for COVID-19 or \"negative\" for anything else\n",
    "train_file = open(\"train_split.txt\",'w') \n",
    "for sample in train:\n",
    "    if len(sample) == 4:\n",
    "        if sample[2] == 'COVID-19':\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'positive' + ' ' + sample[3] + '\\n'\n",
    "        else:\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'negative' + ' ' + sample[3] + '\\n'\n",
    "    else:\n",
    "        if sample[2] == 'COVID-19':\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'positive' + '\\n'\n",
    "        else:\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'negative' + '\\n'\n",
    "    train_file.write(info)\n",
    "\n",
    "train_file.close()\n",
    "\n",
    "test_file = open(\"test_split.txt\", 'w')\n",
    "for sample in final_test:\n",
    "    if len(sample) == 4:\n",
    "        if sample[2] == 'COVID-19':\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'positive' + ' ' + sample[3] + '\\n'\n",
    "        else:\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'negative' + ' ' + sample[3] + '\\n'\n",
    "    else:\n",
    "        if sample[2] == 'COVID-19':\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'positive' + '\\n'\n",
    "        else:\n",
    "            info = str(sample[0]) + ' ' + sample[1] + ' ' + 'negative' + '\\n'\n",
    "    test_file.write(info)\n",
    "\n",
    "test_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}