download_from_pmc_ftp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download from PMC FTP server"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "from ftplib import FTP\n",
    "from fnmatch import fnmatch\n",
    "from urllib.parse import urlparse\n",
    "from tqdm import tqdm\n",
    "\n",
    "# You can also integrate tqdm into Jupyter but it may require more setup\n",
    "#from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_path = 'data'\n",
    "\n",
    "# articles\n",
    "articles_output_dir = os.path.join(data_path, 'FTP_PUBMED_papers')\n",
    "articles_ftp_url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/oa_bulk/'\n",
    "articles_file_pattern = '*.xml.tar.gz'\n",
    "# Or you could start by just downloading one file\n",
    "#file_pattern = 'non_comm_use.A-B.xml.tar.gz'\n",
    "\n",
    "# pmc ids\n",
    "pmc_ids_output_dir = os.path.join(data_path, 'PMC_ids')\n",
    "pmc_ids_ftp_url = 'ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/PMC-ids.csv.gz'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "already downloaded: data/FTP_PUBMED_papers/comm_use.C-H.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/comm_use.I-N.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/comm_use.O-Z.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/comm_use.A-B.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/non_comm_use.C-H.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/non_comm_use.I-N.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/non_comm_use.O-Z.xml.tar.gz\n",
      "already downloaded: data/FTP_PUBMED_papers/non_comm_use.A-B.xml.tar.gz\n",
      "already downloaded: data/PMC_ids/PMC-ids.csv.gz\n"
     ]
    }
   ],
   "source": [
    "def download_ftp_file(ftp_client, filename, output_filename):\n",
    "    # use temp files so we know the file is not complete\n",
    "    temp_output_filename = output_filename + '.part'\n",
    "    if os.path.isfile(output_filename):\n",
    "        os.remove(output_filename)\n",
    "    with open(temp_output_filename, 'wb') as f:\n",
    "        # need to change the type to binary for the size command to succeed\n",
    "        ftp_client.voidcmd('TYPE I')\n",
    "        total = ftp_client.size(filename)\n",
    "\n",
    "        with tqdm(total=total, unit='B', unit_scale=True, leave=False, file=sys.stdout) as pbar:\n",
    "            pbar.set_description(filename)\n",
    "\n",
    "            def cb(data):\n",
    "                pbar.update(len(data))\n",
    "                f.write(data)\n",
    "\n",
    "            ftp_client.retrbinary('RETR {}'.format(filename), cb)\n",
    "    os.rename(temp_output_filename, output_filename)\n",
    "\n",
    "def download_files_from_ftp(ftp_url, file_pattern, output_dir):\n",
    "    os.makedirs(output_dir, exist_ok=True)\n",
    "    parsed_ftp_url = urlparse(ftp_url)\n",
    "    with FTP(parsed_ftp_url.hostname) as ftp_client:\n",
    "        ftp_client.login()\n",
    "        ftp_client.cwd(parsed_ftp_url.path)\n",
    "        filenames = ftp_client.nlst()\n",
    "        matching_filenames = [fn for fn in filenames if fnmatch(fn, file_pattern)]\n",
    "        for filename in matching_filenames:\n",
    "            output_filename = os.path.join(output_dir, filename)\n",
    "            if os.path.isfile(output_filename):\n",
    "                print('already downloaded:', output_filename)\n",
    "            else:\n",
    "                download_ftp_file(ftp_client, filename, output_filename)\n",
    "                print('downloaded:', output_filename)\n",
    "\n",
    "def download_file_from_ftp(ftp_url, output_dir):\n",
    "    download_files_from_ftp(\n",
    "        os.path.dirname(ftp_url),\n",
    "        os.path.basename(ftp_url),\n",
    "        output_dir\n",
    "    )\n",
    "    \n",
    "\n",
    "download_files_from_ftp(articles_ftp_url, articles_file_pattern, articles_output_dir)\n",
    "\n",
    "download_file_from_ftp(pmc_ids_ftp_url, pmc_ids_output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/project/citation-context'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}