xnat_data_extractor.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XNAT database data extractor\n",
    "This notebook walk through all objects in a XNAT database to extract all possible fields and all possible values. This notebook does not just extract unique values but the whole database, hence you should expect it will take a while and quite some storage space on big databases.\n",
    "\n",
    "Please edit login.cfg with your credentials before executing this script.\n",
    "\n",
    "Before (re-)running this script, please clear output, shutdown and relaunch kernel, close down and reopen your browser, and then (re-)launch all the cells! Else the memory is not correctly freed and you will get very fast a MemoryError (this is a bug in ipywidgets or jupyter notebook).\n",
    "\n",
    "Also note that you can stop (KeyboardInterrupt) at any moment, the current state (up to before the current project) will be saved in a pickle file and can be reloaded to continue from this point on. This is particularly useful if you do not use the out of memory argument, the dict might become too big for memory. You can then stop, restart the kernel, and restart the cell to continue from where you stopped, with a fresh memory. The state saving will also kick in automatically if any kind of error/exception arise."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Init and helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#\n",
    "# Creation: 08/2017 by Stephen Larroque\n",
    "#\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import lxml\n",
    "import xml.etree.ElementTree as ET\n",
    "import pyxnat\n",
    "import re\n",
    "import traceback\n",
    "\n",
    "from time import gmtime, strftime\n",
    "from libs.tqdm import tqdm_notebook\n",
    "\n",
    "try:\n",
    "    import cPickle as pickle\n",
    "except ImportError as exc:\n",
    "    import pickle\n",
    "\n",
    "#try:\n",
    "import ujson as json # fast json lib\n",
    "#except ImportError:\n",
    "#    import json # native json lib\n",
    "\n",
    "# For out-of-core computing (ie, to store the dict on disk and thus avoid MemoryOverflow error)\n",
    "from fdict import sfdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#### HELPER FUNCTIONS\n",
    "from copy import deepcopy\n",
    "from libs.xmlpp import get_pprint as xml_pprint\n",
    "def get_raw_xml(elements_list):\n",
    "    '''Get the source xml of a list of lxml elements or pyxnat objects'''\n",
    "    # Convert to a list of elements if it's a single element (to ease looping)\n",
    "    if not isinstance(elements_list, list):\n",
    "        elements_list = [elements_list]\n",
    "\n",
    "    out = ''\n",
    "    for i, element in enumerate(elements_list):\n",
    "        out += '\\n=== Element %i\\n' % i\n",
    "        # If this is an XML element\n",
    "        if isinstance(element, lxml.etree._Element):\n",
    "            # Make a copy of the element because we will modify it\n",
    "            e = deepcopy(element)\n",
    "            # Strip comments, else lxml does not know how to print the XML\n",
    "            lxml.etree.strip_tags(e, lxml.etree.Comment)\n",
    "            # Add the XML of this element to the output\n",
    "            out += xml_pprint(lxml.etree.tostring(e, pretty_print=True))\n",
    "            #print(lxml.etree.tostring(e, pretty_print=True)) #debug\n",
    "        # pyxnat object, we just fetch the xml from the server\n",
    "        if isinstance(element, pyxnat.core.resources.EObject):\n",
    "            out += element.get()\n",
    "        # Print differently if this is any other type\n",
    "        else:\n",
    "            out += repr(element)\n",
    "    return out\n",
    "\n",
    "def pprint_xml(obj):\n",
    "    print(xml_pprint(get_raw_xml(obj)))\n",
    "\n",
    "#### HELPER GLOBALS\n",
    "# XNAT namespace (to use with lxml xpath queries)\n",
    "xnatns = {'arc': 'http://nrg.wustl.edu/arc',\n",
    " 'cat': 'http://nrg.wustl.edu/catalog',\n",
    " 'ext': 'http://nrg.wustl.edu/ext',\n",
    " 'pipe': 'http://nrg.wustl.edu/pipe',\n",
    " 'prov': 'http://www.nbirn.net/prov',\n",
    " 'scr': 'http://nrg.wustl.edu/scr',\n",
    " 'val': 'http://nrg.wustl.edu/val',\n",
    " 'wrk': 'http://nrg.wustl.edu/workflow',\n",
    " 'xdat': 'http://nrg.wustl.edu/security',\n",
    " 'xnat': 'http://nrg.wustl.edu/xnat',\n",
    " 'xnat_a': 'http://nrg.wustl.edu/xnat_assessments',\n",
    " 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Connection to XNAT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Loading login infos\n",
    "cfgpath = os.path.join(os.getcwd(), 'login.cfg')\n",
    "with open(cfgpath) as f:\n",
    "    login_infos = json.load(f)\n",
    "\n",
    "# Connect to XNAT db\n",
    "central = pyxnat.Interface(server=\"http://tbixnat.incf.org:8080\", user=login_infos['username'], password=login_infos['password'], cachedir='/tmp')\n",
    "# Add schemas (allows to use .attrs() to get list of attributes)\n",
    "central.manage.schemas.add('xnat/xnat.xsd')\n",
    "\n",
    "# Get list of all centers\n",
    "centers = central.select.projects()\n",
    "print(centers.get())\n",
    "\n",
    "# Select center (constraining to one center for the moment)\n",
    "# TODO: loop over all centers\n",
    "#cULgData_Liege_project = central.select.project('LIE')\n",
    "\n",
    "# Show structure of project\n",
    "central.inspect.structure()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "count = sum(1 for _ in centers)\n",
    "print('Total number of centers/projects: %i' % count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "obj = centers[0]\n",
    "assert get_raw_xml(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def walkthrough_data(xnatobjlist, namespaces=None, outofcore=False, outofcore_filename=None, firstonly=False, level=0, progress_maxlevel=2, initial=None, debug=False):\n",
    "    if not hasattr(xnatobjlist, '__iter__'):\n",
    "        xnatobjlist = [xnatobjlist]\n",
    "    # Initialization of the out object\n",
    "    passuntilproject = None\n",
    "    if initial is not None and level == 0:\n",
    "        # Restart from previous state (ie, after a bug etc)\n",
    "        if isinstance(initial, str):\n",
    "            with open(initial, 'rb') as f:\n",
    "                initial = pickle.load(f)\n",
    "        if isinstance(initial['object'], str):\n",
    "            # Was an out of core object, we just initialize from the out of core db file\n",
    "            topshelf = sfdict(filename=initial['object'], nodel=True)\n",
    "        else:\n",
    "            # Else it was in RAM memory, and it was pickled in the state file, we recover\n",
    "            topshelf = initial['object']\n",
    "        out = topshelf['projects']\n",
    "        passuntilproject = initial['lastproject']\n",
    "    else:\n",
    "        # Initialization from scratch\n",
    "        if outofcore and level == 0:\n",
    "            # Out of core computing using fdict/shelve to store on disk rather than in-memory (to avoid MemoryOverflow error)\n",
    "            topshelf = sfdict(filename=outofcore_filename, nodel=True)\n",
    "            topshelf['projects'] = {}\n",
    "            out = topshelf['projects']\n",
    "        else:\n",
    "            # Store in memory\n",
    "            if level == 0:\n",
    "                # Fake a topshelf to be compatible with out-of-core\n",
    "                topshelf = {}\n",
    "                topshelf['projects'] = {}\n",
    "                out = topshelf['projects']\n",
    "            else:\n",
    "                out = {}\n",
    "\n",
    "    # prepare namespaces for attribute search\n",
    "    if namespaces:\n",
    "        namespaces_filt = dict()\n",
    "        for val, key in namespaces.items():\n",
    "            namespaces_filt['{'+key+'}'] = val+':'\n",
    "\n",
    "    # prepare for progress display\n",
    "    # count\n",
    "    count = sum(1 for _ in xnatobjlist) # count total number of items (to predict time and display progress)\n",
    "    if firstonly and count > firstonly:\n",
    "        count = firstonly\n",
    "    # current object type name\n",
    "    try:\n",
    "        if hasattr(xnatobjlist, 'tag') and isinstance(xnatobjlist.tag, str):\n",
    "            curtype = re.sub('{.*}', '', xnatobjlist.tag)\n",
    "        else:\n",
    "            obj = str(type(xnatobjlist[0]))\n",
    "            curtype = obj[obj.rfind('.')+1:obj.rfind(\"'\")]\n",
    "    except StopIteration as exc:\n",
    "        curtype = ''\n",
    "        pass\n",
    "\n",
    "    # Limit the progress display, because there is currently a memory leak of ipywidgets, old widgets stay in memory\n",
    "    if progress_maxlevel <= 0:\n",
    "        xnatobjlist_iterator = xnatobjlist\n",
    "    else:\n",
    "        if level < progress_maxlevel:\n",
    "            xnatobjlist_iterator = tqdm_notebook(xnatobjlist, total=count, desc=curtype, position=level, leave=False)\n",
    "        else:\n",
    "            xnatobjlist_iterator = xnatobjlist\n",
    "\n",
    "    i = 0\n",
    "    lastproj = None\n",
    "    # Main loop: for each item in the list\n",
    "    try:\n",
    "        for obj in xnatobjlist_iterator:\n",
    "            # Continue to the next object?\n",
    "            if firstonly:\n",
    "                if i >= firstonly:\n",
    "                    break\n",
    "            # Debug print\n",
    "            if debug: print(obj)\n",
    "\n",
    "            # Find label of current object\n",
    "            if hasattr(obj, 'label'):\n",
    "                # Use label as key to access this object if available\n",
    "                objlabel = obj.label()\n",
    "                if level == 0:\n",
    "                    lastproj = obj.label()\n",
    "            else:\n",
    "                # Else we just use a number\n",
    "                objlabel = str(i)\n",
    "\n",
    "            # Skip if we want to restart on a specific project\n",
    "            if passuntilproject is not None and level == 0:\n",
    "                if lastproj != passuntilproject:\n",
    "                    i += 1\n",
    "                    continue\n",
    "                else:\n",
    "                    # We reached the last project, we disable the \"continue\" flag\n",
    "                    passuntilproject = None\n",
    "\n",
    "            # Create an entry for the current object\n",
    "            if count > 1:\n",
    "                # Create the a subdict for the current element (because there will be several entries of the same type)\n",
    "                if not objlabel in out:\n",
    "                    out[objlabel] = {}\n",
    "                outelt = out[objlabel]  # we use only this subdict from now on\n",
    "            else:\n",
    "                # Only one element to process, we do not need to create an index\n",
    "                outelt = out\n",
    "            # Add label, id, datatype and content\n",
    "            outelt['label'] = objlabel\n",
    "            if hasattr(obj, 'id'):\n",
    "                outelt['id'] = obj.id()\n",
    "            if hasattr(obj, 'datatype'):\n",
    "                outelt['datatype'] = obj.datatype()\n",
    "            if hasattr(obj, 'text') and obj.text:\n",
    "                outelt['text'] = obj.text.strip()\n",
    "            # Only if not a resource nor a file, else it's only about files so no xml content\n",
    "            if not isinstance(obj, (pyxnat.core.resources.Resource, pyxnat.core.resources.File)):\n",
    "                # Attributes\n",
    "                attrs = obj.xpath('@*')\n",
    "                for attr in attrs:\n",
    "                    # Get attribute's xml name and value\n",
    "                    attrname = attr.attrname\n",
    "                    if namespaces: # if namespaces is provided, we can use that to replace the prefix (else attributes don't provide the prefix, only subelements do)\n",
    "                        #if attrname.startswith('{'): print(attrname)\n",
    "                        for key, val in namespaces_filt.items():\n",
    "                            #if attrname.startswith('{'): print(key, val)\n",
    "                            attrname = attrname.replace(key, val)\n",
    "                        #print('lala'+attrname)\n",
    "                    attrval = str(attr)\n",
    "                    # Add this value\n",
    "                    attrnamedict = '@'+attrname\n",
    "                    outelt[attrnamedict] = attrval\n",
    "                # Subelements\n",
    "                subelts = obj.xpath('*')\n",
    "                for subelt in subelts:\n",
    "                    # Get subelement's xml name\n",
    "                    if subelt.prefix:\n",
    "                        prefix = (subelt.prefix + ':')\n",
    "                    else:\n",
    "                        prefix = ''\n",
    "                    subeltname = prefix + re.sub('{.*}', '', subelt.tag)\n",
    "                    # Get subelement's value\n",
    "                    if hasattr(subelt, 'text'):\n",
    "                        subeltval = subelt.text\n",
    "                    else:\n",
    "                        subeltval = str(subelt)\n",
    "                    if subeltval: # remove useless chars at the start and end\n",
    "                        subeltval = subeltval.strip()\n",
    "                    # Recursive call if it has children\n",
    "                    if subelt.getchildren() or subelt.xpath('@*') or (subeltname in outelt and isinstance(outelt[subeltname], dict)): # subelt.getchildren() == subelt.xpath('*')\n",
    "                        # Create a dict for this subelement\n",
    "                        if not subeltname in outelt:\n",
    "                            outelt[subeltname] = {}\n",
    "                        if isinstance(subeltval, (str, list, set)) and subeltval: # value is not an xml element we can walk, but out subelt is a dict, so we have a conflict, we still add the value\n",
    "                            cval = subeltval\n",
    "                        else:\n",
    "                            cval = walkthrough_data([subelt], namespaces=namespaces, firstonly=firstonly, level=level+1, progress_maxlevel=progress_maxlevel, debug=debug)\n",
    "                        # Merge with our dict\n",
    "                        outelt[subeltname] = cval\n",
    "                    else:\n",
    "                        # Add this value since it is a singleton\n",
    "                        try:\n",
    "                            outelt[subeltname] = subeltval\n",
    "                        except AttributeError as exc:\n",
    "                            print(subeltname, subeltval, type(outelt[subeltname]), outelt[subeltname])\n",
    "                            raise\n",
    "            # Children\n",
    "            if hasattr(obj, 'children'):\n",
    "                for childname in obj.children():\n",
    "                    # Call the method to retrieve children from child name\n",
    "                    child = getattr(obj, childname)()\n",
    "                    # Recursive call\n",
    "                    cres = walkthrough_data(child, namespaces=namespaces, firstonly=firstonly, level=level+1, progress_maxlevel=progress_maxlevel, debug=debug)\n",
    "                    # Merge with our dict\n",
    "                    outelt[childname] = cres\n",
    "            # Flush to disk at every project and subject iteration (to unload from memory)\n",
    "            if outofcore and level <= 1:\n",
    "                topshelf.sync()\n",
    "            # Increment counter\n",
    "            i += 1\n",
    "\n",
    "        # Return the dict for this level\n",
    "        if level == 0:\n",
    "            return topshelf\n",
    "        else:\n",
    "            return out\n",
    "    except (Exception, KeyboardInterrupt, UnboundLocalError) as exc:\n",
    "        if level == 0:\n",
    "            # Level is 0, we save the current state before stopping\n",
    "            curstate = {'lastproject': lastproj, 'exception': exc, 'trace': traceback.format_exc()}\n",
    "            if outofcore:\n",
    "                # Cannot pickle an out-of-core object, but anyway we can reload from the file directly, so we store the filename\n",
    "                curstate['object'] = topshelf.filename\n",
    "            else:\n",
    "                curstate['object'] = topshelf\n",
    "            curtime = strftime(\"%Y-%m-%d_%H-%M-%S\", gmtime())\n",
    "            dumpfilename = 'xnat_data_extractor_dump_%s.pickle' % curtime\n",
    "            with open(dumpfilename, 'wb') as f:\n",
    "                pickle.dump(curstate, f)\n",
    "            with open('dbdata_lastdump.pickle', 'wb') as f2:\n",
    "                pickle.dump(curstate, f2)\n",
    "            print('Dump saved in dbdata_lastdump.pickle and %s' % dumpfilename)\n",
    "        # Propagate the exception in any case\n",
    "        raise\n",
    "# If you modify the functions above, please restart kernel and clear output before relaunching a walkthrough_data, else the code changes might not take effect.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload_last = 'dbdata_lastdump.pickle' # if you want to restart from a previous state, put here the dump filename as a string\n",
    "#outofcore_filename = os.path.join(os.getcwd(), 'xnat_data_extractor.db')  # if out of core, where to store the database (set to None if you want to create a new temporary file everytime! If not, be aware that the db will be reused even if you restart from scratch, this can lead to issues, so ensure the db file is deleted if you restart from scratch!)\n",
    "outofcore_filename = None\n",
    "\n",
    "if not os.path.isfile(reload_last):\n",
    "    print('The supplied reload dump file does not exist! Start from scratch...')\n",
    "    reload_last = None\n",
    "out = walkthrough_data(centers, namespaces=xnatns, firstonly=3, outofcore=False, outofcore_filename=outofcore_filename, progress_maxlevel=1, initial=reload_last, debug=False)\n",
    "print('All Done!')\n",
    "# TIP: when testing on your database, do a first run with firstonly=3 (this will process only the first 3 elements at any level) just to quickly see if everything runs alright (you should also check the generated file). Then to go to production mode, set firstonly=None.\n",
    "# TIP2: set progress_maxlevel=1 when using in production to minimize memory overhead but still get to see progress updates.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Save the result into a json file\n",
    "import pickle as pk\n",
    "\n",
    "filename = 'xnat_data_extract.json'\n",
    "filename_flat = 'xnat_data_extract_flat.json'\n",
    "\n",
    "if isinstance(out, sfdict):\n",
    "    with open(filename, 'w') as f:\n",
    "        while 4:  # retry 4 times to save\n",
    "            try:\n",
    "                json.dump(out.to_dict_nested(), f, ensure_ascii=False, indent=4, sort_keys=True)\n",
    "                break  # break if successful\n",
    "            except KeyError:\n",
    "                pass\n",
    "    with open(filename_flat, 'w') as f:\n",
    "        json.dump(out.to_dict(), f, ensure_ascii=False, indent=4, sort_keys=True)\n",
    "#elif isinstance(out, Chest):\n",
    "#    with open(out.key_to_filename('projects'), 'rb') as e:\n",
    "#        with open('db_unique_values.json', 'w') as f:\n",
    "#            json.dump({'projects': pk.load(e)}, f, ensure_ascii=False, indent=4, sort_keys=True)\n",
    "else:\n",
    "    with open(filename, 'w') as f:\n",
    "        json.dump(out, f, ensure_ascii=False, indent=4, sort_keys=True)\n",
    "\n",
    "print('Results saved in %s.' % filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Just a little sanity check\n",
    "#try:\n",
    "#    assert len(out['projects']['children']['subjects']['children']['experiments']['children']['scans']['subelements']['xnat:parameters']['subelements']['xnat:voxelRes']['attributes']['x']) > 1\n",
    "#    print('Sanity check OK!')\n",
    "#except AssertionError as exc:\n",
    "#    print('ERROR: xnat:parameters has only 1 sample set of values, probably something went wrong (or you used firstonly=1)')\n",
    "#    out['projects']['children']['subjects']['children']['experiments']['children']['scans']['subelements']['xnat:parameters']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------\n",
    "### Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def dict_xpath(obj, path):\n",
    "    a = obj\n",
    "    for p in path.split('/'):\n",
    "        a = a[p]\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dict_xpath(out, 'projects/children/subjects/subelements/xnat:experiments/subelements/xnat:experiment/subelements')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Show the result\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load results from last dump (if there was a bug)\n",
    "with open('dbdata_lastdump.pickle', 'rb') as f:\n",
    "    out = pickle.load(f)\n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------\n",
    "### Unused code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "\n",
    "class SortedDict(OrderedDict):\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        super(SortedDict, self).__init__()\n",
    "\n",
    "        for key, value in sorted(kwargs.items()):\n",
    "            if isinstance(value, dict):\n",
    "                self[key] = SortedDict(**value)\n",
    "            else:\n",
    "                self[key] = value\n",
    "\n",
    "sorted_dict = SortedDict(**a)\n",
    "sorted_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "b = obj.xpath('*')\n",
    "b2 = b[0]\n",
    "b2.text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "key, val = ('{http://www.w3.org/2001/XMLSchema-instance}', 'xsi:')\n",
    "s = '{http://www.w3.org/2001/XMLSchema-instance}schemaLocation'\n",
    "s.replace(key, val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "obj.id()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "g = obj.subject('CTBI_S00638').experiment('CTBI_E02818').scan('2').resource('3144').file('DTI.bval')\n",
    "g.attributes()\n",
    "type(g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scans = central.select.project('HEL').subject('CTBI_S00800').experiment('CTBI_E04816').scans()\n",
    "scan = scans[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pprint_xml(scan)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "param = scan.xpath('*')[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "param_children = param.getchildren()\n",
    "param_children"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "param.xpath('*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "walkthrough_data(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pprint_xml(param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from libs.xmltodict import xmltodict\n",
    "xmltodict.parse(get_raw_xml(param).encode('utf-8'), process_namespaces=True, namespaces=xnatns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "a = {'a': set([1, 2]), 'b': {1, 2}, 'c': {'d': set([1, 2])}}\n",
    "b = {'a': set([1, 4]), 'b': {1, 3}, 'c': {'d': set([1, 3, 4])}}\n",
    "c = merge(a, b)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c['a'].add(4)\n",
    "c['b'].add(3)\n",
    "c['b'].add(4)\n",
    "c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "e = set([1, 2])\n",
    "e.update(set([1, 3]))\n",
    "e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from libs.sqlite_object import SqliteDict\n",
    "a = SqliteDict(filename='test.sqlite3', persist=True)\n",
    "a['label'] = set([1,2,3])\n",
    "a['id'] = a['label']\n",
    "a"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}