enquetes.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Takes the results of surveys generated by mod/questionaire
# and processes them, generates graphs and writes the results
# to a tab-separated file
#
# Author: Ewout ter Haar <ewout@usp.br>
# License: Apache 

import sys, os, hashlib
from optparse import OptionParser
import config
import mdlib

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
import pandas as pd
from collections import Counter

DATA_DIR = '/home/ewout/Dropbox/ATP/Pesquisa/Data/2011 - Cursistas/'



def add_curso_e_grupo(df):
    ''
    users = pd.read_table(DATA_DIR+'username-course-group-redefor-11.csv',sep=',',index_col=0)
    
    def find_username(username,detail):
        try:
            user = users.xs(username)
            return user[detail]
        except KeyError:
            return 'Nenhum'

    username_course = lambda username: find_username(username,'course1')
    username_group = lambda username: find_username(username,'group1')
    username_role = lambda username: find_username(username,'role1')

    username = df['Nome de usuário']
    df['Curso'] = username.map(username_course)
    df['Grupo'] = username.map(username_group)
    df['Papel'] = username.map(username_role)

    return df

def add_sexo(df):
    ''
    def codpes_sexo(codpes):
        try:
            codpes = int(codpes)
            return mdlib.pessoa(codpes)['sexpes']
        except ValueError:
            return

    codpes = df['NumeroUSP']
    df['Sexo'] = codpes.map(codpes_sexo) 
    return df

def add_nasc(df):
    ''
    def codpes_nasc(codpes):
        try:
            codpes = int(codpes)
            return mdlib.pessoa(codpes)['dtanas']
        except ValueError:
            return
    
    codpes = df['NumeroUSP']
    df['DataNasc'] =  codpes.map(codpes_nasc) 
    return df


def graph_cb(df,ax):
    ''
    cb = df['classe_cb']
    c = Counter(cb)
    plt.pie(c.values(),labels=c.keys(),autopct='%1.0f%%')
    ax.set_title(u'Critério Brasil')
    return ax

def graph_cb_bar(df,ax):
    ''
    cb = df['classe_cb']
    c = Counter(cb)
    labels = sorted(c.keys())
    values = [c[val] for val in labels]
    N = len(labels)
    x = np.arange(N)
    width = 0.6
    ax.bar(x,values,width,color='r')

    ax.set_title(u'Critério Brasil')
    ax.set_xticks(x+width)
    ax.set_xticklabels(labels)

def graph_ldi(df,ax):
    ''
    ldi = df['ldi15']
    plt.hist(ldi,rwidth=0.8)
    ax.set_title(u'Indice de Literacia Digital\nEscala de 15 itens (0-4)')
    mean = unicode(round(ldi.mean()))
    std = unicode(round(ldi.std()))
    ax.text(0.6,0.85,u'Média = '+ mean + u'\nDesvio Padrão = ' + std,transform=ax.transAxes)

    return ax

def make_graphs(df,filename):
    ''
    name,ext = os.path.splitext(filename)
    fig = plt.figure(figsize =(6,10))
    fig.text(0.02,0.95,unicode(name,'utf8'),fontsize=20)
    ax1 = plt.subplot(211)
    #ax1.set_aspect('equal')
    ax1 = graph_cb_bar(df,ax1)
    ax2 = plt.subplot(212)
    ax2 = graph_ldi(df,ax2)

    fig.subplots_adjust(hspace=0.5)
    
    plt.savefig(filename)

def criterio_brasil(df):
    ''
    itens_pontos = {'tv':[0,1,2,3,4],
                    'radio':[0,1,2,3,4],
                    'banheiro':[0,4,5,6,7],
                    'auto':[0,4,7,9,9],
                    'empregada':[0,3,4,4,4],
                    'maqlavar':[0,2,2,2,2],
                    'dvd':[0,2,2,2,2],
                    'geladeira':[0,4,4,4,4],
                    'freezer':[0,2,2,2,2],
                    'chefe':[0,1,2,3,8]}

    def cb1(item):
        return lambda quantidade: itens_pontos[item][quantidade]

    def pontos_classe(pontos):
        if(pontos < 8):
            return 'E'
        elif(pontos < 14):
            return 'D'
        elif(pontos < 18):
            return 'C2'
        elif(pontos < 23):
            return 'C1'
        elif(pontos < 29):
            return 'B2'
        elif(pontos < 35):
            return 'B1'
        elif(pontos < 42):
            return 'A2'
        else:
            return 'A1'

    tv = df['Q02_critério brasil 1->Televisão em cores']-1
    radio = df['Q02_critério brasil 1->radio']-1
    banheiro = df['Q02_critério brasil 1->banheiro'] -1
    auto = df['Q02_critério brasil 1->automovel']-1
    empregada = df['Q02_critério brasil 1->empregada']-1
    maqlavar = df['Q02_critério brasil 1->maquinalavar']-1
    dvd = df['Q02_critério brasil 1->vcoudvd']-1
    geladeira = df['Q02_critério brasil 1->geladeira']-1
    freezer = df['Q02_critério brasil 1->freezer']-1
    chefe = df['Q03_critério brasil 2'].map(lambda x: int(x[0]))-1 # only first character
    pontos = tv.apply(cb1('tv')) + radio.apply(cb1('radio')) + banheiro.apply(cb1('banheiro')) + auto.apply(cb1('auto')) + empregada.apply(cb1('empregada')) + maqlavar.apply(cb1('maqlavar')) + dvd.apply(cb1('dvd')) + freezer.apply(cb1('freezer')) + geladeira.apply(cb1('geladeira')) + chefe.apply(cb1('chefe'))

    df['pontos_cb'] = pontos
    df['classe_cb'] = pontos.apply(pontos_classe)

    return df

def calc_lit_digital_index(df):
    'Veja http://webuse.org/p/a34'
    
    ldi6 = df['Q08_Literacia digital->Busca Avançada'] + df['Q08_Literacia digital->PDF']  + df['Q08_Literacia digital->Spyware'] + df['Q10_Literacia Digital II->Wiki'] + df['Q10_Literacia Digital II->Cache'] + df['Q10_Literacia Digital II->Phishing'] - 6
    ldi10 = ldi6 + df['Q10_Literacia Digital II->Palavras-chave'] + df['Q08_Literacia digital->JPEG'] + df['Q08_Literacia digital->Blog'] + df['Q10_Literacia Digital II->Vírus'] - 4
    ldi15 = ldi10 + df['Q08_Literacia digital->Preferências'] + df['Q10_Literacia Digital II->Abas no Navegador'] + df['Q10_Literacia Digital II->Firewall'] + df['Q10_Literacia Digital II->Podcast'] + df['Q10_Literacia Digital II->Feeds da Web'] - 5
    df['ldi6'] = ldi6
    df['ldi10'] = ldi10
    df['ldi15'] = ldi15
    return df

def anonimizar(df):
    '''Filter personally identifying information like name, idnumbers, etc.

    We map the Moodle userid to make another ID which will allow us to
    follow a user between surveys.

    But we must be realist: it is very dificult to anonimize data
    without making it useless. Vigilance is required!
    '''

    def RFID_map(seed,N):
        'Return dict with some permutation of range(N)'
        import random
        random.seed(seed)
        ids = range(N)
        random.shuffle(ids)
        return dict(zip(range(N),ids))
    
    del df['Nome de usuário']
    del df['Nome completo']
    del df['NumeroUSP']
    # Aqui indexamos o dataframe com um número mapeado 1-1 com
    # o Moodle ID. Assim podemos comparar usuários entre
    # enquetes. Deixamos o número USP por agora, para fazer testes.
    mapping = RFID_map(config.seed,100000)
    df.index = df['ID'].map(mapping)
    del df['ID']
    del df['Instituição']
    del df['Departamento']
    
    return df

def convert2df(filename):
    ''
    df = pd.read_table(filename,sep='\t',index_col=0)
    return df

def deduplicar(df,field):
    ''
    grouped = df.groupby(field)
    index = [gp_keys[0] for gp_keys in grouped.groups.values()]
    return df.reindex(index)

def dividir(df,field, include_keys=None):
    '''return list  of (value,dataframe) tuples, where dataframes
    contain only rows grouped by values of field'''
    grouped = df.groupby(field)
    if np.iterable(include_keys):
        dfs = [(key,df.reindex(index)) for key, index in grouped.groups.iteritems() if key in include_keys]
    else:
        dfs = [(key,df.reindex(index)) for key, index in grouped.groups.iteritems()]
    return dfs


def process(df,enq_no):
    ''
    if enq_no == 1:
        df = criterio_brasil(df)
        df = calc_lit_digital_index(df)
    elif enq_no == 2:
        df = add_curso_e_grupo(df)
    elif enq_no == 3:
        df = add_curso_e_grupo(df)

    df = add_sexo(df)
    df = add_nasc(df)
    df = deduplicar(df,'ID')
    df = anonimizar(df)
    return df

def writeprocessed(df,filename):
    ''
    print "writing to ", filename
    rec = df.to_records()
    mlab.rec2csv(rec,filename,delimiter='\t')
    #df.to_csv(filename,index=False)

def joinfiles(filenames):
    ''

    dfs = []
    for filename in filenames:
        print "converting " + filename
        dfs.append(convert2df(filename))

    dftotal = dfs[0]
    samecolumns = dfs[0].columns == dfs[1].columns
    try:
        if samecolumns.all():
            print "Same columns: trying append vertically to join"
            if np.intersect1d(dfs[0].index.tolist(),dfs[1].index.tolist()).any():
                print "Warning: some indexes of first two files are the same!"

            for i in range(1,len(dfs)):
                dftotal = dftotal.append(dfs[i])
            
    except AttributeError:
        if not samecolumns:    
            print "Diferent columns: trying to join horizontally"
            for i in range(1,len(dfs)):
                dftotal = dftotal.join(dfs[i],how='outer',lsuffix='_left')
                #dftotal = dftotal.join(dfs[i],how='inner',lsuffix='_left')

    return dftotal



def main(options,filenames):
    ''

    if options.join:
       if not options.outfile:
            print "Need outfile"
            return 1
 
       df = joinfiles(filenames)
       print "Saving to %s" % options.outfile
       
       writeprocessed(df,options.outfile)
       return 0
    
    for filename in filenames:

        print "processing: %s" % filename
        df = convert2df(filename)
        df = process(df,options.enq_no)

        if options.splitfield:
            dfs = dividir(df,options.splitfield)
            for name, df in dfs:
                root, ext = os.path.splitext(filename)
                outfile = root + '-'+ name + '-processed.csv'
                print "Saving to %s" % outfile
                writeprocessed(df,outfile)
        else:
            root, ext = os.path.splitext(filename)
            outfile = root + '-processed.csv'
            print "Saving to %s" % outfile
            writeprocessed(df,outfile)
        
        if options.graph:
            root, ext = os.path.splitext(filename)
            outfile = root + '-graph.png'
            print "Making graphs and saving to ", outfile
            make_graphs(df,outfile)
    
    return 0

    

if __name__ == "__main__":

    usage = "usage: %prog [options] [filename]"
    parser = OptionParser(usage=usage)

    parser.add_option('--dividir', '-d',
                      help   ='Dividir em N arquivos, por Papel ou Curso',
                      type   = 'string',
                      dest   = 'splitfield',
                      action = 'store')

    parser.add_option('--join', '-j',
                      help   =u'Juntar N arquivos, alignando os índices',
                      action = 'store_true')


    parser.add_option('--graph', '-g',
                      help   = 'Make the graphs',
                      action = 'store_true')

    parser.add_option('--outfile', '-o',
                      help   = 'Name of the output graph / join file',
                      action = 'store',
                      dest   = 'outfile')


    parser.add_option('--no', '-n',
                      type   = 'int',
                      help   = 'Number of the Enquete',
                      action = 'store',
                      dest   = 'enq_no')

    

    (options, args) = parser.parse_args()
    if(len(args) == 0):
        parser.error("Especifique pelo menos um (1) arquivo")

    sys.exit(main(options,args))