stated oop-ification

iky_server/actions.py → ikyActions/actions.py

@@ -1,33 +1,9 @@
 from __future__ import print_function
 
-def buy_pizza(tagged_json):
-    return "function buy_pizza(%s)" % (tagged_json)
-
-
-def book_ticket(tagged_json):
-    return "Flight from %s to %s on %s.Done!" % (tagged_json['FROM'], tagged_json['TO'], tagged_json['DATE'])
-
-
-def create_user(tagged_json):
-    return "function create_user(%s)" % (tagged_json)
-
-
-def getMyTxnCount(tagged_json):
-    return "function getMyTxnCount(%s)" % (tagged_json)
-
-
-def check_status(tagged_json):
-    return "function check_status(%s)" % (tagged_json)
-
-def alarm(tagged_json):
-    result = "Alarm set at %s" % (tagged_json["date"])
-    return result
-
-
 def checkTransactionStatus(tagged_json):
     return "Transanction Number : %s is completed." % tagged_json["lr_no"]
 
-def reminder2(tagged_json):
+def addEventToGoogleCalender(tagged_json):
     from apiclient.discovery import build
     from httplib2 import Http
     from oauth2client import file, client, tools
@@ -46,7 +22,7 @@ def reminder2(tagged_json):
     creds = store.get()
 
     if not creds or creds.invalid:
-        flow = client.flow_from_clientsecrets('iky_server/client_secret.json', SCOPES)
+        flow = client.flow_from_clientsecrets('ikyWebServer/client_secret.json', SCOPES)
         creds = tools.run_flow(flow, store, flags) \
             if flags else tools.run(flow, store)
 
@@ -68,6 +44,3 @@ def reminder2(tagged_json):
     return ('''added Event : %r,
             at: %s''' % (e['summary'].encode('utf-8'),
                          e['start']['dateTime']))
-
-def reminder(tagged_json):
-    return "Reminder set!"

ikyCore/crf_train.py

@@ -0,0 +1,49 @@
+import ast
+
+import pycrfsuite
+
+from intentClassifier import Intent_classifier
+#from bson.json_util import loads, dumps
+#from bson.objectid import ObjectId
+
+from ikyWareHouse.mongo import _retrieve
+from featuresExtractor import extractFeatures
+
+def sentToFeatures(sent):
+    return [extractFeatures(sent, i) for i in range(len(sent))]
+
+
+def sentToLabels(sent):
+    return [label for token, postag, label in sent]
+
+
+def sentToTokens(sent):
+    return [token for token, postag, label in sent]
+
+def buildStoryModel(storyId):
+
+    cursor = _retrieve("labled_queries", {"story_id": storyId})
+
+    train_sents = []
+    for item in cursor:
+        train_sents.append(ast.literal_eval(item["item"].encode('ascii', 'ignore')))
+
+    X_train = [_sent2features(s) for s in train_sents]
+    y_train = [_sent2labels(s) for s in train_sents]
+
+    trainer = pycrfsuite.Trainer(verbose=False)
+    for xseq, yseq in zip(X_train, y_train):
+        trainer.append(xseq, yseq)
+
+    trainer.set_params({
+        'c1': 1.0,  # coefficient for L1 penalty
+        'c2': 1e-3,  # coefficient for L2 penalty
+        'max_iterations': 50,  # stop earlier
+
+        # include transitions that are possible, but not observed
+        'feature.possible_transitions': True
+    })
+    trainer.train('models/%s.model' % story_id)
+
+    Intent_classifier().train()
+    return "1"

ikyCore/featuresExtractor.py

@@ -0,0 +1,42 @@
+# NER support functions for Feature extration
+def extractFeatures(sent, i):
+    word = sent[i][0]
+    postag = sent[i][1]
+    features = [
+        'bias',
+        'word.lower=' + word.lower(),
+        'word[-3:]=' + word[-3:],
+        'word[-2:]=' + word[-2:],
+        'word.isupper=%s' % word.isupper(),
+        'word.istitle=%s' % word.istitle(),
+        'word.isdigit=%s' % word.isdigit(),
+        'postag=' + postag,
+        'postag[:2]=' + postag[:2],
+    ]
+    if i > 0:
+        word1 = sent[i - 1][0]
+        postag1 = sent[i - 1][1]
+        features.extend([
+            '-1:word.lower=' + word1.lower(),
+            '-1:word.istitle=%s' % word1.istitle(),
+            '-1:word.isupper=%s' % word1.isupper(),
+            '-1:postag=' + postag1,
+            '-1:postag[:2]=' + postag1[:2],
+        ])
+    else:
+        features.append('BOS')
+
+    if i < len(sent) - 1:
+        word1 = sent[i + 1][0]
+        postag1 = sent[i + 1][1]
+        features.extend([
+            '+1:word.lower=' + word1.lower(),
+            '+1:word.istitle=%s' % word1.istitle(),
+            '+1:word.isupper=%s' % word1.isupper(),
+            '+1:postag=' + postag1,
+            '+1:postag[:2]=' + postag1[:2],
+        ])
+    else:
+        features.append('EOS')
+
+    return features

iky_server/intent_classifier.py → ikyCore/intentClassifier.py

@@ -1,55 +1,55 @@
-# scikit-learn
+import ikyWareHouse.mongo
+
+from bson.json_util import loads
+
 import numpy as np
-from sklearn.pipeline import Pipeline
+from sklearn import preprocessing
+from sklearn.externals import joblib
 from sklearn.feature_extraction.text import CountVectorizer
-from sklearn.svm import LinearSVC
 from sklearn.feature_extraction.text import TfidfTransformer
 from sklearn.multiclass import OneVsRestClassifier
-from sklearn import preprocessing
-from sklearn.externals import joblib
-
-from bson.json_util import loads, dumps
-import mongo
+from sklearn.pipeline import Pipeline
+from sklearn.svm import LinearSVC
 
 
-class Intent_classifier(object):
+class IntentClassifier(object):
     def __init__(self):
         self.lb = preprocessing.MultiLabelBinarizer()
-        self.labeled_stories = mongo._retrieve("labled_queries", {"user_id": "1"})
+        self.labeledStories = ikyWareHouse.mongo._retrieve("labled_queries", {"user_id": "1"})
 
-        y_train_text = []
+        trainLabels = []
 
-        for story in self.labeled_stories:
-            y_train_text.append([story['story_id']])
+        for story in self.labeledStories:
+            trainLabels.append([story['story_id']])
 
-        self.Y = self.lb.fit_transform(y_train_text)
+        self.Y = self.lb.fit_transform(trainLabels)
 
-    def context_train(self):
-        x_train = []
+    def train(self):
+        trainFeatures = []
 
-        for story in self.labeled_stories:
+        for story in self.labeledStories:
             lq = ""
             for i, token in enumerate(loads(story["item"])):
                 if i != 0:
                     lq += " " + token[0]
                 else:
                     lq = token[0]
-            x_train.append(lq)
+            trainFeatures.append(lq)
 
-        self.X_train = np.array(x_train)
+        self.X = np.array(trainFeatures)
 
         classifier = Pipeline([
             ('vectorizer', CountVectorizer()),
             ('tfidf', TfidfTransformer()),
             ('clf', OneVsRestClassifier(LinearSVC(C=0.4)))])
 
-        classifier.fit(self.X_train, self.Y)
+        classifier.fit(self.X, self.Y)
 
         # dump generated model to file
         joblib.dump(classifier, 'models/intent.pkl', compress=3)
         return True
 
-    def context_check(self, user_say):
+    def predict(self, user_say):
         try:
             # Prediction using Model
             classifier = joblib.load('models/intent.pkl')

iky_server/interface.py → ikyCore/interface.py

@@ -1,6 +1,7 @@
-import actions
 import re
 
+from ikyActions import actions
+
 
 # ORACLE database connection
 # import cx_Oracle

iky_server/nlp.py → ikyCore/nlp.py

@@ -1,5 +1,5 @@
 from flask import request, jsonify, Response
-from iky_server import app
+from ikyWebServer import app
 
 # NLP stuff
 from nltk.tag.perceptron import PerceptronTagger

iky_server/predict.py → ikyCore/predict.py

@@ -2,7 +2,7 @@
 
 # Iky's tools
 
-from intent_classifier import Intent_classifier
+from intentClassifier import Intent_classifier
 from functions import datefromstring
 
 # NLP stuff
@@ -12,10 +12,8 @@
 from crf_train import _sent2features
 
 # DB stuff
-from bson.json_util import loads, dumps
 from bson.objectid import ObjectId
-from mongo import _retrieve,_insert
-import ast
+from ikyWareHouse.mongo import _retrieve,_insert
 
 
 # Extract Labeles from BIO tagged sentence
@@ -43,7 +41,7 @@ def extract_labels(tagged):
 def predict(user_say):
     # query = request.args.get('query')
     begin = time()
-    story_id = Intent_classifier().context_check(user_say)
+    story_id = Intent_classifier().predict(user_say)
 
     if not story_id:
         return {"error_code": "0", "error_msg": "can't identify the intent"}

iky_server/celeryconfig.py → ikyMainframe/celeryconfig.py

@@ -11,7 +11,7 @@
 
 CELERYBEAT_SCHEDULE = {
     'every-minute': {
-        'task': 'iky_server.listener.listen',
+        'task': 'ikyWebServer.listener.listen',
         'schedule': timedelta(seconds=5)
     },
 }

iky_server/listener.py → ikyMainframe/listener.py

@@ -1,9 +1,9 @@
-from iky_server import app
-from interface import execute_action
-from mail import emailManager
-from predict import predict
 from celery import Celery
+from ikyWebServer.interface import execute_action
+
 import celeryconfig
+from ikyCore.predict import predict
+from ikyWebServer.mail import emailManager
 
 app.config.from_object(celeryconfig)
 

iky_server/mongo.py → ikyWareHouse/mongo.py

@@ -1,4 +1,4 @@
-from iky_server import app
+from ikyWebServer import app
 from pymongo import MongoClient
 from flask import request
 

ikyWebServer/__init__.py

@@ -0,0 +1,6 @@
+from flask import Flask
+app = Flask(__name__)
+
+import ikyWebServer.home
+import ikyWebServer.ui
+import ikyWebServer.api

iky_server/api.py → ikyWebServer/api.py

@@ -1,9 +1,12 @@
-from iky_server import app
-from flask import Flask, jsonify, render_template, request, Response
 import json
-from predict import predict
+
+from flask import request, Response
 from interface import execute_action
 
+from ikyCore.predict import predict
+from ikyWebServer import app
+
+
 @app.route('/ikyParseAndExecute',methods=['POST','GET'])
 def ikyParseAndExecute():
     if request.method == 'POST':

iky_server/main.py → ikyWebServer/home.py

@@ -1,10 +1,8 @@
-from iky_server import app
-from flask import Flask, render_template, request, jsonify
-import os
-
-from mongo import _retrieve
 from bson.objectid import ObjectId
-from bson.json_util import loads, dumps
+from flask import render_template, request
+
+from ikyWareHouse.mongo import _retrieve
+from ikyWebServer import app
 
 
 # Index

iky_server/mis.py → ikyWebServer/mis.py

@@ -1,4 +1,4 @@
-from iky_server import app
+from ikyWebServer import app
 from flask import request, jsonify, Response
 import json
 from sklearn import linear_model

iky_server/ui.py → ikyWebServer/ui.py

@@ -2,14 +2,14 @@
 
 from flask import request
 
-from iky_server import app
+from ikyWebServer import app
 # DB stuff
 from bson.objectid import ObjectId
 from bson.json_util import dumps
-from mongo import _insert, _retrieve, _delete,_update
+from ikyWareHouse.mongo import _insert, _retrieve, _delete,_update
 
 # Iky's tools
-from intent_classifier import Intent_classifier
+from ikyCore.intentClassifier import Intent_classifier
 
 
 @app.route('/_insert_tagged', methods=['POST'])
@@ -62,7 +62,7 @@ def delete_story():
     query = {"story_id": request.form['story_id']}
     _delete("labled_queries", query);
 
-    Intent_classifier().context_train()
+    Intent_classifier().train()
 
     try:
         os.remove("models/%s.model" % request.form['story_id'])