Lot of progress today. General polish, colors on visualization, bug f…

…ixes, added missing values test.

Backend/Classes/Data.py

@@ -16,6 +16,8 @@ class DataManager():
 
     def __init__(self, redis):
         self.redis = redis
+        # store important locations
+        self.redis.set('session_files', '/SessionFiles/')
 
     #in case file location becomes more complicated
     def retrieveFileLoc(self, sessionId):

Backend/Classes/Flagging.py

@@ -2,7 +2,7 @@
 
 
 class Flag:
-    def __init__(self, config='../SessionFiles/Flagging_Config.json'):
+    def __init__(self, config='/SessionFiles/Flagging_Config.json'):
         self.config = config
         self.flagCodes = self.fetchFlagConfig();
 
@@ -15,9 +15,19 @@ def fetchFlagConfig(self):
 
         return flagCodes;
 
+    def returnAllFlagsAsArr(self):
+        flags = []
+        for flag in self.flagCodes.keys():
+            flags.append({"key": flag, "code":self.flagCodes[flag]})
+
+        return flags
+
     def returnGoodFlag(self):
         return self.flagCodes['None']
 
+    def returnFlag(self, flagName):
+        return self.flagCodes[flagName]
+
     def flag(self, failed, test):
         if(not failed):
             try:

Backend/Classes/Testing.py

@@ -1,6 +1,8 @@
 from abc import ABCMeta, abstractmethod
 from Backend.Classes.Flagging import Flag
 import numpy as np
+import pandas as pd
+from datetime import timedelta
 
 '''
       {'Type':'Basic Outlier Test',
@@ -17,12 +19,17 @@
         {'Name':'Comparision Data', 'Data Type': 'TimeSeries'}, //column name from the same file
         {'Name':'Difference Threshold (%)', 'Data Type': 'Integer'}  //user will be able to select a different column
                                                           //from thier dataset
-      ] },
+      ]},
       {'Type': 'Machine Learning', 'Parameters':[
         {'Name': 'Training Set', 'Data Type': 'TimeSeries'}, //column name
         {'Name': 'Percentage Training Data', 'Data Type' : 'Integer'},
         {'Name': 'Percentage Test Data', 'Data Type' : 'Integer'}
       ]}
+      {'Type': 'Missing Value Test', 'Parameters':[
+        {'Name': 'Missing Value Alias', "Data Type": 'Float'},
+        {'Name': 'Time Step', 'Data Type': 'Integer'},
+        {'Name': 'Time Step Resolution', 'Data Type': 'Time Resolution', 'Options':['minutes', 'hours', 'days', 'weeks']}
+      ]}
 '''
 
 def flagTests(value, standard="", test=""):
@@ -50,14 +57,43 @@ def __init__ (self, testId = 1, **kwargs):
             self.id = testId
             self.column = kwargs["Column"]
             self.testName = kwargs["Type"]
+            self.isAlias = False
+
+            for parameter in kwargs["Parameters"]:
+                if parameter['Name'] == 'Missing Value Alias':
+                    if 'Value' in parameter and parameter['Value'] is not '':
+                        self.mva = parameter['Value']
+                        self.isAlias = True
+                        break
+                elif parameter['Name'] == 'Time Step':
+                    if 'Value' in parameter:
+                        self.step = parameter['Value']
+                elif parameter['Name'] == 'Time Step Resolution':
+                    self.tsr = parameter['Value']
+
+            if self.isAlias is False:
+                if self.tsr == 'minutes':
+                    self.timeStep = pd.Timedelta(minutes=int(self.step))
+                elif self.tsr == 'hours':
+                    self.timeStep = pd.Timedelta(hours=int(self.step))
+                elif self.tsr == 'days':
+                    self.timeStep = pd.Timedelta(days=int(self.step))
+                elif self.tsr == 'weeks':
+                    self.timeStep = pd.Timedelta(weeks=int(self.step))
+
 
 
         # data must be a float list
         # returns a set of boolean flags
         def runTest (self, dataframe):
             # needs flagging
-            outdf = dataframe.isna()[self.column]
-            return outdf.apply(lambda x: self.flag.flag(x, self.testName))
+            if self.isAlias:
+                outdf = np.invert(dataframe[self.column] == float(self.mva))
+                return outdf.apply(lambda x: self.flag.flag(x, self.testName)), dataframe.index
+
+            dataframe = dataframe.reindex(pd.date_range(start=dataframe.index[0], end=dataframe.index[-1], freq=self.timeStep))
+            outdf = dataframe.notna()[self.column]
+            return outdf.apply(lambda x: self.flag.flag(x, self.testName)), dataframe.index
 
 class RangeTest(Test):
     def __init__ (self, testId = 1, **kwargs):

Backend/Classes/Vis.py

@@ -1,55 +1,63 @@
 from bokeh.plotting import figure, output_file, show, save
 from bokeh.embed import components
 from bokeh.models import BoxAnnotation
+from bokeh.palettes import Category20
+from Backend.Classes.Flagging import *
 
 class VisBuilder():
     def __init__(self):
-        # FlagMgr = Flag('../SessionFiles/Flagging_Config.json')
+        self.FlagMgr = Flag()
         pass
 
     def rmvMissingValues(self, y, flags=None):
         for i, val in enumerate(y):
-            if val == -9999.0:
+            if flags[i] == self.FlagMgr.returnFlag('Missing Value Test'):
                 y[i] = float('nan')
 
         return y
 
 
     def BuildLineChart(self, x, y, flags=None):
+        # build color palette
+        flagCodes = self.FlagMgr.returnAllFlagsAsArr()
+        colorMap = Category20[len(flagCodes)]
 
         # remove missing values
-        y = self.rmvMissingValues(y)
+        if flags is not None:
+            y = self.rmvMissingValues(y, flags)
 
         p = figure(title="Line", x_axis_type='datetime', plot_width=700, plot_height=400)
         p.line(x=x, y=y)
 
         i = 0
-
-        print("gets here?")
-
         while i < len(x)-1:
             if 'OK' in flags[i]:
                 i += 1
 
             else:
                 badrng_st = x[i]
                 st = i
-
-                while i < len(x)-2 and 'OK' not in flags[i]:
+                while i < len(x)-2 and self.FlagMgr.returnGoodFlag() not in flags[i]:
                     i += 1
-
                 badrng_end = x[i]
                 end = i
 
-
                 if (end - st) <= 1:
                     badrng_st = x[st-1]
                     badrng_end = x[end+1]
 
+                color = None
+                for c, flag in enumerate(flagCodes):
+                    print(flag['code'], flags[st], flush=True)
+                    if flag['code'] == flags[st]:
+                        color = colorMap[c]
+                        break
+
                 i += 1
 
+                print(color, flush=True)
 
-                badbox = BoxAnnotation(left=badrng_st, right=badrng_end, fill_alpha=0.4, fill_color='red')
+                badbox = BoxAnnotation(left=badrng_st, right=badrng_end, fill_alpha=0.4, fill_color=color)
                 p.add_layout(badbox)
 
         return components(p)

Backend/Services/ConfigManager/ConfigManager.py

@@ -16,9 +16,7 @@ def index():
 	# Try
 	try:
 		testconfig = TestConfiguration("../config/tests.config")
-
-		print(testconfig.TestParameters)
-
+
 		return("working")
 
 	# Except

Backend/Services/TestingFlagging/TestConsumer.py

@@ -4,6 +4,7 @@
 from redis import Redis
 from Backend.Classes.Testing import *
 from Backend.Classes.Data import *
+import math
 import time, sys, traceback
 
 class TestConusmer():
@@ -12,6 +13,7 @@ def __init__(self, boostrapServers=['kafka:29092'], groupId='test001'):
         self.redis = Redis(host='redis', port=6379)
         self.DM = DataManager(self.redis)
         self.flag = Flag()
+        self.liveIndex = None
 
     def combineFlagCols(self, s1, s2):
         goodFlag = self.flag.returnGoodFlag()
@@ -20,6 +22,8 @@ def combineFlagCols(self, s1, s2):
                 s1[col] = s2[col]
             elif s1[col] == goodFlag and s2[col] != goodFlag:
                 s1[col] = s2[col]
+            elif type(s1[col]) is type(0.0) and math.isnan(s1[col]):
+                s1[col] = s2[col]
         return s1
 
     def receive(self):
@@ -38,7 +42,9 @@ def receive(self):
 
                 print(self.redis.get(tests['sessionId']).decode('utf-8'))
                 df = pd.read_csv(self.redis.get(tests['sessionId']).decode('utf-8'))
-
+                df[timeIndex] = pd.to_datetime(df[timeIndex])
+                df = df.set_index(timeIndex)
+                self.liveIndex = df.index
 
                 outdf = pd.DataFrame()
 
@@ -65,16 +71,19 @@ def receive(self):
                                 testrunner = SpatialInconsistencyTest(3, **test)
                                 flags = testrunner.runTest(df)
 
-                            # elif test['Type'] == 'Missing Value Test':
-                            #     # run missingvaltest by default
-                            #     print("MissingValTest")
-                            #     # testrunner = MissingValTest(4, **test)
-                            #     # flags = testrunner.runTest(df)
-
+                            elif test['Type'] == 'Missing Value Test':
+                                # run missingvaltest by default
+                                testrunner = MissingValTest(4, **test)
+                                flags, self.liveIndex = testrunner.runTest(df)
 
+                            print(outdf, flags, flush=True)
+                            outdf = outdf.reindex(self.liveIndex)
+                            flags = flags.reindex(self.liveIndex)
                             if flags.name+"_flags" not in outdf.columns:
+                                # rebase here for alignment if mvt has been run
                                 outdf[flags.name+"_flags"] = flags
                             else:
+                                # rebase here for alignment if
                                 outdf[flags.name+"_flags"] = self.combineFlagCols(outdf[flags.name+"_flags"], flags)
 
 
@@ -88,7 +97,8 @@ def receive(self):
 
 
                 # set datetime as the index of our flags
-                outdf[timeIndex] = pd.to_datetime(df[timeIndex])
+                outdf[timeIndex] = self.liveIndex
+
                 outdf = outdf.set_index(timeIndex)
 
 

Backend/SessionFiles/Flagging_Config.json

@@ -2,5 +2,5 @@
 "Basic Outlier Test": "Outlier",
 "Repeat Value Test": "Repeat Val",
 "Spatial Inconsistency": "Spatially Inconsistent",
-"Missing Value": "Missing Measurement", 
+"Missing Value Test": "Missing Measurement", 
 "Machine Learning": "4"}

Frontend/HQ_Frontend/main.py

@@ -50,6 +50,7 @@ def FetchFlagReview(sessionId):
     VB = VisBuilder();
     # get the name of our current time series column
     col = request.args.get('colName')
+    indx = request.args.get('indexCol')
     # get flags
     fp = redis.get(sessionId+'outputcsv')
     flags = dataManager.getOutputAsDf(sessionId, fp)
@@ -61,15 +62,24 @@ def FetchFlagReview(sessionId):
 
 
     flagobjs = []
-    for i, flag in enumerate(singleSeries):
+    i = 0
+    while i < len(singleSeries)-1:
         obj = {}
-        obj['code'] = flag
-        obj['datetime'] = indexCol[i]
+        bgn = indexCol[i]
+        while i < len(singleSeries)-1 and singleSeries[i] is singleSeries[i+1]:
+            i+=1
+
+        obj['code'] = singleSeries[i]
+        obj['datetime'] = bgn
+
         flagobjs.append(obj)
+        i += 1
 
     # build visualization
     fileName = dataManager.retrieveFileLoc(sessionId)
-    dataCol = dataManager.retrieveOnlyDataCols(fileName, [col], None)[col]
+    tmpCol = dataManager.retrieveOnlyDataCols(fileName, [col], indexName=indx)
+    dataCol = tmpCol.reindex(flags.index)[col]
+
 
     script, div = VB.BuildLineChart(indexCol, dataCol, singleSeries)
 

Frontend/HQ_Frontend/src/AddTestModal.jsx

@@ -121,11 +121,12 @@ class AddTestModal extends React.Component {
     }
   }
 
+
   /* Function that manages the output of the correct tags for the data type
   of a particular parameter. Ultimiately will include error checking.
   */
   handleRenderDifferentParamInputs(parameter){
-   let otherColumns;
+   let otherColumns, options;
 
    switch(parameter['Data Type']){
       case 'TimeSeries':
@@ -136,9 +137,9 @@ class AddTestModal extends React.Component {
           if(dataStream !== this.props.selectedDS){
                if(first){
                  first = false;
-                 for(let parameter in this.props.testJSON['Parameters']){
-                   if(this.props.testJSON['Parameters'][parameter]['Name'] === "Comparision Data"){
-                     this.props.testJSON['Parameters'][parameter]['Value'] = dataStream;
+                 for(let p in this.props.testJSON['Parameters']){
+                   if(this.props.testJSON['Parameters'][p]['Name'] === parameter['Name']){
+                     this.props.testJSON['Parameters'][p]['Value'] = dataStream;
                    }
                  }
                  return <option key={uuidv4()} select="selected">{dataStream}</option>
@@ -151,8 +152,29 @@ class AddTestModal extends React.Component {
         return(<select key={uuidv4()} className="form-control" parameter-name={parameter['Name']} onChange={this.handleValueInput}>{otherColumns}</select>)
         break;
 
+      case 'Time Resolution':
+        first = true;
+        options = parameter['Options'].map( (option) => {
+
+          if(first){
+            first = false;
+            for(let p in this.props.testJSON['Parameters']){
+              if(this.props.testJSON['Parameters'][p]['Name'] === parameter['Name']){
+                this.props.testJSON['Parameters'][p]['Value'] = option;
+              }
+            }
+            return <option key={uuidv4()} select="selected">{option}</option>
+          } else{
+            return <option key={uuidv4()}>{option}</option>
+          }
+
+        })
+
+        return(<select key={uuidv4()} className="form-control" parameter-name={parameter['Name']} onChange={this.handleValueInput}>{options}</select>)
+        break;
+
       default:
-        return(<input key={uuidv4()} className="form-control" parameter-name={parameter['Name']} placeholder="0" onChange={this.handleValueInput}/>)
+        return(<input key={uuidv4()} className="form-control" parameter-name={parameter['Name']} placeholder="-" onChange={this.handleValueInput}/>)
         break;
    }