Pie charts with confidence intervals (vmware-archive#587)

bin/deploy.py

@@ -26,6 +26,7 @@ def generate_script(config, rh, template):
     filename = template.replace("-template", "")
     lines = [variables if "REPLACE_WITH_VARIABLES" in x else x for x in lines]
     with open(filename, "w") as f:
+        f.write("# Automatically generated from " + template)
         for l in lines:
             f.write(l)
     os.chmod(filename, 0o770)

bin/frontend-start.sh

@@ -12,6 +12,6 @@ export JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=
 # If you want GRPC logging uncomment the following line.
 # LOGGING=" -Djava.util.logging.config.file=logging.properties"
 export JAVA_OPTS="$JAVA_OPTS$LOGGING"
-
+export CATALINA_PID="catalina.pid"
 cd ${mydir}/..
 ./apache-tomcat-${TOMCATVERSION}/bin/catalina.sh run

bin/hillview-webserver-manager-template.sh

@@ -12,12 +12,13 @@ start() {
         rm -f hillview-web.log hillview-web.log.* hillview-web.log*.lck
     fi
     export WEB_CLUSTER_DESCRIPTOR=serverlist
-    nohup ./${TOMCAT}/bin/startup.sh &
+    export CATALINA_PID=catalina.pid
+    ./${TOMCAT}/bin/startup.sh
 }
 
 stop() {
     if pgrep -f tomcat; then
-        ${SERVICE_DIRECTORY}"/"${TOMCAT}/bin/shutdown.sh
+        ${SERVICE_DIRECTORY}"/"${TOMCAT}/bin/shutdown.sh -force
         echo Stopped
     else
         echo "Web server already stopped"

data/ontime_private/gen_metadata.py

@@ -20,12 +20,13 @@ def get_metadata(cn):
     elif cn == "DepDelay" or cn == "ArrDelay":
         (g, gMin, gMax) = (1, -100, 1000)
     elif cn == "Cancelled":
-        (g, gMin, gMax) = (1, 0, 2)
+        (g, gMin, gMax) = (1, 0, 1)
     elif cn == "ActualElapsedTime":
-        (g, gMin, gMax) = (1, 15, 700)
+        (g, gMin, gMax) = (1, 15, 800)
     elif cn == "Distance":
-        (g, gMin, gMax) = (10, 0, 5000)
+        (g, gMin, gMax) = (10, 0, 5100)
     elif cn == "FlightDate":
+        # cluster values: (86400000, 852076800000, 1561852800000)
         (g, gMin, gMax) = (86400000, 1451635200000, 1456732800000)
     else:
         raise Exception("Unexpected column " + cn)

docs/userManual.md

@@ -21,7 +21,7 @@ performs all operations using a class of very efficient algorithms,
 called “sketches”, which are constrained to compute with bounded
 memory over distributed data.
 
-Updated on 2019 Dec 20.
+Updated on 2019 Dec 23.
 
 # Contents
 |Section|Reference|
@@ -1178,8 +1178,8 @@ The "View" menu from a histogram display has the following functions:
   current histogram.
 
 * pie chart/histogram: switch the displayed view between a histogram
-  or a pie chart.  This is only enabled for non-numeric columns.  The following
-  image shows a typical pie chart view of the data in a histogram.
+  or a pie chart.  This is only enabled for categorical or integer columns.
+  The following image shows a typical pie chart view of the data in a histogram.
 
   ![Pie chart](pie-chart.png)
 

docs/userManual.src

@@ -1117,8 +1117,8 @@ The "View" menu from a histogram display has the following functions:
   current histogram.
 
 * pie chart/histogram: switch the displayed view between a histogram
-  or a pie chart.  This is only enabled for non-numeric columns.  The following
-  image shows a typical pie chart view of the data in a histogram.
+  or a pie chart.  This is only enabled for categorical or integer columns.
+  The following image shows a typical pie chart view of the data in a histogram.
 
   ![Pie chart](pie-chart.png)
 

web/src/main/java/org/hillview/dataStructures/PrivateHistogram.java

@@ -15,6 +15,8 @@
 @SuppressWarnings("MismatchedReadAndWriteOfArray")
 public class PrivateHistogram extends HistogramPrefixSum implements IJson {
     private int[] confidence;
+    private int missingConfidence;  // Confidence for the missing value
+    // TODO(pratiksha): compute the missing value confidence
     private final double epsilon;
     private SecureLaplace laplace;
 

web/src/main/webapp/dataViews/heatmapView.ts

@@ -304,10 +304,10 @@ export class HeatmapView extends ChartView {
                 histogram: heatmap.histogramMissingX,
                 cdfBuckets: null,
                 confidence: null,
+                missingConfidence: null
             };
 
             this.xHistoPlot.setHistogram(augHist, this.samplingRate,
-                heatmap.histogramMissingX.missingData,
                 this.xAxisData, null,
                 this.page.dataset.isPrivate());
             this.xHistoPlot.draw();

web/src/main/webapp/dataViews/histogramView.ts

@@ -214,7 +214,8 @@ export class HistogramView extends HistogramViewBase /*implements IScrollTarget*
     public setAxes(xAxisData: AxisData): void {
         this.xAxisData = xAxisData;
         const submenu = this.menu.getSubmenu("View");
-        submenu.enable("pie chart/histogram", kindIsString(this.xAxisData.description.kind));
+        submenu.enable("pie chart/histogram", kindIsString(this.xAxisData.description.kind) ||
+            this.xAxisData.description.kind === "Integer");
     }
 
     /**
@@ -243,7 +244,7 @@ export class HistogramView extends HistogramViewBase /*implements IScrollTarget*
 
         const counts = h.buckets;
         this.bucketCount = counts.length;
-        this.plot.setHistogram(augmentedHistogram, this.samplingRate, h.missingData,
+        this.plot.setHistogram(augmentedHistogram, this.samplingRate,
                                this.xAxisData, maxYAxis, this.page.dataset.isPrivate());
         this.plot.draw();
 

web/src/main/webapp/dataViews/spectrumView.ts

@@ -168,9 +168,10 @@ export class SpectrumView extends ChartView {
             histogram: h,
             cdfBuckets: null,
             confidence: null,
+            missingConfidence: null
         };
 
-        this.plot.setHistogram(augHist, 1, h.missingData,
+        this.plot.setHistogram(augHist, 1,
             axisData, null, this.page.dataset.isPrivate());
         this.plot.draw();
 

web/src/main/webapp/dataViews/trellisHistogramView.ts

@@ -371,7 +371,6 @@ export class TrellisHistogramView extends TrellisChartView {
             };
 
             plot.setHistogram(augHist, this.samplingRate,
-                data.histogramMissingX.buckets[i],
                 this.xAxisData,
                 max, this.page.dataset.isPrivate());
             plot.displayAxes = false;

web/src/main/webapp/javaBridge.ts

@@ -254,9 +254,8 @@ export interface AugmentedHistogram {
     cdfBuckets?: number[];
     /// confidence interval for each histogram bucket
     confidence?: number[];
-    // Confidence values around the missing count
-    missingMin?: number;
-    missingMax?: number;
+    // Confidence for the missing count
+    missingConfidence?: number;
 }
 
 // This is actually a union of two java classes

web/src/main/webapp/ui/IBarPlot.ts

@@ -6,7 +6,7 @@ import {AxisData} from "../dataViews/axisData";
 import {D3Scale} from "./ui";
 
 export interface IBarPlot {
-    setHistogram(bars: AugmentedHistogram, samplingRate: number, missingCount: number,
+    setHistogram(bars: AugmentedHistogram, samplingRate: number,
                  axisData: AxisData, maxYAxis: number | null, isPrivate: boolean): void;
     draw(): void;
     getYScale(): D3Scale;

web/src/main/webapp/ui/helpUrl.ts

@@ -20,15 +20,15 @@
  * Maps each ViewKind to a url anchor in the github userManual.
  */
 export const helpUrl = {
-    "Table": "(#32-table-views)",
-    "Histogram": "(#34-uni-dimensional-histogram-views)",
-    "2DHistogram": "(#35-two-dimensional-histogram-views)",
-    "Heatmap": "(#36-heatmap-views)",
+    "Table": "(#33-table-views)",
+    "Histogram": "(#35-uni-dimensional-histogram-views)",
+    "2DHistogram": "(#36-two-dimensional-histogram-views)",
+    "Heatmap": "(#37-heatmap-views)",
     "TrellisHistogram": "(#41-trellis-plots-of-1d-histograms)",
     "TrellisHeatmap": "(#43-trellis-plots-of-heatmaps)",
     "Trellis2DHistogram": "(#42-trellis-plots-of-2d-histograms)",
-    "HeavyHitters": "(#33-frequent-elements-views)",
-    "Schema": "(#31-schema-views)",
+    "HeavyHitters": "(#34-frequent-elements-views)",
+    "Schema": "(#32-schema-views)",
     "Load": "(#23-loading-data)",
-    "SVD Spectrum": "(#37-singular-value-spectrum-views)"
+    "SVD Spectrum": "(#38-singular-value-spectrum-views)"
 };

web/src/main/webapp/ui/histogramPlot.ts

@@ -61,13 +61,13 @@ export class HistogramPlot extends Plot implements IBarPlot {
      * Set the histogram that we want to draw.
      * @param bars          Description of the histogram bars.
      * @param samplingRate  Sampling rate used to compute this histogram.
-     * @param missingCount  Number of values missing.  TODO: display it graphically.
      * @param axisData      Description of the X axis.
      * @param maxYAxis      If present it is used to scale the maximum value for the Y axis.
      * @param isPrivate     True if we are plotting private data.
      */
-    public setHistogram(bars: AugmentedHistogram, samplingRate: number, missingCount: number,
+    public setHistogram(bars: AugmentedHistogram, samplingRate: number,
                         axisData: AxisData, maxYAxis: number | null, isPrivate: boolean): void {
+        // TODO: display missing data graphically.
         this.histogram = bars;
         this.samplingRate = samplingRate;
         this.xAxisData = axisData;
@@ -88,7 +88,6 @@ export class HistogramPlot extends Plot implements IBarPlot {
         const counts = this.histogram.histogram.buckets.map((x) => Math.max(x, 0));
         this.max = Math.max(...counts);
         const displayMax = this.maxYAxis == null ? this.max : this.maxYAxis;
-
         const chartWidth = this.getChartWidth();
         const chartHeight = this.getChartHeight();
         const confidence = this.isPrivate ? this.histogram.confidence :

web/src/main/webapp/ui/piePlot.ts

@@ -22,7 +22,7 @@ import {Plot} from "./plot";
 import {PlottingSurface} from "./plottingSurface";
 import {D3Scale, SpecialChars} from "./ui";
 import {IBarPlot} from "./IBarPlot";
-import {cloneArray, formatNumber, percent, significantDigits} from "../util";
+import {cloneArray, formatNumber, makeInterval, percent, significantDigits, valueWithConfidence} from "../util";
 
 interface ValueAndIndex {
     value: number;
@@ -42,7 +42,6 @@ export class PiePlot extends Plot implements IBarPlot {
      */
     public samplingRate: number;
     public isPrivate: boolean;
-    private missingCount: number;
     public maxYAxis: number;
 
     public constructor(protected plottingSurface: PlottingSurface) {
@@ -53,18 +52,16 @@ export class PiePlot extends Plot implements IBarPlot {
      * Set the histogram that we want to draw.
      * @param bars          Description of the histogram bars.
      * @param samplingRate  Sampling rate used to compute this histogram.
-     * @param missingCount  Number of missing values.
      * @param axisData      Description of the X axis.
      * @param maxYAxis      Not used for pie chart.
      * @param isPrivate     True if we are plotting private data.
      */
-    public setHistogram(bars: AugmentedHistogram, samplingRate: number, missingCount: number,
+    public setHistogram(bars: AugmentedHistogram, samplingRate: number,
                         axisData: AxisData, maxYAxis: number | null, isPrivate: boolean): void {
         this.histogram = bars;
         this.samplingRate = samplingRate;
         this.xAxisData = axisData;
         this.isPrivate = isPrivate;
-        this.missingCount = missingCount;
         this.maxYAxis = maxYAxis;
     }
 
@@ -82,24 +79,60 @@ export class PiePlot extends Plot implements IBarPlot {
         return this.xAxisData.bucketDescription(bucketIndex, 40);
     }
 
-    private countAsString(count: number): string {
-        if (this.samplingRate < 1) {
-            return SpecialChars.approx + significantDigits(count);
+    private countAsString(count: number, confidence: number, sum: number): string {
+        let result;
+        if (this.isPrivate) {
+            result = SpecialChars.approx + makeInterval(valueWithConfidence(count, confidence));
+        } else if (this.samplingRate < 1) {
+            result = SpecialChars.approx + significantDigits(count);
         } else {
-            return formatNumber(count);
+            result = formatNumber(count);
         }
+        result += ", ";
+        if (this.isPrivate) {
+            const min = Math.max(0, count - confidence);
+            const percLow = percent(min/sum);
+            const percHigh = percent((count + confidence) / sum);
+            if (percLow === percHigh)
+                result += percLow;
+            else
+                result += percent(min / sum) + ":" + percent((count + confidence) / sum);
+        } else {
+            result += percent(count / sum);
+        }
+        return result;
+    }
+
+    private static clamp(value: number, min: number, max: number): number {
+        if (min > max) {
+            // This should not happen.
+            return min;
+        } else {
+            if (value < min)
+                return min;
+            else if (value > max)
+                return max;
+        }
+        return value;
     }
 
     private drawPie(): void {
-        const counts = cloneArray(this.histogram.histogram.buckets);
-        counts.push(this.missingCount);
+        const counts = this.histogram.histogram.buckets.map((x) => Math.max(x, 0));
+        counts.push(this.histogram.histogram.missingData);
         const pie = d3pie().sort(null);
         const chartWidth = this.getChartWidth();
         const chartHeight = this.getChartHeight();
         const radius = Math.min(chartWidth, chartHeight) / 2.2;
         const arc = d3arc()
             .innerRadius(0)
             .outerRadius(radius);
+        let confidence;
+        if (this.isPrivate) {
+            confidence = cloneArray(this.histogram.confidence);
+            confidence.push(this.histogram.missingConfidence);
+        } else {
+            confidence = new Array(this.histogram.histogram.buckets.length + 1);
+        }
 
         const sum = counts.reduce((a,b) => a + b);
         const sum2 = sum / 2;   // sum2 is the half of the total count
@@ -116,7 +149,7 @@ export class PiePlot extends Plot implements IBarPlot {
             .attr("d", arc)
             .attr('fill', (d,i) => this.color(i, counts.length))
             .append("svg:title")
-            .text((d,i) => this.label(i) + ":" + this.countAsString(counts[i]) + ", " + percent(counts[i] / sum))
+            .text((d,i) => this.label(i) + ":" + this.countAsString(counts[i], confidence[i], sum))
             .exit();
 
         let total = 0;
@@ -154,16 +187,7 @@ export class PiePlot extends Plot implements IBarPlot {
             const min = Math.max(i * spacingSize, previousPosition + spacingSize);
             const max = chartHeight - (rightPosition.length - i - 1) * spacingSize; // -1 for the start of the next label
             let ideal = chartHeight / 2 - Math.cos(rightPosition[i].value * Math.PI) * radius;
-            let labelPosition = ideal;
-            if (min > max) {
-                // This should not happen.
-                labelPosition = min;
-            } else {
-                if (ideal < min)
-                    labelPosition = min;
-                else if (ideal > max)
-                    labelPosition = max;
-            }
+            let labelPosition = PiePlot.clamp(ideal, min, max);
             rightLabelPosition.push({ value: labelPosition, index: rightPosition[i].index });
             previousPosition = labelPosition;
         }
@@ -173,16 +197,7 @@ export class PiePlot extends Plot implements IBarPlot {
             const min = (leftPosition.length - i) * spacingSize;
             const max = Math.min(previousPosition - spacingSize, chartHeight - (i + 1) * spacingSize);
             const ideal = chartHeight / 2 - Math.cos(leftPosition[i].value * Math.PI) * radius;
-            let labelPosition = ideal;
-            if (min > max) {
-                // This should not happen.
-                labelPosition = max;
-            } else {
-                if (ideal < min)
-                    labelPosition = min;
-                else if (ideal > max)
-                    labelPosition = max;
-            }
+            let labelPosition = PiePlot.clamp(ideal, min, max);
             leftLabelPosition.push({ value: labelPosition, index: leftPosition[i].index });
             previousPosition = labelPosition;
         }
@@ -200,7 +215,7 @@ export class PiePlot extends Plot implements IBarPlot {
             .attr("text-anchor", "end")
             .text((d) => this.label(d.index))
             .append("svg:title")
-            .text((d) => this.countAsString(counts[d.index]) + ", " + percent(counts[d.index] / sum));
+            .text((d) => this.countAsString(counts[d.index], confidence[d.index], sum));
         drawing.data(rightLabelPosition)
             .enter()
             .append("text")
@@ -210,7 +225,7 @@ export class PiePlot extends Plot implements IBarPlot {
             .attr("text-anchor", "start")
             .text((d) => this.label(d.index))
             .append("svg:title")
-            .text((d) => this.countAsString(counts[d.index]) + ", " + percent(counts[d.index] / sum));
+            .text((d) => this.countAsString(counts[d.index], confidence[d.index], sum));
         // lines connecting labels to pie segments
         drawing.data(rightLabelPosition)
             .enter()

web/src/main/webapp/util.ts

@@ -153,7 +153,7 @@ export function makeInterval(value: [number, number]): string {
     if (value[0] >= value[1])
         return significantDigits(value[0]);
     else
-        return significantDigits(value[0]) + " : " + significantDigits(value[1]);
+        return significantDigits(value[0]) + ":" + significantDigits(value[1]);
 }
 
 /**