Skip to content

Commit

Permalink
allow use of different time columns (fixes box#63)
Browse files Browse the repository at this point in the history 
Many reports are based on aggregating by date or hour.
Currently these columns are calculated on the fly,
but grouping on them is very slow.  By allowing
for alternate time columns we can index the table
more efficiently for common reports and gain a huge
improvement in speed.

Also fixed one bug in graph zoom related to filtering
  • Loading branch information
@gtowey
gtowey authored and Gavin Towey committed Dec 28, 2013
1 parent 45104af commit fa9c0f7
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 8 deletions.
55 changes: 53 additions & 2 deletions lib/Anemometer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,17 @@ function __construct($conf)
return;
}


$this->conf = $conf;
$this->data_model = new AnemometerModel($conf);
if (array_key_exists('time_columns', $this->conf))
{
$this->time_columns = $this->conf['time_columns'];
}
else
{
$this->time_columns = array();
}
$datasource = get_var('datasource');
if (isset($datasource)) {
$this->data_model->set_data_source($datasource);
Expand Down Expand Up @@ -147,7 +156,14 @@ private function setup_data_for_graph_search($data=null)

$data['datasource'] = get_var('datasource');

$data['time_field_name'] = $time = $this->data_model->get_field_name('time');
// to maintain backwards config file compatability, we try to guess
// the time field being used for this report
$time = $this->get_time_field_from_report_defaults('graph_defaults');
if (!isset($time))
{
$time = $this->data_model->get_field_name('time');
}
$data['time_field_name'] = $time;
$data['hostname_field_name'] = $this->data_model->get_field_name('hostname');
$data['checksum_field_name'] = $this->data_model->get_field_name('checksum');

Expand Down Expand Up @@ -180,7 +196,7 @@ private function setup_data_for_graph_search($data=null)
$this->set_search_defaults('report_defaults', array('dimension-'.$time.'_start', 'dimension-'.$time.'_end', $data['checksum_field_name']));

$_GET['fact-order'] = get_var('plot_field') . ' DESC';
$data['ajax_table_request_url_base'] = site_url() . '?action=api&output=table&noheader=1&datasource=' . $data['datasource']. '&' . $this->report_obj->get_search_uri(array( 'dimension-'.$data['time_field_name']));
$data['ajax_table_request_url_base'] = site_url() . '?action=api&output=table&noheader=1&datasource=' . $data['datasource']. '&' . $this->report_obj->get_search_uri(array( 'dimension-'.$data['time_field_name'], 'dimension-ts_min'));
$data['table_url_time_start_param'] = 'dimension-'.$data['time_field_name'].'_start';
$data['table_url_time_end_param'] = 'dimension-'.$data['time_field_name'].'_end';
$data['timezone_offset'] = timezone_offset_get( new DateTimeZone( ini_get('date.timezone' )), new DateTime());
Expand Down Expand Up @@ -287,6 +303,7 @@ public function report() {
$this->footer();
}


/**
* Show query samples for a specific checksum
*
Expand Down Expand Up @@ -468,6 +485,7 @@ public function show_query() {
// just set some form fields and call report
// maybe convert to ajax call ...
$this->init_report();
$this->clear_all_time_values();
$this->set_search_defaults('history_defaults', array());
$_GET['fact-checksum'] = $checksum;
//print_r($_GET);
Expand Down Expand Up @@ -614,6 +632,7 @@ private function init_report() {
$conf['tables'],
$this->data_model->get_report($conf['source_type'])
);
$this->report_obj->set_non_aggregate_columns($this->time_columns);
}
}

Expand All @@ -633,6 +652,38 @@ private function set_search_defaults($type = 'report_defaults', $override = null
}
}

private function clear_all_time_values()
{
foreach ($this->time_columns as $col)
{
$start = "dimension-{$col}_start";
$end = "dimension-{$col}_end";
foreach (array($start, $end) as $form_field_name)
{
if (array_key_exists($form_field_name, $_GET))
{
unset($_GET[$form_field_name]);
}
}
}
}

private function get_time_field_from_report_defaults($type)
{
$defaults = $this->data_model->get_report_defaults($type);
foreach ($this->time_columns as $col)
{
$start = "dimension-{$col}_start";
$end = "dimension-{$col}_end";

if (array_key_exists($start, $defaults) or array_key_exists($end, $defaults))
{
return $col;
}
}
return null;
}

private function bchexdec($hex) {
static $hexdec = array(
"0" => 0,
Expand Down
12 changes: 12 additions & 0 deletions lib/MySQLTableReport.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ class MySQLTableReport {
private $order;
private $limit;
private $raw_where;
private $non_aggregate_columns = array();

private static $CONNECT_TIMEOUT = 5;

Expand Down Expand Up @@ -158,6 +159,12 @@ private function init_report()
$this->limit = null;
$this->raw_where = null;
$this->form_data_processed = false;
$this->non_aggregate_columns = array();
}

public function set_non_aggregate_columns(array $columns)
{
$this->non_aggregate_columns = $columns;
}

/**
Expand Down Expand Up @@ -715,6 +722,11 @@ private function get_column_aggregate_function($name) {
if (strpos($name, '.') !== false)
{
$name = substr($name, strpos($name, '.')+1, strlen($name));
}

if (in_array($name, $this->non_aggregate_columns))
{
return null;
}

if (!preg_match("/^([^_]+)_?.*_([^_]+)$/", $name, $regs)) {
Expand Down
13 changes: 8 additions & 5 deletions views/graph_search.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@
*/
function new_plot_data(data) {
// flot requires milliseconds, so convert the timestamp from seconds to milliseconds
new_data = new Array();
DATA = new Array();
for ( var i = 0; i < data.length; i++ )
{
var y_sum = 0; // to check for an empty series.
Expand All @@ -137,13 +137,12 @@ function new_plot_data(data) {
{
delete data[i];
} else {
new_data.push(data[i]);
DATA.push(data[i]);
}

}
//console.log(data);
var theplot = $("#theplot"); // get the graph div
plot_obj = $.plot(theplot, new_data, FLOT_OPTS);
plot_obj = $.plot(theplot, DATA, FLOT_OPTS);
setup_selection(theplot);
}

Expand Down Expand Up @@ -230,7 +229,8 @@ function setup_selection(theplot) {
new_plot_opts['yaxis'] = { min: yrange[0], max: yrange[1]* 1.2 };
}
var plot = $.plot(theplot, DATA, $.extend ( true, {}, FLOT_OPTS, new_plot_opts));

console.log(plot);

// need a date object to shove timestamp into for conversion to ANSI-type date string
d = new Date();

Expand All @@ -253,6 +253,7 @@ function setup_selection(theplot) {
$('#permalink_btn').attr('href', GRAPH_PERMALINK_URL + new_url_start_end_params);

// Get the data for the table and re-populate it!
console.log("Refreshing Table Data: " + new_table_data_url);
$.ajax({
url: new_table_data_url,
method: 'GET',
Expand Down Expand Up @@ -369,6 +370,7 @@ function toggle_autoscale_y()
});

// kick off the initial AJAX call to get the data to plot for the graph
console.log("Fetching graph results: " + GRAPH_DATA_URL + url_start_end_params);
$.ajax({
url: GRAPH_DATA_URL + url_start_end_params,
method: 'GET',
Expand All @@ -377,6 +379,7 @@ function toggle_autoscale_y()
});

// kick off the initial AJAX call to get the data for the table below the graph
console.log("Fetching initial table results: "+table_url_now);
$.ajax({
url: table_url_now,
method: 'GET',
Expand Down
25 changes: 24 additions & 1 deletion views/report_json2.php
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,36 @@
<?php

$group = get_var('fact-group');
$debug = get_var('debug');
if ($debug)
{
print "<pre>".$sql."</pre>";
}

$series = array_filter($columns, function ($x) use ($group) { return $x == $group ? false : true; });
$wtfdata = array();
if (count($result))
{
if (preg_match("/^\d+$/", $result[0][$group]))
{
$is_datetime = false;
}
else
{
$is_datetime = true;
}
}
foreach ($result as $row)
{
foreach ($series as $col)
{
$wtfdata[$col][] = array( $row[$group], $row[$col]);
$date = $row[$group];
if ($is_datetime)
{
$parts = strptime($date, "%Y-%m-%d %H:%M:%S");
$date = mktime($parts['tm_hour'], $parts['tm_min'],$parts['tm_sec'], $parts['tm_mon']+1, $parts['tm_mday'], $parts['tm_year']+1900);
}
$wtfdata[$col][] = array( $date, $row[$col]);
}
}

Expand Down

0 comments on commit fa9c0f7

Please sign in to comment.