Skip to content

Commit

Permalink
Fixes for MarkovPathEstimator. We need to make sure that always execu…
Browse files Browse the repository at this point in the history 
…te callback() in order to populate the path correctly. This may make the estimates run a little bit slower.
  • Loading branch information
@apavlo
apavlo committed Nov 21, 2012
1 parent 6ffadb9 commit af0bc55
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 29 deletions.
10 changes: 3 additions & 7 deletions log4j.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,6 @@ log4j.logger.edu.brown.hstore.conf=INFO
log4j.logger.edu.brown.catalog=INFO
log4j.logger.edu.brown.utils=INFO

log4j.logger.edu.brown.markov=INFO
log4j.logger.edu.brown.hstore.estimators=INFO
log4j.logger.edu.brown.hstore.estimators.markov=INFO
log4j.logger.edu.brown.hstore.specexec=INFO

## VoltDB Stuff
log4j.logger.org.voltdb.VoltProcedure=INFO
log4j.logger.org.voltdb.client=INFO
Expand All @@ -39,8 +34,9 @@ log4j.logger.org.voltdb.sysprocs.LoadMultipartitionTable=INFO
log4j.logger.org.voltdb.sysprocs=INFO

## Research Projects
log4j.logger.edu.brown.catalog.FixCatalog=INFO
log4j.logger.edu.brown.catalog.special.VerticalPartitionColumn=INFO
log4j.logger.edu.brown.hstore.estimators=INFO
log4j.logger.edu.brown.hstore.estimators.markov=INFO
log4j.logger.edu.brown.hstore.specexec=INFO
log4j.logger.edu.brown.statistics=INFO
log4j.logger.edu.brown.markov=INFO
log4j.logger.edu.brown.costmodel=INFO
Expand Down
3 changes: 1 addition & 2 deletions src/frontend/edu/brown/hstore/estimators/markov/MarkovPathEstimator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,6 @@ protected void populate_children(Children<MarkovVertex> children, MarkovVertex e
Collection<MarkovVertex> next_vertices = markov.getSuccessors(element);
if (next_vertices == null || next_vertices.isEmpty()) {
if (t) LOG.trace("No succesors were found for " + element + ". Halting traversal");
this.stop();
return;
}
if (t) LOG.trace("Successors: " + next_vertices);
Expand Down Expand Up @@ -500,6 +499,7 @@ else if (t) {

@Override
protected void callback(MarkovVertex v) {
this.estimate.path.add(v);
if (v.isQueryVertex() == false) {
if (v.isCommitVertex()) {
if (t) LOG.trace("Reached COMMIT. Stopping...");
Expand All @@ -509,7 +509,6 @@ protected void callback(MarkovVertex v) {
this.stop();
}
}
this.estimate.path.add(v);
}


Expand Down
41 changes: 21 additions & 20 deletions tests/frontend/edu/brown/hstore/estimators/markov/TestMarkovPathEstimator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@

import java.io.File;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.voltdb.VoltProcedure;
import org.voltdb.benchmark.tpcc.procedures.neworder;
Expand All @@ -14,17 +12,15 @@

import edu.brown.BaseTestCase;
import edu.brown.catalog.CatalogUtil;
import edu.brown.hstore.estimators.markov.MarkovEstimate;
import edu.brown.hstore.estimators.markov.MarkovPathEstimator;
import edu.brown.mappings.ParameterMappingsSet;
import edu.brown.markov.MarkovGraph;
import edu.brown.markov.MarkovVertex;
import edu.brown.markov.containers.MarkovGraphContainersUtil;
import edu.brown.markov.containers.MarkovGraphsContainer;
import edu.brown.utils.CollectionUtil;
import edu.brown.utils.MathUtil;
import edu.brown.utils.PartitionSet;
import edu.brown.utils.ProjectType;
import edu.brown.utils.StringUtil;
import edu.brown.workload.TransactionTrace;
import edu.brown.workload.Workload;
import edu.brown.workload.filters.BasePartitionTxnFilter;
Expand All @@ -45,10 +41,10 @@ public class TestMarkovPathEstimator extends BaseTestCase {

private static Workload workload;
private static MarkovGraphsContainer markovs;
private static ParameterMappingsSet correlations;
private static ParameterMappingsSet mappings;
private static TransactionTrace singlep_trace;
private static TransactionTrace multip_trace;
private static final Set<Integer> multip_partitions = new HashSet<Integer>();
private static final PartitionSet multip_partitions = new PartitionSet();
private static final List<MarkovVertex> multip_path = new ArrayList<MarkovVertex>();

private MarkovPathEstimator pathEstimator;
Expand All @@ -63,8 +59,8 @@ public void setUp() throws Exception {

if (markovs == null) {
File file = this.getParameterMappingsFile(ProjectType.TPCC);
correlations = new ParameterMappingsSet();
correlations.load(file, catalog_db);
mappings = new ParameterMappingsSet();
mappings.load(file, catalog_db);

// Workload Filter:
// (1) Only include TARGET_PROCEDURE traces
Expand Down Expand Up @@ -150,13 +146,13 @@ public void testMarkovEstimate() throws Exception {
pathEstimator.traverse(this.graph.getStartVertex());

List<MarkovVertex> visitPath = pathEstimator.getVisitPath();
System.err.println(StringUtil.columns(StringUtil.join("\n", visitPath), this.estimate.toString()));
// System.err.println(StringUtil.columns(StringUtil.join("\n", visitPath), this.estimate.toString()));

assertFalse(singlep_trace.isAborted());
assertFalse(visitPath.contains(this.graph.getAbortVertex()));

// System.err.println(singlep_trace.debug(catalog_db));
System.err.println("Base Partition = " + p_estimator.getBasePartition(singlep_trace));
// System.err.println("Base Partition = " + p_estimator.getBasePartition(singlep_trace));
// for (QueryTrace qtrace : singlep_trace.getQueries()) {
// System.err.println(qtrace.debug(catalog_db) + " => " + p_estimator.getAllPartitions(qtrace, BASE_PARTITION));
// }
Expand Down Expand Up @@ -191,18 +187,20 @@ public void testSinglePartition() throws Exception {
MarkovVertex commit = this.graph.getCommitVertex();
MarkovVertex abort = this.graph.getAbortVertex();

// MarkovPathEstimator.LOG.setLevel(Level.DEBUG);
pathEstimator.init(this.graph, this.estimate, BASE_PARTITION, singlep_trace.getParams());
pathEstimator.setForceTraversal(true);
pathEstimator.traverse(this.graph.getStartVertex());
ArrayList<MarkovVertex> path = new ArrayList<MarkovVertex>(pathEstimator.getVisitPath());
assertTrue(estimate.isConfidenceCoefficientSet());
float confidence = this.estimate.getConfidenceCoefficient();

// System.err.println("INITIAL PATH:\n" + StringUtil.join("\n", path));
// System.err.println("CONFIDENCE: " + confidence);
// System.err.println("DUMPED FILE: " + MarkovUtil.exportGraphviz(this.graph, false, this.graph.getPath(path)).writeToTempFile());

// System.err.println(singlep_trace.debug(catalog_db));
// System.err.println(StringUtil.columns(StringUtil.join("\n", path), this.estimate.toString()));

ArrayList<MarkovVertex> path = new ArrayList<MarkovVertex>(this.estimate.getMarkovPath());
assertEquals(path, new ArrayList<MarkovVertex>(pathEstimator.getVisitPath()));
assertEquals(start, CollectionUtil.first(path));
assertEquals(commit, CollectionUtil.last(path));
assertFalse(path.contains(abort));
Expand All @@ -214,8 +212,6 @@ public void testSinglePartition() throws Exception {
assertEquals(1, v.getPartitions().size());
assert(v.getPartitions().contains(BASE_PARTITION));
} // FOR
// MarkovPathEstimator.LOG.setLevel(Level.INFO);

// GraphvizExport<Vertex, Edge> gv = MarkovUtil.exportGraphviz(this.graph, true, this.graph.getPath(path));
// FileUtil.writeStringToFile("/tmp/dump.dot", gv.export(this.graph.getProcedure().getName()));
}
Expand All @@ -233,16 +229,21 @@ public void testMultiPartition() throws Exception {
pathEstimator.init(this.graph, this.estimate, BASE_PARTITION, multip_trace.getParams());
pathEstimator.setForceTraversal(true);
pathEstimator.traverse(this.graph.getStartVertex());
ArrayList<MarkovVertex> path = new ArrayList<MarkovVertex>(pathEstimator.getVisitPath());

// System.err.println("INITIAL PATH:\n" + StringUtil.join("\n", path));
// System.err.println("INITIAL PATH:\n" + StringUtil.join("\n", path));
// System.err.println("CONFIDENCE: " + confidence);
// System.err.println("DUMPED FILE: " + MarkovUtil.exportGraphviz(this.graph, false, this.graph.getPath(path)).writeToTempFile());
// System.err.println(multip_trace.debug(catalog_db));
// System.err.println(StringUtil.columns(StringUtil.join("\n", path), this.estimate.toString()));

ArrayList<MarkovVertex> path = new ArrayList<MarkovVertex>(this.estimate.getMarkovPath());
assertEquals(path, new ArrayList<MarkovVertex>(pathEstimator.getVisitPath()));
assertEquals(start, CollectionUtil.first(path));
assertEquals(commit, CollectionUtil.last(path));
assertFalse(path.contains(abort));

// All of the vertices should only have the base partition in their partition set
Set<Integer> touched_partitions = new HashSet<Integer>();
PartitionSet touched_partitions = new PartitionSet();
for (MarkovVertex v : path) {
touched_partitions.addAll(v.getPartitions());
} // FOR
Expand All @@ -252,6 +253,6 @@ public void testMultiPartition() throws Exception {

// this.writeGraphviz(multip_path);
// this.writeGraphviz(path);
// assertEquals(multip_partitions, touched_partitions);
assertEquals(multip_partitions, touched_partitions);
}
}

0 comments on commit af0bc55

Please sign in to comment.