Fixed a bug in the cost-based optimizer that caused slow performance …

…(KeyValueStore.size() fixed).
avitorovic · Sep 20, 2016 · 47daaf4 · 47daaf4
1 parent 7f6a2bd
commit 47daaf4
Show file tree

Hide file tree

Showing 6 changed files with 30 additions and 23 deletions.
diff --git a/squall-core/src/main/java/ch/epfl/data/squall/storage/KeyValueStore.java b/squall-core/src/main/java/ch/epfl/data/squall/storage/KeyValueStore.java
@@ -35,12 +35,10 @@
 import ch.epfl.data.squall.utilities.SystemParameters;
 
 public class KeyValueStore<K, V> extends BasicStore<V> {
-
-    /**
-	 * 
-	 */
     private static final long serialVersionUID = 1L;
 
+    private int _msize = 0; // size that we maintain, rather than asking the underlying _memstore 
+
     private static Logger LOG = Logger.getLogger(KeyValueStore.class);
     private Type _tc = null;
     private HashMap<K, ArrayList<V>> _memstore;
@@ -206,6 +204,7 @@ public void onInsert(Object... data) {
 	    values = this._memstore.get(key);
 	    values.add(value);
 	}
+	_msize++;
     }
 
     @Override
@@ -255,6 +254,7 @@ public void purgeState(long tillTimeStamp) {
 		}
 	    }
 	    // removed !!!! use DST_TUPLE_STORAGE
+	    // TODO: _msize is not maintained here
 	}
     }
 
@@ -268,13 +268,15 @@ public void setTypeConversion(Type tc) {
     }
 
     public int size() {
-	int size = 0;
-	final Object[] x = _memstore.values().toArray();
-	for (int i = 0; i < x.length; i++) {
-	    final ArrayList<V> entry = (ArrayList<V>) x[i];
-	    size += entry.size();
-	}
-	return size;
+// This piece of code causes 1-2 orders of magnitude slowdown	
+//	int size = 0;
+//	final Object[] x = _memstore.values().toArray();
+//	for (int i = 0; i < x.length; i++) {
+//	    final ArrayList<V> entry = (ArrayList<V>) x[i];
+//	    size += entry.size();
+//	}
+//	return size;
+	return _msize;
     }
 
     @Override
@@ -287,4 +289,4 @@ public void setSingleEntry(boolean singleEntry) {
 
     }
 
-}
+}
diff --git a/test/squall/confs/cluster/1G_hyracks → test/squall/confs/cluster/10G_hyracks b/test/squall/confs/cluster/1G_hyracks → test/squall/confs/cluster/10G_hyracks
@@ -1,13 +1,16 @@
 DIP_DISTRIBUTED true
 DIP_QUERY_NAME hyracks
-
 DIP_TOPOLOGY_NAME_PREFIX username
-DIP_DATA_ROOT /data/squall_blade/data/tpchdb/
+
+DIP_DATA_ROOT /data/lab/squall_data/tpchdb/Z0/
 DIP_SQL_ROOT ../test/squall/sql_queries/
 DIP_SCHEMA_PATH ../test/squall/schemas/tpch.txt
-
 # DIP_DB_SIZE is in GBs
-DIP_DB_SIZE 1 
+DIP_DB_SIZE 10
+
+# the following two are optional, by default they use topology.workers and topology.ackers from storm.yaml
+# DIP_NUM_WORKERS 2
+# DIP_NUM_ACKERS 0
 
 ########################################
 #DIP_OPTIMIZER_TYPE INDEX_SIMPLE
@@ -27,17 +30,19 @@ DIP_DB_SIZE 1
 #DIP_TOTAL_SRC_PAR 20
 
 DIP_OPTIMIZER_TYPE NAME_COST_LEFTY
-DIP_TOTAL_SRC_PAR 20
+DIP_TOTAL_SRC_PAR 9
 
 ########################################
 
+DIP_OUTPUT_FREQ_PRINT 200000
+
 #below are unlikely to change
 DIP_EXTENSION .tbl
 DIP_READ_SPLIT_DELIMITER \|
 DIP_GLOBAL_ADD_DELIMITER |
 DIP_GLOBAL_SPLIT_DELIMITER \|
 
-DIP_ACK_EVERY_TUPLE true
+DIP_ACK_EVERY_TUPLE false
 DIP_KILL_AT_THE_END true
 
 # Storage manager parameters

diff --git a/...unner/confs/cluster/10G_dbtoaster_hyracks → ...er/confs/cluster/10G_dbtoaster_z1_hyracks b/...unner/confs/cluster/10G_dbtoaster_hyracks → ...er/confs/cluster/10G_dbtoaster_z1_hyracks
diff --git a/...uall_plan_runner/confs/cluster/1G_hyracks → ..._plan_runner/confs/cluster/10G_z0_hyracks b/...uall_plan_runner/confs/cluster/1G_hyracks → ..._plan_runner/confs/cluster/10G_z0_hyracks
@@ -8,7 +8,7 @@ DIP_TOPOLOGY_NAME_PREFIX username
 # DIP_NUM_WORKERS 2
 # DIP_NUM_ACKERS 0
 
-DIP_DATA_PATH /data/lab/squall_data/tpchdb/1G/
+DIP_DATA_PATH /data/lab/squall_data/tpchdb/Z0/10G/
 
 CUSTOMER_PAR 8
 ORDERS_PAR 8

diff --git a/...uall_plan_runner/confs/cluster/1G_tpch3_4 → ..._plan_runner/confs/cluster/10G_z0_tpch3_4 b/...uall_plan_runner/confs/cluster/1G_tpch3_4 → ..._plan_runner/confs/cluster/10G_z0_tpch3_4
@@ -6,7 +6,7 @@ DIP_TOPOLOGY_NAME_PREFIX username
 #DIP_NUM_WORKERS 176
 #DIP_NUM_ACKERS 0
 
-DIP_DATA_PATH /data/squall_blade/data/tpchdb/1G
+DIP_DATA_PATH /data/lab/squall_data/tpchdb/Z0/10G/
 
 CUSTOMER_PAR 4
 ORDERS_PAR 4
@@ -29,4 +29,4 @@ STORAGE_LOCAL_DIR /tmp/ramdisk
 # Storage directory for cluster runs
 STORAGE_CLUSTER_DIR /data/squall_zone/storage
 STORAGE_COLD_START true
-STORAGE_MEMORY_SIZE_MB 4096
+STORAGE_MEMORY_SIZE_MB 4096
diff --git a/...uall_plan_runner/confs/cluster/1G_tpch7_4 → ..._plan_runner/confs/cluster/10G_z0_tpch7_4 b/...uall_plan_runner/confs/cluster/1G_tpch7_4 → ..._plan_runner/confs/cluster/10G_z0_tpch7_4
@@ -6,7 +6,7 @@ DIP_TOPOLOGY_NAME_PREFIX username
 #DIP_NUM_WORKERS 176
 #DIP_NUM_ACKERS 0
 
-DIP_DATA_PATH /data/squall_blade/data/tpchdb/1G
+DIP_DATA_PATH /data/lab/squall_data/tpchdb/Z0/10G
 
 NATION1_PAR 1
 NATION2_PAR 1
@@ -35,4 +35,4 @@ STORAGE_LOCAL_DIR /tmp/ramdisk
 # Storage directory for cluster runs
 STORAGE_CLUSTER_DIR /data/squall_zone/storage
 STORAGE_COLD_START true
-STORAGE_MEMORY_SIZE_MB 4096
+STORAGE_MEMORY_SIZE_MB 4096