Skip to content

Commit

Permalink
add scripts to start and stop tapir servers, modify script to run ycs…
Browse files Browse the repository at this point in the history 
…b test (can run ycsb clients on several machines, specify workload type and number of threads), add workload property files
  • Loading branch information
@olesya13
olesya13 committed Apr 23, 2018
1 parent 2a98b9f commit 8f914c7
Show file tree
Hide file tree
Showing 6 changed files with 272 additions and 15 deletions.
68 changes: 53 additions & 15 deletions ycsb-t/run-tapir.sh
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,58 @@
#! /usr/bin/env bash

# Copy the shared library to libs folder.
mkdir -p libs
cp ../libtapir/libtapir.so ./libs/
##
#Start Tapir servers start_servers.sh before you run this script
##

# Make the tapir binding using maven
mvn clean package

ycsb_machines=("172.31.42.111" "172.31.29.240" "172.31.10.55")
nhosts=${#ycsb_machines[@]}

closest_replica=(0 1 2)
ycsb_wdir="/home/ubuntu/DCAP-Tapir/ycsb-t"
nserv=9

nshards=8 # number of shards, must match number in start_servers.sh
reccount=1000000 # number of records to load
opcount=300000 # number of txn requests to make
read_per=50 # read percentage, defines type of workload
nthreads=(1) # number of threads, use one value at a time

need_setup=true # true - load records and run workloads; false - run workloads (if you have key-value store running)



if [[ "$need_setup" == true ]]; then
# Load the records in Tapir
java -cp tapir-interface/target/tapir-interface-0.1.4.jar:core/target/core-0.1.4.jar:tapir/target/tapir-binding-0.1.4.jar:javacpp/target/javacpp.jar \
-Djava.library.path=libs/ com.yahoo.ycsb.Client -P workloads/workloada \
-load -db com.yahoo.ycsb.db.TapirClient \
-p tapir.configpath=../store/tools/shard -p tapir.nshards=1 -p tapir.closestreplica=0 > load.log 2>&1

# Run the YCSB workload
java -cp tapir-interface/target/tapir-interface-0.1.4.jar:core/target/core-0.1.4.jar:tapir/target/tapir-binding-0.1.4.jar:javacpp/target/javacpp.jar \
-Djava.library.path=libs/ com.yahoo.ycsb.Client -P workloads/workloada \
-t -db com.yahoo.ycsb.db.TapirClient \
-p tapir.configpath=../store/tools/shard -p tapir.nshards=1 -p tapir.closestreplica=0 > run.log 2>&1
echo "Loading records"
java -cp tapir-interface/target/tapir-interface-0.1.4.jar:core/target/core-0.1.4.jar:tapir/target/tapir-binding-0.1.4.jar:javacpp/target/javacpp.jar \
-Djava.library.path=libs/ com.yahoo.ycsb.Client -P workloads/workload100 \
-load -db com.yahoo.ycsb.db.TapirClient \
-p tapir.configpath=../store/tools/shard -p tapir.nshards=$nshards -p tapir.closestreplica=0 \
-p measurement.type=histogram -p histogram.buckets=25 -p recordcount=$reccount > load.log 2>&1
fi

# Run the YCSB workloads
for i in ${nthreads[@]}
do
echo "$i threads"

echo "Running workload: read percentage - $read_per"
for j in "${!ycsb_machines[@]}"
do
echo " java -cp tapir-interface/target/tapir-interface-0.1.4.jar:core/target/core-0.1.4.jar:tapir/target/tapir-binding-0.1.4.jar:javacpp/target/javacpp.jar \
-Djava.library.path=libs/ com.yahoo.ycsb.Client -P workloads/workload$read_per \
-t -db com.yahoo.ycsb.db.TapirClient \
-p tapir.configpath=../store/tools/shard -p tapir.nshards=$nshards -p tapir.closestreplica=${closest_replica[$j]} \
-p measurement.type=histogram -p histogram.buckets=25 -threads $i -p operationcount=$opcount > logs/run-rper$read_per-hst$nhosts-srv$nserv-thr$i-this$j.log 2>&1"

ssh ${ycsb_machines[$j]} "cd $ycsb_wdir;\
java -cp tapir-interface/target/tapir-interface-0.1.4.jar:core/target/core-0.1.4.jar:tapir/target/tapir-binding-0.1.4.jar:javacpp/target/javacpp.jar \
-Djava.library.path=libs/ com.yahoo.ycsb.Client -P workloads/workload$read_per \
-t -db com.yahoo.ycsb.db.TapirClient \
-p tapir.configpath=../store/tools/shard -p tapir.nshards=$nshards -p tapir.closestreplica=${closest_replica[$j]} \
-p measurement.type=histogram -p histogram.buckets=25 -threads $i -p operationcount=$opcount > logs/run-rper$read_per-hst$nhosts-srv$nserv-thr$i-this$j.log 2>&1 &"

done
done

91 changes: 91 additions & 0 deletions ycsb-t/start_servers.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
#!/bin/bash


# Paths to source code and logfiles.
srcdir="/home/ubuntu/DCAP-Tapir"
logdir="/home/ubuntu/logs/tapir"
configdir="/home/ubuntu/DCAP-Tapir/store/tools"

this="172.31.42.111"
clients=("172.31.29.240" "172.31.10.55")

# Machines on which replicas are running.
replicas=("172.31.43.8" "172.31.20.205" "172.31.8.131" "172.31.32.118" "172.31.26.209" "172.31.12.41" "172.31.35.12" "172.31.30.218" "172.31.10.193")


store="tapirstore" # Which store (strongstore, weakstore, tapirstore)
mode="txn-l" # Mode for storage system.


port=50000 # port to start with, use if you generate config files by script

nkeys=0 # number of keys to use
nshard=8 # number of shards


# Print out configuration being used.
echo "Configuration:"
echo "Shards: $nshard"
echo "Keys: $nkeys"
echo "Store: $store"
echo "Mode: $mode"

# Make Tapir code if needed
for server in ${replicas[@]}
do
ssh $server "cd $srcdir; make"
done
for host in ${clients[@]}
do
ssh $host "cd $srcdir; make"
done

# Make java code if needed
for host in ${clients[@]}
do
ssh $host "cd $srcdir/ycsb-t; mkdir -p libs; cp ../libtapir/libtapir.so ./libs/; mvn clean package"
done



# Generate config files for replicas and timeserver
# !Use this block only if you have all shards on one server!
#echo "Generating config files.."
#$srcdir/store/tools/generate_configs.sh $configdir $nshard $port ${replicas[@]}
#for server in ${replicas[@]}
#do
# ssh $server "$srcdir/store/tools/generate_configs.sh $configdir $nshard $port ${replicas[@]}"
#done
#for host in ${clients[@]}
#do
# ssh $host "$srcdir/store/tools/generate_configs.sh $configdir $nshard $port ${replicas[@]}"
#done



# Copy config files and workloads
for host in ${clients[@]}
do
scp $configdir/*.config ubuntu@$host:$configdir
scp $srcdir/ycsb-t/workloads/* ubuntu@$host:$srcdir/ycsb-t/workloads
done
for server in ${replicas[@]}
do
scp $configdir/*.config ubuntu@$server:$configdir
done



# Start all replicas and timestamp servers
echo "Starting TimeStampServer replicas.."
$srcdir/store/tools/start_replica.sh tss $srcdir/store/tools/shard.tss.config \
"$srcdir/timeserver/timeserver" $logdir

for ((i=0; i<$nshard; i++))
do
echo "Starting shard$i replicas.."
$srcdir/store/tools/start_replica.sh shard$i $srcdir/store/tools/shard$i.config \
"$srcdir/store/$store/server -m $mode -f $srcdir/store/tools/keys -k $nkeys -n $i -N $nshard" $logdir
done


17 changes: 17 additions & 0 deletions ycsb-t/stop_servers.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/bash

# Script to stop nodes for Tapir cluster

replicas=("172.31.43.8" "172.31.20.205" "172.31.8.131" "172.31.32.118" "172.31.26.209" "172.31.12.41"\
"172.31.35.12" "172.31.30.218" "172.31.10.193")

# Stop node
for server in ${replicas[@]}
do
ssh $server "killall -9 server"
ssh $server "killall -9 timeserver"
done




37 changes: 37 additions & 0 deletions ycsb-t/workloads/workload0
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Copyright (c) 2010 Yahoo! Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.


# Yahoo! Cloud System Benchmark
#
# Read/update ratio: 0/100
# Data size: 1 field - 100 bytes, plus key
# Request distribution: zipfian

recordcount=1000000
operationcount=1000000
workload=com.yahoo.ycsb.workloads.CoreWorkload

readallfields=true

readproportion=0
updateproportion=1
scanproportion=0
insertproportion=0

requestdistribution=zipfian
dotransactions=true

fieldcount=1
37 changes: 37 additions & 0 deletions ycsb-t/workloads/workload100
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Copyright (c) 2010 Yahoo! Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.


# Yahoo! Cloud System Benchmark
#
# Read/update ratio: 100/0
# Data size: 1 field - 100 bytes, plus key
# Request distribution: zipfian

recordcount=1000000
operationcount=1000000
workload=com.yahoo.ycsb.workloads.CoreWorkload

readallfields=true

readproportion=1
updateproportion=0
scanproportion=0
insertproportion=0

requestdistribution=zipfian
dotransactions=true

fieldcount=1
37 changes: 37 additions & 0 deletions ycsb-t/workloads/workload50
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Copyright (c) 2010 Yahoo! Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you
# may not use this file except in compliance with the License. You
# may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License. See accompanying
# LICENSE file.


# Yahoo! Cloud System Benchmark
#
# Read/update ratio: 50/50
# Data size: 1 field - 100 bytes, plus key
# Request distribution: zipfian

recordcount=1000000
operationcount=1000000
workload=com.yahoo.ycsb.workloads.CoreWorkload

readallfields=true

readproportion=0.5
updateproportion=0.5
scanproportion=0
insertproportion=0

requestdistribution=zipfian
dotransactions=true

fieldcount=1

0 comments on commit 8f914c7

Please sign in to comment.