First experiment: memcached w/ QPS scanning

.gitignore

+data

configs/base.sh

 IS_CUSTOM_KERNEL=false
 CPUPOWER=cpupower
 X86_ENERGY_PERF_POLICY=x86_energy_perf_policy
+
+# Execute all SSH commands under this user
+# so that we don't need ssh keys for root
+SSH_USER=p5cai

configs/memcached.sh

+MEMCACHED=/home/p5cai/workspace/memcached/memcached
+THREADS=8 # default

configs/mutilate.sh

+# The server IP (the current machine)
+MEMSERVER_IP=192.168.199.1
+# List of Mutilate clients
+MEMCLIENTS=tilly03,tilly04,tilly05,tilly06,tilly07,tilly08
+# Command shorthands
+MUTILATE="ssh tilly02 taskset -c 0-7 mutilate -T8"
+MUTILATE+=" -s $MEMSERVER_IP -d1 -K fb_key -V fb_value -i fb_ia -r1000000"
+MUTCLIENTS=$(for m in $(echo $MEMCLIENTS|tr , ' '); do echo " -a $m";done)
+
+QPS_START_PER_THREAD=40000
+QPS_END_PER_THREAD=110000
+QPS_STEP_PER_THREAD=5000
+CONNS=160 # Per mutilate agent thread

experiments/memcached.shared.sh

+load_config() {
+    source "$EXP_ROOT/configs/memcached.sh"
+    source "$EXP_ROOT/configs/mutilate.sh"
+}
+
+cleanup_mutilate() {
+    echo "Killing all mutilate processes on clients"
+    sudo -u $SSH_USER pdsh -R ssh -w $MEMCLIENTS pkill mutilate || true
+}
+
+cleanup_memcached() {
+    echo "Killing memcached process"
+    kill $MEMCACHED_PID
+}
+
+start_memcached() {
+    echo "Starting memcached"
+    taskset -c 0-$THREADS $MEMCACHED -u root -t $THREADS -b 16384 -c 32768 -m 10240 -o hashpower=24,no_lru_maintainer,no_lru_crawler $@ 2>&1 > /dev/null &
+    MEMCACHED_PID=$!
+}
+
+start_mutilate() {
+    echo "Starting mutilate"
+    # Mutilate client agents always run on 8 cores
+    sudo -u $SSH_USER pdsh -R ssh -w $MEMCLIENTS taskset -c 0-7 mutilate -A -T 8 2>/dev/null & sleep 1
+}
+
+warmup() {
+    echo "Warming up Memcached..."
+    sudo -u $SSH_USER $MUTILATE --noload -c100 $MUTCLIENTS -t30 -u 0
+}
+
+run_mutilate() {
+    echo "Running mutilate with $CONNS connections per agent (core) and QPS = $QPS"
+    sudo -u $SSH_USER $MUTILATE --noload -c$CONNS $MUTCLIENTS -t30 -u 0 -q $QPS | tee $DATA_OUT/$1.txt
+}

experiments/memcached_qps_scan.sh

+source "$EXP_ROOT/experiments/memcached.shared.sh"
+
+pre_start() {
+    load_config
+
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            -t)
+                [ -z "$2" ] && echo "# of threads must not be empty" && exit 1
+                THREADS="$2"
+                shift
+                shift
+                ;;
+            *)
+                break
+                ;;
+        esac
+    done
+
+    EXPERIMENT_NAME_EXT="t$THREADS.c$CONNS"
+
+    cleanup_mutilate    
+}
+
+run_exp() {
+    start_memcached
+    start_mutilate
+    warmup
+
+    QPS_START=$((QPS_START_PER_THREAD * THREADS))
+    QPS_END=$((QPS_END_PER_THREAD * THREADS))
+    QPS_STEP=$((QPS_STEP_PER_THREAD * THREADS))
+
+    for QPS in $(seq $QPS_START $QPS_STEP $QPS_END); do
+        export QPS
+        run_mutilate $QPS
+    done    
+
+    cleanup_mutilate
+    cleanup_memcached
+}
+
+post_start() {
+    cleanup_memcached
+    cleanup_mutilate
+}

run.sh

@@ -7,6 +7,8 @@ if [ "$UID" -ne 0 ]; then
     exit 1
 fi
 
+export EXP_ROOT="$PWD"
+
 source ./configs/base.sh
 
 if [ -e "./configs/$(uname -r).sh" ]; then
@@ -15,3 +17,44 @@ if [ -e "./configs/$(uname -r).sh" ]; then
 fi
 
 source ./tunings/base.sh
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -e)
+            EXPERIMENT="$2"
+            shift
+            shift
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+if [ -z "$EXPERIMENT" ]; then
+    echo "Must specify an experiment to run"
+    exit 1
+fi
+
+if [ ! -f "./experiments/$EXPERIMENT.sh" ]; then
+     echo "$EXPERIMENT does not exist"
+     exit 1
+fi
+
+source "./experiments/$EXPERIMENT.sh"
+
+pre_start
+
+EXPERIMENT_NAME="$EXPERIMENT"
+if [ ! -z "$EXPERIMENT_NAME_EXT" ]; then
+    EXPERIMENT_NAME="$EXPERIMENT.$EXPERIMENT_NAME_EXT"
+fi
+
+DATA_OUT="$EXP_ROOT/data/$EXPERIMENT_NAME"
+
+rm -rf "$DATA_OUT"
+mkdir -p "$DATA_OUT"
+
+run_exp
+
+post_start

tunings/base.sh

@@ -13,4 +13,4 @@ echo 0 > /proc/sys/kernel/nmi_watchdog              # perf cache tracing
 
 # CPU frequency
 $X86_ENERGY_PERF_POLICY --turbo-enable 0
-$CPUPOWER frequency-set -f 2.7GHz
+$CPUPOWER frequency-set -f 2.7GHz || true # This can throw out errors but it's generally harmless