From 610d68147867c03657476f17abc2aae75c6197d5 Mon Sep 17 00:00:00 2001
From: Peter Cai <peter.cai@uwaterloo.ca>
Date: Tue, 21 Mar 2023 17:38:24 -0400
Subject: [PATCH] Add first CPU offset for NUMA
---
experiments/extra.shared.sh | 4 ++--
experiments/memcached.shared.sh | 4 ++--
run.sh | 9 +++++++++
tunings/irq_on_ht.sh | 5 +++++
tunings/irq_packing.sh | 4 ++--
tunings/normalize_queues.sh | 4 ++--
6 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/experiments/extra.shared.sh b/experiments/extra.shared.sh
index 62fa1cf..ff5158c 100644
--- a/experiments/extra.shared.sh
+++ b/experiments/extra.shared.sh
@@ -1,7 +1,7 @@
perf_stat_start() {
local opt_args=""
if [ "$PERF_STAT_NO_CORES" != "true" ]; then
- opt_args="$opt_args -C 0-$((THREADS - 1))"
+ opt_args="$opt_args -C $FIRST_CPU-$((FIRST_CPU + THREADS - 1))"
fi
$PERF stat $PERF_EXTRA_ARGS $opt_args -e LLC-load-misses:u,LLC-load-misses:k,LLC-loads:u,LLC-loads:k,LLC-store-misses:u,LLC-store-misses:k,LLC-stores:u,LLC-stores:k,task-clock,cycles:u,cycles:k,instructions:u,instructions:k -o "$DATA_OUT/stat_$1$EXP_AUX_SUFFIX.txt" &
PERF_STAT_PID=$!
@@ -14,7 +14,7 @@ perf_stat_stop() {
perf_record_start() {
local opt_args=""
if [ "$PERF_STAT_NO_CORES" != "true" ]; then
- opt_args="$opt_args -C 0-$((THREADS - 1))"
+ opt_args="$opt_args -C $FIRST_CPU-$((FIRST_CPU + THREADS - 1))"
fi
pushd "$DATA_OUT"
diff --git a/experiments/memcached.shared.sh b/experiments/memcached.shared.sh
index d6423f1..f0a550c 100644
--- a/experiments/memcached.shared.sh
+++ b/experiments/memcached.shared.sh
@@ -28,8 +28,8 @@ cleanup_memcached() {
start_memcached() {
echo "Starting memcached"
- echo "taskset -c 0-$((THREADS - 1 - THREAD_TAIL_OFFSET)) $MEMCACHED $MEMCACHED_ARGS -u root -t $((THREADS - THREAD_TAIL_OFFSET)) -b 16384 -c 32768 -m 10240 -o hashpower=24,no_lru_maintainer,no_lru_crawler"
- taskset -c 0-$((THREADS - 1 - THREAD_TAIL_OFFSET)) $MEMCACHED $MEMCACHED_ARGS -u root -t $((THREADS - THREAD_TAIL_OFFSET)) -b 16384 -c 32768 -m 10240 -o hashpower=24,no_lru_maintainer,no_lru_crawler &
+ echo "taskset -c $FIRST_CPU-$((FIRST_CPU + THREADS - 1 - THREAD_TAIL_OFFSET)) $MEMCACHED $MEMCACHED_ARGS -u root -t $((THREADS - THREAD_TAIL_OFFSET)) -b 16384 -c 32768 -m 10240 -o hashpower=24,no_lru_maintainer,no_lru_crawler"
+ taskset -c $FIRST_CPU-$((FIRST_CPU + THREADS - 1 - THREAD_TAIL_OFFSET)) $MEMCACHED $MEMCACHED_ARGS -u root -t $((THREADS - THREAD_TAIL_OFFSET)) -b 16384 -c 32768 -m 10240 -o hashpower=24,no_lru_maintainer,no_lru_crawler &
MEMCACHED_PID=$!
if [ "$PERF_STAT_ONLY_MEMCACHED" == "true" ]; then
PERF_EXTRA_ARGS_ORIG="$PERF_EXTRA_ARGS"
diff --git a/run.sh b/run.sh
index 57560b7..f05ab92 100755
--- a/run.sh
+++ b/run.sh
@@ -8,6 +8,7 @@ if [ "$UID" -ne 0 ]; then
fi
EXTRA_TUNING=()
+FIRST_CPU=0
export EXP_ROOT="$PWD"
@@ -38,6 +39,11 @@ while [[ $# -gt 0 ]]; do
shift
shift
;;
+ --first-cpu)
+ FIRST_CPU=$2
+ shift
+ shift
+ ;;
--fstack)
FSTACK=true
USERSTACK=true
@@ -122,6 +128,9 @@ if [ "${#EXTRA_TUNING[@]}" -gt 0 ]; then
EXPERIMENT_NAME="$EXPERIMENT_NAME.$tuning"
done
fi
+if [ $FIRST_CPU -gt 0 ]; then
+ EXPERIMENT_NAME="$EXPERIMENT_NAME.first$FIRST_CPU"
+fi
if [ "$FSTACK" == "true" ]; then
EXPERIMENT_NAME="$EXPERIMENT_NAME.fstack"
fi
diff --git a/tunings/irq_on_ht.sh b/tunings/irq_on_ht.sh
index e8e351b..893a57b 100644
--- a/tunings/irq_on_ht.sh
+++ b/tunings/irq_on_ht.sh
@@ -1,2 +1,7 @@
+if [ "$FIRST_CPU" -ne 0 ]; then
+ echo "FIRST_CPU must be 0"
+ exit 1
+fi
+
$EXP_ROOT/irq.sh set 16 $((THREADS + 16 - 1))
$EXP_ROOT/irq.sh set rx 16 $((THREADS + 16 - 1))
diff --git a/tunings/irq_packing.sh b/tunings/irq_packing.sh
index f0d4e1c..d7635bd 100644
--- a/tunings/irq_packing.sh
+++ b/tunings/irq_packing.sh
@@ -1,5 +1,5 @@
-if [ "$THREADS" -ne 8 ]; then
- echo "IRQ packing can only be done with 8 threads"
+if [ "$THREADS" -ne 8 ] || [ "$FIRST_CPU" -ne 0 ]; then
+ echo "IRQ packing can only be done with 8 threads and FIRST_CPU must be 0"
exit 1
fi
diff --git a/tunings/normalize_queues.sh b/tunings/normalize_queues.sh
index 48e878a..d457310 100644
--- a/tunings/normalize_queues.sh
+++ b/tunings/normalize_queues.sh
@@ -1,3 +1,3 @@
ethtool -L $NET_IF rx $THREADS tx $THREADS || true
-$EXP_ROOT/irq.sh set 0 $((THREADS - 1))
-$EXP_ROOT/irq.sh set rx 0 $((THREADS - 1))
+$EXP_ROOT/irq.sh set $FIRST_CPU $((FIRST_CPU + THREADS - 1))
+$EXP_ROOT/irq.sh set rx $FIRST_CPU $((FIRST_CPU + THREADS - 1))
--
GitLab