NotAShelf/nixir
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

tests/benchmark: rename runner script; compare compilation with native eval

Browse source 
Signed-off-by: NotAShelf <raf@notashelf.dev>
Change-Id: I6ef30732f875ab134a35282eb2cd66a36a6a6964
This commit is contained in:
raf 2026-02-22 23:27:57 +03:00
commit 68873352f9
Signed by: NotAShelf
GPG key ID: 29D95B64378DB4BF
2 changed files with 119 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

119
tests/benchmark/run.sh Executable file
View file

@ -0,0 +1,119 @@
#!/usr/bin/env bash
set -e
echo "# Running benchmarks..."
echo ""
BENCH_DIR="$(pwd)/tests/benchmark"
IRC_BIN="$(pwd)/build/nix-irc"
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[0;33m'
NC='\033[0m'
get_ms() {
local time_str="$1"
if [[ $time_str =~ ([0-9]+)m([0-9.]+)s ]]; then
local mins="${BASH_REMATCH[1]}"
local secs="${BASH_REMATCH[2]}"
local ms
ms=$(awk "BEGIN {printf \"%.1f\", ($mins * 60000) + ($secs * 1000)}")
echo "$ms"
else
echo "0"
fi
}
run_benchmark() {
local name="$1"
local file="$2"
echo -e "${BLUE}=== $name ===${NC}"
echo ""
# Measure compilation time
echo -n " Compilation time: "
local compile_start
compile_start=$(date +%s%N)
"$IRC_BIN" "$file" /tmp/bench.nixir >/dev/null 2>&1
local compile_end
compile_end=$(date +%s%N)
local compile_ms=$(((compile_end - compile_start) / 1000000))
echo -e "${YELLOW}${compile_ms}ms${NC}"
# Source and IR sizes
local src_size
src_size=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null)
local ir_size
ir_size=$(stat -c%s /tmp/bench.nixir 2>/dev/null || stat -f%z /tmp/bench.nixir 2>/dev/null)
local ratio=$((ir_size * 100 / src_size))
echo -e " Source size: ${src_size}B"
echo -e " IR bundle size: ${ir_size}B (${ratio}% of source)"
echo ""
# Native Nix evaluation (baseline)
echo -n " Native Nix eval: "
local native_total=0
for _ in {1..5}; do
local t
t=$( (time nix-instantiate --eval --strict "$file" >/dev/null 2>&1) 2>&1 | grep "real" | awk '{print $2}')
local ms
ms=$(get_ms "$t")
native_total=$(awk "BEGIN {print $native_total + $ms}")
done
local native_avg
native_avg=$(awk "BEGIN {printf \"%.1f\", $native_total / 5}")
echo -e "${GREEN}${native_avg}ms${NC} avg (5 runs)"
echo ""
}
echo "Measuring IR compilation speed and bundle size characteristics."
echo ""
run_benchmark "Simple Expression" "$BENCH_DIR/simple.nix"
run_benchmark "Medium Complexity" "$BENCH_DIR/medium.nix"
run_benchmark "Large Expression" "$BENCH_DIR/large.nix"
# Overall statistics
echo -e "${BLUE}=== Overall Statistics ===${NC}"
echo ""
testdir=$(mktemp -d)
total_nix=0
total_ir=0
total_compile_time=0
for f in "$BENCH_DIR"/*.nix; do
nixsize=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null)
base=$(basename "$f" .nix)
irfile="${testdir}/${base}.nixir"
start=$(date +%s%N)
"$IRC_BIN" "$f" "$irfile" >/dev/null 2>&1
end=$(date +%s%N)
compile_time=$(((end - start) / 1000000))
if [ -f "$irfile" ]; then
irsize=$(stat -c%s "$irfile" 2>/dev/null || stat -f%z "$irfile" 2>/dev/null)
total_nix=$((total_nix + nixsize))
total_ir=$((total_ir + irsize))
total_compile_time=$((total_compile_time + compile_time))
fi
done
total_ratio=$((total_ir * 100 / total_nix))
avg_compile_time=$((total_compile_time / 3))
# TBH those are entirely unnecessary. However, I'm a sucker for data
# and those are trivial to compile. Might as well. Who knows, maybe it'll
# come in handy in the future.
echo " Total source size: ${total_nix}B"
echo " Total IR size: ${total_ir}B"
echo " Compression ratio: ${total_ratio}% of source"
echo " Average compile time: ${avg_compile_time}ms"
echo ""
rm -rf "$testdir"

69
tests/benchmark/run_benchmarks.sh
View file

@ -1,69 +0,0 @@
#!/usr/bin/env bash
set -e
echo ""
PLUGIN_PATH="$(pwd)/build/nix-ir-plugin.so"
BENCH_DIR="$(pwd)/tests/benchmark"
IRC_BIN="$(pwd)/build/nix-irc"
# Colors for output
GREEN='\033[0;32m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
run_benchmark() {
local name="$1"
local file="$2"
echo -e "${BLUE}Benchmark: $name${NC}"
echo "----------------------------------------"
# 1. Parse + Compile Time (nix-irc)
echo -n " Parse + Compile: "
local compile_output=$( (time "$IRC_BIN" "$file" /tmp/bench.nixir 2>&1) 2>&1)
local compile_time=$(echo "$compile_output" | grep "real" | awk '{print $2}')
echo -e "${GREEN}$compile_time${NC}"
# 2. IR Size
if [ -f /tmp/bench.nixir ]; then
local ir_size=$(stat -f%z /tmp/bench.nixir 2>/dev/null || stat -c%s /tmp/bench.nixir 2>/dev/null)
echo -e " IR Bundle Size: ${GREEN}${ir_size} bytes${NC}"
fi
# 3. Native Nix evaluation time
echo -n " Native Eval: "
local native_time=$( (time nix-instantiate --eval --strict "$file" >/dev/null 2>&1) 2>&1 | grep "real" | awk '{print $2}')
echo -e "${GREEN}$native_time${NC}"
# 4. With Plugin evaluation time
echo -n " Plugin Eval: "
local plugin_time=$( (time nix-instantiate --plugin-files "$PLUGIN_PATH" --eval --strict "$file" >/dev/null 2>&1) 2>&1 | grep "real" | awk '{print $2}')
echo -e "${GREEN}$plugin_time${NC}"
echo ""
}
echo "# Running benchmarks..."
echo ""
run_benchmark "Simple Expressions" "$BENCH_DIR/simple.nix"
run_benchmark "Medium Complexity" "$BENCH_DIR/medium.nix"
run_benchmark "Large/Complex" "$BENCH_DIR/large.nix"
# File size comparison
echo -e "${BLUE}File Size Comparison${NC}"
echo "----------------------------------------"
testdir=$(mktemp -d)
for f in "$BENCH_DIR"/*.nix; do
nixsize=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null)
base=$(basename "$f" .nix)
irfile="${testdir}/${base}.nixir"
$IRC_BIN "$f" "$irfile" >/dev/null 2>&1
if [ -f "$irfile" ]; then
irsize=$(stat -c%s "$irfile" 2>/dev/null || stat -f%z "$irfile" 2>/dev/null)
ratio=$((irsize * 100 / nixsize))
echo " $base: ${nixsize}B => ${irsize}B (${ratio}% of source)"
fi
done