#!/usr/bin/env bash set -e echo "# Running benchmarks..." echo "" BENCH_DIR="$(pwd)/tests/benchmark" IRC_BIN="$(pwd)/build/nix-irc" GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[0;33m' NC='\033[0m' get_ms() { local time_str="$1" if [[ $time_str =~ ([0-9]+)m([0-9.]+)s ]]; then local mins="${BASH_REMATCH[1]}" local secs="${BASH_REMATCH[2]}" local ms ms=$(awk "BEGIN {printf \"%.1f\", ($mins * 60000) + ($secs * 1000)}") echo "$ms" else echo "0" fi } run_benchmark() { local name="$1" local file="$2" echo -e "${BLUE}=== $name ===${NC}" echo "" # Measure compilation time only echo -n " Compilation only: " local compile_start compile_start=$(date +%s%N) "$IRC_BIN" "$file" /tmp/bench.nixir >/dev/null 2>&1 local compile_end compile_end=$(date +%s%N) local compile_ms=$(((compile_end - compile_start) / 1000000)) echo -e "${YELLOW}${compile_ms}ms${NC}" # Measure IR loading only (deserialization + evaluation) echo -n " IR load only: " PLUGIN_PATH="$(pwd)/build/nix-ir-plugin.so" if [ ! -f "$PLUGIN_PATH" ]; then echo -e "${YELLOW}skipped${NC} (plugin not built)" else # Pre-compile the IR "$IRC_BIN" "$file" /tmp/bench.nixir >/dev/null 2>&1 # Measure just the loading (average of 10 runs to reduce noise) local total_load_us=0 for _ in {1..10}; do local load_output load_output=$(nix-instantiate --plugin-files "$PLUGIN_PATH" --eval --expr "builtins.nixIR_loadIR \"/tmp/bench.nixir\"" 2>&1 >/dev/null | grep "nixIR timing" | grep -oP 'total=\K[0-9]+') total_load_us=$((total_load_us + load_output)) done local avg_load_us=$((total_load_us / 10)) local avg_load_ms_frac=$(awk "BEGIN {printf \"%.3f\", $avg_load_us / 1000}") echo -e "${GREEN}${avg_load_ms_frac}ms${NC} avg (10 runs)" fi # Measure full pipeline (compile + nix-instantiate overhead + IR load) echo -n " Full pipeline: " if [ ! -f "$PLUGIN_PATH" ]; then echo -e "${YELLOW}skipped${NC}" else local pipeline_start pipeline_start=$(date +%s%N) "$IRC_BIN" "$file" /tmp/bench.nixir >/dev/null 2>&1 nix-instantiate --plugin-files "$PLUGIN_PATH" --eval --expr "builtins.nixIR_loadIR \"/tmp/bench.nixir\"" >/dev/null 2>&1 local pipeline_end pipeline_end=$(date +%s%N) local pipeline_ms=$(((pipeline_end - pipeline_start) / 1000000)) echo -e "${YELLOW}${pipeline_ms}ms${NC}" fi # Source and IR sizes local src_size src_size=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null) local ir_size ir_size=$(stat -c%s /tmp/bench.nixir 2>/dev/null || stat -f%z /tmp/bench.nixir 2>/dev/null) local ratio=0 if [[ "$src_size" -gt 0 ]]; then ratio=$((ir_size * 100 / src_size)) fi echo -e " Source size: ${src_size}B" echo -e " IR bundle size: ${ir_size}B (${ratio}% of source)" echo "" # Native Nix evaluation (baseline) echo -n " Native Nix eval: " local native_total=0 for _ in {1..5}; do local t t=$( (time nix-instantiate --eval --strict "$file" >/dev/null 2>&1) 2>&1 | grep "real" | awk '{print $2}') local ms ms=$(get_ms "$t") native_total=$(awk "BEGIN {print $native_total + $ms}") done local native_avg native_avg=$(awk "BEGIN {printf \"%.1f\", $native_total / 5}") echo -e "${GREEN}${native_avg}ms${NC} avg (5 runs)" echo "" } echo "Measuring IR compilation speed and bundle size characteristics." echo "" run_benchmark "Simple Expression" "$BENCH_DIR/simple.nix" run_benchmark "Medium Complexity" "$BENCH_DIR/medium.nix" run_benchmark "Large Expression" "$BENCH_DIR/large.nix" # Overall statistics echo -e "${BLUE}=== Overall Statistics ===${NC}" echo "" testdir=$(mktemp -d) total_nix=0 total_ir=0 total_compile_time=0 for f in "$BENCH_DIR"/*.nix; do nixsize=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null) base=$(basename "$f" .nix) irfile="${testdir}/${base}.nixir" start=$(date +%s%N) "$IRC_BIN" "$f" "$irfile" >/dev/null 2>&1 end=$(date +%s%N) compile_time=$(((end - start) / 1000000)) if [ -f "$irfile" ]; then irsize=$(stat -c%s "$irfile" 2>/dev/null || stat -f%z "$irfile" 2>/dev/null) total_nix=$((total_nix + nixsize)) total_ir=$((total_ir + irsize)) total_compile_time=$((total_compile_time + compile_time)) fi done total_ratio=$((total_ir * 100 / total_nix)) avg_compile_time=$((total_compile_time / 3)) # TBH those are entirely unnecessary. However, I'm a sucker for data # and those are trivial to compile. Might as well. Who knows, maybe it'll # come in handy in the future. echo " Total source size: ${total_nix}B" echo " Total IR size: ${total_ir}B" echo " Compression ratio: ${total_ratio}% of source" echo " Average compile time: ${avg_compile_time}ms" echo "" rm -rf "$testdir"