diff --git a/.gitignore b/.gitignore index 221b15a..ebb181e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ dto-test* libdto.so* +build*/ +tests/baselines*/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..ef14d03 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,208 @@ +cmake_minimum_required(VERSION 3.5...3.31) +project(DTO VERSION 1.0 LANGUAGES C) + +# Default to RelWithDebInfo when no build type is specified. +# (Skip for multi-config generators, which ignore CMAKE_BUILD_TYPE.) +get_property(_is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) +if(NOT _is_multi_config AND NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING + "Choose the type of build" FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS + Debug Release RelWithDebInfo MinSizeRel) + message(STATUS "No build type specified, defaulting to RelWithDebInfo") +endif() + +include(GNUInstallDirs) +set(CMAKE_INSTALL_LIBDIR lib) + +# Options to enable/disable library support (on by default) +option(DTO_ACCEL_CONFIG_SUPPORT "Enable accel-config for automatic DSA discovery (requires libaccel-config)" ON) +option(DTO_NUMA_SUPPORT "Enable NUMA awareness support (requires libnuma)" ON) + +# Options to statically link dependencies into libdto.so +option(DTO_STATIC_ACCEL_CONFIG "Statically link accel-config into libdto.so" OFF) +option(DTO_STATIC_NUMA "Statically link numa into libdto.so (requires PIC-compiled libnuma.a)" OFF) + +# Build the shared library +add_library(dto SHARED dto.c) +add_library(DTO::dto ALIAS dto) +target_include_directories(dto + PUBLIC + $ + $) + +# set gnu source everywhere +add_compile_definitions(_GNU_SOURCE) + +# Add the -DDTO_STATS_SUPPORT preprocessor definition +target_compile_definitions(dto PRIVATE DTO_STATS_SUPPORT) + +# Link libraries +set(DTO_STATIC_LIBS "") +set(DTO_DYNAMIC_LIBS dl) + +# Handle accel-config support +if(DTO_ACCEL_CONFIG_SUPPORT) + target_compile_definitions(dto PRIVATE DTO_ACCEL_CONFIG_SUPPORT) + message(STATUS "accel-config support: enabled") + + if(DTO_STATIC_ACCEL_CONFIG) + find_library(ACCEL_CONFIG_STATIC NAMES libaccel-config.a PATHS /usr/lib /usr/local/lib /usr/lib/x86_64-linux-gnu) + if(NOT ACCEL_CONFIG_STATIC) + message(FATAL_ERROR "libaccel-config.a not found. Install static accel-config or disable DTO_STATIC_ACCEL_CONFIG.") + endif() + message(STATUS "Statically linking accel-config: ${ACCEL_CONFIG_STATIC}") + list(APPEND DTO_STATIC_LIBS ${ACCEL_CONFIG_STATIC}) + else() + list(APPEND DTO_DYNAMIC_LIBS accel-config) + endif() +else() + message(STATUS "accel-config support: disabled (DTO_WQ_LIST env var required at runtime)") +endif() + +# Handle numa support +if(DTO_NUMA_SUPPORT) + target_compile_definitions(dto PRIVATE DTO_NUMA_SUPPORT) + message(STATUS "NUMA support: enabled") + + if(DTO_STATIC_NUMA) + find_library(NUMA_STATIC NAMES libnuma.a PATHS /usr/lib /usr/local/lib /usr/lib/x86_64-linux-gnu) + if(NOT NUMA_STATIC) + message(FATAL_ERROR "libnuma.a not found. Install static numa or disable DTO_STATIC_NUMA.") + endif() + message(STATUS "Statically linking numa: ${NUMA_STATIC}") + message(STATUS " (Note: libnuma.a must be compiled with -fPIC)") + list(APPEND DTO_STATIC_LIBS ${NUMA_STATIC}) + else() + list(APPEND DTO_DYNAMIC_LIBS numa) + endif() +else() + message(STATUS "NUMA support: disabled") +endif() + +# Link everything together +if(DTO_STATIC_LIBS) + target_link_libraries(dto + -Wl,--whole-archive + ${DTO_STATIC_LIBS} + -Wl,--no-whole-archive + ${DTO_DYNAMIC_LIBS} + ) +else() + target_link_libraries(dto ${DTO_DYNAMIC_LIBS}) +endif() + +include(CheckCCompilerFlag) +check_c_compiler_flag("-mwaitpkg" HAS_WAITPKG) +if (HAS_WAITPKG) + target_compile_options(dto PRIVATE -mwaitpkg -march=native) +endif() + +# Build dto-test and dto-test-wodto +add_executable(dto-test dto-test.c) +target_link_libraries(dto-test PRIVATE DTO::dto pthread) + +add_executable(dto-test-wodto dto-test.c) +target_link_libraries(dto-test-wodto PRIVATE pthread) + +# ---- Test Suite ---- +option(DTO_BUILD_TESTS "Build the DTO test suite" ON) + +if(DTO_BUILD_TESTS) + enable_testing() + + # Functional correctness tests (safe to run in CI without DSA hardware) + add_executable(dto-test-functional tests/test_functional.c) + target_link_libraries(dto-test-functional PRIVATE DTO::dto pthread) + + add_test(NAME functional COMMAND dto-test-functional) + set_tests_properties(functional PROPERTIES + LABELS "functional" + TIMEOUT 120 + ) + + # Build current dto.c as object file with renamed symbols (cur_memcpy, etc.) + set(PERF_BASELINE_DIR "${CMAKE_BINARY_DIR}/perf_baseline") + + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/dto_current.o + COMMAND ${CMAKE_C_COMPILER} -c -O3 -DNDEBUG -march=native -mwaitpkg -fPIC + -D_GNU_SOURCE -DDTO_STATS_SUPPORT + ${CMAKE_SOURCE_DIR}/dto.c + -o ${CMAKE_BINARY_DIR}/dto_current_raw.o + COMMAND objcopy --redefine-sym memcpy=cur_memcpy + --redefine-sym memset=cur_memset + --redefine-sym memcmp=cur_memcmp + --redefine-sym memmove=cur_memmove + ${CMAKE_BINARY_DIR}/dto_current_raw.o + ${CMAKE_BINARY_DIR}/dto_current.o + DEPENDS ${CMAKE_SOURCE_DIR}/dto.c + COMMENT "Building dto_current.o with renamed symbols" + ) + + # Build baseline dto.o from baseline branch (cached by SHA) + add_custom_command( + OUTPUT ${PERF_BASELINE_DIR}/dto_baseline.o + COMMAND bash ${CMAKE_SOURCE_DIR}/tests/build_baseline.sh + ${CMAKE_SOURCE_DIR} ${PERF_BASELINE_DIR} + DEPENDS ${CMAKE_SOURCE_DIR}/tests/build_baseline.sh + COMMENT "Building dto_baseline.o from baseline branch" + ) + + add_executable(dto-test-perf-combined + tests/test_perf_combined.c + ${CMAKE_BINARY_DIR}/dto_current.o + ${PERF_BASELINE_DIR}/dto_baseline.o + ) + target_link_libraries(dto-test-perf-combined PRIVATE m dl accel-config numa) + + # Common env vars for perf tests + # Pin frequency to 2000 MHz by default; override with -DPERF_FREQ_MHZ=... + if(NOT DEFINED PERF_FREQ_MHZ) + set(PERF_FREQ_MHZ 2000) + endif() + + set(PERF_COMMON_ENV + "RESULTS_DIR=${CMAKE_BINARY_DIR}/perf_results" + "PERF_FREQ_MHZ=${PERF_FREQ_MHZ}" + ) + + # --- Combined A/B test (all configs + page sizes in one run) --- + add_test(NAME perf_combined COMMAND dto-test-perf-combined) + set_tests_properties(perf_combined PROPERTIES + LABELS "perf" + TIMEOUT 2400 + ENVIRONMENT "${PERF_COMMON_ENV}" + ) +endif() + +# Install and export the library +install(TARGETS dto + EXPORT DTOTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + +install(FILES dto.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +install(EXPORT DTOTargets + FILE DTOTargets.cmake + NAMESPACE DTO:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/DTO) + +include(CMakePackageConfigHelpers) +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/DTOConfigVersion.cmake" + VERSION ${PROJECT_VERSION} + COMPATIBILITY AnyNewerVersion) + +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/DTOConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/DTOConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/DTO) + +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/DTOConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/DTOConfigVersion.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/DTO) + diff --git a/DTOConfig.cmake.in b/DTOConfig.cmake.in new file mode 100644 index 0000000..56e57bd --- /dev/null +++ b/DTOConfig.cmake.in @@ -0,0 +1,5 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/DTOTargets.cmake") + +check_required_components(DTO) diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..0cd31ed --- /dev/null +++ b/TESTING.md @@ -0,0 +1,396 @@ +# DTO Test Suite + +## Prerequisites + +DSA hardware tests require configured work queues and hugepage allocation: + +```bash +# Configure DSA (run your accel-config setup script) +sudo ./accelConfig.sh 0 yes 0 # Example: 1 work queue, DSA0 enabled + +# Allocate hugepages for 2MB page tests +sudo sh -c 'echo 128 > /proc/sys/vm/nr_hugepages' +``` + +## Build + +```bash +cmake -B build -DDTO_BUILD_TESTS=ON +cmake --build build +cd build +``` + +To disable tests: `-DDTO_BUILD_TESTS=OFF` + +## Running Tests + +```bash +# Run all tests (functional + perf) +ctest + +# Functional tests only (no DSA hardware needed) +ctest --label-regex functional + +# Performance tests only (requires DSA hardware) +ctest --label-regex perf + +# Verbose output (shows stdout for all tests, not just failures) +ctest -V + +# With output on failure only +ctest --output-on-failure +``` + +## Test Labels + +| Label | Description | +|--------------|----------------------------------------| +| `functional` | Correctness tests, no DSA needed | +| `perf` | Performance benchmarks, requires DSA | + +## Functional Tests + +Correctness tests for `memset`, `memcpy`, `memmove`, `memcmp` across buffer +sizes spanning below and above the DSA offload threshold. Includes +unaligned access and multithreaded tests. These pass with or without DSA +hardware -- DTO falls back to CPU transparently. + +```bash +ctest -R functional +``` + +## Performance Tests + +The performance test (`perf_combined`) detects regressions in libdto by +comparing a **baseline** build (from the `perf_baseline` branch on +`github.com/intel/DTO`) against the **current** working tree, both +statically linked into a single +test binary. The test fails if any non-reference benchmark shows a trimmed +mean latency increase exceeding the minimum effect threshold (default 2%). + +```bash +ctest -R perf_combined --output-on-failure +``` + +### How It Works + +1. **Build phase** — CMake compiles two object files: + - `dto_current.o` from the working tree (`cur_memcpy`, `cur_memset`, ...) + - `dto_baseline.o` from the `perf_baseline` branch on `github.com/intel/DTO` + via `tests/build_baseline.sh` (`bl_memcpy`, `bl_memset`, ...) + + Both use `objcopy --redefine-sym` so the two versions coexist in the same + binary with no symbol conflicts. The baseline is cached by git SHA and + only rebuilt when the upstream branch changes. + +2. **Measurement** — For each (benchmark × DTO config × page size) cell: + - Each cell runs `PERF_AB_ROUNDS` (default 3) forked child processes, + each executing 10,000 iterations (configurable via `DEFAULT_ITERS`). + - Each child runs with **randomized interleaving**: a xorshift32 PRNG + decides whether baseline or current runs first on each iteration, + eliminating first-mover bias. + +3. **Analysis** — Samples from all rounds are pooled, sorted, then: + - **Trimmed mean** (10th–90th percentile) computes the central tendency, + excluding outlier tails. + - **Kolmogorov–Smirnov test** on the trimmed distributions reports the + D statistic (max ECDF distance) and p-value as a distribution shape + diagnostic. Note: KS D is sensitive to distribution width, not just + shift—tight distributions can show large D for tiny absolute changes. + - **IQR-based outlier count** uses a shared threshold (Q3 + 1.5×IQR from + the combined baseline+current quartiles) to count outliers per version. + - **Pass/fail** is based solely on the trimmed mean change exceeding the + `PERF_MIN_EFFECT` threshold (default 2%). + +4. **Output** — Two detail tables (one per page size) show per-cell results, + followed by a speedup summary comparing each DTO config against raw CPU. + A `distributions.csv` file is written to `RESULTS_DIR` for offline + plotting with `tests/plot-distributions.R`. + +### A/B Fork Architecture + +The following diagram shows the full lifecycle of one cell (one benchmark × +DTO config × page size combination). The parent process orchestrates +everything; each measurement round runs in a forked child that communicates +results back through shared memory (`MAP_SHARED|MAP_ANONYMOUS`). + +``` +run_cell(benchmark, dto_config, page_config) +│ +│ ┌─────────────────────────────────────────────────────────┐ +│ │ MEASUREMENT PHASE — ab_rounds forked children │ +│ └─────────────────────────────────────────────────────────┘ +│ +├── iters = DEFAULT_ITERS (10,000) +├── alloc pooled arrays: all_bl[rounds×iters], all_cur[rounds×iters] +│ +├── for round = 0 .. ab_rounds-1: +│ │ +│ ├── fork_ab(iters) +│ │ │ +│ │ ├── [parent] setenv(DTO config vars: CSF, AAK, MIN_BYTES, etc.) +│ │ ├── [parent] fork() ─────────────────────────────────────┐ +│ │ │ │ +│ │ │ ┌────────────────────────────────────────────────────▼───┐ +│ │ │ │ CHILD PROCESS (pid == 0) │ +│ │ │ │ │ +│ │ │ │ 1. pthread_atfork child handler fires: │ +│ │ │ │ └── dto.c:child() resets dto_initialized = 0 │ +│ │ │ │ and calls init_dto() which re-reads env vars, │ +│ │ │ │ reopens DSA work queues, resets auto-tune state│ +│ │ │ │ │ +│ │ │ │ 2. Resolve function pointers: │ +│ │ │ │ ├── cpu config: dlopen("libc.so.6") → dlsym │ +│ │ │ │ │ a_memcpy = b_memcpy = libc memcpy (A==B) │ +│ │ │ │ └── dto configs: a_* = bl_* (baseline symbols) │ +│ │ │ │ b_* = cur_* (current symbols) │ +│ │ │ │ │ +│ │ │ │ 3. Initialize src[i] = i & 0xFF, dst = src │ +│ │ │ │ │ +│ │ │ │ 4. Warmup (100 iters of both a and b) │ +│ │ │ │ └── primes IOTLB entries for DSA │ +│ │ │ │ │ +│ │ │ │ 5. Measurement loop (10,000 iters): │ +│ │ │ │ see "Measurement Loop Detail" below │ +│ │ │ │ │ +│ │ │ │ 6. qsort(bl_samples), qsort(cur_samples) │ +│ │ │ │ 7. slot->nsamples, slot->ok = 1 │ +│ │ │ │ 8. exit(0) │ +│ │ │ └────────────────────────────────────────────────────────┘ +│ │ │ +│ │ ├── [parent] waitpid() +│ │ └── [parent] unsetenv(DTO config vars) +│ │ +│ └── [parent] memcpy child samples into pooled all_bl[], all_cur[] +│ +│ ┌─────────────────────────────────────────────────────────┐ +│ │ ANALYSIS PHASE │ +│ └─────────────────────────────────────────────────────────┘ +│ +├── qsort all_bl[], all_cur[] +├── trimmed mean (10th–90th percentile) → bl_ns, cur_ns +├── change = (cur_ns - bl_ns) / bl_ns × 100 +├── KS test on trimmed distributions → D statistic, p-value +├── IQR outlier count (shared Q1/Q3 threshold) +├── regression = change > min_effect (default 2%) +└── append raw samples to distributions.csv +``` + +#### Measurement Loop Detail + +Each iteration measures both baseline and current with randomized ordering +to eliminate first-mover bias (e.g., DSA engine idle advantage). + +``` +seed rng = rdtsc() | 1 ← nonzero seed for xorshift32 + +for i = 0 .. iters-1: +│ +├── xorshift32(rng) → bl_first = (rng & 1) +│ +│ ┌── Measure FIRST ──────────────────────────────┐ +│ │ if cold_cache: clflushopt(src), clflushopt(dst)│ +│ │ lfence │ +│ │ start = rdtsc │ +│ │ execute (bl_first ? BASELINE : CURRENT) │ +│ │ COMPILER_BARRIER │ +│ │ end = rdtscp │ +│ └────────────────────────────────────────────────┘ +│ +│ (memcmp: reset dst = src) +│ +│ ┌── Measure SECOND ─────────────────────────────┐ +│ │ if cold_cache: clflushopt(src), clflushopt(dst)│ +│ │ lfence │ +│ │ start = rdtsc │ +│ │ execute (bl_first ? CURRENT : BASELINE) │ +│ │ COMPILER_BARRIER │ +│ │ end = rdtscp │ +│ └────────────────────────────────────────────────┘ +│ +├── bl_samples[i] = t_bl +└── cur_samples[i] = t_cur +``` + +#### Shared Memory Layout + +The parent allocates one shared memory region (`MAP_SHARED|MAP_ANONYMOUS`) +used by all children sequentially. The child writes directly to this +region; the parent reads it after `waitpid()` returns. + +``` +shm ── ┌──────────────────────────────────────┐ + │ struct shared_result │ + │ .ok (child sets to 1) │ + │ .nsamples (actual count) │ + │ .requested_iters (parent sets) │ + ├──────────────────────────────────────┤ + │ bl_samples[0..DEFAULT_ITERS-1] │ + │ (baseline cycle counts, uint64_t) │ + ├──────────────────────────────────────┤ + │ cur_samples[0..DEFAULT_ITERS-1] │ + │ (current cycle counts, uint64_t) │ + └──────────────────────────────────────┘ + +src ── ┌──────────────────────────────────────┐ + │ Source buffer (MAP_SHARED) │ + │ 4KB pages or 2MB hugepages │ + └──────────────────────────────────────┘ + +dst ── ┌──────────────────────────────────────┐ + │ Destination buffer (MAP_SHARED) │ + │ 4KB pages or 2MB hugepages │ + └──────────────────────────────────────┘ +``` + +All buffers use `MAP_SHARED` so the child (a separate process after `fork()`) +operates on the same physical pages as the parent. This avoids copy-on-write +faults during measurement that would add noise. + +### Noise Reduction + +The test applies several techniques for stable, reproducible measurements: + +- **CPU pinning** (`sched_setaffinity`) to a single core (default core 1) +- **Core and uncore frequency pinning** via sysfs (requires root) +- **Cold-cache** benchmarking with `clflushopt` before each iteration (default) +- **Process isolation** via `fork()` — each A/B round runs in a child process, + triggering DTO's `pthread_atfork` handler to reinitialize DSA state +- **Static linking** — both DTO versions share the same code layout, eliminating + noise from separate shared library mappings +- **Randomized A/B order** — eliminates systematic first-mover advantage + (e.g., idle DSA engine bias) + +### Test Modes + +Four DTO configurations isolate different code paths: + +| Config | What it Measures | +|------------|----------------------------------------------------------| +| `cpu` | Raw libc (via `dlsym`), no DTO — reference baseline | +| `stdc` | DTO linked, forced CPU path (`DTO_USESTDC_CALLS=1`) | +| `dsa` | Pure DSA through DTO (`CSF=0`, no auto-tuning) | +| `dsa_auto` | DSA with CPU+DSA split (`CSF=0.33`, auto-tuning enabled) | + +Each config is tested with both **4KB pages** and **2MB hugepages**. + +DSA-enabled modes set `DTO_MIN_BYTES=4096` so DSA is exercised for all buffer +sizes in the benchmark. + +### Benchmarked Operations + +| Benchmark | Size | Pass/Fail | +|---------------|---------|-----------| +| `memcpy_4k` | 4 KB | Reference only (high CV at small sizes) | +| `memcpy_8k` | 8 KB | Reference only | +| `memcpy_16k` | 16 KB | Yes | +| `memcpy_32k` | 32 KB | Yes | +| `memcpy_64k` | 64 KB | Yes | +| `memcpy_128k` | 128 KB | Yes | +| `memcpy_256k` | 256 KB | Yes | +| `memcpy_512k` | 512 KB | Yes | +| `memcpy_1m` | 1 MB | Yes | +| `memset_64k` | 64 KB | Yes | +| `memset_128k` | 128 KB | Yes | +| `memset_256k` | 256 KB | Yes | +| `memset_1m` | 1 MB | Yes | +| `memcmp_64k` | 64 KB | Yes | +| `memcmp_128k` | 128 KB | Yes | +| `memcmp_1m` | 1 MB | Yes | + +### Example Output + +Note: Your numbers will differ based on hardware, kernel, and DSA +configuration. + +``` +DTO Combined A/B Performance Test (cold cache) +================================================ +Pinned CPU: 1 +TSC freq: 2.000 GHz +Core freq: 2000 MHz (min=2000 max=2000) [pinned] +Uncore freq: min=2000 max=2000 MHz [pinned] +A/B rounds: 3 (interleaved, randomized order) +Min effect: 2.0% +Page sizes: 4KB, 2MB hugepages +Linking: static (bl_* / cur_* in same binary) + + =============================================== + A/B Results — 4KB pages + =============================================== + Test Config Base mean Cur mean Change KS D Result Outliers + ------------- --------- --------- --------- -------- ----- ------- -------- + memcpy_16k cpu 4033 ns 4036 ns +0.1% 0.026 PASS bl=5 cur=3 + stdc 4035 ns 4034 ns -0.0% 0.021 PASS bl=4 cur=6 + dsa 1520 ns 1518 ns -0.1% 0.018 PASS bl=2 cur=3 + dsa_auto 1680 ns 1675 ns -0.3% 0.031 PASS bl=8 cur=5 + ... + + =============================================== + Speedup vs CPU (current library) + =============================================== + + 4KB pages: + Test stdc dsa dsa_auto + -------------- ---------- ---------- ---------- + memcpy_16k 1.00x 2.66x 2.41x + ... +``` + +### Plotting Distributions + +The test writes raw sample data to `distributions.csv`. Use the R script to +generate density, violin, and ECDF plots for each operation: + +```bash +cd build +Rscript ../tests/plot-distributions.R perf_results/distributions.csv [4k|2m] [memcpy|memset|memcmp] +``` + +### Baseline Management + +The baseline is built automatically from the `perf_baseline` branch on +`github.com/intel/DTO` during `cmake --build`. The build script +(`tests/build_baseline.sh`): + +- Fetches the upstream branch and checks the SHA +- Skips the build if the cached `dto_baseline.o` matches the current SHA +- Creates a temporary git worktree to compile the baseline source +- Compiles with the same flags as the current build (`-O3 -DNDEBUG -march=native`) +- Renames symbols via `objcopy` (`memcpy` → `bl_memcpy`, etc.) +- Cleans up the worktree after building + +To force a baseline rebuild, delete `build/perf_baseline/dto_baseline.o` and +rebuild. + +### Environment Variables + +| Variable | Default | Effect | +|---------------------|----------------------------------------|----------------------------------------------------| +| `PERF_CPU` | `1` | CPU core to pin the benchmark thread to | +| `PERF_FREQ_MHZ` | `2000` | Pin core and uncore frequency (MHz); requires root | +| `PERF_MIN_EFFECT` | `2` | Minimum trimmed mean change % to flag a FAIL | +| `PERF_AB_ROUNDS` | `3` | Number of interleaved A/B fork rounds | +| `PERF_COLD_CACHE` | `1` | 1 = `clflushopt` per iteration, 0 = flush once | +| `RESULTS_DIR` | ${CMAKE_BINARY_DIR}/perf_results | Directory for `distributions.csv` output | + +CMake sets `RESULTS_DIR` and `PERF_FREQ_MHZ` automatically when running via +CTest. Override `PERF_FREQ_MHZ` at configure time: + +```bash +cmake -B build -DDTO_BUILD_TESTS=ON -DPERF_FREQ_MHZ=2400 +``` + +### Pin to a Different CPU Core + +```bash +PERF_CPU=3 ctest -R perf_combined +``` + +### Adjust Regression Threshold + +Default is 2%. To relax (e.g., on noisy systems): + +```bash +PERF_MIN_EFFECT=5 ctest -R perf_combined +``` diff --git a/dto.c b/dto.c index 48ef3a7..1f7d360 100644 --- a/dto.c +++ b/dto.c @@ -1813,12 +1813,19 @@ static int dto_memcmp(const void *s1, const void *s2, size_t n, int *result) } if (thr_comp.result) { - /* cmp returned mismatch. determine the return value */ - uint8_t *t1 = (uint8_t *)s1 + thr_bytes_completed; - uint8_t *t2 = (uint8_t *)s2 + thr_bytes_completed; + /* cmp returned mismatch. dsa_execute() already added + * xfer_size to thr_bytes_completed, so subtract it back + * to get the base of the last descriptor's range, then + * add thr_comp.bytes_completed (offset of the first + * differing byte within that range). + */ + uint64_t off = thr_bytes_completed - thr_desc.xfer_size + + thr_comp.bytes_completed; + uint8_t *t1 = (uint8_t *)s1 + off; + uint8_t *t2 = (uint8_t *)s2 + off; cmp_result = *t1 - *t2; - /* Inform the caller than the job is done even though + /* Inform the caller that the job is done even though * we didn't process all the bytes */ thr_bytes_completed = orig_n; diff --git a/tests/build_baseline.sh b/tests/build_baseline.sh new file mode 100755 index 0000000..a4ff6cb --- /dev/null +++ b/tests/build_baseline.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Build the baseline dto_baseline.o from the baseline branch. +# Compiles dto.c to an object file and renames the 4 public symbols +# (memcpy/memset/memcmp/memmove → bl_*) so it can be statically linked +# alongside the current version in the same binary. +# +# Usage: build_baseline.sh +# source_dir - root of the DTO repo (for git operations) +# output_dir - where to put dto_baseline.o + +set -e + +SOURCE_DIR="$1" +OUTPUT_DIR="$2" +BASELINE_OBJ="${OUTPUT_DIR}/dto_baseline.o" +WORKTREE_DIR="${OUTPUT_DIR}/baseline_worktree" +UPSTREAM_URL="https://github.com/intel/DTO.git" +BRANCH="perf_baseline" + +mkdir -p "${OUTPUT_DIR}" + +# Skip if baseline already built and up-to-date +if [ -f "${BASELINE_OBJ}" ]; then + cd "${SOURCE_DIR}" + timeout 20 git fetch "${UPSTREAM_URL}" "${BRANCH}" --quiet 2>/dev/null || true + REMOTE_SHA=$(git rev-parse FETCH_HEAD 2>/dev/null || echo "unknown") + BUILT_SHA="" + if [ -f "${OUTPUT_DIR}/baseline_sha" ]; then + BUILT_SHA=$(cat "${OUTPUT_DIR}/baseline_sha") + fi + if [ "${REMOTE_SHA}" = "${BUILT_SHA}" ]; then + echo "Baseline dto_baseline.o is up-to-date (${UPSTREAM_URL} ${BRANCH} = ${REMOTE_SHA:0:12})" + exit 0 + fi + echo "Baseline outdated, rebuilding..." +fi + +# Clean up any previous worktree +if [ -d "${WORKTREE_DIR}" ]; then + cd "${SOURCE_DIR}" + git worktree remove --force "${WORKTREE_DIR}" 2>/dev/null || rm -rf "${WORKTREE_DIR}" +fi + +# Create a worktree at the baseline branch +cd "${SOURCE_DIR}" +git fetch "${UPSTREAM_URL}" "${BRANCH}" --quiet +echo "Building baseline from ${UPSTREAM_URL} ${BRANCH}..." +git worktree add --detach "${WORKTREE_DIR}" FETCH_HEAD + +# Compile to object file with same flags as CMake Release +gcc -c -O3 -DNDEBUG -march=native -mwaitpkg -fPIC \ + -D_GNU_SOURCE -DDTO_STATS_SUPPORT \ + "${WORKTREE_DIR}/dto.c" -o "${OUTPUT_DIR}/dto_baseline_raw.o" \ + >>"${OUTPUT_DIR}/build.log" 2>&1 + +# Rename public symbols: memcpy→bl_memcpy, etc. +objcopy --redefine-sym memcpy=bl_memcpy \ + --redefine-sym memset=bl_memset \ + --redefine-sym memcmp=bl_memcmp \ + --redefine-sym memmove=bl_memmove \ + "${OUTPUT_DIR}/dto_baseline_raw.o" "${BASELINE_OBJ}" + +rm -f "${OUTPUT_DIR}/dto_baseline_raw.o" + +# Record the SHA we built +BUILT_SHA=$(git rev-parse FETCH_HEAD) +echo "${BUILT_SHA}" > "${OUTPUT_DIR}/baseline_sha" + +# Clean up worktree +cd "${SOURCE_DIR}" +git worktree remove --force "${WORKTREE_DIR}" 2>/dev/null || true + +echo "Baseline built: ${BASELINE_OBJ} (${UPSTREAM_URL} ${BRANCH} = ${BUILT_SHA:0:12})" diff --git a/tests/dto_test_utils.h b/tests/dto_test_utils.h new file mode 100644 index 0000000..9f1319b --- /dev/null +++ b/tests/dto_test_utils.h @@ -0,0 +1,166 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * dto_test_utils.h - Lightweight test utilities for DTO test suite. + * + * Provides: + * - Assertion macros (return 1 from calling function on failure) + * - Test runner with pass/fail reporting + * - Timing helpers for performance benchmarks + * - Baseline load/save for performance regression testing + ******************************************************************************/ + +#ifndef DTO_TEST_UTILS_H +#define DTO_TEST_UTILS_H + +#include +#include +#include +#include +#include + +/* ---- Assertion macros ---- */ + +#define ASSERT_TRUE(cond) do { \ + if (!(cond)) { \ + fprintf(stderr, " FAIL: %s (at %s:%d)\n", \ + #cond, __FILE__, __LINE__); \ + return 1; \ + } \ +} while (0) + +#define ASSERT_EQ(a, b) do { \ + long long _a = (long long)(a), _b = (long long)(b); \ + if (_a != _b) { \ + fprintf(stderr, " FAIL: %s == %s (%lld != %lld) at %s:%d\n", \ + #a, #b, _a, _b, __FILE__, __LINE__); \ + return 1; \ + } \ +} while (0) + +/* ---- Test runner ---- */ + +typedef int (*test_fn)(void); + +struct dto_test { + const char *name; + test_fn fn; +}; + +#define TEST_ENTRY(fn) { #fn, fn } + +static inline int run_tests(struct dto_test *tests) +{ + int total = 0, passed = 0, failed = 0; + + for (int i = 0; tests[i].name != NULL; i++) { + total++; + printf(" %-50s ", tests[i].name); + fflush(stdout); + if (tests[i].fn() == 0) { + printf("PASS\n"); + passed++; + } else { + failed++; + } + } + + printf("\nResults: %d/%d passed", passed, total); + if (failed > 0) + printf(" (%d FAILED)", failed); + printf("\n"); + + return failed > 0 ? 1 : 0; +} + +/* ---- Timing helpers ---- */ + +static inline uint64_t time_ns(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec; +} + +/* Prevent dead code elimination of benchmark operations */ +#define COMPILER_BARRIER() __asm__ volatile("" ::: "memory") + +/* ---- Aligned allocation ---- */ + +static inline void *alloc_aligned(size_t size) +{ + void *ptr = NULL; + + if (posix_memalign(&ptr, 4096, size) != 0) + return NULL; + return ptr; +} + +/* ---- Performance baseline support ---- */ + +#define MAX_BASELINES 64 +#define BASELINE_NAME_LEN 64 + +struct perf_baseline { + char name[BASELINE_NAME_LEN]; + double latency_ns; +}; + +static inline int load_baselines(const char *path, + struct perf_baseline *baselines, int max) +{ + FILE *f = fopen(path, "r"); + int count = 0; + char line[256]; + + if (!f) + return 0; + + while (fgets(line, sizeof(line), f) && count < max) { + if (line[0] == '#' || line[0] == '\n') + continue; + if (sscanf(line, "%63s %lf", baselines[count].name, + &baselines[count].latency_ns) == 2) + count++; + } + fclose(f); + return count; +} + +static inline int save_baselines(const char *path, + struct perf_baseline *baselines, int count) +{ + FILE *f = fopen(path, "w"); + time_t now; + + if (!f) { + fprintf(stderr, "Failed to open %s for writing\n", path); + return 1; + } + + now = time(NULL); + fprintf(f, "# DTO Performance Baselines (latency in ns)\n"); + fprintf(f, "# Re-generate with: UPDATE_BASELINES=1 ctest --label-regex perf\n"); + fprintf(f, "# Generated: %s", ctime(&now)); + for (int i = 0; i < count; i++) + fprintf(f, "%-24s %.1f\n", baselines[i].name, + baselines[i].latency_ns); + + fclose(f); + return 0; +} + +static inline double find_baseline(struct perf_baseline *baselines, int count, + const char *name) +{ + for (int i = 0; i < count; i++) { + if (strcmp(baselines[i].name, name) == 0) + return baselines[i].latency_ns; + } + return -1.0; +} + +#endif /* DTO_TEST_UTILS_H */ diff --git a/tests/perf_summary.c b/tests/perf_summary.c new file mode 100644 index 0000000..5f02231 --- /dev/null +++ b/tests/perf_summary.c @@ -0,0 +1,629 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * perf_summary.c - Mann-Whitney U based performance regression detection. + * + * Reads per-mode result files and sample distributions from RESULTS_DIR + * (written by test_perf_combined.c or test_perf.c), prints a side-by-side + * summary table, then runs the Mann-Whitney U test comparing current sample + * distributions against baseline distributions. + * + * A regression is detected when the distribution of latencies has shifted + * significantly (p < 0.05) AND the median got worse (higher latency). + * + * Expected files in RESULTS_DIR: + * {cpu,stdc,dsa,dsa_auto}_{4k,2m}.dat - median latencies + * {cpu,stdc,dsa,dsa_auto}_{4k,2m}_*.samples - raw sample distributions + * + * Baseline sample files in BASELINE_DIR: + * {cpu,stdc,dsa,dsa_auto}_{4k,2m}_*.samples + * + * Environment: + * RESULTS_DIR - directory with current result files (required) + * BASELINE_DIR - directory with baseline sample files (required) + * UPDATE_BASELINES - if set, copy current samples to baseline dir + * PERF_MIN_EFFECT - minimum median change %% to count as regression (default: 2) + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#define MAX_TESTS 32 +#define NAME_LEN 64 +#define MAX_MODES 8 + +/* ---- Result loading (same format as test_perf.c output) ---- */ + +struct result { + char name[NAME_LEN]; + double latency_ns; +}; + +struct mode { + const char *suffix; + const char *header; + struct result tests[MAX_TESTS]; + int count; +}; + +static int load_results(const char *path, struct result *results, int max) +{ + FILE *f = fopen(path, "r"); + int count = 0; + char line[256]; + + if (!f) + return 0; + + while (fgets(line, sizeof(line), f) && count < max) { + if (line[0] == '#' || line[0] == '\n') + continue; + if (sscanf(line, "%63s %lf", results[count].name, + &results[count].latency_ns) == 2) + count++; + } + fclose(f); + return count; +} + +static double find_latency(struct mode *m, const char *name) +{ + for (int i = 0; i < m->count; i++) { + if (strcmp(m->tests[i].name, name) == 0) + return m->tests[i].latency_ns; + } + return -1.0; +} + +/* ---- Sample distribution loading ---- */ + +struct sample_dist { + uint64_t *ticks; + int count; + double tsc_ghz; +}; + +static int load_samples(const char *dir, const char *mode, + const char *pagesize, const char *testname, + struct sample_dist *dist) +{ + char path[4096]; + FILE *f; + uint32_t n; + + dist->ticks = NULL; + dist->count = 0; + dist->tsc_ghz = 1.0; + + snprintf(path, sizeof(path), "%s/%s_%s_%s.samples", + dir, mode, pagesize, testname); + f = fopen(path, "rb"); + if (!f) + return -1; + + if (fread(&n, sizeof(n), 1, f) != 1 || n == 0) { + fclose(f); + return -1; + } + if (fread(&dist->tsc_ghz, sizeof(dist->tsc_ghz), 1, f) != 1) { + fclose(f); + return -1; + } + + dist->ticks = malloc((size_t)n * sizeof(uint64_t)); + if (!dist->ticks) { + fclose(f); + return -1; + } + + if (fread(dist->ticks, sizeof(uint64_t), n, f) != n) { + free(dist->ticks); + dist->ticks = NULL; + fclose(f); + return -1; + } + + dist->count = n; + fclose(f); + return 0; +} + +static void free_samples(struct sample_dist *dist) +{ + free(dist->ticks); + dist->ticks = NULL; + dist->count = 0; +} + +/* ---- Two-sample Kolmogorov-Smirnov test ---- */ + +struct ks_result { + double d; /* max ECDF distance (0..1) */ + double p; /* p-value */ +}; + +static struct ks_result ks_test(const uint64_t *a, int n1, + const uint64_t *b, int n2) +{ + struct ks_result r = {0.0, 1.0}; + int i = 0, j = 0; + double d_max = 0.0; + + while (i < n1 || j < n2) { + uint64_t val; + double diff; + + if (i < n1 && (j >= n2 || a[i] <= b[j])) + val = a[i]; + else + val = b[j]; + + while (i < n1 && a[i] == val) i++; + while (j < n2 && b[j] == val) j++; + + diff = fabs((double)i / n1 - (double)j / n2); + if (diff > d_max) + d_max = diff; + } + + r.d = d_max; + + double ne = (double)n1 * n2 / (n1 + n2); + double lambda = (sqrt(ne) + 0.12 + 0.11 / sqrt(ne)) * d_max; + double sum = 0.0; + + for (int k = 1; k <= 100; k++) { + double term = exp(-2.0 * k * k * lambda * lambda); + + if (k % 2 == 1) + sum += term; + else + sum -= term; + if (term < 1e-12) + break; + } + r.p = 2.0 * sum; + if (r.p < 0) r.p = 0; + if (r.p > 1) r.p = 1; + + return r; +} + +/* + * Print overlaid ASCII histogram of two sorted sample distributions. + * Shows the 5th-95th percentile range to exclude extreme outliers. + * 'B' = baseline only, 'C' = current only, '#' = overlap. + */ +#define HIST_BINS 40 +#define HIST_HEIGHT 12 + +static void print_histogram(const struct sample_dist *bl, + const struct sample_dist *cur) +{ + /* Use 5th and 95th percentile across both for range */ + double bl_p5 = (double)bl->ticks[(int)(bl->count * 0.05)] / bl->tsc_ghz; + double bl_p95 = (double)bl->ticks[(int)(bl->count * 0.95)] / bl->tsc_ghz; + double cur_p5 = (double)cur->ticks[(int)(cur->count * 0.05)] / cur->tsc_ghz; + double cur_p95 = (double)cur->ticks[(int)(cur->count * 0.95)] / cur->tsc_ghz; + + double lo = bl_p5 < cur_p5 ? bl_p5 : cur_p5; + double hi = bl_p95 > cur_p95 ? bl_p95 : cur_p95; + + if (hi <= lo) + return; + + double bin_width = (hi - lo) / HIST_BINS; + int bl_bins[HIST_BINS] = {0}; + int cur_bins[HIST_BINS] = {0}; + int max_count = 0; + + /* Bin the baseline samples */ + for (int i = 0; i < bl->count; i++) { + double ns = (double)bl->ticks[i] / bl->tsc_ghz; + int bin = (int)((ns - lo) / bin_width); + + if (bin < 0) bin = 0; + if (bin >= HIST_BINS) bin = HIST_BINS - 1; + bl_bins[bin]++; + } + + /* Bin the current samples */ + for (int i = 0; i < cur->count; i++) { + double ns = (double)cur->ticks[i] / cur->tsc_ghz; + int bin = (int)((ns - lo) / bin_width); + + if (bin < 0) bin = 0; + if (bin >= HIST_BINS) bin = HIST_BINS - 1; + cur_bins[bin]++; + } + + for (int b = 0; b < HIST_BINS; b++) { + if (bl_bins[b] > max_count) max_count = bl_bins[b]; + if (cur_bins[b] > max_count) max_count = cur_bins[b]; + } + + if (max_count == 0) + return; + + /* Print top-down */ + for (int row = HIST_HEIGHT; row >= 1; row--) { + double threshold = (double)row / HIST_HEIGHT * max_count; + + printf(" %s", row == HIST_HEIGHT ? " " : " "); + for (int b = 0; b < HIST_BINS; b++) { + int has_bl = (bl_bins[b] >= threshold); + int has_cur = (cur_bins[b] >= threshold); + + if (has_bl && has_cur) + putchar('#'); + else if (has_bl) + putchar('B'); + else if (has_cur) + putchar('C'); + else + putchar(' '); + } + printf("\n"); + } + + /* X-axis */ + printf(" "); + for (int b = 0; b < HIST_BINS + 2; b++) + putchar('-'); + printf("\n"); + printf(" %-20.0f ns %*s %18.0f ns\n", + lo, HIST_BINS - 22, "", hi); + printf(" B=baseline C=current #=overlap\n\n"); +} + +/* Parse size from test name suffix: memcpy_128k -> 131072 */ +static size_t parse_size(const char *name) +{ + const char *s = strrchr(name, '_'); + char *end; + long val; + + if (!s) + return 0; + s++; + + val = strtol(s, &end, 10); + if (*end == 'k' || *end == 'K') + return (size_t)val * 1024; + if (*end == 'm' || *end == 'M') + return (size_t)val * 1048576; + return (size_t)val; +} + +/* Skip 4k/8k entries (ref-only in benchmarks) */ +static int is_ref_only(const char *name) +{ + const char *s = strrchr(name, '_'); + + if (!s) + return 0; + s++; + return (strcmp(s, "4k") == 0 || strcmp(s, "8k") == 0); +} + +/* ---- Mode definitions ---- */ + +static struct { + const char *suffix; + const char *header; +} mode_defs[] = { + {"cpu", "CPU (no DTO)"}, + {"stdc", "DTO+STDC"}, + {"dsa", "DTO+DSA"}, + {"dsa_auto", "DTO+DSA+Auto"}, +}; +#define NUM_MODE_DEFS (sizeof(mode_defs) / sizeof(mode_defs[0])) + +/* + * Process one page size: load results, print summary table, run + * Mann-Whitney U tests against baseline sample distributions. + * + * Returns number of regressions detected (0 = all passed). + */ +static int process_pagesize(const char *results_dir, const char *baseline_dir, + const char *pagesize, const char *pagesize_label, + int update_mode, double min_effect_pct) +{ + struct mode modes[MAX_MODES]; + int num_modes = 0; + int cpu_idx = -1; + int ref; + int failures = 0; + + /* Load available result files for this page size */ + for (int i = 0; i < (int)NUM_MODE_DEFS; i++) { + char path[4096]; + + snprintf(path, sizeof(path), "%s/%s_%s.dat", + results_dir, mode_defs[i].suffix, pagesize); + + modes[num_modes].count = load_results( + path, modes[num_modes].tests, MAX_TESTS); + if (modes[num_modes].count > 0) { + modes[num_modes].suffix = mode_defs[i].suffix; + modes[num_modes].header = mode_defs[i].header; + if (strcmp(mode_defs[i].suffix, "cpu") == 0) + cpu_idx = num_modes; + num_modes++; + } + } + + if (num_modes == 0) { + printf(" No results found for %s\n\n", pagesize_label); + return 0; + } + + /* Use mode with most tests as reference for row names */ + ref = 0; + for (int m = 1; m < num_modes; m++) { + if (modes[m].count > modes[ref].count) + ref = m; + } + + /* ---- Summary table (informational) ---- */ + printf("=========================================================="); + printf("===========================================================\n"); + printf(" DTO Performance Summary (%s)\n", pagesize_label); + printf("=========================================================="); + printf("===========================================================\n\n"); + + printf(" %-14s", "Test"); + for (int m = 0; m < num_modes; m++) + printf(" | %12s %8s", modes[m].header, "GB/s"); + if (cpu_idx >= 0) + printf(" | Speedup"); + printf("\n"); + + printf(" %-14s", "--------------"); + for (int m = 0; m < num_modes; m++) + printf("-|-%12s-%8s", "------------", "--------"); + if (cpu_idx >= 0) + printf("-|--------"); + printf("\n"); + + for (int t = 0; t < modes[ref].count; t++) { + const char *name = modes[ref].tests[t].name; + size_t size = parse_size(name); + double cpu_ns = -1, dsa_ns = -1; + + printf(" %-14s", name); + + for (int m = 0; m < num_modes; m++) { + double ns = find_latency(&modes[m], name); + + if (ns > 0 && size > 0) { + double gbps = (double)size / ns; + + printf(" | %10.1f ns %8.4f", ns, gbps); + + if (m == cpu_idx) + cpu_ns = ns; + if (strcmp(modes[m].suffix, "dsa") == 0) + dsa_ns = ns; + } else { + printf(" | %12s %8s", "N/A", "N/A"); + } + } + + if (cpu_ns > 0 && dsa_ns > 0) + printf(" | %5.2fx", cpu_ns / dsa_ns); + else if (cpu_idx >= 0) + printf(" | N/A"); + + if (is_ref_only(name)) + printf(" (ref)"); + printf("\n"); + } + + /* ---- Mann-Whitney U tests ---- */ + printf("\n ---- Mann-Whitney U Distribution Tests ----\n"); + + /* Auto-generate baselines if none exist */ + if (!update_mode) { + struct sample_dist probe; + + if (load_samples(baseline_dir, modes[0].suffix, pagesize, + modes[ref].tests[0].name, &probe) < 0) { + printf(" No baseline samples found in %s\n" + " Auto-generating baselines on this run\n\n", + baseline_dir); + update_mode = 1; + } else { + free_samples(&probe); + } + } + + if (update_mode) { + printf(" UPDATE MODE: copying current samples to baseline\n\n"); + mkdir(baseline_dir, 0755); + + for (int t = 0; t < modes[ref].count; t++) { + const char *name = modes[ref].tests[t].name; + + for (int m = 0; m < num_modes; m++) { + char src_path[4096], dst_path[4096]; + FILE *sf, *df; + char buf[8192]; + size_t n; + + snprintf(src_path, sizeof(src_path), + "%s/%s_%s_%s.samples", + results_dir, modes[m].suffix, + pagesize, name); + snprintf(dst_path, sizeof(dst_path), + "%s/%s_%s_%s.samples", + baseline_dir, modes[m].suffix, + pagesize, name); + + sf = fopen(src_path, "rb"); + if (!sf) + continue; + df = fopen(dst_path, "wb"); + if (!df) { + fclose(sf); + continue; + } + while ((n = fread(buf, 1, sizeof(buf), sf)) > 0) + fwrite(buf, 1, n, df); + fclose(sf); + fclose(df); + } + } + printf(" Baseline samples saved to %s\n\n", baseline_dir); + return 0; + } + + /* Comparison mode */ + { + int mw_tested = 0, mw_sig = 0; + + printf(" %-14s %-12s %20s %20s %10s %8s %10s %s\n", + "Test", "Mode", "baseline trimmed", + "current trimmed", "Change", + "KS D", "p-value", "Result"); + printf(" %-14s %-12s %20s %20s %10s %8s %10s %s\n", + "--------------", "------------", + "--------------------", "--------------------", + "----------", "--------", "----------", "------"); + + for (int t = 0; t < modes[ref].count; t++) { + const char *name = modes[ref].tests[t].name; + int ref_only = is_ref_only(name); + + for (int m = 0; m < num_modes; m++) { + struct sample_dist bl_dist, cur_dist; + double bl_ns, cur_ns, change_pct; + struct ks_result ks; + int regression; + + if (load_samples(baseline_dir, + modes[m].suffix, pagesize, + name, &bl_dist) < 0) + continue; + if (load_samples(results_dir, + modes[m].suffix, pagesize, + name, &cur_dist) < 0) { + free_samples(&bl_dist); + continue; + } + + /* + * Trimmed mean (5th-95th pctl) and + * Mann-Whitney on trimmed range. + */ + { + int blo = bl_dist.count / 10; + int bhi = bl_dist.count * 9 / 10; + int clo = cur_dist.count / 10; + int chi = cur_dist.count * 9 / 10; + int bn = bhi - blo, cn = chi - clo; + double bs = 0, cs = 0; + + for (int k = blo; k < bhi; k++) + bs += (double)bl_dist.ticks[k] + / bl_dist.tsc_ghz; + for (int k = clo; k < chi; k++) + cs += (double)cur_dist.ticks[k] + / cur_dist.tsc_ghz; + bl_ns = bs / bn; + cur_ns = cs / cn; + ks = ks_test(bl_dist.ticks + blo, bn, + cur_dist.ticks + clo, cn); + } + change_pct = ((cur_ns - bl_ns) / + bl_ns) * 100.0; + + regression = change_pct > min_effect_pct; + + printf(" %-14s %-12s %18.1f ns %18.1f ns " + "%+8.1f%% D=%.4f p=%.2e ", + name, modes[m].suffix, + bl_ns, cur_ns, + change_pct, ks.d, ks.p); + + if (ref_only) + printf("REF\n"); + else if (regression) + printf("FAIL ***\n"); + else if (change_pct < -min_effect_pct) + printf("IMPROVED\n"); + else + printf("PASS\n"); + + mw_tested++; + if (regression && !ref_only) { + mw_sig++; + failures++; + } + + /* Show distribution for significant changes */ + if (!ref_only && + (change_pct > min_effect_pct || + change_pct < -min_effect_pct)) + print_histogram(&bl_dist, &cur_dist); + + free_samples(&bl_dist); + free_samples(&cur_dist); + } + } + + printf("\n"); + if (mw_tested > 0) { + if (mw_sig > 0) + printf(" %d/%d tests show significant " + "regression (p < 0.05)\n\n", + mw_sig, mw_tested); + else + printf(" All %d tests passed " + "(no significant regressions)\n\n", + mw_tested); + } else { + printf(" No sample distributions found — run " + "UPDATE_BASELINES=1 first\n\n"); + } + } + + return failures; +} + +int main(void) +{ + const char *results_dir = getenv("RESULTS_DIR"); + const char *baseline_dir = getenv("BASELINE_DIR"); + int update_mode = (getenv("UPDATE_BASELINES") != NULL); + const char *effect_env = getenv("PERF_MIN_EFFECT"); + double min_effect_pct = effect_env ? atof(effect_env) : 2.0; + int failures = 0; + + if (!results_dir) { + fprintf(stderr, "RESULTS_DIR not set\n"); + return 1; + } + if (!baseline_dir) { + fprintf(stderr, "BASELINE_DIR not set\n"); + return 1; + } + + printf("\n Regression threshold: p < 0.05 AND median change > " + "+%.1f%%\n", min_effect_pct); + + failures += process_pagesize(results_dir, baseline_dir, + "4k", "4KB Pages", + update_mode, min_effect_pct); + failures += process_pagesize(results_dir, baseline_dir, + "2m", "2MB Hugepages", + update_mode, min_effect_pct); + + return failures > 0 ? 1 : 0; +} diff --git a/tests/plot-distributions.R b/tests/plot-distributions.R new file mode 100644 index 0000000..048066c --- /dev/null +++ b/tests/plot-distributions.R @@ -0,0 +1,110 @@ +#!/usr/bin/env Rscript +# Plot baseline vs current latency distributions from perf_combined output. +# +# Usage: Rscript tests/plot-distributions.R [results_dir] [pagesize] +# results_dir - directory with distributions_*.csv (default: build/perf_results) +# pagesize - "4k" or "2m" (default: "4k") +# +# Generates: +# distributions_{pagesize}_density.png - overlaid density curves +# distributions_{pagesize}_violin.png - violin plots per test/config +# distributions_{pagesize}_ecdf.png - empirical CDF (most sensitive) + +library(ggplot2) +library(dplyr) + +args <- commandArgs(trailingOnly = TRUE) +results_dir <- if (length(args) >= 1) args[1] else "build/perf_results" +pagesize <- if (length(args) >= 2) args[2] else "4k" +optype <- if (length(args) >= 3) args[3] else "memcpy" + +csv_path <- file.path(results_dir, "distributions.csv") +if (!file.exists(csv_path)) { + stop("CSV not found: ", csv_path, + "\nRun: ctest -R perf_combined --output-on-failure") +} + +d <- read.csv(csv_path, stringsAsFactors = FALSE) + +# Filter to requested pagesize and op type +d <- d[d$pagesize == pagesize, ] +d <- d[grepl(paste("^",optype,sep=""), d$test), ] +if (nrow(d) == 0) { + stop("No data for pagesize '", pagesize, "' in ", csv_path) +} +cat("Loaded", nrow(d), "samples from", csv_path, "\n") + +# Trim outliers per group (keep 1st-99th percentile) for cleaner plots +d <- d %>% + group_by(test, config, version) %>% + filter(latency_ns >= quantile(latency_ns, 0.01), + latency_ns <= quantile(latency_ns, 0.99)) %>% + ungroup() + +d$version <- factor(d$version, levels = c("baseline", "current")) +d$config <- factor(d$config, levels = c("cpu", "stdc", "dsa", "dsa_auto")) + +# Compute median per group for annotation +medians <- d %>% + group_by(test, config, version) %>% + summarise(med = median(latency_ns), .groups = "drop") + +# --- 1. Density plot: overlaid distributions --- +p1 <- ggplot(d, aes(x = latency_ns, fill = version, color = version)) + + geom_density(alpha = 0.3, linewidth = 0.5) + + geom_vline(data = medians, aes(xintercept = med, color = version), + linetype = "dashed", linewidth = 0.4) + + facet_wrap(~ test + config, scales = "free", ncol = 3, + labeller = labeller(.multi_line = FALSE)) + + scale_fill_manual(values = c(baseline = "#2166ac", current = "#b2182b")) + + scale_color_manual(values = c(baseline = "#2166ac", current = "#b2182b")) + + labs(title = paste("Latency Distributions: Baseline vs Current (", pagesize, ",", optype, ")"), + x = "Latency (ns)", y = "Density", + fill = "Version", color = "Version") + + theme_minimal(base_size = 10) + + theme(strip.text = element_text(face = "bold", size = 7), + legend.position = "bottom", + axis.text.x = element_text(angle = 45, hjust = 1)) + +outfile <- file.path(results_dir, + paste0("distributions_", pagesize, "_", optype, "_density.png")) +ggsave(outfile, p1, width = 16, height = 20, dpi = 150) +cat("Saved", outfile, "\n") + +# --- 2. Violin plot: shape comparison --- +p2 <- ggplot(d, aes(x = version, y = latency_ns, fill = version)) + + geom_violin(alpha = 0.6, draw_quantiles = c(0.25, 0.5, 0.75)) + + facet_wrap(~ test + config, scales = "free_y", ncol = 3, + labeller = labeller(.multi_line = FALSE)) + + scale_fill_manual(values = c(baseline = "#2166ac", current = "#b2182b")) + + labs(title = paste("Latency Violins: Baseline vs Current (", pagesize, ",", optype, ")"), + x = "", y = "Latency (ns)") + + theme_minimal(base_size = 10) + + theme(strip.text = element_text(face = "bold", size = 7), + legend.position = "none") + +outfile <- file.path(results_dir, + paste0("distributions_", pagesize, "_", optype, "_violin.png")) +ggsave(outfile, p2, width = 16, height = 20, dpi = 150) +cat("Saved", outfile, "\n") + +# --- 3. ECDF plot: most sensitive to distribution shifts --- +p3 <- ggplot(d, aes(x = latency_ns, color = version)) + + stat_ecdf(linewidth = 0.5) + + facet_wrap(~ test + config, scales = "free_x", ncol = 3, + labeller = labeller(.multi_line = FALSE)) + + scale_color_manual(values = c(baseline = "#2166ac", current = "#b2182b")) + + labs(title = paste("Empirical CDF: Baseline vs Current (", pagesize, "," , optype, ")"), + x = "Latency (ns)", y = "Cumulative Probability", + color = "Version") + + theme_minimal(base_size = 10) + + theme(strip.text = element_text(face = "bold", size = 7), + legend.position = "bottom", + axis.text.x = element_text(angle = 45, hjust = 1)) + +outfile <- file.path(results_dir, + paste0("distributions_", pagesize, "_", optype, "_ecdf.png")) +ggsave(outfile, p3, width = 16, height = 20, dpi = 150) +cat("Saved", outfile, "\n") + +cat("\nDone. All plots in", results_dir, "\n") diff --git a/tests/test_functional.c b/tests/test_functional.c new file mode 100644 index 0000000..e3802a0 --- /dev/null +++ b/tests/test_functional.c @@ -0,0 +1,502 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * test_functional.c - Functional correctness tests for DTO. + * + * Tests memset, memcpy, memmove, memcmp across a range of buffer sizes + * spanning below and above the DSA offload threshold (64KB default). + * + * When linked against libdto, these functions are intercepted by DTO. + * In CI (no DSA hardware), DTO falls back to the CPU path transparently. + * All tests should pass regardless of whether DSA hardware is present. + * + * Verification is done byte-by-byte to avoid relying on DTO's own memcmp + * for checking results. + * + * Labels: ci (safe to run in GitHub Actions without DSA) + ******************************************************************************/ + +#include "dto_test_utils.h" +#include +#include + +/* Buffer sizes spanning below and above the DSA threshold (65536) */ +static const size_t test_sizes[] = { + 0, 1, 7, 15, 16, 31, 32, 63, 64, + 127, 128, 255, 256, 511, 512, 1023, 1024, + 4096, 8192, 16384, 32768, + 65536, /* DTO_DEFAULT_MIN_SIZE - DSA threshold */ + 131072, /* 128K - above threshold, triggers DSA */ + 262144, /* 256K */ +}; +#define NUM_SIZES (sizeof(test_sizes) / sizeof(test_sizes[0])) +#define MAX_SIZE 262144 + +/* + * Helper functions for verification. + * + * These use byte-by-byte loops so the compiler cannot optimize them + * into memset/memcmp calls (which would be intercepted by DTO, defeating + * the purpose of independent verification). + */ +static int __attribute__((optimize("no-tree-loop-distribute-patterns"))) +verify_set(const uint8_t *buf, uint8_t val, size_t n) +{ + for (size_t i = 0; i < n; i++) { + if (buf[i] != val) { + fprintf(stderr, " byte[%zu] = 0x%02x, expected 0x%02x\n", + i, buf[i], val); + return 0; + } + } + return 1; +} + +static int __attribute__((optimize("no-tree-loop-distribute-patterns"))) +verify_equal(const uint8_t *a, const uint8_t *b, size_t n) +{ + for (size_t i = 0; i < n; i++) { + if (a[i] != b[i]) { + fprintf(stderr, " byte[%zu]: got 0x%02x, expected 0x%02x\n", + i, a[i], b[i]); + return 0; + } + } + return 1; +} + +static void __attribute__((optimize("no-tree-loop-distribute-patterns"))) +fill_pattern(uint8_t *buf, size_t n) +{ + for (size_t i = 0; i < n; i++) + buf[i] = (uint8_t)(i & 0xFF); +} + +static void __attribute__((optimize("no-tree-loop-distribute-patterns"))) +clear_buf(uint8_t *buf, size_t n) +{ + for (size_t i = 0; i < n; i++) + buf[i] = 0; +} + +/* ---- memset tests ---- */ + +static int test_memset_correctness(void) +{ + uint8_t *buf = alloc_aligned(MAX_SIZE); + + ASSERT_TRUE(buf != NULL); + + for (size_t s = 0; s < NUM_SIZES; s++) { + size_t n = test_sizes[s]; + + /* Test with 0xAA */ + clear_buf(buf, n); + memset(buf, 0xAA, n); + if (!verify_set(buf, 0xAA, n)) { + fprintf(stderr, " memset(0xAA) failed at size %zu\n", n); + free(buf); + return 1; + } + + /* Test with 0x00 */ + fill_pattern(buf, n); + memset(buf, 0x00, n); + if (!verify_set(buf, 0x00, n)) { + fprintf(stderr, " memset(0x00) failed at size %zu\n", n); + free(buf); + return 1; + } + + /* Test with 0xFF */ + clear_buf(buf, n); + memset(buf, 0xFF, n); + if (!verify_set(buf, 0xFF, n)) { + fprintf(stderr, " memset(0xFF) failed at size %zu\n", n); + free(buf); + return 1; + } + } + + free(buf); + return 0; +} + +/* ---- memcpy tests ---- */ + +static int test_memcpy_correctness(void) +{ + uint8_t *src = alloc_aligned(MAX_SIZE); + uint8_t *dst = alloc_aligned(MAX_SIZE); + + ASSERT_TRUE(src != NULL && dst != NULL); + + fill_pattern(src, MAX_SIZE); + + for (size_t s = 0; s < NUM_SIZES; s++) { + size_t n = test_sizes[s]; + + clear_buf(dst, n); + memcpy(dst, src, n); + if (!verify_equal(dst, src, n)) { + fprintf(stderr, " memcpy failed at size %zu\n", n); + free(src); + free(dst); + return 1; + } + } + + free(src); + free(dst); + return 0; +} + +/* ---- memmove tests ---- */ + +static int test_memmove_nonoverlap(void) +{ + uint8_t *src = alloc_aligned(MAX_SIZE); + uint8_t *dst = alloc_aligned(MAX_SIZE); + + ASSERT_TRUE(src != NULL && dst != NULL); + + fill_pattern(src, MAX_SIZE); + + for (size_t s = 0; s < NUM_SIZES; s++) { + size_t n = test_sizes[s]; + + clear_buf(dst, n); + memmove(dst, src, n); + if (!verify_equal(dst, src, n)) { + fprintf(stderr, " memmove (non-overlap) failed at size %zu\n", n); + free(src); + free(dst); + return 1; + } + } + + free(src); + free(dst); + return 0; +} + +static int test_memmove_overlap_forward(void) +{ + /* Overlapping forward: dst > src, dst within [src, src+n) */ + size_t buf_size = MAX_SIZE + 4096; + uint8_t *buf = alloc_aligned(buf_size); + uint8_t *ref = alloc_aligned(buf_size); + + ASSERT_TRUE(buf != NULL && ref != NULL); + + size_t overlap_sizes[] = {128, 1024, 4096, 65536, 131072}; + int num = sizeof(overlap_sizes) / sizeof(overlap_sizes[0]); + + for (int s = 0; s < num; s++) { + size_t n = overlap_sizes[s]; + size_t offset = n / 4; + + fill_pattern(buf, n + offset); + fill_pattern(ref, n + offset); + + /* Reference: manual backward copy (correct for forward overlap) */ + for (size_t i = n; i > 0; i--) + ref[offset + i - 1] = ref[i - 1]; + + memmove(buf + offset, buf, n); + + if (!verify_equal(buf + offset, ref + offset, n)) { + fprintf(stderr, " memmove (overlap forward) failed at size %zu\n", n); + free(buf); + free(ref); + return 1; + } + } + + free(buf); + free(ref); + return 0; +} + +static int test_memmove_overlap_backward(void) +{ + /* Overlapping backward: dst < src, src within [dst, dst+n) */ + size_t buf_size = MAX_SIZE + 4096; + uint8_t *buf = alloc_aligned(buf_size); + uint8_t *ref = alloc_aligned(buf_size); + + ASSERT_TRUE(buf != NULL && ref != NULL); + + size_t overlap_sizes[] = {128, 1024, 4096, 65536, 131072}; + int num = sizeof(overlap_sizes) / sizeof(overlap_sizes[0]); + + for (int s = 0; s < num; s++) { + size_t n = overlap_sizes[s]; + size_t offset = n / 4; + + fill_pattern(buf, n + offset); + fill_pattern(ref, n + offset); + + /* Reference: forward copy (correct for backward overlap) */ + for (size_t i = 0; i < n; i++) + ref[i] = ref[offset + i]; + + memmove(buf, buf + offset, n); + + if (!verify_equal(buf, ref, n)) { + fprintf(stderr, " memmove (overlap backward) failed at size %zu\n", n); + free(buf); + free(ref); + return 1; + } + } + + free(buf); + free(ref); + return 0; +} + +/* ---- memcmp tests ---- */ + +static int test_memcmp_equal(void) +{ + uint8_t *a = alloc_aligned(MAX_SIZE); + uint8_t *b = alloc_aligned(MAX_SIZE); + + ASSERT_TRUE(a != NULL && b != NULL); + + fill_pattern(a, MAX_SIZE); + fill_pattern(b, MAX_SIZE); + + for (size_t s = 0; s < NUM_SIZES; s++) { + size_t n = test_sizes[s]; + + if (memcmp(a, b, n) != 0) { + fprintf(stderr, " memcmp(equal) returned non-zero at size %zu\n", n); + free(a); + free(b); + return 1; + } + } + + free(a); + free(b); + return 0; +} + +static int test_memcmp_differ(void) +{ + uint8_t *a = alloc_aligned(MAX_SIZE); + uint8_t *b = alloc_aligned(MAX_SIZE); + + ASSERT_TRUE(a != NULL && b != NULL); + + for (size_t s = 0; s < NUM_SIZES; s++) { + size_t n = test_sizes[s]; + + if (n == 0) + continue; + + fill_pattern(a, n); + fill_pattern(b, n); + + /* Make last byte different */ + b[n - 1] = (uint8_t)(~a[n - 1]); + + int result = memcmp(a, b, n); + + if (result == 0) { + fprintf(stderr, " memcmp returned 0 for different buffers at size %zu\n", n); + free(a); + free(b); + return 1; + } + + /* Verify sign matches expectation */ + int expected_sign = (a[n - 1] > b[n - 1]) ? 1 : -1; + int actual_sign = (result > 0) ? 1 : -1; + + if (actual_sign != expected_sign) { + fprintf(stderr, " memcmp sign mismatch at size %zu: got %d, expected %d\n", + n, actual_sign, expected_sign); + free(a); + free(b); + return 1; + } + } + + free(a); + free(b); + return 0; +} + +/* ---- Edge case tests ---- */ + +static int test_zero_length(void) +{ + uint8_t a[16], b[16], save_a[16]; + + fill_pattern(a, 16); + clear_buf(b, 16); + + /* Save original a */ + for (int i = 0; i < 16; i++) + save_a[i] = a[i]; + + /* memset with n=0 should not modify buffer */ + memset(a, 0xFF, 0); + ASSERT_TRUE(verify_equal(a, save_a, 16)); + + /* memcpy with n=0 should not modify dst */ + memcpy(b, a, 0); + ASSERT_TRUE(verify_set(b, 0, 16)); + + /* memmove with n=0 should not modify dst */ + memmove(b, a, 0); + ASSERT_TRUE(verify_set(b, 0, 16)); + + /* memcmp with n=0 should return 0 */ + ASSERT_EQ(memcmp(a, b, 0), 0); + + return 0; +} + +static int test_memcpy_unaligned(void) +{ + size_t buf_size = MAX_SIZE + 64; + uint8_t *src_base = alloc_aligned(buf_size); + uint8_t *dst_base = alloc_aligned(buf_size); + + ASSERT_TRUE(src_base != NULL && dst_base != NULL); + + int offsets[] = {1, 3, 7, 13, 31, 63}; + int num_offsets = sizeof(offsets) / sizeof(offsets[0]); + size_t sizes[] = {64, 1024, 4096, 65536, 131072}; + int num_sizes = sizeof(sizes) / sizeof(sizes[0]); + + for (int o = 0; o < num_offsets; o++) { + uint8_t *src = src_base + offsets[o]; + uint8_t *dst = dst_base + offsets[o]; + + fill_pattern(src, MAX_SIZE); + + for (int s = 0; s < num_sizes; s++) { + size_t n = sizes[s]; + + clear_buf(dst, n); + memcpy(dst, src, n); + + if (!verify_equal(dst, src, n)) { + fprintf(stderr, " memcpy (unaligned offset=%d) failed at size %zu\n", + offsets[o], n); + free(src_base); + free(dst_base); + return 1; + } + } + } + + free(src_base); + free(dst_base); + return 0; +} + +/* ---- Multithreaded correctness test ---- */ + +#define MT_NUM_THREADS 4 +#define MT_BUF_SIZE (128 * 1024) +#define MT_ITERS 100 + +struct mt_result { + int passed; + int thread_id; +}; + +static void *mt_worker(void *arg) +{ + struct mt_result *result = (struct mt_result *)arg; + uint8_t *src = alloc_aligned(MT_BUF_SIZE); + uint8_t *dst = alloc_aligned(MT_BUF_SIZE); + + result->passed = 1; + + if (!src || !dst) { + result->passed = 0; + free(src); + free(dst); + return NULL; + } + + for (int i = 0; i < MT_ITERS; i++) { + uint8_t pattern = (uint8_t)(i & 0xFF); + + memset(src, pattern, MT_BUF_SIZE); + if (!verify_set(src, pattern, MT_BUF_SIZE)) { + fprintf(stderr, " thread %d: memset failed at iter %d\n", + result->thread_id, i); + result->passed = 0; + break; + } + + memcpy(dst, src, MT_BUF_SIZE); + if (!verify_equal(dst, src, MT_BUF_SIZE)) { + fprintf(stderr, " thread %d: memcpy failed at iter %d\n", + result->thread_id, i); + result->passed = 0; + break; + } + } + + free(src); + free(dst); + return NULL; +} + +static int test_multithread(void) +{ + pthread_t threads[MT_NUM_THREADS]; + struct mt_result results[MT_NUM_THREADS]; + + for (int i = 0; i < MT_NUM_THREADS; i++) { + results[i].thread_id = i; + pthread_create(&threads[i], NULL, mt_worker, &results[i]); + } + + for (int i = 0; i < MT_NUM_THREADS; i++) + pthread_join(threads[i], NULL); + + for (int i = 0; i < MT_NUM_THREADS; i++) { + if (!results[i].passed) { + fprintf(stderr, " Thread %d failed\n", i); + return 1; + } + } + + return 0; +} + +/* ---- Test runner ---- */ + +int main(void) +{ + printf("DTO Functional Tests\n"); + printf("====================\n\n"); + + struct dto_test tests[] = { + TEST_ENTRY(test_memset_correctness), + TEST_ENTRY(test_memcpy_correctness), + TEST_ENTRY(test_memmove_nonoverlap), + TEST_ENTRY(test_memmove_overlap_forward), + TEST_ENTRY(test_memmove_overlap_backward), + TEST_ENTRY(test_memcmp_equal), + TEST_ENTRY(test_memcmp_differ), + TEST_ENTRY(test_zero_length), + TEST_ENTRY(test_memcpy_unaligned), + TEST_ENTRY(test_multithread), + {NULL, NULL} + }; + + return run_tests(tests); +} diff --git a/tests/test_perf.c b/tests/test_perf.c new file mode 100644 index 0000000..b185079 --- /dev/null +++ b/tests/test_perf.c @@ -0,0 +1,915 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * test_perf.c - Performance regression tests for DTO. + * + * Measures throughput (GB/s) and per-operation latency (ns) for memset, + * memcpy, memcmp using cold-cache methodology (clflushopt before each + * operation). Results are compared against checked-in baselines to detect + * performance regressions from incoming DTO library changes. + * + * The benchmark thread is pinned to a single CPU core (PERF_CPU env var, + * default: core 1) and elevated to SCHED_FIFO priority to reduce run-to-run + * variance. Per-iteration timings are collected and the median is reported + * rather than the mean, making results robust against outlier spikes from + * interrupts or transient system activity. + * + * Four modes of operation (separate CTest entries): + * + * 1. Raw CPU: Built without libdto (dto-test-perf-cpu). + * Measures baseline libc performance. + * + * 2. DTO + stdc: Built with libdto, DTO_USESTDC_CALLS=1. + * Measures DTO interception overhead without DSA. + * + * 3. DTO + DSA: Built with libdto, DTO_CPU_SIZE_FRACTION=0, + * DTO_AUTO_ADJUST_KNOBS=0. + * Measures pure DSA performance through DTO. + * + * 4. DTO + DSA Built with libdto, DTO_CPU_SIZE_FRACTION=0.33, + * (auto): DTO_AUTO_ADJUST_KNOBS=1. + * Measures DSA with CPU+DSA auto-tuning. + * + * Environment variables: + * BASELINE_DIR - Directory containing baseline .dat files + * RESULTS_DIR - Directory to write result files for summary + * UPDATE_BASELINES - If set, overwrite baseline file with measured results + * PERF_TOLERANCE - Allowed %% drop below baseline before failing (default: 0) + * PERF_CPU - CPU core to pin benchmark thread to (default: 1) + * PERF_PASSES - Full passes per benchmark; median-of-medians when >1 (default: 1) + * PERF_COLD_CACHE - Flush caches every iteration (1, default) or once per size (0) + * PERF_FREQ_MHZ - Pin core and uncore frequency to this value in MHz (e.g., 2000) + * DTO_PERF_HUGE - If set, allocate with 2MB hugepages + * PERF_LABEL - Label for file prefix (cpu/stdc/dsa/dsa_auto) + * + * Labels: hardware, perf (DSA mode requires configured DSA work queues) + ******************************************************************************/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include "dto_test_utils.h" + +#include +#include +#include +#include +#include +#include + +/* Warmup iterations before timing */ +#define WARMUP_ITERS 10000 + +/* Default tolerance: fail on any regression */ +#define DEFAULT_TOLERANCE 2.0 + +#define DEFAULT_CPU 1 + +/* Default number of full passes per benchmark (median-of-medians when > 1) */ +#define DEFAULT_PASSES 11 + +/* Sink for memcmp results to prevent dead code elimination */ +static volatile int benchmark_sink; + +static int use_hugepages; +static int cold_cache; + +/* TSC frequency in ticks per nanosecond (GHz), calibrated at startup */ +static double tsc_ghz; + +/* ---- rdtsc helpers ---- */ + +static inline uint64_t rdtsc_start(void) +{ + __asm__ volatile("lfence"); + return __rdtsc(); +} + +static inline uint64_t rdtsc_end(void) +{ + unsigned int aux; + + return __rdtscp(&aux); +} + +/* + * Calibrate TSC frequency by measuring ticks over a known wall-clock + * interval. Busy-waits for ~50ms to get a stable ratio. + */ +static double calibrate_tsc(void) +{ + struct timespec t0, t1; + uint64_t tsc0, tsc1; + uint64_t elapsed_ns; + + clock_gettime(CLOCK_MONOTONIC, &t0); + tsc0 = rdtsc_end(); + + /* Busy-wait ~50ms */ + do { + clock_gettime(CLOCK_MONOTONIC, &t1); + elapsed_ns = (t1.tv_sec - t0.tv_sec) * 1000000000ULL + + (t1.tv_nsec - t0.tv_nsec); + } while (elapsed_ns < 50000000ULL); + + tsc1 = rdtsc_end(); + + return (double)(tsc1 - tsc0) / (double)elapsed_ns; +} + +/* ---- CPU pinning ---- */ + +static int pin_to_cpu(int cpu) +{ + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask) < 0) { + perror("sched_setaffinity"); + return -1; + } + return 0; +} + +/* ---- Frequency pinning ---- */ + +static struct { + int active; + int cpu; + unsigned long orig_core_min_khz; + unsigned long orig_core_max_khz; + char uncore_dir[256]; + unsigned long orig_uncore_min_khz; + unsigned long orig_uncore_max_khz; +} freq_state; + +static int read_sysfs_ulong(const char *path, unsigned long *val) +{ + FILE *f = fopen(path, "r"); + + if (!f) + return -1; + if (fscanf(f, "%lu", val) != 1) { + fclose(f); + return -1; + } + fclose(f); + return 0; +} + +static int write_sysfs_ulong(const char *path, unsigned long val) +{ + FILE *f = fopen(path, "w"); + + if (!f) + return -1; + fprintf(f, "%lu", val); + fclose(f); + return 0; +} + +/* + * Set core frequency scaling_min and scaling_max for a given CPU. + * Handles ordering: drops min first so max can be lowered, then sets both. + */ +static int set_core_freq(int cpu, unsigned long khz) +{ + char path[256]; + unsigned long abs_min; + + /* Read hw minimum so we can drop scaling_min safely */ + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_min_freq", + cpu); + if (read_sysfs_ulong(path, &abs_min) < 0) + return -1; + + /* Drop min to absolute minimum to allow max to go anywhere */ + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", + cpu); + if (write_sysfs_ulong(path, abs_min) < 0) + return -1; + + /* Set max */ + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_max_freq", + cpu); + if (write_sysfs_ulong(path, khz) < 0) + return -1; + + /* Set min to match */ + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", + cpu); + return write_sysfs_ulong(path, khz); +} + +/* + * Find the uncore frequency sysfs directory for the package that owns + * the given CPU. Returns 0 on success with dir filled in. + */ +static int find_uncore_dir(int cpu, char *dir, size_t len) +{ + char path[512]; + unsigned long pkg, die; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", + cpu); + if (read_sysfs_ulong(path, &pkg) < 0) + return -1; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/die_id", cpu); + if (read_sysfs_ulong(path, &die) < 0) + die = 0; + + snprintf(dir, len, + "/sys/devices/system/cpu/intel_uncore_frequency/" + "package_%02lu_die_%02lu", pkg, die); + + /* Verify it exists */ + snprintf(path, sizeof(path), "%s/min_freq_khz", dir); + unsigned long tmp; + + return read_sysfs_ulong(path, &tmp); +} + +static int set_uncore_freq(const char *dir, unsigned long khz) +{ + char path[512]; + unsigned long abs_min; + + snprintf(path, sizeof(path), "%s/initial_min_freq_khz", dir); + if (read_sysfs_ulong(path, &abs_min) < 0) { + /* Fallback: read current min */ + snprintf(path, sizeof(path), "%s/min_freq_khz", dir); + if (read_sysfs_ulong(path, &abs_min) < 0) + return -1; + } + + /* Drop min first */ + snprintf(path, sizeof(path), "%s/min_freq_khz", dir); + write_sysfs_ulong(path, abs_min); + + snprintf(path, sizeof(path), "%s/max_freq_khz", dir); + if (write_sysfs_ulong(path, khz) < 0) + return -1; + + snprintf(path, sizeof(path), "%s/min_freq_khz", dir); + return write_sysfs_ulong(path, khz); +} + +static void restore_freq(void) +{ + if (!freq_state.active) + return; + + set_core_freq(freq_state.cpu, freq_state.orig_core_max_khz); + /* Restore original min (may be lower than max) */ + { + char path[512]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", + freq_state.cpu); + write_sysfs_ulong(path, freq_state.orig_core_min_khz); + } + + if (freq_state.uncore_dir[0]) { + set_uncore_freq(freq_state.uncore_dir, + freq_state.orig_uncore_max_khz); + char path[512]; + + snprintf(path, sizeof(path), "%s/min_freq_khz", + freq_state.uncore_dir); + write_sysfs_ulong(path, freq_state.orig_uncore_min_khz); + } +} + +static int pin_freq(int cpu, unsigned long mhz) +{ + unsigned long khz = mhz * 1000; + char path[512]; + + /* Save and set core frequency */ + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", + cpu); + if (read_sysfs_ulong(path, &freq_state.orig_core_min_khz) < 0) { + fprintf(stderr, "WARNING: cannot read core freq for CPU %d " + "(run as root, check cpufreq driver)\n", cpu); + return -1; + } + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_max_freq", + cpu); + read_sysfs_ulong(path, &freq_state.orig_core_max_khz); + + if (set_core_freq(cpu, khz) < 0) { + fprintf(stderr, "WARNING: could not pin core freq to %lu MHz " + "(run as root)\n", mhz); + return -1; + } + + freq_state.cpu = cpu; + freq_state.active = 1; + atexit(restore_freq); + + /* Save and set uncore frequency */ + if (find_uncore_dir(cpu, freq_state.uncore_dir, + sizeof(freq_state.uncore_dir)) == 0) { + snprintf(path, sizeof(path), "%s/min_freq_khz", + freq_state.uncore_dir); + read_sysfs_ulong(path, &freq_state.orig_uncore_min_khz); + snprintf(path, sizeof(path), "%s/max_freq_khz", + freq_state.uncore_dir); + read_sysfs_ulong(path, &freq_state.orig_uncore_max_khz); + + if (set_uncore_freq(freq_state.uncore_dir, khz) < 0) + fprintf(stderr, "WARNING: could not pin uncore freq " + "to %lu MHz\n", mhz); + } else { + fprintf(stderr, "WARNING: intel_uncore_frequency sysfs not " + "found, uncore freq not pinned\n"); + freq_state.uncore_dir[0] = '\0'; + } + + return 0; +} + +/* ---- Cache flush helpers (matches bench_batch_vs_single) ---- */ + +static inline void clflushopt(volatile void *p) +{ + asm volatile("clflushopt (%0)" :: "r"(p) : "memory"); +} + +static void flush_buffer(void *buf, size_t size) +{ + char *p = buf; + + for (size_t i = 0; i < size; i += 64) + clflushopt(p + i); + _mm_sfence(); +} + +enum perf_op { + OP_MEMSET, + OP_MEMCPY, + OP_MEMCMP, +}; + +struct perf_test { + const char *name; + enum perf_op op; + size_t size; + int iterations; + int batch; /* ops per timed sample (amortizes rdtsc for small sizes) */ + int ref_only; /* if 1, report results but skip baseline pass/fail check */ +}; + +/* + * Benchmark configurations. + * + * Sizes range from 4KB to 1MB covering below and above the DSA offload + * threshold. Iterations scale inversely with buffer size: small buffers + * (4-8KB) use 100K iterations to compensate for their short per-op time, + * while large buffers (256KB+) use 10K. + * + * Small sizes (4-8KB) use batch=10: each timed sample covers 10 + * back-to-back operations, amortizing rdtsc and per-call overhead. + * The recorded sample is divided by batch to get per-op ticks. + */ + +#define ITERATIONS 20000 +#define SMALL_ITERATIONS 50000 +static struct perf_test benchmarks[] = { + /* name op size iters batch ref */ + {"memcpy_4k", OP_MEMCPY, 4096, SMALL_ITERATIONS, 1, 1}, + {"memcpy_8k", OP_MEMCPY, 8192, SMALL_ITERATIONS, 1, 1}, + {"memcpy_16k", OP_MEMCPY, 16384, ITERATIONS, 1, 0}, + {"memcpy_32k", OP_MEMCPY, 32768, ITERATIONS, 1, 0}, + {"memcpy_64k", OP_MEMCPY, 65536, ITERATIONS, 1, 0}, + {"memcpy_128k", OP_MEMCPY, 131072, ITERATIONS, 1, 0}, + {"memcpy_256k", OP_MEMCPY, 262144, ITERATIONS, 1, 0}, + {"memcpy_512k", OP_MEMCPY, 524288, ITERATIONS, 1, 0}, + {"memcpy_1m", OP_MEMCPY, 1048576, ITERATIONS, 1, 0}, + + {"memset_64k", OP_MEMSET, 65536, ITERATIONS, 1, 0}, + {"memset_128k", OP_MEMSET, 131072, ITERATIONS, 1, 0}, + {"memset_256k", OP_MEMSET, 262144, ITERATIONS, 1, 0}, + {"memset_1m", OP_MEMSET, 1048576, ITERATIONS, 1, 0}, + + {"memcmp_64k", OP_MEMCMP, 65536, ITERATIONS, 1, 0}, + {"memcmp_128k", OP_MEMCMP, 131072, ITERATIONS, 1, 0}, + {"memcmp_1m", OP_MEMCMP, 1048576, ITERATIONS, 1, 0}, + + {NULL, 0, 0, 0, 0, 0} +}; + +static void *alloc_buffer(size_t size) +{ + void *p; + + if (use_hugepages) { + size_t alloc = (size + (2 << 20) - 1) & ~((2 << 20) - 1UL); + + p = mmap(NULL, alloc, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | + MAP_POPULATE | (21 << MAP_HUGE_SHIFT), + -1, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "mmap hugepage failed (need: " + "echo 128 > /proc/sys/vm/nr_hugepages)\n"); + return NULL; + } + } else { + p = mmap(NULL, size, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, + -1, 0); + if (p == MAP_FAILED) { + perror("mmap"); + return NULL; + } + } + return p; +} + +static void free_buffer(void *p, size_t size) +{ + if (use_hugepages) { + size_t alloc = (size + (2 << 20) - 1) & ~((2 << 20) - 1UL); + + munmap(p, alloc); + } else { + munmap(p, size); + } +} + +static int cmp_u64(const void *a, const void *b) +{ + uint64_t va = *(const uint64_t *)a; + uint64_t vb = *(const uint64_t *)b; + + return (va > vb) - (va < vb); +} + +static int cmp_double(const void *a, const void *b) +{ + double va = *(const double *)a; + double vb = *(const double *)b; + + return (va > vb) - (va < vb); +} + +/* + * Sample distribution file format (binary): + * uint32_t count - number of samples + * double tsc_ghz - TSC ticks-per-ns for conversion + * uint64_t samples[] - raw TSC ticks, sorted ascending + */ +static int save_samples(const char *dir, const char *label, + const char *pagesize, const char *testname, + uint64_t *samples, int count) +{ + char path[4096]; + FILE *f; + uint32_t n = count; + + snprintf(path, sizeof(path), "%s/%s_%s_%s.samples", + dir, label, pagesize, testname); + f = fopen(path, "wb"); + if (!f) + return -1; + + fwrite(&n, sizeof(n), 1, f); + fwrite(&tsc_ghz, sizeof(tsc_ghz), 1, f); + fwrite(samples, sizeof(uint64_t), count, f); + fclose(f); + return 0; +} + +/* + * Run a single pass of the benchmark. Returns GB/s (or -1 on error). + * If out_samples / out_nsamples are non-NULL, the caller takes ownership + * of the samples array (ticks, not ns) and must free() it. + */ +static double run_benchmark(struct perf_test *test, double *out_avg_ns, + uint64_t **out_samples, int *out_nsamples) +{ + uint8_t *src, *dst; + uint64_t *samples; + uint64_t start, end; + + if (out_samples) + *out_samples = NULL; + if (out_nsamples) + *out_nsamples = 0; + + src = alloc_buffer(test->size); + dst = alloc_buffer(test->size); + if (!src || !dst) { + if (src) free_buffer(src, test->size); + if (dst) free_buffer(dst, test->size); + return -1.0; + } + + samples = malloc(test->iterations * sizeof(uint64_t)); + if (!samples) { + free_buffer(src, test->size); + free_buffer(dst, test->size); + return -1.0; + } + + /* Initialize source data */ + for (size_t i = 0; i < test->size; i++) + src[i] = (uint8_t)(i & 0xFF); + /* For memcmp, dst must match src */ + for (size_t i = 0; i < test->size; i++) + dst[i] = src[i]; + + /* Warmup: prime IOTLB and DSA work queues */ + for (int i = 0; i < WARMUP_ITERS; i++) { + switch (test->op) { + case OP_MEMSET: + memset(dst, 0xAA, test->size); + break; + case OP_MEMCPY: + memcpy(dst, src, test->size); + break; + case OP_MEMCMP: + benchmark_sink = memcmp(dst, src, test->size); + break; + } + } + + /* Re-init dst for memcmp (warmup may have overwritten via memset) */ + if (test->op == OP_MEMCMP) { + for (size_t i = 0; i < test->size; i++) + dst[i] = src[i]; + } + + /* + * Timed run. + * + * Cold-cache mode (default): flush both buffers before each + * iteration so every operation measures DRAM latency. + * Warm-cache mode (PERF_COLD_CACHE=0): flush once before the + * loop; subsequent iterations hit L1/L2 — more representative + * for small buffers that stay cache-resident in real workloads. + */ + if (!cold_cache) { + flush_buffer(src, test->size); + flush_buffer(dst, test->size); + } + + int batch = test->batch > 0 ? test->batch : 1; + + for (int i = 0; i < test->iterations; i++) { + if (cold_cache) { + flush_buffer(src, test->size); + flush_buffer(dst, test->size); + } + + start = rdtsc_start(); + + for (int b = 0; b < batch; b++) { + + switch (test->op) { + case OP_MEMSET: + memset(dst, 0xAA, test->size); + break; + case OP_MEMCPY: + memcpy(dst, src, test->size); + break; + case OP_MEMCMP: + benchmark_sink = memcmp(dst, src, test->size); + break; + } + COMPILER_BARRIER(); + } + + end = rdtsc_end(); + samples[i] = (end - start) / batch; + } + + qsort(samples, test->iterations, sizeof(uint64_t), cmp_u64); + double median_ns = (double)samples[test->iterations / 2] / tsc_ghz; + double gbps = (double)test->size / median_ns; + + *out_avg_ns = median_ns; + + if (out_samples) { + *out_samples = samples; /* caller owns it now */ + if (out_nsamples) + *out_nsamples = test->iterations; + } else { + free(samples); + } + + /* + * Post-benchmark verification: prove the full buffer was operated on. + * Poison dst, run one more operation, then check first/mid/last bytes. + */ + if (test->op == OP_MEMCPY && test->size > 0) { + /* Poison entire dst with 0xDE */ + for (size_t i = 0; i < test->size; i++) + dst[i] = 0xDE; + + flush_buffer(src, test->size); + flush_buffer(dst, test->size); + memcpy(dst, src, test->size); + COMPILER_BARRIER(); + + /* Check first, middle, and last bytes */ + size_t checks[] = {0, test->size / 2, test->size - 1}; + for (int c = 0; c < 3; c++) { + size_t idx = checks[c]; + + if (dst[idx] != src[idx]) { + fprintf(stderr, + "\n VERIFY FAIL: %s byte[%zu] " + "dst=0x%02x src=0x%02x\n", + test->name, idx, dst[idx], src[idx]); + free_buffer(src, test->size); + free_buffer(dst, test->size); + return -1.0; + } + } + /* Spot-check that poison was fully overwritten */ + if (dst[test->size - 1] == 0xDE) { + fprintf(stderr, + "\n VERIFY FAIL: %s last byte still " + "poisoned (0xDE) — copy may be incomplete\n", + test->name); + free_buffer(src, test->size); + free_buffer(dst, test->size); + return -1.0; + } + } + + if (test->op == OP_MEMSET && test->size > 0) { + /* Clear dst, run one memset, verify last byte */ + for (size_t i = 0; i < test->size; i++) + dst[i] = 0x00; + + flush_buffer(dst, test->size); + memset(dst, 0xAA, test->size); + COMPILER_BARRIER(); + + if (dst[test->size - 1] != 0xAA) { + fprintf(stderr, + "\n VERIFY FAIL: %s last byte = 0x%02x, " + "expected 0xAA\n", + test->name, dst[test->size - 1]); + free_buffer(src, test->size); + free_buffer(dst, test->size); + return -1.0; + } + } + + free_buffer(src, test->size); + free_buffer(dst, test->size); + return gbps; +} + +int main(void) +{ + const char *results_dir = getenv("RESULTS_DIR"); + const char *label = getenv("PERF_LABEL"); + const char *cpu_env = getenv("PERF_CPU"); + const char *passes_env = getenv("PERF_PASSES"); + int cpu = cpu_env ? atoi(cpu_env) : DEFAULT_CPU; + int passes = passes_env ? atoi(passes_env) : DEFAULT_PASSES; + struct perf_baseline results[MAX_BASELINES]; + int num_results = 0; + int errors = 0; + + use_hugepages = (getenv("DTO_PERF_HUGE") != NULL); + cold_cache = getenv("PERF_COLD_CACHE") ? atoi(getenv("PERF_COLD_CACHE")) : 1; + + if (!label) + label = "dsa"; + if (passes < 1) + passes = 1; + + /* Real-time priority to minimize scheduler preemption */ + { + struct sched_param sp = { .sched_priority = 1 }; + + if (sched_setscheduler(0, SCHED_FIFO, &sp) < 0) + fprintf(stderr, "WARNING: could not set SCHED_FIFO " + "(run as root for lower variance)\n"); + } + + /* Pin benchmark thread to a single core for stable results */ + if (pin_to_cpu(cpu) < 0) + fprintf(stderr, "WARNING: could not pin to CPU %d, " + "results may have higher variance\n", cpu); + + /* Pin core + uncore frequency if requested (before TSC calibration) */ + { + const char *freq_env = getenv("PERF_FREQ_MHZ"); + + if (freq_env) + pin_freq(cpu, strtoul(freq_env, NULL, 10)); + } + + /* Calibrate TSC after pinning so we measure the pinned core's freq */ + tsc_ghz = calibrate_tsc(); + + printf("DTO Performance Tests (%s cache) [%s]\n", + cold_cache ? "cold" : "warm", label); + printf("==========================================\n"); + printf("Page size: %s\n", + use_hugepages ? "2MB hugepages" : "4KB"); + printf("Pinned to CPU: %d\n", cpu); + printf("TSC frequency: %.3f GHz\n", tsc_ghz); + if (freq_state.active) { + const char *mhz = getenv("PERF_FREQ_MHZ"); + + printf("Core frequency: %s MHz (pinned)\n", mhz); + if (freq_state.uncore_dir[0]) + printf("Uncore frequency: %s MHz (pinned)\n", + mhz); + } + printf("Cache mode: %s\n", + cold_cache ? "cold (flush every iteration)" : + "warm (flush once per size)"); + printf("Passes per benchmark: %d%s\n", passes, + passes > 1 ? " (median-of-medians)" : ""); + printf("DTO_CPU_SIZE_FRACTION: %s\n", + getenv("DTO_CPU_SIZE_FRACTION") ? : "(default)"); + printf("DTO_AUTO_ADJUST_KNOBS: %s\n", + getenv("DTO_AUTO_ADJUST_KNOBS") ? : "(default)"); + printf("DTO_USESTDC_CALLS: %s\n\n", + getenv("DTO_USESTDC_CALLS") ? : "(default)"); + + /* Table header */ + printf(" %-20s %12s %10s", "Test", "med ns/op", "GB/s"); + if (passes > 1) + printf(" %8s", "CV%%"); + printf("\n"); + printf(" %-20s %12s %10s", "----", "-----", "----"); + if (passes > 1) + printf(" %8s", "----"); + printf("\n"); + + const char *pagesize = use_hugepages ? "2m" : "4k"; + + /* Run benchmarks */ + for (int i = 0; benchmarks[i].name != NULL; i++) { + double avg_ns; + double gbps; + double cv_pct = -1.0; + int iters = benchmarks[i].iterations; + + /* Accumulate all raw samples across passes for distribution saving */ + uint64_t *all_samples = NULL; + int total_samples = 0; + + if (results_dir) { + all_samples = malloc((size_t)passes * iters * + sizeof(uint64_t)); + } + + if (passes == 1) { + uint64_t *pass_samples = NULL; + int pass_n = 0; + + gbps = run_benchmark(&benchmarks[i], &avg_ns, + all_samples ? &pass_samples : NULL, + &pass_n); + if (gbps >= 0 && pass_samples) { + memcpy(all_samples, pass_samples, + pass_n * sizeof(uint64_t)); + total_samples = pass_n; + free(pass_samples); + } + } else { + double *pass_medians; + int ok = 1; + + pass_medians = malloc(passes * sizeof(double)); + if (!pass_medians) { + printf(" %-20s %12s\n", + benchmarks[i].name, "ERROR"); + errors++; + free(all_samples); + continue; + } + + for (int p = 0; p < passes; p++) { + double ns; + uint64_t *pass_samples = NULL; + int pass_n = 0; + + gbps = run_benchmark(&benchmarks[i], &ns, + all_samples ? &pass_samples : NULL, + &pass_n); + if (gbps < 0) { + ok = 0; + free(pass_samples); + break; + } + pass_medians[p] = ns; + + if (pass_samples) { + memcpy(all_samples + total_samples, + pass_samples, + pass_n * sizeof(uint64_t)); + total_samples += pass_n; + free(pass_samples); + } + } + if (!ok) { + free(pass_medians); + free(all_samples); + printf(" %-20s %12s\n", + benchmarks[i].name, "ERROR"); + errors++; + continue; + } + + qsort(pass_medians, passes, sizeof(double), + cmp_double); + avg_ns = pass_medians[passes / 2]; + gbps = (double)benchmarks[i].size / avg_ns; + + /* Coefficient of variation of pass medians */ + { + double sum = 0.0, sum_sq = 0.0, mean, var; + + for (int p = 0; p < passes; p++) + sum += pass_medians[p]; + mean = sum / passes; + for (int p = 0; p < passes; p++) { + double d = pass_medians[p] - mean; + + sum_sq += d * d; + } + var = sum_sq / passes; + cv_pct = (sqrt(var) / mean) * 100.0; + } + free(pass_medians); + } + + /* Save sample distribution */ + if (all_samples && total_samples > 0 && gbps >= 0) { + qsort(all_samples, total_samples, sizeof(uint64_t), + cmp_u64); + save_samples(results_dir, label, pagesize, + benchmarks[i].name, all_samples, + total_samples); + } + free(all_samples); + + if (gbps < 0) { + printf(" %-20s %12s\n", benchmarks[i].name, "ERROR"); + errors++; + continue; + } + + /* Store latency as the baseline metric */ + snprintf(results[num_results].name, BASELINE_NAME_LEN, + "%s", benchmarks[i].name); + results[num_results].latency_ns = avg_ns; + num_results++; + + printf(" %-20s %10.1f ns %10.4f", benchmarks[i].name, + avg_ns, gbps); + if (cv_pct >= 0) + printf(" %7.2f%%", cv_pct); + if (benchmarks[i].ref_only) + printf(" (ref)"); + printf("\n"); + } + + /* Write results for summary/ratio comparison */ + if (results_dir) { + char results_path[4096]; + + mkdir(results_dir, 0755); + snprintf(results_path, sizeof(results_path), "%s/%s_%s.dat", + results_dir, label, + use_hugepages ? "2m" : "4k"); + save_baselines(results_path, results, num_results); + printf("\n Results saved to %s\n", results_path); + } + + printf("\n"); + if (errors > 0) { + printf(" %d measurement error(s)\n", errors); + return 1; + } + + printf(" Measurement complete\n"); + return 0; +} diff --git a/tests/test_perf_combined.c b/tests/test_perf_combined.c new file mode 100644 index 0000000..5213fb2 --- /dev/null +++ b/tests/test_perf_combined.c @@ -0,0 +1,1069 @@ +/******************************************************************************* + * Copyright (C) 2023 Intel Corporation + * + * SPDX-License-Identifier: MIT + * + * test_perf_combined.c - Statically-linked A/B performance comparison. + * + * Both the baseline and current versions of libdto are compiled into this + * binary as object files with renamed symbols: + * - Baseline: bl_memcpy, bl_memset, bl_memcmp, bl_memmove + * - Current: cur_memcpy, cur_memset, cur_memcmp, cur_memmove + * + * For each benchmark size, tests all DTO configs (stdc, dsa, dsa_auto) + * with both 4KB and 2MB page sizes, running interleaved A/B comparisons + * in forked children. + * + * Environment variables: + * PERF_MIN_EFFECT - Minimum trimmed mean change %% to FAIL (default: 2) + * PERF_AB_ROUNDS - Number of interleaved A/B rounds (default: 3) + * RESULTS_DIR - Directory to write sample + CSV files + * PERF_CPU - CPU core to pin to (default: 1) + * PERF_COLD_CACHE - 1=flush per iteration (default), 0=flush once + * PERF_FREQ_MHZ - Pin core+uncore frequency + * PERF_OP - Run only this op type: memcpy, memset, or memcmp + * PERF_SIZE - Run only this size (e.g. 4k, 64k, 128k, 1m) + ******************************************************************************/ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include "dto_test_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +/* ---- Extern declarations for statically-linked DTO versions ---- */ + +extern void *bl_memcpy(void *dest, const void *src, size_t n); +extern void *bl_memset(void *s, int c, size_t n); +extern int bl_memcmp(const void *s1, const void *s2, size_t n); + +extern void *cur_memcpy(void *dest, const void *src, size_t n); +extern void *cur_memset(void *s, int c, size_t n); +extern int cur_memcmp(const void *s1, const void *s2, size_t n); + +/* ---- Configuration ---- */ + +#define WARMUP_ITERS 100 +#define DEFAULT_CPU 1 + +static int cold_cache; +static double tsc_ghz; + +/* ---- rdtsc helpers ---- */ + +static inline uint64_t rdtsc_start(void) +{ + __asm__ volatile("lfence"); + return __rdtsc(); +} + +static inline uint64_t rdtsc_end(void) +{ + unsigned int aux; + + return __rdtscp(&aux); +} + +static double calibrate_tsc(void) +{ + struct timespec t0, t1; + uint64_t tsc0, tsc1, elapsed_ns; + + clock_gettime(CLOCK_MONOTONIC, &t0); + tsc0 = rdtsc_end(); + do { + clock_gettime(CLOCK_MONOTONIC, &t1); + elapsed_ns = (t1.tv_sec - t0.tv_sec) * 1000000000ULL + + (t1.tv_nsec - t0.tv_nsec); + } while (elapsed_ns < 50000000ULL); + tsc1 = rdtsc_end(); + return (double)(tsc1 - tsc0) / (double)elapsed_ns; +} + +/* ---- CPU pinning ---- */ + +static int pin_to_cpu(int cpu) +{ + cpu_set_t mask; + + CPU_ZERO(&mask); + CPU_SET(cpu, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask) < 0) { + perror("sched_setaffinity"); + return -1; + } + return 0; +} + +/* ---- Frequency pinning ---- */ + +static struct { + int active; + int cpu; + unsigned long orig_core_min_khz, orig_core_max_khz; + char uncore_dir[256]; + unsigned long orig_uncore_min_khz, orig_uncore_max_khz; +} freq_state; + +static int read_sysfs_ulong(const char *path, unsigned long *val) +{ + FILE *f = fopen(path, "r"); + + if (!f) return -1; + if (fscanf(f, "%lu", val) != 1) { fclose(f); return -1; } + fclose(f); + return 0; +} + +static int write_sysfs_ulong(const char *path, unsigned long val) +{ + FILE *f = fopen(path, "w"); + + if (!f) return -1; + fprintf(f, "%lu", val); + fclose(f); + return 0; +} + +static int set_core_freq(int cpu, unsigned long khz) +{ + char path[256]; + unsigned long abs_min; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/cpuinfo_min_freq", cpu); + if (read_sysfs_ulong(path, &abs_min) < 0) return -1; + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", cpu); + if (write_sysfs_ulong(path, abs_min) < 0) return -1; + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_max_freq", cpu); + if (write_sysfs_ulong(path, khz) < 0) return -1; + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", cpu); + return write_sysfs_ulong(path, khz); +} + +static void restore_freq(void) +{ + char path[512]; + + if (!freq_state.active) return; + set_core_freq(freq_state.cpu, freq_state.orig_core_max_khz); + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", + freq_state.cpu); + write_sysfs_ulong(path, freq_state.orig_core_min_khz); + + if (freq_state.uncore_dir[0]) { + snprintf(path, sizeof(path), "%s/min_freq_khz", + freq_state.uncore_dir); + write_sysfs_ulong(path, freq_state.orig_uncore_min_khz); + snprintf(path, sizeof(path), "%s/max_freq_khz", + freq_state.uncore_dir); + write_sysfs_ulong(path, freq_state.orig_uncore_max_khz); + } +} + +static int pin_freq(int cpu, unsigned long mhz) +{ + unsigned long khz = mhz * 1000; + char path[512]; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", cpu); + if (read_sysfs_ulong(path, &freq_state.orig_core_min_khz) < 0) { + fprintf(stderr, "ERROR: cannot read core freq for CPU %d\n", cpu); + return -1; + } + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_max_freq", cpu); + read_sysfs_ulong(path, &freq_state.orig_core_max_khz); + if (set_core_freq(cpu, khz) < 0) { + fprintf(stderr, "ERROR: could not pin core freq to %lu MHz\n", mhz); + return -1; + } + freq_state.cpu = cpu; + freq_state.active = 1; + atexit(restore_freq); + + /* Pin uncore */ + unsigned long pkg = 0, die = 0; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu); + read_sysfs_ulong(path, &pkg); + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/topology/die_id", cpu); + if (read_sysfs_ulong(path, &die) < 0) die = 0; + + snprintf(freq_state.uncore_dir, sizeof(freq_state.uncore_dir), + "/sys/devices/system/cpu/intel_uncore_frequency/" + "package_%02lu_die_%02lu", pkg, die); + + snprintf(path, sizeof(path), "%s/min_freq_khz", freq_state.uncore_dir); + if (read_sysfs_ulong(path, &freq_state.orig_uncore_min_khz) == 0) { + snprintf(path, sizeof(path), "%s/max_freq_khz", + freq_state.uncore_dir); + read_sysfs_ulong(path, &freq_state.orig_uncore_max_khz); + + snprintf(path, sizeof(path), "%s/min_freq_khz", + freq_state.uncore_dir); + write_sysfs_ulong(path, freq_state.orig_uncore_min_khz); + snprintf(path, sizeof(path), "%s/max_freq_khz", + freq_state.uncore_dir); + if (write_sysfs_ulong(path, khz) < 0) { + fprintf(stderr, "WARNING: could not pin uncore freq\n"); + freq_state.uncore_dir[0] = '\0'; + } else { + snprintf(path, sizeof(path), "%s/min_freq_khz", + freq_state.uncore_dir); + write_sysfs_ulong(path, khz); + } + } else { + fprintf(stderr, "WARNING: uncore sysfs not found\n"); + freq_state.uncore_dir[0] = '\0'; + } + + return 0; +} + +static void print_freq_info(int cpu) +{ + char path[512]; + unsigned long cur_freq, min_freq, max_freq; + + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_cur_freq", cpu); + if (read_sysfs_ulong(path, &cur_freq) == 0) { + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_min_freq", cpu); + read_sysfs_ulong(path, &min_freq); + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_max_freq", cpu); + read_sysfs_ulong(path, &max_freq); + printf("Core freq: %lu MHz (min=%lu max=%lu)%s\n", + cur_freq / 1000, min_freq / 1000, max_freq / 1000, + (min_freq == max_freq) ? " [pinned]" : ""); + } + if (freq_state.uncore_dir[0]) { + snprintf(path, sizeof(path), "%s/min_freq_khz", + freq_state.uncore_dir); + if (read_sysfs_ulong(path, &min_freq) == 0) { + snprintf(path, sizeof(path), "%s/max_freq_khz", + freq_state.uncore_dir); + read_sysfs_ulong(path, &max_freq); + printf("Uncore freq: min=%lu max=%lu MHz%s\n", + min_freq / 1000, max_freq / 1000, + (min_freq == max_freq) ? " [pinned]" : ""); + } + } +} + +/* ---- Cache flush ---- */ + +static inline void clflushopt_addr(volatile void *p) +{ + asm volatile("clflushopt (%0)" :: "r"(p) : "memory"); +} + +static void flush_buffer(void *buf, size_t size) +{ + char *p = buf; + + for (size_t i = 0; i < size; i += 64) + clflushopt_addr(p + i); + _mm_sfence(); +} + +/* ---- Benchmark definitions ---- */ + +enum perf_op { OP_MEMSET, OP_MEMCPY, OP_MEMCMP }; + +struct perf_test { + const char *name; + enum perf_op op; + size_t size; + int ref_only; + int pf_pct; /* page-fault probability: 0=none, 1=1%, etc. */ +}; + +static struct perf_test benchmarks[] = { + {"memcpy_4k", OP_MEMCPY, 4096, 1, 0}, + {"memcpy_8k", OP_MEMCPY, 8192, 1, 0}, + {"memcpy_16k", OP_MEMCPY, 16384, 0, 0}, + {"memcpy_32k", OP_MEMCPY, 32768, 0, 0}, + {"memcpy_64k", OP_MEMCPY, 65536, 0, 0}, + {"memcpy_128k", OP_MEMCPY, 131072, 0, 0}, + {"memcpy_256k", OP_MEMCPY, 262144, 0, 0}, + {"memcpy_512k", OP_MEMCPY, 524288, 0, 0}, + {"memcpy_1m", OP_MEMCPY, 1048576, 0, 0}, + {"memcpy_1m_pf50", OP_MEMCPY, 1048576, 0, 50}, + {"memset_64k", OP_MEMSET, 65536, 0, 0}, + {"memset_128k", OP_MEMSET, 131072, 0, 0}, + {"memset_256k", OP_MEMSET, 262144, 0, 0}, + {"memset_1m", OP_MEMSET, 1048576, 0, 0}, + {"memcmp_64k", OP_MEMCMP, 65536, 0, 0}, + {"memcmp_128k", OP_MEMCMP, 131072, 0, 0}, + {"memcmp_1m", OP_MEMCMP, 1048576, 0, 0}, + {NULL, 0, 0, 0, 0} +}; + +#define DEFAULT_ITERS 10000 + +/* ---- DTO config sets ---- */ + +struct dto_config { + const char *label; + int use_libc; /* 1 = bypass DTO, use raw libc via dlsym */ + struct { const char *key; const char *val; } envs[8]; +}; + +static struct dto_config dto_configs[] = { + {"cpu", 1, {{NULL, NULL}}}, + {"stdc", 0, {{"DTO_USESTDC_CALLS", "1"}, {NULL, NULL}}}, + {"dsa", 0, {{"DTO_CPU_SIZE_FRACTION", "0"}, + {"DTO_AUTO_ADJUST_KNOBS", "0"}, + {"DTO_MIN_BYTES", "4096"}, {NULL, NULL}}}, + {"dsa_auto", 0, {{"DTO_CPU_SIZE_FRACTION", "0"}, + {"DTO_AUTO_ADJUST_KNOBS", "1"}, + {"DTO_MIN_BYTES", "4096"}, {NULL, NULL}}}, +}; +#define NUM_DTO_CONFIGS (sizeof(dto_configs) / sizeof(dto_configs[0])) +#define CPU_CONFIG_IDX 0 + +/* Page size configs */ +struct page_config { + const char *label; + int hugepage; +}; + +static struct page_config page_configs[] = { + {"4k", 0}, + {"2m", 1}, +}; +#define NUM_PAGE_CONFIGS (sizeof(page_configs) / sizeof(page_configs[0])) + +/* ---- Shared memory for child results ---- */ + +struct shared_result { + int ok; + int nsamples; + int requested_iters; /* parent sets this before fork */ +}; + +#define MAX_CHILD_SAMPLES DEFAULT_ITERS +#define SHARED_SLOT_SIZE (sizeof(struct shared_result) + \ + 2 * MAX_CHILD_SAMPLES * sizeof(uint64_t)) + +static uint64_t *get_bl_samples(struct shared_result *slot) +{ + return (uint64_t *)(slot + 1); +} + +static uint64_t *get_cur_samples(struct shared_result *slot) +{ + return (uint64_t *)(slot + 1) + MAX_CHILD_SAMPLES; +} + +/* ---- Buffer allocation ---- */ + +static size_t buf_alloc_size(size_t size, int hugepage) +{ + if (hugepage) + return (size + (2 << 20) - 1) & ~((2 << 20) - 1UL); + return size; +} + +static void *alloc_shared_buffer(size_t size, int hugepage) +{ + int flags = MAP_SHARED | MAP_ANONYMOUS | MAP_POPULATE; + size_t alloc = buf_alloc_size(size, hugepage); + + if (hugepage) + flags |= MAP_HUGETLB | (21 << MAP_HUGE_SHIFT); + + void *p = mmap(NULL, alloc, PROT_READ | PROT_WRITE, flags, -1, 0); + + return (p == MAP_FAILED) ? NULL : p; +} + +static void free_shared_buffer(void *p, size_t size, int hugepage) +{ + munmap(p, buf_alloc_size(size, hugepage)); +} + +/* ---- Sort ---- */ + +static int cmp_u64(const void *a, const void *b) +{ + uint64_t va = *(const uint64_t *)a; + uint64_t vb = *(const uint64_t *)b; + + return (va > vb) - (va < vb); +} + +static volatile int benchmark_sink; + +/* ---- KS test ---- */ + +struct ks_result { + double d; + double p; +}; + +static struct ks_result ks_test(const uint64_t *a, int n1, + const uint64_t *b, int n2) +{ + struct ks_result r = {0.0, 1.0}; + int i = 0, j = 0; + double d_max = 0.0; + + while (i < n1 || j < n2) { + uint64_t val; + double diff; + + if (i < n1 && (j >= n2 || a[i] <= b[j])) + val = a[i]; + else + val = b[j]; + + while (i < n1 && a[i] == val) i++; + while (j < n2 && b[j] == val) j++; + + diff = fabs((double)i / n1 - (double)j / n2); + if (diff > d_max) + d_max = diff; + } + + r.d = d_max; + + double ne = (double)n1 * n2 / (n1 + n2); + double lambda = (sqrt(ne) + 0.12 + 0.11 / sqrt(ne)) * d_max; + double sum = 0.0; + + for (int k = 1; k <= 100; k++) { + double term = exp(-2.0 * k * k * lambda * lambda); + + if (k % 2 == 1) + sum += term; + else + sum -= term; + if (term < 1e-12) + break; + } + r.p = 2.0 * sum; + if (r.p < 0) r.p = 0; + if (r.p > 1) r.p = 1; + + return r; +} + +/* ---- CSV export ---- */ + +static void append_csv(const char *dir, const char *testname, + const char *config, const char *pagesz, + const char *version, + const uint64_t *ticks, int count, double ghz) +{ + char path[4096]; + FILE *f; + int need_header; + + snprintf(path, sizeof(path), "%s/distributions.csv", dir); + f = fopen(path, "r"); + need_header = (f == NULL); + if (f) fclose(f); + + f = fopen(path, "a"); + if (!f) return; + + if (need_header) + fprintf(f, "test,config,pagesize,version,latency_ns\n"); + + for (int i = 0; i < count; i++) + fprintf(f, "%s,%s,%s,%s,%.1f\n", + testname, config, pagesz, version, + (double)ticks[i] / ghz); + fclose(f); +} + +/* ---- Result for one (config, pagesize) cell in the table ---- */ + +struct cell_result { + int valid; + int iters; /* iterations per round (from pilot) */ + int n_total; /* total samples (rounds × iters) */ + int bl_outliers;/* baseline outliers (outside trimmed range) */ + int cur_outliers; + double bl_ns; /* baseline trimmed mean (10th-90th pctl) */ + double cur_ns; /* current trimmed mean */ + double change; /* percent change */ + double ks_d; + double ks_p; + int regression; + int improved; +}; + +/* ---- Function pointer types for cpu/libc mode ---- */ + +typedef void *(*fn_memcpy_t)(void *, const void *, size_t); +typedef void *(*fn_memset_t)(void *, int, size_t); +typedef int (*fn_memcmp_t)(const void *, const void *, size_t); + +/* ---- Child: interleaved A/B benchmark ---- */ + +static void child_run_ab(struct perf_test *test, + struct shared_result *slot, + uint8_t *src, uint8_t *dst, + int use_libc) +{ + uint64_t *bl_s = get_bl_samples(slot); + uint64_t *cur_s = get_cur_samples(slot); + uint64_t start, end; + int iters = slot->requested_iters; + + /* + * Function pointers: for cpu mode, resolve libc directly. + * For DTO modes, use the statically-linked bl_/cur_ symbols. + */ + fn_memcpy_t a_memcpy, b_memcpy; + fn_memset_t a_memset, b_memset; + fn_memcmp_t a_memcmp, b_memcmp; + + if (use_libc) { + void *libc = dlopen("libc.so.6", RTLD_NOW | RTLD_NOLOAD); + + if (!libc) libc = dlopen("libc.so.6", RTLD_NOW); + if (!libc) _exit(1); + a_memcpy = b_memcpy = (fn_memcpy_t)dlsym(libc, "memcpy"); + a_memset = b_memset = (fn_memset_t)dlsym(libc, "memset"); + a_memcmp = b_memcmp = (fn_memcmp_t)dlsym(libc, "memcmp"); + if (!a_memcpy || !a_memset || !a_memcmp) _exit(1); + } else { + a_memcpy = (fn_memcpy_t)bl_memcpy; + b_memcpy = (fn_memcpy_t)cur_memcpy; + a_memset = (fn_memset_t)bl_memset; + b_memset = (fn_memset_t)cur_memset; + a_memcmp = bl_memcmp; + b_memcmp = cur_memcmp; + } + + if (iters < 1) iters = 1; + if (iters > MAX_CHILD_SAMPLES) + iters = MAX_CHILD_SAMPLES; + + for (size_t i = 0; i < test->size; i++) { + src[i] = (uint8_t)(i & 0xFF); + dst[i] = (uint8_t)(i & 0xFF); + } + + /* Quick warmup phase */ + for (int w = 0; w < WARMUP_ITERS; w++) { + switch (test->op) { + case OP_MEMSET: a_memset(dst, 0xAA, test->size); + b_memset(dst, 0xAA, test->size); break; + case OP_MEMCPY: a_memcpy(dst, src, test->size); + b_memcpy(dst, src, test->size); break; + case OP_MEMCMP: benchmark_sink = a_memcmp(dst, src, test->size); + benchmark_sink = b_memcmp(dst, src, test->size); break; + } + } + + /* Page fault injection setup */ + size_t page_size = sysconf(_SC_PAGESIZE); + size_t num_pages = test->size / page_size; + uintptr_t dst_base = (uintptr_t)dst & ~(page_size - 1); + + /* Randomized interleaved measurement */ + uint32_t rng = (uint32_t)rdtsc_end() | 1; /* must be nonzero for xorshift */ + + for (int i = 0; i < iters; i++) { + uint64_t t_bl, t_cur; + + rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; + int bl_first = (rng & 1); + + /* First */ + if (cold_cache) { + flush_buffer(src, test->size); + flush_buffer(dst, test->size); + } + if (test->pf_pct && num_pages > 0) { + rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; + if ((rng % 100) < (uint32_t)test->pf_pct) { + rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; + size_t pg = rng % num_pages; + madvise((void *)(dst_base + pg * page_size), + page_size, MADV_DONTNEED); + } + } + start = rdtsc_start(); + switch (test->op) { + case OP_MEMSET: (bl_first ? a_memset : b_memset)(dst, 0xAA, test->size); break; + case OP_MEMCPY: (bl_first ? a_memcpy : b_memcpy)(dst, src, test->size); break; + case OP_MEMCMP: benchmark_sink = (bl_first ? a_memcmp : b_memcmp)(dst, src, test->size); break; + } + COMPILER_BARRIER(); + end = rdtsc_end(); + if (bl_first) t_bl = end - start; else t_cur = end - start; + + if (test->op == OP_MEMCMP) + for (size_t j = 0; j < test->size; j++) + dst[j] = src[j]; + + /* Second */ + if (cold_cache) { + flush_buffer(src, test->size); + flush_buffer(dst, test->size); + } + if (test->pf_pct && num_pages > 0) { + rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; + if ((rng % 100) < (uint32_t)test->pf_pct) { + rng ^= rng << 13; rng ^= rng >> 17; rng ^= rng << 5; + size_t pg = rng % num_pages; + madvise((void *)(dst_base + pg * page_size), + page_size, MADV_DONTNEED); + } + } + start = rdtsc_start(); + switch (test->op) { + case OP_MEMSET: (!bl_first ? a_memset : b_memset)(dst, 0xAA, test->size); break; + case OP_MEMCPY: (!bl_first ? a_memcpy : b_memcpy)(dst, src, test->size); break; + case OP_MEMCMP: benchmark_sink = (!bl_first ? a_memcmp : b_memcmp)(dst, src, test->size); break; + } + COMPILER_BARRIER(); + end = rdtsc_end(); + if (!bl_first) t_bl = end - start; else t_cur = end - start; + + bl_s[i] = t_bl; + cur_s[i] = t_cur; + + if (test->op == OP_MEMCMP) + for (size_t j = 0; j < test->size; j++) + dst[j] = src[j]; + } + + qsort(bl_s, iters, sizeof(uint64_t), cmp_u64); + qsort(cur_s, iters, sizeof(uint64_t), cmp_u64); + slot->nsamples = iters; + slot->ok = 1; + exit(0); +} + +static int fork_ab(struct perf_test *test, struct shared_result *slot, + uint8_t *src, uint8_t *dst, struct dto_config *cfg, + int iters) +{ + pid_t pid; + int status; + + slot->ok = 0; + slot->requested_iters = iters; + + for (int e = 0; cfg->envs[e].key; e++) + setenv(cfg->envs[e].key, cfg->envs[e].val, 1); + + pid = fork(); + if (pid < 0) { perror("fork"); return -1; } + + if (pid == 0) + child_run_ab(test, slot, src, dst, cfg->use_libc); + + waitpid(pid, &status, 0); + + for (int e = 0; cfg->envs[e].key; e++) + unsetenv(cfg->envs[e].key); + + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + return -1; + return slot->ok ? 0 : -1; +} + +static struct cell_result run_cell(struct perf_test *test, + struct dto_config *cfg, + struct page_config *pg, + void *shm, int ab_rounds, + double min_effect, + const char *results_dir) +{ + struct cell_result res = {0}; + int iters; + int max_total; + uint64_t *all_bl, *all_cur; + int n_bl = 0, n_cur = 0; + + iters = DEFAULT_ITERS; + max_total = ab_rounds * iters; + + uint8_t *src = alloc_shared_buffer(test->size, pg->hugepage); + uint8_t *dst = alloc_shared_buffer(test->size, pg->hugepage); + + if (!src || !dst) { + if (src) free_shared_buffer(src, test->size, pg->hugepage); + if (dst) free_shared_buffer(dst, test->size, pg->hugepage); + return res; + } + + all_bl = malloc(max_total * sizeof(uint64_t)); + all_cur = malloc(max_total * sizeof(uint64_t)); + if (!all_bl || !all_cur) { + free(all_bl); free(all_cur); + free_shared_buffer(src, test->size, pg->hugepage); + free_shared_buffer(dst, test->size, pg->hugepage); + return res; + } + + for (int r = 0; r < ab_rounds; r++) { + struct shared_result *slot = shm; + + memset(slot, 0, SHARED_SLOT_SIZE); + if (fork_ab(test, slot, src, dst, cfg, iters) < 0) + goto out; + + memcpy(all_bl + n_bl, get_bl_samples(slot), + slot->nsamples * sizeof(uint64_t)); + n_bl += slot->nsamples; + memcpy(all_cur + n_cur, get_cur_samples(slot), + slot->nsamples * sizeof(uint64_t)); + n_cur += slot->nsamples; + } + + qsort(all_bl, n_bl, sizeof(uint64_t), cmp_u64); + qsort(all_cur, n_cur, sizeof(uint64_t), cmp_u64); + + /* Trimmed mean 10th-90th */ + { + int blo = n_bl / 10, bhi = n_bl * 9 / 10; + int clo = n_cur / 10, chi = n_cur * 9 / 10; + int bn = bhi - blo, cn = chi - clo; + double bs = 0, cs = 0; + + for (int s = blo; s < bhi; s++) + bs += (double)all_bl[s] / tsc_ghz; + for (int s = clo; s < chi; s++) + cs += (double)all_cur[s] / tsc_ghz; + + res.bl_ns = bs / bn; + res.cur_ns = cs / cn; + res.change = ((res.cur_ns - res.bl_ns) / res.bl_ns) * 100.0; + + struct ks_result ks = ks_test(all_bl + blo, bn, + all_cur + clo, cn); + res.ks_d = ks.d; + res.ks_p = ks.p; + } + + res.regression = (res.change > min_effect); + res.improved = (res.change < -min_effect); + res.iters = iters; + res.n_total = n_bl; + + /* Count outliers using shared IQR threshold */ + { + double q1_bl = (double)all_bl[n_bl / 4] / tsc_ghz; + double q3_bl = (double)all_bl[3 * n_bl / 4] / tsc_ghz; + double q1_cur = (double)all_cur[n_cur / 4] / tsc_ghz; + double q3_cur = (double)all_cur[3 * n_cur / 4] / tsc_ghz; + double q1 = q1_bl < q1_cur ? q1_bl : q1_cur; + double q3 = q3_bl > q3_cur ? q3_bl : q3_cur; + double thresh_ns = q3 + 1.5 * (q3 - q1); + uint64_t thresh_t = (uint64_t)(thresh_ns * tsc_ghz); + + res.bl_outliers = 0; + res.cur_outliers = 0; + for (int s = n_bl - 1; s >= 0 && all_bl[s] > thresh_t; s--) + res.bl_outliers++; + for (int s = n_cur - 1; s >= 0 && all_cur[s] > thresh_t; s--) + res.cur_outliers++; + } + res.valid = 1; + + /* Save CSV */ + if (results_dir) { + append_csv(results_dir, test->name, cfg->label, + pg->label, "baseline", all_bl, n_bl, tsc_ghz); + append_csv(results_dir, test->name, cfg->label, + pg->label, "current", all_cur, n_cur, tsc_ghz); + } + +out: + free(all_bl); + free(all_cur); + free_shared_buffer(src, test->size, pg->hugepage); + free_shared_buffer(dst, test->size, pg->hugepage); + return res; +} + +int main(void) +{ + const char *results_dir = getenv("RESULTS_DIR"); + const char *cpu_env = getenv("PERF_CPU"); + const char *effect_env = getenv("PERF_MIN_EFFECT"); + const char *rounds_env = getenv("PERF_AB_ROUNDS"); + int cpu = cpu_env ? atoi(cpu_env) : DEFAULT_CPU; + double min_effect = effect_env ? atof(effect_env) : 2.0; + int ab_rounds = rounds_env ? atoi(rounds_env) : 3; + int errors = 0, failures = 0; + + cold_cache = getenv("PERF_COLD_CACHE") ? + atoi(getenv("PERF_COLD_CACHE")) : 1; + int filter_op = -1; + size_t filter_size = 0; + { + const char *op_env = getenv("PERF_OP"); + if (op_env) { + if (strcasecmp(op_env, "memcpy") == 0) filter_op = OP_MEMCPY; + else if (strcasecmp(op_env, "memset") == 0) filter_op = OP_MEMSET; + else if (strcasecmp(op_env, "memcmp") == 0) filter_op = OP_MEMCMP; + else fprintf(stderr, "WARNING: unknown PERF_OP=%s, running all\n", op_env); + } + const char *size_env = getenv("PERF_SIZE"); + if (size_env) { + char *end; + filter_size = strtoul(size_env, &end, 10); + if (*end == 'k' || *end == 'K') + filter_size *= 1024; + else if (*end == 'm' || *end == 'M') + filter_size *= 1024 * 1024; + } + } + if (ab_rounds < 1) ab_rounds = 1; + + /* Setup */ + { + struct sched_param sp = { .sched_priority = 1 }; + + if (sched_setscheduler(0, SCHED_FIFO, &sp) < 0) + fprintf(stderr, "WARNING: could not set SCHED_FIFO\n"); + } + if (pin_to_cpu(cpu) < 0) + fprintf(stderr, "WARNING: could not pin to CPU %d\n", cpu); + { + const char *freq_env = getenv("PERF_FREQ_MHZ"); + + if (freq_env) { + if (pin_freq(cpu, strtoul(freq_env, NULL, 10)) < 0) { + fprintf(stderr, + "ERROR: PERF_FREQ_MHZ=%s but could not " + "pin. Run as root.\n", freq_env); + } + } + } + + tsc_ghz = calibrate_tsc(); + + printf("DTO Combined A/B Performance Test (%s cache)\n", + cold_cache ? "cold" : "warm"); + printf("================================================\n"); + printf("Pinned CPU: %d\n", cpu); + printf("TSC freq: %.3f GHz\n", tsc_ghz); + print_freq_info(cpu); + printf("A/B rounds: %d (interleaved, randomized order)\n", ab_rounds); + printf("Min effect: %.1f%%\n", min_effect); + fflush(stdout); + + /* Shared memory for child results */ + void *shm = mmap(NULL, SHARED_SLOT_SIZE, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (shm == MAP_FAILED) { perror("mmap shm"); return 1; } + + if (results_dir) { + char csv_path[4096]; + + mkdir(results_dir, 0755); + snprintf(csv_path, sizeof(csv_path), + "%s/distributions.csv", results_dir); + remove(csv_path); + } + + /* Count benchmarks */ + int num_benchmarks = 0; + + while (benchmarks[num_benchmarks].name) + num_benchmarks++; + + /* Store all results: [benchmark][config][pagesize] */ + struct cell_result (*all_results)[NUM_DTO_CONFIGS][NUM_PAGE_CONFIGS]; + + all_results = calloc(num_benchmarks, + sizeof(*all_results)); + if (!all_results) { + perror("calloc results"); + return 1; + } + + /* + * Run all benchmarks and collect results. + * Then print two detail tables (one per page size) and a summary. + */ + for (int b = 0; b < num_benchmarks; b++) { + struct perf_test *test = &benchmarks[b]; + + if (filter_op >= 0 && test->op != filter_op) + continue; + if (filter_size && test->size != filter_size) + continue; + + for (int c = 0; c < (int)NUM_DTO_CONFIGS; c++) { + for (int p = 0; p < (int)NUM_PAGE_CONFIGS; p++) { + all_results[b][c][p] = run_cell( + test, &dto_configs[c], &page_configs[p], + shm, ab_rounds, min_effect, + results_dir); + + if (!all_results[b][c][p].valid) + errors++; + else if (all_results[b][c][p].regression && + !test->ref_only) + failures++; + + printf(" [done] %-14s %-10s %-4s\n", + test->name, dto_configs[c].label, + page_configs[p].label); + fflush(stdout); + } + } + } + + munmap(shm, SHARED_SLOT_SIZE); + + /* + * ---- Detail tables: one per page size ---- + * + * Each table has a row per transaction size, with sub-rows + * for each DTO config (cpu, stdc, dsa, dsa_auto). + */ + for (int p = 0; p < (int)NUM_PAGE_CONFIGS; p++) { + printf("\n ===============================" + "=======================================\n"); + printf(" A/B Results — %s pages\n", + page_configs[p].hugepage ? "2MB" : "4KB"); + printf(" ===============================" + "=======================================\n"); + printf(" %-13s %-9s %-9s %-9s %-8s %-5s %-7s %s\n", + "Test", "Config", + "Base mean", "Cur mean", + "Change", "KS D", "Result", "Outliers"); + printf(" %-13s %-9s %-9s %-9s %-8s %-5s %-7s %s\n", + "-------------", "---------", + "---------", "---------", + "--------", "-----", "-------", + "--------"); + for (int b = 0; b < num_benchmarks; b++) { + if (filter_op >= 0 && benchmarks[b].op != filter_op) + continue; + if (filter_size && benchmarks[b].size != filter_size) + continue; + for (int c = 0; c < (int)NUM_DTO_CONFIGS; c++) { + struct cell_result *r = + &all_results[b][c][p]; + + if (c == 0) + printf(" %-13s", benchmarks[b].name); + else + printf(" %-13s", ""); + + printf(" %-9s", dto_configs[c].label); + + if (!r->valid) { + printf(" %6s ns %6s ns %8s %-5s ERROR\n", + "", "", "", ""); + } else { + const char *result; + + if (benchmarks[b].ref_only) + result = "REF"; + else if (r->regression) + result = "FAIL ***"; + else if (r->improved) + result = "IMPROVED"; + else + result = "PASS"; + + printf(" %6.0f ns %6.0f ns " + "%+7.1f%% %-5.3f %-7s " + "bl=%d cur=%d\n", + r->bl_ns, r->cur_ns, + r->change, r->ks_d, + result, + r->bl_outliers, + r->cur_outliers); + } + } + } + } + + /* + * ---- Summary: current library speedup vs CPU ---- + */ + printf("\n ===============================" + "=======================================\n"); + printf(" Speedup vs CPU (current library)\n"); + printf(" ===============================" + "=======================================\n"); + + for (int p = 0; p < (int)NUM_PAGE_CONFIGS; p++) { + printf("\n %s pages:\n", + page_configs[p].hugepage ? "2MB" : "4KB"); + printf(" %-14s", "Test"); + for (int c = 1; c < (int)NUM_DTO_CONFIGS; c++) + printf(" %10s", dto_configs[c].label); + printf("\n"); + printf(" %-14s", "--------------"); + for (int c = 1; c < (int)NUM_DTO_CONFIGS; c++) + printf(" %10s", "----------"); + printf("\n"); + + for (int b = 0; b < num_benchmarks; b++) { + if (filter_op >= 0 && benchmarks[b].op != filter_op) + continue; + if (filter_size && benchmarks[b].size != filter_size) + continue; + if (benchmarks[b].ref_only) + continue; + + struct cell_result *cpu_r = + &all_results[b][CPU_CONFIG_IDX][p]; + + if (!cpu_r->valid) + continue; + + printf(" %-14s", benchmarks[b].name); + + for (int c = 1; c < (int)NUM_DTO_CONFIGS; c++) { + struct cell_result *r = + &all_results[b][c][p]; + + if (!r->valid) { + printf(" %10s", "N/A"); + } else { + printf(" %9.2fx", + cpu_r->cur_ns / r->cur_ns); + } + } + printf("\n"); + } + } + + /* Final summary */ + if (failures > 0) + printf(" %d regression(s) detected (> +%.1f%%)\n", + failures, min_effect); + else + printf(" No regressions (threshold: +%.1f%%)\n", min_effect); + if (errors > 0) + printf(" %d error(s)\n", errors); + + free(all_results); + return (failures > 0 || errors > 0) ? 1 : 0; +}