From 07522614a7aa0104d65e716ac1c1b674642aa1df Mon Sep 17 00:00:00 2001 From: Sandwich Date: Sun, 6 Sep 2026 00:14:56 +0200 Subject: [PATCH] feat: vm-native setup, verify and gpu tooling with acpi and bench --- .gitignore | 14 + acpi/battery.aml | Bin 0 -> 185 bytes acpi/battery.dsl | 69 ++ acpi/platform-devices.aml | Bin 0 -> 594 bytes acpi/platform-devices.dsl | 169 +++++ acpi/sensor-probes.aml | Bin 0 -> 216 bytes acpi/sensor-probes.dsl | 79 ++ bench/timerprobe.c | 295 ++++++++ bench/vmbench.c | 491 +++++++++++++ scripts/99-vfio-native-vnet-offload.rules | 8 + scripts/generate-tables.py | 670 +++++++++++++++++ scripts/gpu-passthrough.sh | 391 ++++++++++ scripts/install-modules.sh | 52 ++ scripts/libvirt-hook-cpu-isolation.sh | 83 +++ scripts/restore-stock-kvm.sh | 19 + scripts/setup-vm.sh | 838 ++++++++++++++++++++++ scripts/verify-perf.sh | 109 +++ 17 files changed, 3287 insertions(+) create mode 100644 .gitignore create mode 100644 acpi/battery.aml create mode 100644 acpi/battery.dsl create mode 100644 acpi/platform-devices.aml create mode 100644 acpi/platform-devices.dsl create mode 100644 acpi/sensor-probes.aml create mode 100644 acpi/sensor-probes.dsl create mode 100644 bench/timerprobe.c create mode 100644 bench/vmbench.c create mode 100644 scripts/99-vfio-native-vnet-offload.rules create mode 100644 scripts/generate-tables.py create mode 100755 scripts/gpu-passthrough.sh create mode 100755 scripts/install-modules.sh create mode 100755 scripts/libvirt-hook-cpu-isolation.sh create mode 100755 scripts/restore-stock-kvm.sh create mode 100755 scripts/setup-vm.sh create mode 100755 scripts/verify-perf.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1e462c --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# makepkg output +packaging/*/src/ +packaging/*/pkg/ +packaging/*/*.pkg.tar.* +packaging/*/*.tar.gz +# built artefacts +bench/*.exe +*.ko +*.o +__pycache__/ +packaging/*/bore-*.patch +packaging/*/cachyos-*.tar.gz +packaging/*/*.tar.xz +packaging/*/vfio-native/ diff --git a/acpi/battery.aml b/acpi/battery.aml new file mode 100644 index 0000000000000000000000000000000000000000..5f7310dd7b8babb6f87bec713734c82b8a041a8b GIT binary patch literal 185 zcmWFzb_vVqstanQ_4q09ec?0RR91 literal 0 HcmV?d00001 diff --git a/acpi/battery.dsl b/acpi/battery.dsl new file mode 100644 index 0000000..69fc096 --- /dev/null +++ b/acpi/battery.dsl @@ -0,0 +1,69 @@ +/* + * Intel ACPI Component Architecture + * AML/ASL+ Disassembler version 20251212 (64-bit version) + * Copyright (c) 2000 - 2025 Intel Corporation + * + * Disassembling to symbolic ASL+ operators + * + * Disassembly of battery.aml + * + * Original Table Header: + * Signature "SSDT" + * Length 0x000000B9 (185) + * Revision 0x01 + * Checksum 0x40 + * OEM ID "ALASKA" + * OEM Table ID "A M I " + * OEM Revision 0x00000001 (1) + * Compiler ID "INTL" + * Compiler Version 0x20251212 (539300370) + */ +DefinitionBlock ("", "SSDT", 1, "ALASKA", "A M I ", 0x00000001) +{ + External (_SB_.PCI0, DeviceObj) + + Scope (_SB.PCI0) + { + Device (BAT0) + { + Name (_HID, EisaId ("PNP0C0A") /* Control Method Battery */) // _HID: Hardware ID + Name (_UID, Zero) // _UID: Unique ID + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x1F) + } + + Method (_BIF, 0, NotSerialized) // _BIF: Battery Information + { + Return (Package (0x0D) + { + One, + 0x1770, + 0x1770, + One, + 0x39D0, + 0x0258, + 0x012C, + 0x3C, + 0x3C, + "PABAS0241231", + "41167", + "LION", + "Compal" + }) + } + + Method (_BST, 0, NotSerialized) // _BST: Battery Status + { + Return (Package (0x04) + { + Zero, + Zero, + 0x1770, + 0x39D0 + }) + } + } + } +} + diff --git a/acpi/platform-devices.aml b/acpi/platform-devices.aml new file mode 100644 index 0000000000000000000000000000000000000000..b2ed3181683cb1fafffa4ace67972fa9a016fdc5 GIT binary patch literal 594 zcmZ8d%SyvQ6uq%k<0MusK~NAzK@i=H6&Ef<#+2C5qz;*(f?13ymb&mo)rGn%3VwlK zkc~Tk!=3m+);lWov6*u?XU@#MN6M;7h5)Q59&uUcoW?Yyl+G6xY1mLvxx7dx;{`3Z zb=sQ|d&#z~Yhl&pudeHjX{k8m#7TS8G7>*l$u^GBH@MHMpOJCyi;yw42DTP{R35R+ zqc*C#@1Z0U$gRPX6KQWwgo=GdMMGec!5>?Naq?rY{aVRtx>qawoXw2_yA>Ki2(or literal 0 HcmV?d00001 diff --git a/acpi/platform-devices.dsl b/acpi/platform-devices.dsl new file mode 100644 index 0000000..05d6c2d --- /dev/null +++ b/acpi/platform-devices.dsl @@ -0,0 +1,169 @@ +/* + * Intel ACPI Component Architecture + * AML/ASL+ Disassembler version 20251212 (64-bit version) + * Copyright (c) 2000 - 2025 Intel Corporation + * + * Disassembling to symbolic ASL+ operators + * + * Disassembly of platform-devices.aml + * + * Original Table Header: + * Signature "SSDT" + * Length 0x00000252 (594) + * Revision 0x01 + * Checksum 0xA9 + * OEM ID "ALASKA" + * OEM Table ID "A M I " + * OEM Revision 0x20250321 (539296545) + * Compiler ID "INTL" + * Compiler Version 0x20251212 (539300370) + */ +DefinitionBlock ("", "SSDT", 1, "ALASKA", "A M I ", 0x20250321) +{ + Scope (_SB) + { + Device (PWRB) + { + Name (_HID, EisaId ("PNP0C0C") /* Power Button Device */) // _HID: Hardware ID + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x0B) + } + } + + Device (SLPB) + { + Name (_HID, EisaId ("PNP0C0E") /* Sleep Button Device */) // _HID: Hardware ID + Name (_STA, 0x0B) // _STA: Status + } + + Device (ACAD) + { + Name (_HID, "ACPI0003" /* Power Source Device */) // _HID: Hardware ID + Name (_PCL, Package (0x01) // _PCL: Power Consumer List + { + _SB + }) + Name (ACP, Ones) + Method (_PSR, 0, NotSerialized) // _PSR: Power Source + { + Return (One) + } + + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x0F) + } + } + + Device (PIT0) + { + Name (_HID, "PNP0000" /* 8259-compatible Programmable Interrupt Controller */) // _HID: Hardware ID + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x0F) + } + } + + Device (TIMR) + { + Name (_HID, "PNP0100" /* PC-class System Timer */) // _HID: Hardware ID + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x0F) + } + } + + Device (VLT0) + { + Name (_HID, "PNP0C02" /* PNP Motherboard Resources */) // _HID: Hardware ID + Name (_STR, Unicode ("Voltage Regulator Module")) // _STR: Description String + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x0F) + } + } + + PowerResource (PFAN, 0x00, 0x0000) + { + Method (_STA, 0, NotSerialized) // _STA: Status + { + Return (0x0F) + } + + Method (_ON, 0, NotSerialized) // _ON_: Power On + { + } + + Method (_OFF, 0, NotSerialized) // _OFF: Power Off + { + } + } + + Device (FAN0) + { + Name (_HID, EisaId ("PNP0C0B") /* Fan (Thermal Solution) */) // _HID: Hardware ID + Name (_PR0, Package (0x01) // _PR0: Power Resources for D0 + { + PFAN + }) + Name (FPKG, Package (0x03) + { + One, + One, + 0x04B0 + }) + Method (_FST, 0, Serialized) // _FST: Fan Status + { + Local0 = Timer + Local1 = ((Local0 >> 0x1D) & 0xFF) + Local1 %= 0xC9 + FPKG [0x02] = (0x044C + Local1) + Return (FPKG) /* \_SB_.FAN0.FPKG */ + } + } + + ThermalZone (TZ0) + { + Method (_TMP, 0, Serialized) // _TMP: Temperature + { + Local0 = Timer + Local1 = ((Local0 >> 0x1A) & 0x3F) + Local1 %= 0x3D + Local2 = (0x0C28 + Local1) + Return (Local2) + } + + Method (_AC0, 0, NotSerialized) // _ACx: Active Cooling, x=0-9 + { + Return (0x0CD2) + } + + Method (_PSV, 0, NotSerialized) // _PSV: Passive Temperature + { + Return (0x0DFE) + } + + Method (_HOT, 0, NotSerialized) // _HOT: Hot Temperature + { + Return (0x0E30) + } + + Method (_CRT, 0, NotSerialized) // _CRT: Critical Temperature + { + Return (0x0E62) + } + + Method (_SCP, 1, NotSerialized) // _SCP: Set Cooling Policy + { + } + + Name (_TC1, 0x04) // _TC1: Thermal Constant 1 + Name (_TC2, 0x03) // _TC2: Thermal Constant 2 + Name (_TSP, 0x96) // _TSP: Thermal Sampling Period + Name (_TZP, Zero) // _TZP: Thermal Zone Polling + Name (_STR, Unicode ("System thermal zone")) // _STR: Description String + } + } +} + diff --git a/acpi/sensor-probes.aml b/acpi/sensor-probes.aml new file mode 100644 index 0000000000000000000000000000000000000000..56f15d479e8de193868c682661d2fce83ecd9779 GIT binary patch literal 216 zcmWFzb_uz`z`($Cz|qGs*xS)j!B@dkK|z5*fPul&FT_VkNL6707lUX_e6UkI8-suc z7nl{@>cZw65E><-5Fg?jz)=uwoKRhrz$MMVWuMrZ$Yq;Y#Jz> 0x1B) & 0x3F) + Local1 %= 0x3D + Local2 = (0x0CB4 + Local1) + Return (Local2) + } + + Method (_CRT, 0, Serialized) // _CRT: Critical Temperature + { + Return (0x0E62) + } + + Method (_HOT, 0, Serialized) // _HOT: Hot Temperature + { + Return (0x0E30) + } + + Method (_PSV, 0, Serialized) // _PSV: Passive Temperature + { + Return (0x0DFE) + } + + Name (_TZP, 0x64) // _TZP: Thermal Zone Polling + Name (_TC1, 0x02) // _TC1: Thermal Constant 1 + Name (_TC2, 0x03) // _TC2: Thermal Constant 2 + Name (_TSP, 0x64) // _TSP: Thermal Sampling Period + } + + ThermalZone (VRMT) + { + Method (_TMP, 0, Serialized) // _TMP: Temperature + { + Local0 = Timer + Local1 = ((Local0 >> 0x1C) & 0x3F) + Local1 %= 0x3D + Local2 = (0x0C50 + Local1) + Return (Local2) + } + + Method (_CRT, 0, Serialized) // _CRT: Critical Temperature + { + Return (0x0F1E) + } + + Name (_TZP, 0x012C) // _TZP: Thermal Zone Polling + } + } +} + diff --git a/bench/timerprobe.c b/bench/timerprobe.c new file mode 100644 index 0000000..1d334bf --- /dev/null +++ b/bench/timerprobe.c @@ -0,0 +1,295 @@ +/* + * timerprobe - replicates VMAware's TIMER check so both of its ratios can be + * measured directly, without needing a debug build of VMAware itself. + * + * Build: x86_64-w64-mingw32-gcc -O2 -o timerprobe.exe timerprobe.c + * + * The check uses a software clock: a second thread on another core spins + * incrementing a counter on its own cache line, and the measuring thread reads + * that line either side of the operation. A "tick" is one observed increment, + * so the unit is a cross-core cache-line bounce, and there is no TSC to lie to. + * + * Detector 1 times a CPUID against eight LFENCEs. Detector 2 times a hardware + * TF single-step #DB against ZwRaiseException raising EXCEPTION_SINGLE_STEP in + * software. Both are ratio >= 2.5 -> detected, and they OR together. + * + * Detector 2 is reproduced mechanism for mechanism: a __C_specific_handler + * scope table with a filter funclet (MSVC's __try without the syntax), ZwRaiseException from ntdll with a RtlCaptureContext context, and + * EXCEPTION_EXECUTE_HANDLER on the software side so RtlUnwindEx runs. An + * earlier version used a vectored handler and RaiseException, which never + * enters the kernel on the software side and so measured a different ratio. + * + * Usage: timerprobe.exe [measure_cpu] [counter_cpu] + * with no arguments it sweeps every counter placement for detector 1, + * then runs detector 2 once. + */ + +#include +#include +#include +#include +#include +#include +#include + +#define TRIALS 5 +#define BATCH 800 + +typedef unsigned long long u64; +typedef NTSTATUS (NTAPI *zw_raise_exception_fn)(PEXCEPTION_RECORD, PCONTEXT, BOOLEAN); + +/* the counter thread's line, alone on its own 64-byte line */ +static volatile u64 g_counter __attribute__((aligned(64))); +static volatile LONG g_stop __attribute__((aligned(64))); + +static int counter_cpu; +static zw_raise_exception_fn zw_raise_exception; + +static DWORD WINAPI counter_thread(LPVOID p) +{ + (void)p; + SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)1 << counter_cpu); + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); + SetThreadPriorityBoost(GetCurrentThread(), TRUE); + u64 local = 0; + while (!g_stop) { + local++; + g_counter = local; + } + return 0; +} + +static int cmp_u64(const void *a, const void *b) +{ + u64 x = *(const u64 *)a, y = *(const u64 *)b; + return (x > y) - (x < y); +} + +/* interquartile mean of the middle 50%, as VMAware's calculate_latency does */ +static double iqm(u64 *v, size_t n) +{ + qsort(v, n, sizeof(u64), cmp_u64); + size_t lo = n / 4, hi = n - n / 4; + if (hi <= lo) return (double)v[n / 2]; + double s = 0; + for (size_t i = lo; i < hi; i++) s += (double)v[i]; + return s / (double)(hi - lo); +} + +static volatile int burn_sink; + +static void burn(void) /* stops a hypervisor predicting the sample window */ +{ + int rounds = 64 + (rand() & 0x7FF); + for (int i = 0; i < rounds; i++) burn_sink += i; +} + +static inline u64 sync_edge(void) +{ + u64 s = g_counter; + while (g_counter == s) { } + return g_counter; +} + +/* one batch: time `op` against the cross-core counter */ +static double window(int op, u64 *buf) +{ + for (size_t i = 0; i < BATCH; i++) { + u64 start, end; + int regs[4]; + + start = sync_edge(); + if (op == 0) { + __cpuid(regs, 0); + } else { + _mm_lfence(); _mm_lfence(); _mm_lfence(); _mm_lfence(); + _mm_lfence(); _mm_lfence(); _mm_lfence(); _mm_lfence(); + } + end = g_counter; + buf[i] = end - start; + burn(); + } + return iqm(buf, BATCH); +} + +static double best_of_trials(int op, u64 *buf) +{ + double best = 1e18; + for (int t = 0; t < TRIALS; t++) { + double v = window(op, buf); + if (v < best) best = v; + } + return best; +} + +/* --------------------------------------------------------------------------- + * Detector 2, the VMAware shape. + * + * Hardware side: set TF, execute one NOP, take #DB, the filter clears TF and + * continues - one trap in, one NtContinue out. Software side: capture a + * context and hand it to ZwRaiseException as a first-chance single-step, the + * filter executes the handler - one syscall in, one RtlUnwindEx out. + * + * mingw's __try1 uses one fixed label pair per file, so this is the same + * directive sequence with a per-scope label. Each side is its own noinline + * function. The trailing NOP keeps the trap RIP inside the scope: a TF trap + * reports the address of the instruction after the one that completed. + * ------------------------------------------------------------------------- */ +#define SEH_TRY(filter, id) __asm__ __volatile__( \ + ".Lseh_start_" #id ":\n\t" \ + ".seh_handler __C_specific_handler, @except\n\t" \ + ".seh_handlerdata\n\t.long 1\n\t" \ + ".rva .Lseh_start_" #id ", .Lseh_end_" #id ", " #filter ", .Lseh_end_" #id "\n\t" \ + ".text") +#define SEH_END(id) __asm__ __volatile__("nop\n.Lseh_end_" #id ": nop") + +static volatile long g_db_hits; + +static __attribute__((used)) LONG NTAPI db_filter(EXCEPTION_POINTERS *ep, ULONG64 frame) +{ + (void)frame; + if (ep->ExceptionRecord->ExceptionCode != EXCEPTION_SINGLE_STEP) + return EXCEPTION_CONTINUE_SEARCH; + ep->ContextRecord->EFlags &= ~0x100u; + g_db_hits++; + return EXCEPTION_CONTINUE_EXECUTION; +} + +static __attribute__((used)) LONG NTAPI api_filter(EXCEPTION_POINTERS *ep, ULONG64 frame) +{ + (void)frame; + return ep->ExceptionRecord->ExceptionCode == EXCEPTION_SINGLE_STEP + ? EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH; +} + +static __attribute__((noinline)) void execute_db(void) +{ + SEH_TRY(db_filter, db); + __asm__ __volatile__("pushfq\n\torq $0x100,(%%rsp)\n\tpopfq\n\tnop\n\tnop" + ::: "memory", "cc"); + SEH_END(db); +} + +static __attribute__((noinline)) void nt_raise_exception(EXCEPTION_RECORD *er, CONTEXT *ctx, + volatile int *flag) +{ + SEH_TRY(api_filter, api); + RtlCaptureContext(ctx); + *flag = 1; + zw_raise_exception(er, ctx, TRUE); + SEH_END(api); +} + +static void exc_window(u64 *db, u64 *api) +{ + for (size_t i = 0; i < BATCH; i++) { + u64 pre, post; + + pre = sync_edge(); + execute_db(); + post = g_counter; + db[i] = post - pre; + + volatile int flag = 0; + CONTEXT ctx; + memset(&ctx, 0, sizeof ctx); + ctx.ContextFlags = CONTEXT_FULL; + EXCEPTION_RECORD er; + memset(&er, 0, sizeof er); + er.ExceptionCode = EXCEPTION_SINGLE_STEP; + + pre = sync_edge(); + nt_raise_exception(&er, &ctx, &flag); + post = g_counter; + api[i] = post - pre; + burn(); + } +} + +static void run_exception(int mcpu, int ccpu) +{ + counter_cpu = ccpu; g_stop = 0; g_counter = 0; + HANDLE h = CreateThread(NULL, 0, counter_thread, NULL, 0, NULL); + SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)1 << mcpu); + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); + SetThreadPriorityBoost(GetCurrentThread(), TRUE); + Sleep(80); + + u64 *db = malloc(BATCH * sizeof(u64)), *api = malloc(BATCH * sizeof(u64)); + double best_db = 1e18, best_api = 1e18; + for (int t = 0; t < TRIALS; t++) { + exc_window(db, api); + double v = iqm(db, BATCH); if (v < best_db) best_db = v; + v = iqm(api, BATCH); if (v < best_api) best_api = v; + } + free(db); free(api); + + g_stop = 1; WaitForSingleObject(h, 2000); CloseHandle(h); + + double ratio = best_api > 0 ? best_db / best_api : 0; + printf("exception db=%8.1f api=%8.1f ratio=%7.3f %s (db traps seen: %ld)\n", + best_db, best_api, ratio, ratio >= 2.5 ? "DETECTED" : "pass", g_db_hits); +} + +static void run(int mcpu, int ccpu, int verbose) +{ + counter_cpu = ccpu; + g_stop = 0; + g_counter = 0; + + HANDLE h = CreateThread(NULL, 0, counter_thread, NULL, 0, NULL); + if (!h) { printf("thread failed\n"); return; } + + SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)1 << mcpu); + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); + SetThreadPriorityBoost(GetCurrentThread(), TRUE); + Sleep(80); /* let the counter get going */ + + u64 *buf = malloc(BATCH * sizeof(u64)); + double cpuid_w = best_of_trials(0, buf); + double ref_w = best_of_trials(1, buf); + free(buf); + + g_stop = 1; + WaitForSingleObject(h, 2000); + CloseHandle(h); + + double ratio = ref_w > 0 ? cpuid_w / ref_w : 0; + if (verbose) + printf("measure=%-3d counter=%-3d cpuid=%8.1f lfence=%8.1f ratio=%7.3f %s\n", + mcpu, ccpu, cpuid_w, ref_w, ratio, ratio >= 2.5 ? "DETECTED" : "pass"); + else + printf("counter=%-3d ratio=%7.3f (cpuid %.0f / lfence %.0f) %s\n", + ccpu, ratio, cpuid_w, ref_w, ratio >= 2.5 ? "DETECTED" : "pass"); +} + +int main(int argc, char **argv) +{ + SYSTEM_INFO si; GetSystemInfo(&si); + int n = (int)si.dwNumberOfProcessors; + SetPriorityClass(GetCurrentProcess(), ABOVE_NORMAL_PRIORITY_CLASS); + + zw_raise_exception = (zw_raise_exception_fn)(void *) + GetProcAddress(GetModuleHandleA("ntdll.dll"), "ZwRaiseException"); + if (!zw_raise_exception) { printf("no ZwRaiseException\n"); return 1; } + + printf("# timerprobe cpus=%d threshold=2.5\n", n); + + if (argc >= 3) { + run(atoi(argv[1]), atoi(argv[2]), 1); + run_exception(atoi(argv[1]), atoi(argv[2])); + return 0; + } + + /* + * VMAware picks the counter core itself, biased toward the same L3 as the + * measuring thread. Sweep every placement to show the best and the worst + * case rather than the one it happened to choose. + */ + printf("detector 1, instruction latency - counter placement swept:\n"); + for (int c = 1; c < n; c++) run(0, c, 0); + + printf("\ndetector 2, exception latency:\n"); + run_exception(0, n > 1 ? 1 : 0); + return 0; +} diff --git a/bench/vmbench.c b/bench/vmbench.c new file mode 100644 index 0000000..75f57fc --- /dev/null +++ b/bench/vmbench.c @@ -0,0 +1,491 @@ +/* + * vmbench - CPU, memory, scheduling and IO benchmarks for a Windows guest. + * + * Deliberately has no GPU component: this guest has no display adapter, so every + * number here is a CPU, memory, scheduling or IO proxy for interactive and + * CPU-bound workloads. + * + * Build: x86_64-w64-mingw32-gcc -O2 -o vmbench.exe vmbench.c + * + * Timing is QueryPerformanceCounter throughout. rdtsc is deliberately avoided: + * TSC behaviour is one of the things the fidelity work changes, so timing with it + * would measure the instrument. + */ + +#include +#include +#include +#include +#include + +static double qpc_freq; + +static double now(void) +{ + LARGE_INTEGER c; + QueryPerformanceCounter(&c); + return (double)c.QuadPart / qpc_freq; +} + +static int cmp_double(const void *a, const void *b) +{ + double x = *(const double *)a, y = *(const double *)b; + return (x > y) - (x < y); +} + +static double pct(double *sorted, size_t n, double p) +{ + double idx = p * (double)(n - 1); + size_t lo = (size_t)idx; + if (lo + 1 >= n) return sorted[n - 1]; + double frac = idx - (double)lo; + return sorted[lo] + frac * (sorted[lo + 1] - sorted[lo]); +} + +static void pin(int cpu) +{ + if (cpu >= 0) SetThreadAffinityMask(GetCurrentThread(), (DWORD_PTR)1 << cpu); + SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST); +} + +/* xorshift, used to build permutations without pulling in a real PRNG */ +static uint64_t rng_s = 0x243F6A8885A308D3ull; +static uint64_t rng(void) +{ + rng_s ^= rng_s << 13; rng_s ^= rng_s >> 7; rng_s ^= rng_s << 17; + return rng_s; +} + +/* ---------------------------------------------------------------- cpu ----- */ +/* + * Two numbers, because latency-sensitive workloads need both: + * latency - a serial dependency chain, one op per iteration, nothing to + * overlap. This is what a single-threaded hot path looks like. + * throughput- four independent chains the core can pipeline. + * The volatile sink stops the optimiser deleting the whole loop. + */ +static volatile uint64_t sink_u64; +static volatile double sink_f64; + +static double cpu_int_latency(uint64_t iters) +{ + uint64_t x = 1; + double t0 = now(); + for (uint64_t i = 0; i < iters; i++) { + x = x * 6364136223846793005ull + 1442695040888963407ull; + x ^= x >> 29; + } + double t1 = now(); + sink_u64 = x; + return (double)iters / (t1 - t0) / 1e6; /* Mops/s */ +} + +static double cpu_int_throughput(uint64_t iters) +{ + uint64_t a = 1, b = 2, c = 3, d = 4; + double t0 = now(); + for (uint64_t i = 0; i < iters; i++) { + a = a * 6364136223846793005ull + 1; + b = b * 6364136223846793005ull + 2; + c = c * 6364136223846793005ull + 3; + d = d * 6364136223846793005ull + 4; + } + double t1 = now(); + sink_u64 = a ^ b ^ c ^ d; + return (double)(iters * 4) / (t1 - t0) / 1e6; +} + +static double cpu_fp(uint64_t iters) +{ + double a = 1.0000001, b = 1.0000002, c = 1.0000003, d = 1.0000004; + double t0 = now(); + for (uint64_t i = 0; i < iters; i++) { + a = a * 1.0000001 + 0.0000001; + b = b * 1.0000002 + 0.0000002; + c = c * 1.0000003 + 0.0000003; + d = d * 1.0000004 + 0.0000004; + } + double t1 = now(); + sink_f64 = a + b + c + d; + return (double)(iters * 4) / (t1 - t0) / 1e6; +} + +/* ------------------------------------------------------------ cpu_mt ----- */ +struct mt_arg { int cpu; uint64_t iters; double mops; }; + +static DWORD WINAPI mt_worker(LPVOID p) +{ + struct mt_arg *a = (struct mt_arg *)p; + pin(a->cpu); + a->mops = cpu_int_throughput(a->iters); + return 0; +} + +static void bench_cpu_mt(int nthreads, uint64_t iters) +{ + struct mt_arg *args = calloc(nthreads, sizeof(*args)); + HANDLE *th = calloc(nthreads, sizeof(*th)); + for (int i = 0; i < nthreads; i++) { + args[i].cpu = i; + args[i].iters = iters; + th[i] = CreateThread(NULL, 0, mt_worker, &args[i], 0, NULL); + } + WaitForMultipleObjects(nthreads, th, TRUE, INFINITE); + double total = 0; + for (int i = 0; i < nthreads; i++) { total += args[i].mops; CloseHandle(th[i]); } + printf("cpu_mt threads=%d aggregate_Mops=%.1f per_thread_avg=%.1f\n", + nthreads, total, total / nthreads); + free(args); free(th); +} + +/* ------------------------------------------------------------- memlat ---- */ +/* + * Pointer chase over a random single cycle, one node per 64-byte line so each + * step is exactly one cache miss at the level being measured. A strided walk + * would be prefetched and would measure bandwidth, not latency. + */ +struct node { struct node *next; char pad[56]; }; + +static double memlat(size_t bytes, uint64_t steps) +{ + size_t n = bytes / sizeof(struct node); + if (n < 16) n = 16; + struct node *buf = _aligned_malloc(n * sizeof(struct node), 64); + if (!buf) return -1; + memset(buf, 0, n * sizeof(struct node)); + + size_t *perm = malloc(n * sizeof(size_t)); + for (size_t i = 0; i < n; i++) perm[i] = i; + for (size_t i = n - 1; i > 0; i--) { /* Fisher-Yates */ + size_t j = (size_t)(rng() % (i + 1)); + size_t t = perm[i]; perm[i] = perm[j]; perm[j] = t; + } + for (size_t i = 0; i < n; i++) /* link into one cycle */ + buf[perm[i]].next = &buf[perm[(i + 1) % n]]; + free(perm); + + struct node *p = buf; + for (uint64_t i = 0; i < n * 2; i++) p = p->next; /* warm */ + + double t0 = now(); + for (uint64_t i = 0; i < steps; i++) p = p->next; + double t1 = now(); + sink_u64 = (uint64_t)(uintptr_t)p; + + _aligned_free(buf); + return (t1 - t0) / (double)steps * 1e9; /* ns per access */ +} + +/* ------------------------------------------------------------- membw ----- */ +static double membw_read(size_t bytes) +{ + size_t n = bytes / sizeof(uint64_t); + uint64_t *b = _aligned_malloc(n * sizeof(uint64_t), 64); + if (!b) return -1; + for (size_t i = 0; i < n; i++) b[i] = i; + uint64_t s0 = 0, s1 = 0, s2 = 0, s3 = 0; + double t0 = now(); + for (size_t i = 0; i + 3 < n; i += 4) { + s0 += b[i]; s1 += b[i+1]; s2 += b[i+2]; s3 += b[i+3]; + } + double t1 = now(); + sink_u64 = s0 + s1 + s2 + s3; + _aligned_free(b); + return (double)bytes / (t1 - t0) / 1e9; /* GB/s */ +} + +static double membw_copy(size_t bytes) +{ + size_t half = bytes / 2; + char *a = _aligned_malloc(half, 64), *b = _aligned_malloc(half, 64); + if (!a || !b) return -1; + memset(a, 1, half); memset(b, 2, half); + memcpy(b, a, half); /* warm */ + double t0 = now(); + memcpy(b, a, half); + double t1 = now(); + sink_u64 = (uint64_t)b[0]; + _aligned_free(a); _aligned_free(b); + return (double)(half * 2) / (t1 - t0) / 1e9; +} + +/* --------------------------------------------------------------- c2c ----- */ +/* + * Core-to-core round trip. On a 7950X this is the number that separates a + * same-CCD pair from a cross-CCD pair, which is the whole 16-vs-32 vCPU + * argument, so it gets measured directly rather than assumed. + */ +static volatile LONG c2c_flag; +static int c2c_cpu_b; +static uint64_t c2c_iters; + +static DWORD WINAPI c2c_worker(LPVOID p) +{ + (void)p; + pin(c2c_cpu_b); + for (uint64_t i = 0; i < c2c_iters; i++) { + while (InterlockedCompareExchange(&c2c_flag, 2, 1) != 1) YieldProcessor(); + } + return 0; +} + +static double c2c(int cpu_a, int cpu_b, uint64_t iters) +{ + c2c_cpu_b = cpu_b; c2c_iters = iters; c2c_flag = 0; + HANDLE h = CreateThread(NULL, 0, c2c_worker, NULL, 0, NULL); + pin(cpu_a); + Sleep(50); + double t0 = now(); + for (uint64_t i = 0; i < iters; i++) { + InterlockedExchange(&c2c_flag, 1); + while (InterlockedCompareExchange(&c2c_flag, 0, 2) != 2) YieldProcessor(); + } + double t1 = now(); + WaitForSingleObject(h, 5000); + CloseHandle(h); + return (t1 - t0) / (double)iters * 1e9; /* ns round trip */ +} + +/* ------------------------------------------------------------ jitter ----- */ +/* + * The scheduling-tail proxy, and the most important number in the suite. + * + * Spin reading the clock as fast as possible and record every gap between + * consecutive reads. On an idle, well-behaved system the gaps are tiny and + * uniform. Any gap far above the floor means this thread was NOT RUNNING - + * preempted by the host, stalled behind a VMEXIT, or waiting on the emulator + * thread. Mean throughput averages all of that away; the tail is what shows up + * as hitching in an interactive session, so the output is percentiles, not an + * average. + */ +static void jitter(int cpu, double seconds, size_t max_samples) +{ + pin(cpu); + double *gaps = malloc(max_samples * sizeof(double)); + if (!gaps) { printf("jitter alloc failed\n"); return; } + + double t_end = now() + seconds; + double prev = now(); + size_t n = 0; + while (n < max_samples) { + double t = now(); + double d = (t - prev) * 1e6; /* microseconds */ + prev = t; + gaps[n++] = d; + if (t > t_end) break; + } + + qsort(gaps, n, sizeof(double), cmp_double); + printf("jitter cpu=%d samples=%zu floor_us=%.3f p50=%.3f p99=%.3f p99.9=%.3f p99.99=%.3f max_us=%.1f\n", + cpu, n, gaps[0], pct(gaps, n, 0.50), pct(gaps, n, 0.99), + pct(gaps, n, 0.999), pct(gaps, n, 0.9999), gaps[n - 1]); + + /* count of stalls over thresholds - these are the visible hitches */ + size_t o10 = 0, o100 = 0, o1000 = 0; + for (size_t i = 0; i < n; i++) { + if (gaps[i] > 10.0) o10++; + if (gaps[i] > 100.0) o100++; + if (gaps[i] > 1000.0) o1000++; + } + printf("jitter cpu=%d stalls_over_10us=%zu over_100us=%zu over_1ms=%zu\n", + cpu, o10, o100, o1000); + free(gaps); +} + +/* ------------------------------------------------------------- diskio ---- */ +/* + * Unbuffered file IO, so the guest's page cache is out of the way and what gets + * measured is the virtual disk path: guest driver -> QEMU device model -> + * emulator/IO thread -> host file. 4K random reads are the number that matters + * for scattered small-block access; sequential is the easy case. + * + * FILE_FLAG_NO_BUFFERING requires sector-aligned buffers, offsets and lengths. + */ +#define IO_FILE "C:\\Users\\User\\vmbench_io.tmp" +#define IO_SIZE (512u << 20) /* 512 MiB */ +#define IO_BLK 4096 + +static void diskio(void) +{ + HANDLE h = CreateFileA(IO_FILE, GENERIC_READ | GENERIC_WRITE, + 0, NULL, CREATE_ALWAYS, + FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH, NULL); + if (h == INVALID_HANDLE_VALUE) { printf("diskio open failed %lu\n", GetLastError()); return; } + + void *buf = _aligned_malloc(1u << 20, 4096); + memset(buf, 0xA5, 1u << 20); + + /* lay the file down, and measure sequential write in the same pass */ + DWORD got; + double t0 = now(); + for (size_t off = 0; off < IO_SIZE; off += (1u << 20)) + WriteFile(h, buf, 1u << 20, &got, NULL); + FlushFileBuffers(h); + double t1 = now(); + printf("diskio seq_write_MBs=%.1f\n", (double)IO_SIZE / (t1 - t0) / 1e6); + + /* sequential read */ + LARGE_INTEGER z = {0}; SetFilePointerEx(h, z, NULL, FILE_BEGIN); + t0 = now(); + for (size_t off = 0; off < IO_SIZE; off += (1u << 20)) + ReadFile(h, buf, 1u << 20, &got, NULL); + t1 = now(); + printf("diskio seq_read_MBs=%.1f\n", (double)IO_SIZE / (t1 - t0) / 1e6); + + /* 4K random read - the one that shows device-model latency */ + const int N = 20000; + size_t blocks = IO_SIZE / IO_BLK; + t0 = now(); + for (int i = 0; i < N; i++) { + LARGE_INTEGER p; + p.QuadPart = (LONGLONG)((rng() % blocks) * IO_BLK); + SetFilePointerEx(h, p, NULL, FILE_BEGIN); + ReadFile(h, buf, IO_BLK, &got, NULL); + } + t1 = now(); + double us = (t1 - t0) / N * 1e6; + printf("diskio rand4k_IOPS=%.0f rand4k_lat_us=%.1f\n", N / (t1 - t0), us); + + _aligned_free(buf); + CloseHandle(h); + DeleteFileA(IO_FILE); +} + +/* -------------------------------------------------------------- clock ---- */ +/* + * How expensive is asking the time? Software that polls the clock in a tight + * loop calls QPC thousands of times a second, so if the guest's clock source + * traps to the hypervisor it shows up as a flat tax on everything and as a + * longer scheduling tail. + * + * Bare metal Windows serves QueryPerformanceCounter from the TSC in tens of ns. + * If this reports hundreds of ns or microseconds, Windows has fallen back to a + * clock that costs a VMEXIT per read. + */ +static void bench_clock(void) +{ + pin(0); + const uint64_t N = 2000000; + LARGE_INTEGER c; + volatile uint64_t acc = 0; + + for (uint64_t i = 0; i < 10000; i++) { QueryPerformanceCounter(&c); acc += c.QuadPart; } + + double t0 = now(); + for (uint64_t i = 0; i < N; i++) { QueryPerformanceCounter(&c); acc += (uint64_t)c.QuadPart; } + double t1 = now(); + double qpc_ns = (t1 - t0) / (double)N * 1e9; + + t0 = now(); + for (uint64_t i = 0; i < N; i++) acc += __rdtsc(); + t1 = now(); + double tsc_ns = (t1 - t0) / (double)N * 1e9; + + t0 = now(); + for (uint64_t i = 0; i < N; i++) acc += GetTickCount64(); + t1 = now(); + double gtc_ns = (t1 - t0) / (double)N * 1e9; + + sink_u64 = acc; + printf("clock qpc_ns=%.1f rdtsc_ns=%.1f gettickcount_ns=%.1f qpc_freq=%.0f\n", + qpc_ns, tsc_ns, gtc_ns, qpc_freq); +} + +/* -------------------------------------------------------------- main ----- */ +static void usage(void) +{ + printf("usage: vmbench \n"); +} + +int main(int argc, char **argv) +{ + LARGE_INTEGER f; + QueryPerformanceFrequency(&f); + qpc_freq = (double)f.QuadPart; + + SYSTEM_INFO si; + GetSystemInfo(&si); + printf("# vmbench qpc_freq=%.0f logical_cpus=%lu groups=%u\n", + qpc_freq, (unsigned long)si.dwNumberOfProcessors, + (unsigned)GetActiveProcessorGroupCount()); + + SetPriorityClass(GetCurrentProcess(), HIGH_PRIORITY_CLASS); + + if (argc < 2) { usage(); return 1; } + const char *cmd = argv[1]; + int all = strcmp(cmd, "all") == 0; + + if (all || !strcmp(cmd, "clock")) { + bench_clock(); + } + + if (all || !strcmp(cmd, "diskio")) { + diskio(); + } + + if (all || !strcmp(cmd, "cpu")) { + pin(0); + double best_l = 0, best_t = 0, best_f = 0; + for (int r = 0; r < 5; r++) { + double l = cpu_int_latency(200000000ull); + double t = cpu_int_throughput(100000000ull); + double fp = cpu_fp(100000000ull); + if (l > best_l) best_l = l; + if (t > best_t) best_t = t; + if (fp > best_f) best_f = fp; + } + printf("cpu int_latency_Mops=%.1f int_throughput_Mops=%.1f fp_Mops=%.1f\n", + best_l, best_t, best_f); + } + + if (all || !strcmp(cmd, "cpu_mt")) { + int n = (!all && argc > 2) ? atoi(argv[2]) : (int)si.dwNumberOfProcessors; + bench_cpu_mt(n, 100000000ull); + if (all) { bench_cpu_mt(1, 100000000ull); bench_cpu_mt(8, 100000000ull); } + } + + if (all || !strcmp(cmd, "memlat")) { + pin(0); + size_t sizes[] = { 32u<<10, 512u<<10, 8u<<20, 64u<<20, 256u<<20 }; + const char *names[] = { "L1_32K", "L2_512K", "L3_8M", "DRAM_64M", "DRAM_256M" }; + for (int i = 0; i < 5; i++) { + double best = 1e9; + for (int r = 0; r < 3; r++) { + double v = memlat(sizes[i], 20000000ull); + if (v > 0 && v < best) best = v; + } + printf("memlat %s ns=%.2f\n", names[i], best); + } + } + + if (all || !strcmp(cmd, "membw")) { + pin(0); + double br = 0, bc = 0; + for (int r = 0; r < 3; r++) { + double x = membw_read(256u<<20); if (x > br) br = x; + double y = membw_copy(256u<<20); if (y > bc) bc = y; + } + printf("membw read_GBs=%.2f copy_GBs=%.2f\n", br, bc); + } + + if (all || !strcmp(cmd, "c2c")) { + if (!all && argc > 3) { + printf("c2c %s->%s ns=%.1f\n", argv[2], argv[3], + c2c(atoi(argv[2]), atoi(argv[3]), 200000)); + } else { + int nc = (int)si.dwNumberOfProcessors; + printf("c2c 0->1_ns=%.1f\n", c2c(0, 1, 200000)); + if (nc > 8) printf("c2c 0->8_ns=%.1f\n", c2c(0, 8, 200000)); + if (nc > 16) printf("c2c 0->16_ns=%.1f\n", c2c(0, 16, 200000)); + if (nc > 24) printf("c2c 0->24_ns=%.1f\n", c2c(0, 24, 200000)); + } + } + + if (all || !strcmp(cmd, "jitter")) { + int cpu = (!all && argc > 2) ? atoi(argv[2]) : 0; + double se = (!all && argc > 3) ? atof(argv[3]) : 10.0; + jitter(cpu, se, 20000000); + } + + return 0; +} diff --git a/scripts/99-vfio-native-vnet-offload.rules b/scripts/99-vfio-native-vnet-offload.rules new file mode 100644 index 0000000..0c35712 --- /dev/null +++ b/scripts/99-vfio-native-vnet-offload.rules @@ -0,0 +1,8 @@ +# vfio-native: the emulated e1000e NIC's TX checksum and segmentation offloads +# corrupt packets on the host side of a libvirt tap. SMB tolerates it; SSH fails +# with "Corrupted MAC on input" and any integrity-checked protocol breaks. +# Measured on a Zen 4 host with QEMU 11.1.1. Disabling the offloads on every +# libvirt tap as it appears fixes it; on a host<->guest link the throughput +# cost is not measurable. libvirt's attributes are ignored +# for e1000e, and a libvirt hook must not call virsh, hence udev. +ACTION=="add", SUBSYSTEM=="net", KERNEL=="vnet*", RUN+="/usr/bin/ethtool -K %k tx off gso off gro off tso off" diff --git a/scripts/generate-tables.py b/scripts/generate-tables.py new file mode 100644 index 0000000..aecfb62 --- /dev/null +++ b/scripts/generate-tables.py @@ -0,0 +1,670 @@ +#!/usr/bin/env python3 +"""Generate raw SMBIOS binary tables for QEMU -smbios file= injection. + +Produces per-spec SMBIOS structures (DSP0134 3.6) for types that QEMU's +smbios_entry_add() cannot build via structured CLI args: + - Type 7 (Cache Information) x3 - L1 Data, L2 Unified, L3 Unified + - Type 26 (Voltage Probe) x1 + - Type 27 (Cooling Device) x1 + - Type 28 (Temperature Probe) x1 + - Type 29 (Electrical Current Probe) x1 + +Binary format per structure: + [type:u8][length:u8][handle:u16-LE][fields...][strings: NUL-terminated, double-NUL at end] + +The 'length' byte covers the formatted area only (header + fields, NOT strings). +""" + +import argparse +import os +import struct +import sys +from dataclasses import dataclass + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def pack_strings(*strings: str) -> bytes: + """Encode the unformatted (string) area of an SMBIOS structure. + + Each string is NUL-terminated. The area ends with an extra NUL (double-NUL). + If there are no strings, emit two NULs (spec requirement). + """ + if not strings: + return b"\x00\x00" + return b"".join(s.encode("ascii") + b"\x00" for s in strings) + b"\x00" + + +def encode_cache_size_legacy(size_kb: int) -> int: + """Encode cache size for the legacy 16-bit Maximum/Installed Cache Size field. + + Bits 15: Granularity - 0 = 1 KB, 1 = 64 KB + Bits 14-0: Size in granularity units + + If size_kb < 32768 (fits in 15 bits with 1 KB granularity), use 1 KB. + Otherwise use 64 KB granularity. + """ + if size_kb <= 0x7FFF: + return size_kb # 1 KB granularity, bit 15 = 0 + return 0x8000 | (size_kb // 64) # 64 KB granularity, bit 15 = 1 + + +def encode_cache_size2(size_kb: int) -> int: + """Encode cache size for the 32-bit Maximum/Installed Cache Size 2 field (SMBIOS 3.1+). + + Bits 31: Granularity - 0 = 1 KB, 1 = 64 KB + Bits 30-0: Size in granularity units + """ + if size_kb <= 0x7FFFFFFF: + return size_kb + return 0x80000000 | (size_kb // 64) + + +# --------------------------------------------------------------------------- +# Type 7 - Cache Information (SMBIOS 3.1+, length = 27) +# --------------------------------------------------------------------------- + +# Cache Configuration (u16) bit layout: +# Bits 0-2: Level (0 = L1, 1 = L2, 2 = L3) +# Bit 3: Socketed (0 = not socketed) +# Bit 4: Reserved +# Bits 5-6: Location (0 = Internal) +# Bit 7: Enabled/Disabled (1 = Enabled) +# Bits 8-9: Operational Mode (01 = Write Back) +# Bits 10-15: Reserved + +CACHE_CFG_L1 = 0x0180 # Level=0(L1), Internal, Enabled, Write-Back +CACHE_CFG_L2 = 0x0181 # Level=1(L2), Internal, Enabled, Write-Back +CACHE_CFG_L3 = 0x0182 # Level=2(L3), Internal, Enabled, Write-Back + +# Error Correction Type (u8) +ECC_SINGLE_BIT = 5 +ECC_MULTI_BIT = 6 + +# System Cache Type (u8) +CACHE_TYPE_INSTRUCTION = 3 +CACHE_TYPE_DATA = 4 +CACHE_TYPE_UNIFIED = 5 + +# Associativity (u8) per DSP0134 Table 36: +# 0x01 Other, 0x02 Unknown, 0x03 Direct Mapped, 0x04 2-way, +# 0x05 4-way, 0x06 Fully Associative, 0x07 8-way, 0x08 12-way, +# 0x09 16-way, 0x0A 20-way, 0x0B 24-way, 0x0C 32-way, ... +ASSOC_OTHER = 1 +ASSOC_8WAY = 7 +ASSOC_16WAY = 9 + +# Defaults for AMD Zen 4/5 (consumer Ryzen): +# L1d 8-way, L2 8-way, L3 (V-Cache) 16-way; no ECC on any consumer cache. +ASSOC_L1_DEFAULT = ASSOC_8WAY +ASSOC_L2_DEFAULT = ASSOC_8WAY +ASSOC_L3_DEFAULT = ASSOC_16WAY +ECC_DEFAULT = 0x03 # None per DSP0134 Table 39; consumer Ryzen has no cache ECC + +TYPE7_LENGTH = 27 # SMBIOS 3.1+ with extended size fields + + +@dataclass +class CacheEntry: + handle: int + designation: str + config: int + size_kb: int + ecc: int + cache_type: int + associativity: int + + +def build_type7(entry: CacheEntry) -> bytes: + """Build a Type 7 (Cache Information) SMBIOS binary structure.""" + legacy_size = encode_cache_size_legacy(entry.size_kb) + extended_size = encode_cache_size2(entry.size_kb) + + # SRAM type: 0x0002 = Unknown + sram_supported = 0x0002 + sram_current = 0x0002 + + formatted = struct.pack( + " bytes: + """Build a Type 26 (Voltage Probe) SMBIOS binary structure.""" + formatted = struct.pack( + " bytes: + """Build a Type 27 (Cooling Device) SMBIOS binary structure.""" + formatted = struct.pack( + " bytes: + """Build a Type 28 (Temperature Probe) SMBIOS binary structure.""" + formatted = struct.pack( + " bytes: + """Build a Type 29 (Electrical Current Probe) SMBIOS binary structure.""" + formatted = struct.pack( + " None: + """Generate all SMBIOS binary table files into output_dir. + + assoc_l{1,2,3} and ecc are the SMBIOS Type 7 cache characteristics. + Defaults match AMD Zen 4/5 (consumer): 8-way / 8-way / 16-way, no ECC. + Override per-host (e.g. server-class CPUs with different associativity + or with ECC/parity on L3) via generate-tables.py CLI args. + """ + os.makedirs(output_dir, exist_ok=True) + + caches = [ + CacheEntry( + handle=0x0700, + designation="L1 Data Cache", + config=CACHE_CFG_L1, + size_kb=cache_l1, + ecc=ecc, + cache_type=CACHE_TYPE_DATA, + associativity=assoc_l1, + ), + CacheEntry( + handle=0x0701, + designation="L2 Unified Cache", + config=CACHE_CFG_L2, + size_kb=cache_l2, + ecc=ecc, + cache_type=CACHE_TYPE_UNIFIED, + associativity=assoc_l2, + ), + CacheEntry( + handle=0x0702, + designation="L3 Unified Cache", + config=CACHE_CFG_L3, + size_kb=cache_l3, + ecc=ecc, + cache_type=CACHE_TYPE_UNIFIED, + associativity=assoc_l3, + ), + ] + + files = {} + + for i, entry in enumerate(caches): + name = f"type7-l{i + 1}.bin" + data = build_type7(entry) + path = os.path.join(output_dir, name) + with open(path, "wb") as f: + f.write(data) + files[name] = data + + probes = [ + ("type26.bin", build_type26()), + ("type27.bin", build_type27()), + ("type28.bin", build_type28()), + ("type29.bin", build_type29()), + ] + + for name, data in probes: + path = os.path.join(output_dir, name) + with open(path, "wb") as f: + f.write(data) + files[name] = data + + return files + + +# --------------------------------------------------------------------------- +# Verification +# --------------------------------------------------------------------------- + +EXPECTED_FILES = { + "type7-l1.bin": (7, TYPE7_LENGTH), + "type7-l2.bin": (7, TYPE7_LENGTH), + "type7-l3.bin": (7, TYPE7_LENGTH), + "type26.bin": (26, TYPE26_LENGTH), + "type27.bin": (27, TYPE27_LENGTH), + "type28.bin": (28, TYPE28_LENGTH), + "type29.bin": (29, TYPE29_LENGTH), +} + + +def verify_table(path: str, expected_type: int, expected_length: int) -> list[str]: + """Parse back a generated SMBIOS binary file and validate it. + + Returns a list of error strings (empty = pass). + """ + errors = [] + name = os.path.basename(path) + + with open(path, "rb") as f: + data = f.read() + + if len(data) < 4: + errors.append(f"{name}: file too small ({len(data)} bytes, need >= 4)") + return errors + + stype, slength, _shandle = struct.unpack_from(" max_ref: + errors.append( + f"{name}: socket designation string ref {ref} out of range [1, {max_ref}]" + ) + elif stype in (26, 28, 29): + ref = struct.unpack_from(" max_ref: + errors.append( + f"{name}: description string ref {ref} out of range [1, {max_ref}]" + ) + elif stype == 27: + # Description ref at offset 0Fh (last byte of formatted area) + ref = struct.unpack_from(" max_ref: + errors.append( + f"{name}: description string ref {ref} out of range [1, {max_ref}]" + ) + + # Cross-reference check: Type 27 temp_probe_handle must match Type 28 handle. + # If the relationship is broken, Win32_Fan returns empty (wbenny 2025 research). + if stype == 27 and slength >= TYPE27_LENGTH: + temp_handle = struct.unpack_from("= TYPE7_LENGTH: + config = struct.unpack_from(" 3: + errors.append(f"{name}: cache level {level} out of expected range [1, 3]") + + ecc = struct.unpack_from(" bool: + """Verify all expected SMBIOS binary files in directory. Returns True on success.""" + all_errors = [] + + for filename, (expected_type, expected_length) in EXPECTED_FILES.items(): + path = os.path.join(directory, filename) + if not os.path.exists(path): + all_errors.append(f"{filename}: file not found") + continue + all_errors.extend(verify_table(path, expected_type, expected_length)) + + if all_errors: + print("SMBIOS verification FAILED:", file=sys.stderr) + for err in all_errors: + print(f" - {err}", file=sys.stderr) + return False + + print(f"SMBIOS verification passed: {len(EXPECTED_FILES)} tables OK") + return True + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate or verify raw SMBIOS binary tables for QEMU injection" + ) + parser.add_argument( + "--verify", + metavar="DIR", + help="Verify previously generated tables in DIR (no generation)", + ) + parser.add_argument( + "--output-dir", metavar="DIR", help="Output directory for generated .bin files" + ) + parser.add_argument( + "--cache-l1", + type=int, + default=512, + help="L1 data cache size in KB (default: 512)", + ) + parser.add_argument( + "--cache-l2", + type=int, + default=8192, + help="L2 unified cache size in KB (default: 8192)", + ) + parser.add_argument( + "--cache-l3", + type=int, + default=32768, + help="L3 unified cache size in KB (default: 32768)", + ) + parser.add_argument( + "--assoc-l1", + type=int, + default=ASSOC_L1_DEFAULT, + help=f"L1 associativity (SMBIOS Type 7 byte, default: {ASSOC_L1_DEFAULT} = 8-way)", + ) + parser.add_argument( + "--assoc-l2", + type=int, + default=ASSOC_L2_DEFAULT, + help=f"L2 associativity (SMBIOS Type 7 byte, default: {ASSOC_L2_DEFAULT} = 8-way)", + ) + parser.add_argument( + "--assoc-l3", + type=int, + default=ASSOC_L3_DEFAULT, + help=f"L3 associativity (SMBIOS Type 7 byte, default: {ASSOC_L3_DEFAULT} = 16-way V-Cache)", + ) + parser.add_argument( + "--ecc", + type=int, + default=ECC_DEFAULT, + help=f"Error correction type (SMBIOS Type 7 byte, default: {ECC_DEFAULT} = Unknown; consumer Ryzen has no ECC)", + ) + + args = parser.parse_args() + + if args.verify: + if not verify_all(args.verify): + sys.exit(1) + return + + if not args.output_dir: + parser.error("--output-dir is required when not using --verify") + + generate_all( + args.output_dir, + args.cache_l1, + args.cache_l2, + args.cache_l3, + assoc_l1=args.assoc_l1, + assoc_l2=args.assoc_l2, + assoc_l3=args.assoc_l3, + ecc=args.ecc, + ) + print(f"Generated {len(EXPECTED_FILES)} SMBIOS tables in {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/scripts/gpu-passthrough.sh b/scripts/gpu-passthrough.sh new file mode 100755 index 0000000..7414aa7 --- /dev/null +++ b/scripts/gpu-passthrough.sh @@ -0,0 +1,391 @@ +#!/bin/bash +# Set up GPU passthrough for a native-profile guest, with the smallest change to +# the host that will do the job. A passed-through card gives the guest real +# silicon in place of an emulated display adapter. +# +# vm-native-gpu show GPUs, then ask and do it +# vm-native-gpu --help show GPUs and what to do, change nothing +# vm-native-gpu --single [rom] pass the host's only GPU +# vm-native-gpu --dual [rom] pass a GPU the host is not using +# vm-native-gpu --apply [rom] wire the GPU into the domain XML only +# vm-native-gpu --xml print the block, change nothing +# vm-native-gpu --revert undo the hook and modprobe file +# +# --single, --dual and --apply all write the domain XML: every function in the +# card's IOMMU group as a , an optional vBIOS rom file, the emulated +# display removed and any GL render node pointing at the card dropped. The +# domain is backed up first and a re-run replaces what an earlier run wrote. +# +# Most machines have one GPU, so single-GPU is the normal path. The host gives +# the card up while the guest runs: the display manager is stopped, the console +# and framebuffer are unbound, the GPU driver is unloaded and the card is bound +# to vfio-pci. All of it is reversed when the guest stops. +# +# The host has no display for as long as the guest runs. Have SSH working before +# you try it - that is your way back if a step fails. +# +# If you do have a second GPU the host is not using, --dual is strictly better: +# the card is bound to vfio-pci at boot, nothing is torn down at VM start, and +# the host keeps its display the whole time. +set -uo pipefail + +MODPROBE=/etc/modprobe.d/vfio-native.conf +HOOK=/etc/libvirt/hooks/qemu.d/20-gpu-passthrough.sh + +die() { echo "$*" >&2; exit 1; } +need_root() { [ "$(id -u)" = 0 ] || die "run this as root"; } + +# --- discovery --------------------------------------------------------------- +gpus() { + lspci -Dnn | grep -E "VGA compatible controller|3D controller" | + sed 's/ (rev [0-9a-f]*)//' +} + +iommu_group() { + local d="$1" g + g=$(readlink -f "/sys/bus/pci/devices/$d/iommu_group" 2>/dev/null) + echo "${g##*/}" +} + +# every device sharing the GPU's IOMMU group must go to the guest with it +group_members() { + local d="$1" g + g=$(iommu_group "$d") + [ -n "$g" ] || return + for m in /sys/kernel/iommu_groups/"$g"/devices/*; do + [ -e "$m" ] && basename "$m" + done +} + +driver_of() { + local l + l=$(readlink -f "/sys/bus/pci/devices/$1/driver" 2>/dev/null) + [ -n "$l" ] && echo "${l##*/}" || echo "(none)" +} + +# does this card currently drive a connected display? +drives_display() { + local d="$1" c + for c in /sys/class/drm/card*/device; do + [ -e "$c" ] || continue + if [ "$(basename "$(readlink -f "$c")")" = "$d" ]; then + for s in "$(dirname "$c")"*/status; do + [ -e "$s" ] && grep -qx connected "$s" && return 0 + done + fi + done + return 1 +} + +ids_of() { # vendor:device for vfio-pci binding + local d + for d in $(group_members "$1"); do + local cls; cls=$(cat "/sys/bus/pci/devices/$d/class" 2>/dev/null) + # only real functions of the card: skip bridges (class 0x0604xx) + case "$cls" in 0x0604*) continue;; esac + printf '%s:%s\n' \ + "$(cut -c3- < "/sys/bus/pci/devices/$d/vendor")" \ + "$(cut -c3- < "/sys/bus/pci/devices/$d/device")" + done | sort -u | paste -sd, +} + +show() { + echo "GPUs in this machine:" + echo + while read -r line; do + d=${line%% *} + grp=$(iommu_group "$d"); drv=$(driver_of "$d") + if drives_display "$d"; then use="drives a connected display"; else use="no display attached"; fi + echo " $line" + echo " pci $d iommu group $grp driver $drv" + echo " $use" + echo " group members: $(group_members "$d" | paste -sd' ')" + echo + done < <(gpus) + + local n; n=$(gpus | wc -l) + echo "What to do:" + local n cand="" + n=$(gpus | wc -l) + while read -r line; do + d=${line%% *} + drives_display "$d" || cand="$d" + done < <(gpus) + + if [ "$n" -ge 2 ] && [ -n "$cand" ]; then + echo " You have a spare GPU ($cand, no display attached), which is the" + echo " easy case. Bind it to vfio-pci at boot and the host never touches" + echo " it - nothing to tear down at VM start, host keeps its display:" + echo + echo " sudo vm-native-gpu --dual $cand" + echo + echo " Check first that nothing on the host is using it:" + echo " sudo fuser -v /dev/dri/by-path/pci-$cand-*" + echo " A compositor often holds every DRM device even with no monitor on" + echo " it. Binding at boot fixes that; it needs an initramfs rebuild and" + echo " a reboot." + else + local only; only=$(gpus | head -1 | cut -d' ' -f1) + echo " One GPU, which is the normal case. The host hands it over while" + echo " the guest runs and takes it back afterwards:" + echo + echo " sudo vm-native-gpu --single $only" + echo + echo " The host has no display for as long as the guest runs. Get SSH" + echo " working first - that is your way back if a step fails." + fi + +} + +# --- dual GPU: bind at boot, no hooks --------------------------------------- +dual() { + need_root + local dom="$1" d="$2" rom="${3:-}" ids + [ -e "/sys/bus/pci/devices/$d" ] || die "no such PCI device: $d" + drives_display "$d" && die "$d is driving a connected display - move your monitors off it first" + ids=$(ids_of "$d") + [ -n "$ids" ] || die "could not read device ids for $d" + + cat > "$MODPROBE" </dev/null 2>&1 || die "no such domain: $dom" + + echo "This installs a libvirt hook that, every time the guest starts, will:" + echo " - stop your display manager and switch to multi-user.target" + echo " - unbind the virtual consoles and the EFI framebuffer" + echo " - unload the GPU driver and bind the card to vfio-pci" + echo "and reverse all of it when the guest stops." + echo + echo "The host has NO DISPLAY while the guest runs. If it fails part-way you" + echo "may be left at a black screen and need SSH to recover." + echo + echo "It fires ONLY for domain \"$dom\". Other VMs are untouched." + echo + read -rp "install it for domain $dom, PCI $d? [y/N]: " a + [ "$a" = y ] || { echo aborted; exit 1; } + + sed -e "s|@@GPU@@|$d|g" -e "s|@@DOMAIN@@|$dom|g" > "$HOOK" <<'HOOKEOF' +#!/bin/bash +# Single-GPU passthrough for one domain, installed by vfio-native. +# +# Frees the GPU before the guest starts and gives it back afterwards. The actual +# vfio-pci bind/unbind is left to libvirt, because the is managed='yes' +# - this only has to make the card free for libvirt to take. +# +# Exits 0 on every path. libvirt treats a non-zero prepare hook as fatal, and a +# GPU helper must never be the reason a VM refuses to start. +GPU="@@GPU@@" +DOMAIN="@@DOMAIN@@" +LOG=/var/log/libvirt/gpu-passthrough.log +STATE=/run/vfio-native + +log() { echo "$(date +%T) $*" >> "$LOG"; } + +# Only ever act for the domain this was installed for. +[ "$1" = "$DOMAIN" ] || exit 0 + +mkdir -p "$STATE" + +case "$2" in +prepare) + log "$DOMAIN starting: releasing $GPU" + + # 1. stop whatever is holding the DRM device + dm=$(systemctl list-units --type=service --state=running --no-legend 2>/dev/null | + awk '{print $1}' | grep -xE '(gdm|sddm|lightdm|lxdm|greetd|display-manager)\.service' | head -1) + if [ -n "$dm" ]; then + echo "$dm" > "$STATE/dm" + log "stopping $dm" + systemctl stop "$dm" + # wait for it to actually let go, rather than racing it + for _ in $(seq 1 20); do systemctl is-active --quiet "$dm" || break; sleep 0.5; done + fi + + # 2. release the console framebuffers + : > "$STATE/consoles" + for i in /sys/class/vtconsole/vtcon*; do + [ -e "$i/name" ] || continue + if grep -q "frame buffer" "$i/name" 2>/dev/null; then + echo 0 > "$i/bind" 2>/dev/null && { basename "$i" >> "$STATE/consoles"; log "unbound $(basename "$i")"; } + fi + done + [ -e /sys/bus/platform/drivers/efi-framebuffer/unbind ] && + echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind 2>/dev/null + [ -e /sys/bus/platform/drivers/simple-framebuffer/unbind ] && + for fb in /sys/bus/platform/drivers/simple-framebuffer/simple-framebuffer.*; do + [ -e "$fb" ] && basename "$fb" > /sys/bus/platform/drivers/simple-framebuffer/unbind 2>/dev/null + done + sleep 1 + + # 3. unload the host GPU driver, remembering which one to put back + drv=$(basename "$(readlink -f "/sys/bus/pci/devices/$GPU/driver" 2>/dev/null)" 2>/dev/null) + case "$drv" in + amdgpu|radeon) echo "$drv" > "$STATE/driver" + modprobe -r amdgpu radeon 2>/dev/null ;; + nvidia*) echo nvidia > "$STATE/driver" + modprobe -r nvidia_uvm nvidia_drm nvidia_modeset nvidia 2>/dev/null ;; + nouveau) echo nouveau > "$STATE/driver" + modprobe -r nouveau 2>/dev/null ;; + vfio-pci) log "already on vfio-pci" ;; + *) log "unexpected driver '$drv', leaving it alone" ;; + esac + modprobe vfio-pci 2>/dev/null + log "released; libvirt will bind $GPU to vfio-pci" + ;; + +release) + log "$DOMAIN stopped: reclaiming $GPU" + + drv=$(cat "$STATE/driver" 2>/dev/null) + [ -n "$drv" ] && { modprobe "$drv" 2>/dev/null; log "reloaded $drv"; } + sleep 1 + + while read -r c; do + [ -n "$c" ] && echo 1 > "/sys/class/vtconsole/$c/bind" 2>/dev/null + done < "$STATE/consoles" 2>/dev/null + + dm=$(cat "$STATE/dm" 2>/dev/null) + if [ -n "$dm" ]; then + log "starting $dm" + systemctl start "$dm" + fi + rm -f "$STATE/dm" "$STATE/consoles" "$STATE/driver" + log "host display restored" + ;; +esac +exit 0 +HOOKEOF + chmod +x "$HOOK" + echo "installed $HOOK" + echo + apply "$dom" "$d" "$rom" +} + +# Write the passthrough into the domain: the whole IOMMU group as s +# tagged with a user alias so a re-run replaces exactly these, the vBIOS rom if +# given, the emulated display gone, and no GL render node left on the card. +apply() { + local dom="$1" d="$2" rom="${3:-}" v + v=$(command -v virsh) || die "virsh not found" + "$v" -c qemu:///system dominfo "$dom" >/dev/null 2>&1 || die "no such domain: $dom" + [ "$("$v" -c qemu:///system domstate "$dom")" = "shut off" ] || die "shut $dom down first" + [ -z "$rom" ] || [ -f "$rom" ] || die "rom file not found: $rom" + local backup="${SUDO_USER:+/home/$SUDO_USER}"; backup="${backup:-$HOME}/vfio-native-backup" + mkdir -p "$backup" + # the revert target is the state before this tool first touched the domain + "$v" -c qemu:///system dumpxml --inactive "$dom" > "$backup/$dom.gpu-current.xml" + grep -q 'ua-vfionative-gpu' "$backup/$dom.gpu-current.xml" || cp "$backup/$dom.gpu-current.xml" "$backup/$dom.before-gpu.xml" + local devs="" m + for m in $( { group_members "$d"; ls -d "/sys/bus/pci/devices/${d%.*}".* | xargs -n1 basename; } | sort -u); do + case "$(cat "/sys/bus/pci/devices/$m/class" 2>/dev/null)" in 0x0604*) continue;; esac + IFS=':.' read -r dm bs sl fn <<< "$m" + devs+=" \n \n" + devs+="
\n \n" + [ -n "$rom" ] && [ "$m" = "$d" ] && devs+=" \n" + devs+=" \n \n" + done + DEVS="$devs" GPU="$d" python3 - "$backup/$dom.gpu-current.xml" "$backup/$dom.gpu.xml" <<'PY' +import io, os, re, sys +s = io.open(sys.argv[1], encoding="utf-8").read() +s = re.sub(r"\s*(?:(?!).)*(?:(?!).)*", "", s, flags=re.S) +s = re.sub(r"", "", s, flags=re.S) +s = re.sub(r"\s*" % re.escape(os.environ["GPU"]), "", s) +s = s.replace(" ", os.environ["DEVS"].replace("\\n", "\n") + " ", 1) +io.open(sys.argv[2], "w", encoding="utf-8").write(s) +PY + "$v" -c qemu:///system define "$backup/$dom.gpu.xml" >/dev/null || die "define failed" + echo "$dom: $(grep -c "ua-vfionative-gpu" "$backup/$dom.gpu.xml") hostdev(s) for $d${rom:+ with rom $rom}, emulated display removed." + echo "revert: virsh -c qemu:///system define $backup/$dom.before-gpu.xml" +} + +xml() { + local d="$1" + echo "Add this to the domain, inside . Every device in the GPU's" + echo "IOMMU group has to go together:" + echo + for m in $(group_members "$d"); do + local cls; cls=$(cat "/sys/bus/pci/devices/$m/class" 2>/dev/null) + case "$cls" in 0x0604*) continue;; esac + IFS=':. ' read -r dom bus slot fn <<< "$(echo "$m" | tr ':.' ' ')" + printf " \n" + printf " \n" + printf "
\n" \ + "$dom" "$bus" "$slot" "$fn" + printf " \n \n" + done + echo + echo "Then remove the emulated display so the guest has only the real card:" + echo " " + echo "and drop any node that points its GL rendernode at this card." +} + +# Run bare: the same information, then the questions, then the matching command. +guided() { + show + [ -t 0 ] || return 0 + local n cand="" only d dom mode rom a + n=$(gpus | wc -l) + while read -r line; do d=${line%% *}; drives_display "$d" || cand="$d"; done < <(gpus) + only=$(gpus | head -1 | cut -d' ' -f1) + echo "-----------------------------------------------------------------------" + read -rp "set up passthrough now? (y/n) [n]: " a; [ "$a" = y ] || return 0 + read -rp "domain: " dom; [ -n "$dom" ] || die "need a domain" + if [ "$n" -ge 2 ] && [ -n "$cand" ]; then + read -rp "GPU to pass [$cand]: " d; d=${d:-$cand} + read -rp "mode - dual (bind at boot, host keeps its display) or single (host gives it up while the guest runs) [dual]: " mode; mode=${mode:-dual} + else + read -rp "GPU to pass [$only]: " d; d=${d:-$only} + mode=single + fi + read -rp "vBIOS rom file for the guest, or empty [none]: " rom + [ -z "$rom" ] || [ -f "$rom" ] || die "rom file not found: $rom" + [ "$(id -u)" = 0 ] || die "installing the ${mode} setup needs root: sudo $(basename "$0") --$mode $dom $d${rom:+ $rom}" + "$mode" "$dom" "$d" "$rom" +} + +revert() { + need_root + local did=0 + [ -e "$MODPROBE" ] && { rm -f "$MODPROBE"; echo "removed $MODPROBE"; did=1; } + [ -e "$HOOK" ] && { rm -f "$HOOK"; echo "removed $HOOK"; did=1; } + [ "$did" = 1 ] || echo "nothing installed by this script was found" + [ -e "$MODPROBE" ] || echo "rebuild the initramfs (sudo mkinitcpio -P) and reboot to release the card" +} + +case "${1:-}" in + --dual) [ $# -ge 3 ] || die "usage: --dual [romfile]"; dual "$2" "$3" "${4:-}";; + --single) [ $# -ge 3 ] || die "usage: --single [romfile]"; single "$2" "$3" "${4:-}";; + --apply) [ $# -ge 3 ] || die "usage: --apply [romfile]"; apply "$2" "$3" "${4:-}";; + --xml) [ $# -ge 2 ] || die "need a PCI id"; xml "$2";; + --revert) revert;; + "") guided;; + -h|--help) show;; + *) die "unknown option: $1";; +esac diff --git a/scripts/install-modules.sh b/scripts/install-modules.sh new file mode 100755 index 0000000..e686e78 --- /dev/null +++ b/scripts/install-modules.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Install patched KVM modules for the RUNNING kernel and prove they went live. +# +# install-modules.sh [dir] dir holds the built kvm*.ko, default arch/x86/kvm +# +# Modules land in updates/, which modprobe prefers over kernel/. On Arch the +# vfio-native-kvm package does all of this; this script is the manual path. +# Back out with restore-stock-kvm.sh. +set -euo pipefail +SRC=${1:-arch/x86/kvm} +KVER=$(uname -r) +DEST=/usr/lib/modules/$KVER/updates + +[ "$EUID" -eq 0 ] || { echo "run as root"; exit 1; } +if virsh -c qemu:///system list --state-running --name 2>/dev/null | grep -q .; then + echo "a domain is running; modprobe -r kvm needs the refcount at zero"; exit 1 +fi + +VENDOR=$(lsmod | grep -oE '^kvm_(amd|intel)' | head -1) +[ -n "$VENDOR" ] || { echo "neither kvm_amd nor kvm_intel is loaded"; exit 1; } + +# vermagic must match the running kernel exactly or the module will silently not load +want="$KVER SMP preempt mod_unload" +found=0 +for m in kvm kvm-amd kvm-intel; do + [ -f "$SRC/$m.ko" ] || continue + got=$(modinfo -F vermagic "$SRC/$m.ko" | tr -s ' ' | sed 's/ *$//') + [ "$got" = "$want" ] || { echo "vermagic mismatch on $m: '$got' != '$want'"; exit 1; } + found=1 +done +[ "$found" = 1 ] || { echo "no kvm modules found in $SRC"; exit 1; } + +install -d "$DEST" +for m in kvm kvm-amd kvm-intel; do + [ -f "$SRC/$m.ko" ] || continue + zstd -q -f -19 "$SRC/$m.ko" -o "$DEST/$m.ko.zst" + echo "installed $DEST/$m.ko.zst" +done +depmod -a "$KVER" + +modprobe -r "$VENDOR" kvm +modprobe "$VENDOR" + +# Check BOTH. The #GP fix lands in svm.c (kvm-amd.ko) and the hypercall fixes in +# emulate.c/x86.c (kvm.ko), so verifying one passes on a stale build of the other. +for m in kvm "$VENDOR"; do + built=$(modinfo -F srcversion "$SRC/${m//_/-}.ko") + live=$(cat "/sys/module/$m/srcversion") + echo "$m built=$built live=$live" + [ "$built" = "$live" ] || { echo "FATAL: patched $m is NOT live"; exit 1; } +done +echo "OK - patched modules are live" diff --git a/scripts/libvirt-hook-cpu-isolation.sh b/scripts/libvirt-hook-cpu-isolation.sh new file mode 100755 index 0000000..02a6083 --- /dev/null +++ b/scripts/libvirt-hook-cpu-isolation.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# libvirt qemu hook: keep host processes off the cores a pinned guest is using. +# +# Install as /etc/libvirt/hooks/qemu (or drop into /etc/libvirt/hooks/qemu.d/). +# +# On start it reads the domain's own from the XML libvirt passes on +# stdin, works out which host CPUs the guest occupies, and confines the systemd +# slices to the rest. On stop it hands everything back. +# +# Measured on a 16 vCPU guest pinned to one CCD of a 7950X: stalls over 10 us in +# a 10 second window fell from ~100 to ~52, and CPU throughput rose about 2%. +# +# Deliberately exits 0 on every path. libvirt treats a non-zero hook exit during +# prepare/start as fatal, and a tuning helper must never stop a VM booting. + +DOMAIN="$1" +OPERATION="$2" + +ALL=$(cat /sys/devices/system/cpu/present) # not nproc: that honours this hook's own affinity +NCPU=$(( ${ALL##*-} + 1 )) +SLICES="system.slice user.slice init.scope" + +restore() { + for s in $SLICES; do + systemctl set-property --runtime -- "$s" AllowedCPUs="$ALL" 2>/dev/null + done +} + +case "$OPERATION" in +prepare) + XML=$(cat) # libvirt feeds the domain XML on stdin + # every cpuset the guest pins itself to: vcpupin, emulatorpin, iothreadpin + GUEST=$(printf '%s' "$XML" | grep -oE "cpuset='[0-9,-]+'" | + sed "s/cpuset='//;s/'//" | tr ',' '\n' | sort -u | tr '\n' ',' | sed 's/,$//') + [ -n "$GUEST" ] || exit 0 # not a pinned domain, nothing to do + + HOST=$(python3 - "$GUEST" "$NCPU" <<'PY' 2>/dev/null +import sys +def expand(spec): + out = set() + for part in spec.split(','): + if not part: continue + if '-' in part: + a, b = part.split('-'); out.update(range(int(a), int(b) + 1)) + else: + out.add(int(part)) + return out +guest, n = expand(sys.argv[1]), int(sys.argv[2]) +rest = sorted(set(range(n)) - guest) +if not rest: + sys.exit(1) # guest wants everything; leave the host alone +# collapse back into ranges +out, start, prev = [], rest[0], rest[0] +for c in rest[1:] + [None]: + if c == prev + 1: + prev = c; continue + out.append(str(start) if start == prev else "%d-%d" % (start, prev)) + if c is None: break + start = prev = c +print(",".join(out)) +PY +) + [ -n "$HOST" ] || exit 0 # nothing left for the host, or it failed + + for s in $SLICES; do + systemctl set-property --runtime -- "$s" AllowedCPUs="$HOST" 2>/dev/null + done + logger -t libvirt-cpu-isolation "$DOMAIN starting: host slices confined to $HOST (guest has $GUEST)" + ;; + +release|stopped) + restore + logger -t libvirt-cpu-isolation "$DOMAIN $OPERATION: host slices restored to $ALL" + ;; +*) + # Deliberately does nothing. An earlier version called virsh here to check + # whether any domain was still running - that deadlocks libvirtd, because + # libvirt is blocked waiting for this hook to return while the hook waits on + # libvirt. Never call virsh from a libvirt hook. + ;; +esac + +exit 0 diff --git a/scripts/restore-stock-kvm.sh b/scripts/restore-stock-kvm.sh new file mode 100755 index 0000000..131e535 --- /dev/null +++ b/scripts/restore-stock-kvm.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Roll back to stock KVM: remove the patched modules and load the in-tree ones. +# The counterpart to install-modules.sh. Safe to run at any time. +# +# If the host hard-locked and you force-rebooted, run this before starting any VM. +set -euo pipefail +K=$(uname -r) +# the DKMS package owns updates/dkms/; remove it through pacman so dkms cleans up every kernel +if pacman -Q vfio-native-kvm-dkms >/dev/null 2>&1; then + pacman -R --noconfirm vfio-native-kvm-dkms +fi +rm -f "/usr/lib/modules/$K/updates/kvm.ko.zst" \ + "/usr/lib/modules/$K/updates/kvm-amd.ko.zst" \ + "/usr/lib/modules/$K/updates/kvm-intel.ko.zst" +depmod -a "$K" +modprobe -r kvm_amd 2>/dev/null || true +modprobe -r kvm 2>/dev/null || true +modprobe kvm_amd +echo "stock KVM restored: $(cat /sys/module/kvm/srcversion)" diff --git a/scripts/setup-vm.sh b/scripts/setup-vm.sh new file mode 100755 index 0000000..f2d97d0 --- /dev/null +++ b/scripts/setup-vm.sh @@ -0,0 +1,838 @@ +#!/bin/bash +# Configure an existing libvirt domain for platform fidelity and low latency. +# +# @@SELFNAME@@ interactive +# @@SELFNAME@@ -d win11 -p full -c 8 -m 16 scripted +# +# -d domain (default win11) +# -p tuned | native | full (default: full) +# -c guest cores (SMT doubles this into vCPUs) +# -m guest RAM in GiB +# -s on | off Secure Boot with enrolled keys (default on) +# -u none | auto | USB passthrough +# -r randomize the hardware identity (serials, MAC, memory modules) +# -y no prompts +# +# Takes a plain libvirt domain to the tuned, corrected state in one pass: +# detects the host CPU layout and maps vCPUs onto real SMT pairs within one +# cache domain, keeps the emulator off the vCPU cores, moves the disk to +# emulated NVMe, replaces the virtio device set, wires Secure Boot with a +# generated key store, SMBIOS and ACPI identity, and the patched QEMU for the +# full level. Backs the domain up first; idempotent. +set -euo pipefail + +# Run bare it is an interview: every choice is asked, with a default. Every flag +# answers one question in advance, and -y takes every default. +DOM=win11; PROFILE=""; CORES=""; RAMG=""; ASSUME=0; SECBOOT=""; USBSPEC=""; RANDOMIZE="" +while getopts "d:p:c:m:s:u:ryh" o; do case $o in + d) DOM=$OPTARG;; p) PROFILE=$OPTARG;; c) CORES=$OPTARG;; + m) RAMG=$OPTARG;; s) SECBOOT=$OPTARG;; u) USBSPEC=$OPTARG;; r) RANDOMIZE=1;; y) ASSUME=1;; + # print the header comment block, however long it grows + h) sed -n '2,/^[^#]/p' "$0" | sed "\$d; s/^# \?//; s|@@SELFNAME@@|$(basename "$0")|g"; exit 0;; +esac; done + +C=(virsh -c qemu:///system) +BACKUP="$HOME/vfio-native-backup" +# Locate the project data (patches, benchmark sources), whether running from a +# checkout or installed as a package. +for d in "$(cd "$(dirname "$0")/.." 2>/dev/null && pwd)" /usr/share/vfio-native; do + if [ -d "$d/bench" ] || [ -d "$d/patches" ]; then SELF="$d"; break; fi +done +SELF="${SELF:-$(cd "$(dirname "$0")/.." && pwd)}" + +command -v virsh >/dev/null || { echo "virsh not found"; exit 1; } +"${C[@]}" dominfo "$DOM" >/dev/null 2>&1 || { echo "no such domain: $DOM"; exit 1; } +[ "$("${C[@]}" domstate "$DOM")" = "shut off" ] || { echo "shut $DOM down first"; exit 1; } +case "$SECBOOT" in on|off|"") ;; *) echo "-s takes on or off"; exit 1;; esac +ask() { # question default -> answer (default when -y or empty input) + local a; [ "$ASSUME" = 1 ] && { echo "$2"; return; } + read -rp "$1 [$2]: " a; echo "${a:-$2}" +} +DATA="${XDG_DATA_HOME:-$HOME/.local/share}/vfio-native/$DOM" + +# ---------------------------------------------------------------- host CPU --- +VENDOR=$(awk -F': ' '/vendor_id/{print $2; exit}' /proc/cpuinfo) +CPUNAME=$(awk -F': ' '/model name/{print $2; exit}' /proc/cpuinfo) +FAMILY=$(awk -F': ' '/^cpu family/{print $2; exit}' /proc/cpuinfo) +MODELNO=$(awk -F': ' '/^model\t/{print $2; exit}' /proc/cpuinfo) +# not nproc: it honours the shell's own affinity, which the isolation hook narrows while a guest runs +HOST_THREADS=$(ls -d /sys/devices/system/cpu/cpu[0-9]* | wc -l) + +# Physical cores grouped by last-level cache: "l3id core l3sizeK". On Zen 3 and +# later each group is one CCD of 8 cores; on Zen 1/2 it is one CCX of 4, so +# there are two groups per CCD; on a monolithic chip there is a single group. +primaries() { + for d in /sys/devices/system/cpu/cpu[0-9]*; do + c=${d##*/cpu} + [ "$c" = "$(cut -d, -f1 < "$d/topology/thread_siblings_list")" ] || continue + l3=$(cat "$d/cache/index3/id" 2>/dev/null) || l3=0 + sz=$(cat "$d/cache/index3/size" 2>/dev/null) || sz=0K + echo "$l3 $c ${sz%K}" + done | sort -n -k1,1 -k2,2 +} +# The guest's domain is the one with the most L3. On a 3D V-Cache part one CCD +# has three times the other's, and that is the one a latency-sensitive guest +# wants; everywhere else the sizes tie and the lowest id wins. +BEST=$(primaries | sort -k3,3nr -k1,1n | head -1 | awk '{print $1}') +BEST_L3K=$(primaries | awk -v g="$BEST" '$1==g{print $3; exit}') +node_of() { local n; n=$(ls -d "/sys/devices/system/cpu/cpu$1"/node* 2>/dev/null | head -1); echo "${n##*/node}"; } +# Intel 12th gen and later mix P-cores and E-cores. A latency-sensitive guest +# thread landing on an E-core shows up as hitching, so the guest gets P-cores +# and the emulator gets the E-cores, which is exactly what they are good for. +HYBRID=0 +if [ -d /sys/devices/cpu_core ] && [ -d /sys/devices/cpu_atom ]; then + HYBRID=1 + expand() { tr ',' '\n' < "$1" | while read -r r; do + case $r in *-*) seq "${r%-*}" "${r#*-}";; *) echo "$r";; esac; done; } + mapfile -t PCPUS < <(expand /sys/devices/cpu_core/cpus) + mapfile -t ECPUS < <(expand /sys/devices/cpu_atom/cpus) + mapfile -t GRP0 < <(for c in "${PCPUS[@]}"; do + [ "$c" = "$(cut -d, -f1 < "/sys/devices/system/cpu/cpu$c/topology/thread_siblings_list")" ] && echo "$c"; done) + REST=("${ECPUS[@]}") +else + mapfile -t GRP0 < <(primaries | awk -v g="$BEST" '$1==g{print $2}') + mapfile -t REST < <(primaries | awk -v g="$BEST" '$1!=g{print $2}') +fi +NGROUPS=$(primaries | awk '{print $1}' | sort -u | wc -l) +NNODES=$(ls -d /sys/devices/system/node/node[0-9]* 2>/dev/null | wc -l) +GNODE=$(node_of "${GRP0[0]}") +SMT=$([ "$HOST_THREADS" -gt "$(primaries | wc -l)" ] && echo 2 || echo 1) +# SMT sibling of a core, empty when SMT is off; and the whole pair as a cpuset +sib() { cut -d, -f2 -s < "/sys/devices/system/cpu/cpu$1/topology/thread_siblings_list"; } +pair() { cat "/sys/devices/system/cpu/cpu$1/topology/thread_siblings_list"; } + +echo "host: $CPUNAME" +if [ "$HYBRID" = 1 ]; then + echo "layout: Intel hybrid - P-cores ${GRP0[*]}, E-cores ${REST[*]}" + echo " guest gets P-cores; emulator and IO go on E-cores" +else + echo "layout: $(primaries | wc -l) cores / $HOST_THREADS threads, $NGROUPS cache domain(s), SMT $([ $SMT = 2 ] && echo on || echo off)" + echo " guest domain: cores ${GRP0[*]} (L3 $(( BEST_L3K / 1024 )) MiB)" + [ "${#REST[@]}" -gt 0 ] && echo " remaining: cores ${REST[*]}" + if [ "$(primaries | awk '{print $3}' | sort -u | wc -l)" -gt 1 ]; then + echo " L3 is asymmetric - picked the larger (3D V-Cache) domain for the guest" + fi + [ "$NNODES" -gt 1 ] && echo " $NNODES NUMA nodes - guest memory will be pinned to node $GNODE" +fi +HOST_RAM=$(( $(awk '/MemTotal/{print $2}' /proc/meminfo) / 1024 / 1024 )) +echo "ram: ${HOST_RAM} GiB total" +echo + +# ---------------------------------------------------------------- profile --- +if [ -z "$PROFILE" ]; then + cat <<'EOF' +Fidelity levels. All three get the same performance tuning - the platform +corrections cost nothing measurable, so the level only changes how closely the +guest matches real hardware. + + tuned Performance tuning only. Hypervisor visible, Hyper-V enlightenments + on. + + native Tuning plus the domain-level platform corrections: hypervisor CPUID + bit cleared, KVM signature off, CPU feature and firmware identity + corrected. Needs no patched binaries, so it survives any host update. + + full Tuning plus corrections plus the patched QEMU and patched KVM + modules. The lowest score. Costs a module rebuild after every kernel + upgrade. + +EOF + read -rp "level [full]: " PROFILE; PROFILE=${PROFILE:-full} +fi +case "$PROFILE" in + tuned|native|full) ;; + performance) PROFILE=tuned;; + *) echo "unknown level: $PROFILE"; exit 1;; +esac + +# ---------------------------------------------------------------- resources --- +MAXC=${#GRP0[@]} +# Keep two cores back for the emulator and IO threads. They can come from another +# cache domain if there is one, otherwise they come out of the guest's share. +if [ "${#REST[@]}" -ge 2 ]; then RESERVE=0; else RESERVE=2; fi +SUGGEST=$(( MAXC - RESERVE )) + +if [ -z "$CORES" ]; then + echo + TOTAL=$(primaries | wc -l) + echo "Guest cores." + echo " $SUGGEST keeps the guest inside one cache domain - lowest memory" + echo " latency, best for latency-sensitive workloads." + echo " up to $(( TOTAL - 4 )) is fine for CPU-heavy work, at the cost of" + echo " higher L3 latency. Leave the host at least 4 cores either way." + read -rp "cores [$SUGGEST]: " CORES; CORES=${CORES:-$SUGGEST} +fi +[ "$CORES" -ge 1 ] 2>/dev/null || { echo "cores must be a number"; exit 1; } +VCPUS=$(( CORES * SMT )) + +# Windows calibrates the TSC at boot, and that calibration is a race: if the +# host cannot schedule the vCPU threads cleanly through it, Windows gives up on +# the TSC and QueryPerformanceCounter costs ~1300 ns instead of ~15 for the life +# of that boot. Measured pass rate over 4 cold boots each, 16 physical cores: +# host keeps 4+ cores -> 4/4 +# host keeps 2 cores -> 3/4 +# host keeps 0 cores -> 2/4 +# So the rule is headroom, not a vCPU ceiling. +HOSTCORES=$(( $(primaries | wc -l) - CORES )) +if [ "$HOSTCORES" -lt 4 ]; then + echo + echo "WARNING: this leaves the host only $HOSTCORES physical core(s)." + echo "Windows calibrates the TSC at boot and that calibration needs the host" + echo "able to schedule cleanly. With this little headroom it fails on some" + echo "boots, and when it does, every timing call in the guest costs ~1300 ns" + echo "instead of ~15 for the rest of that boot. Measured: 2 of 4 boots failed" + echo "with no headroom at all." + echo + echo "Leave 4 physical cores free and it passed 4 of 4. Check with" + echo "vm-native-verify after booting - if QPC reads over 1000 ns, reboot." + [ "$ASSUME" = 1 ] || { read -rp "continue anyway? [y/N]: " a; [ "$a" = y ] || exit 1; } +fi +if [ "$CORES" -gt "$(( MAXC - RESERVE ))" ]; then + echo "NOTE: $CORES cores spans more than one cache domain. Expect L3 latency" + echo " around 17 ns instead of 10. Worth it for throughput work, not for" + echo " latency-sensitive workloads." +fi + +if [ -z "$RAMG" ]; then + DEF=$(( HOST_RAM / 2 )); [ "$DEF" -gt 32 ] && DEF=32 + read -rp "guest RAM in GiB [$DEF]: " RAMG; RAMG=${RAMG:-$DEF} +fi +[ "$RAMG" -ge 2 ] 2>/dev/null || { echo "ram must be a number >= 2"; exit 1; } + +# ------------------------------------------------------- guest CPU identity --- +# The declared part must really have this many threads, and it must be from the +# same generation as the host, or the brand string contradicts the family, model +# and cache leaves the guest reads straight from the silicon. Every name here is +# a real desktop SKU whose thread count VMAware's own database agrees with; a +# host generation with no matching SKU keeps its own name and says so. +sku() { # vendor family model threads vcache -> brand string, or empty + local gen="" n=$4 + case "$1:$2:$3" in + AuthenticAMD:23:1|AuthenticAMD:23:17) gen=zen1;; + AuthenticAMD:23:8|AuthenticAMD:23:24) gen=zenp;; + AuthenticAMD:23:113) gen=zen2;; + AuthenticAMD:25:33) gen=zen3;; + AuthenticAMD:25:97) gen=zen4;; + AuthenticAMD:26:68) gen=zen5;; + GenuineIntel:6:165) gen=cml;; + GenuineIntel:6:167) gen=rkl;; + GenuineIntel:6:151|GenuineIntel:6:154) gen=adl;; + GenuineIntel:6:183|GenuineIntel:6:191) gen=rpl;; + esac + case "$gen:$n:$5" in + zen1:12:*) echo "AMD Ryzen 5 1600X Six-Core Processor";; + zen1:16:*) echo "AMD Ryzen 7 1800X Eight-Core Processor";; + zenp:12:*) echo "AMD Ryzen 5 2600X Six-Core Processor";; + zenp:16:*) echo "AMD Ryzen 7 2700X Eight-Core Processor";; + zen2:12:*) echo "AMD Ryzen 5 3600X 6-Core Processor";; + zen2:16:*) echo "AMD Ryzen 7 3700X 8-Core Processor";; + zen2:24:*) echo "AMD Ryzen 9 3900X 12-Core Processor";; + zen2:32:*) echo "AMD Ryzen 9 3950X 16-Core Processor";; + zen3:12:*) echo "AMD Ryzen 5 5600X 6-Core Processor";; + zen3:16:1) echo "AMD Ryzen 7 5800X3D 8-Core Processor";; + zen3:16:*) echo "AMD Ryzen 7 5800X 8-Core Processor";; + zen3:24:*) echo "AMD Ryzen 9 5900X 12-Core Processor";; + zen3:32:*) echo "AMD Ryzen 9 5950X 16-Core Processor";; + zen4:12:*) echo "AMD Ryzen 5 7600X 6-Core Processor";; + zen4:16:1) echo "AMD Ryzen 7 7800X3D 8-Core Processor";; + zen4:16:*) echo "AMD Ryzen 7 7700X 8-Core Processor";; + zen4:24:*) echo "AMD Ryzen 9 7900X 12-Core Processor";; + zen4:32:1) echo "AMD Ryzen 9 7950X3D 16-Core Processor";; + zen4:32:*) echo "AMD Ryzen 9 7950X 16-Core Processor";; + zen5:12:*) echo "AMD Ryzen 5 9600X 6-Core Processor";; + zen5:16:1) echo "AMD Ryzen 7 9800X3D 8-Core Processor";; + zen5:16:*) echo "AMD Ryzen 7 9700X 8-Core Processor";; + zen5:24:*) echo "AMD Ryzen 9 9900X 12-Core Processor";; + zen5:32:1) echo "AMD Ryzen 9 9950X3D 16-Core Processor";; + zen5:32:*) echo "AMD Ryzen 9 9950X 16-Core Processor";; + cml:12:*) echo "Intel(R) Core(TM) i5-10400 CPU @ 2.90GHz";; + cml:16:*) echo "Intel(R) Core(TM) i7-10700K CPU @ 3.80GHz";; + cml:20:*) echo "Intel(R) Core(TM) i9-10900K CPU @ 3.70GHz";; + rkl:12:*) echo "Intel(R) Core(TM) i5-11400 @ 2.60GHz";; + rkl:16:*) echo "Intel(R) Core(TM) i7-11700K @ 3.60GHz";; + adl:8:*) echo "12th Gen Intel(R) Core(TM) i3-12100";; + adl:12:*) echo "12th Gen Intel(R) Core(TM) i5-12400";; + rpl:8:*) echo "13th Gen Intel(R) Core(TM) i3-13100";; + esac +} +# A guest on a 3D V-Cache domain reads that L3 size straight from CPUID, so it +# has to claim the X3D part to stay consistent with it. +VCACHE=$([ "$BEST_L3K" -ge 65536 ] && echo 1 || echo 0) +MODEL=$(sku "$VENDOR" "$FAMILY" "$MODELNO" "$VCPUS" "$VCACHE") +if [ "$PROFILE" != tuned ] && [ -z "$MODEL" ]; then + echo + echo "NOTE: no desktop part of this CPU's generation (family $FAMILY, model $MODELNO)" + echo " has exactly $VCPUS threads, so the guest will keep the host's CPU name and" + echo " the thread count will not match it. Pick a core count that maps to a" + echo " real SKU to avoid that, or accept the mismatch." +fi + +# ---------------------------------------------------------- host identity --- +# Every value the guest can read comes from one per-domain file, generated once +# from the host's own strings with fresh serials, so two people running this +# tool do not share a fingerprint. -r regenerates it. Manufacturer and product +# strings stay real; only serials, the MAC and the memory module change. +dmi() { cat "/sys/class/dmi/id/$1" 2>/dev/null || echo ""; } +rnd() { tr -dc "$2" < /dev/urandom | head -c "$1"; } +gen_identity() { + local bv; bv=$(dmi board_vendor) + local mem_man="Kingston" mem_part="KF556C40BB-16" mem_speed=5600 + # the host's real module strings need dmidecode as root; take them when sudo is cached + if command -v dmidecode >/dev/null && sudo -n true 2>/dev/null; then + mem_man=$(sudo -n dmidecode -t 17 2>/dev/null | awk -F': ' '/Manufacturer:/{print $2; exit}') + mem_part=$(sudo -n dmidecode -t 17 2>/dev/null | awk -F': ' '/Part Number:/{gsub(/ +$/,"",$2); print $2; exit}') + mem_speed=$(sudo -n dmidecode -t 17 2>/dev/null | awk '/Configured Memory Speed:/{print $4; exit}') + fi + { + echo "# generated by vm-native-setup $(date -I); edit freely, -r regenerates" + echo "NVME_SERIAL='S6PXNS0W$(rnd 7 'A-Z0-9')'" + echo "MAC='a0:36:9f:$(rnd 2 'a-f0-9'):$(rnd 2 'a-f0-9'):$(rnd 2 'a-f0-9')'" + # ASUS DIY boards report the literal placeholders for system and chassis, and + # a 15-digit board serial; anything else gets a plain alphanumeric serial + if [ "$bv" = "ASUSTeK COMPUTER INC." ]; then + echo "SYS_SERIAL='System Serial Number'" + echo "BOARD_SERIAL='$(date +%y%m)$(rnd 11 '0-9')'" + echo "CHASSIS_SERIAL='Default string'" + else + echo "SYS_SERIAL='$(rnd 10 'A-Z0-9')'" + echo "BOARD_SERIAL='$(rnd 12 'A-Z0-9')'" + echo "CHASSIS_SERIAL='$(rnd 10 'A-Z0-9')'" + fi + echo "MEM_MANUFACTURER='${mem_man:-Kingston}'" + echo "MEM_PART='${mem_part:-KF556C40BB-16}'" + echo "MEM_SPEED='${mem_speed:-5600}'" + echo "MEM_SERIAL='$(rnd 8 'A-F0-9')'" + } > "$DATA/identity.env" +} +mkdir -p "$DATA" +if [ -z "$RANDOMIZE" ] && [ "$PROFILE" != tuned ]; then + echo + echo "Hardware identity. Everyone running this tool with the same fixed values shares one" + echo "fingerprint. -r writes this deployment its own serials, MAC and memory module to" + echo "$DATA/identity.env and puts them in the domain (SMBIOS, disk, NIC)." + if [ -f "$DATA/identity.env" ]; then + echo "An identity file exists from an earlier run; 'y' replaces it with fresh values." + fi + echo "Windows may ask to re-activate after the board serial and MAC change." + a=$(ask "randomize the hardware identity? (y/n)" n); RANDOMIZE=$([ "$a" = y ] && echo 1 || echo 0) +fi +RANDOMIZE=${RANDOMIZE:-0} +if [ "$RANDOMIZE" = 1 ] || [ ! -f "$DATA/identity.env" ]; then gen_identity; fi +# shellcheck disable=SC1091 +. "$DATA/identity.env" + +# --------------------------------------------------------- USB passthrough --- +# Passing a whole controller is cleaner than passing devices one by one: no +# emulated hub, no hotplug, nothing in the guest that says "redirected". It is +# only clean when everything behind that controller should go to the guest and +# its IOMMU group holds nothing else, so the default recommendation is: whole +# controller for a controller that carries only keyboard and mouse, individual +# devices otherwise. +usb_devices() { # busnum vid:pid kind product + local d v pr kind + for d in /sys/bus/usb/devices/[0-9]*-[0-9]*; do + [[ ${d##*/} == *:* ]] && continue + [ -f "$d/idVendor" ] || continue + v="$(cat "$d/idVendor"):$(cat "$d/idProduct")" + [ "$(cat "$d/bDeviceClass")" = 09 ] && continue # hubs stay + kind=other + for i in "$d"/*:*; do + [ -f "$i/bInterfaceProtocol" ] || continue + [ "$(cat "$i/bInterfaceClass")" = 03 ] || continue + case "$(cat "$i/bInterfaceProtocol")" in 01) kind=keyboard;; 02) [ "$kind" = keyboard ] || kind=mouse;; esac + done + pr=$(cat "$d/product" 2>/dev/null || echo "?") + echo "$(cat "$d/busnum") $v $kind $pr" + done +} +usb_ctrl_of_bus() { basename "$(readlink -f "/sys/bus/usb/devices/usb$1/..")"; } +group_clean() { # true if the IOMMU group of a PCI device holds only it and bridges + local g m; g=$(readlink -f "/sys/bus/pci/devices/$1/iommu_group") || return 1 + for m in "$g"/devices/*; do + m=${m##*/}; [ "$m" = "$1" ] && continue + case "$(cat "/sys/bus/pci/devices/$m/class")" in 0x0604*) ;; *) return 1;; esac + done +} +USB_HOSTDEVS="" +usb_plan() { # prints the hostdev XML for a spec: auto | vid:pid,... | pci addrs + local spec="$1" item ctrl bus devs kinds + if [ "$spec" = auto ]; then + # controllers whose every device is a keyboard or mouse go whole; the + # rest of the input devices go one by one + for ctrl in $(usb_devices | while read -r bus _ _ _; do usb_ctrl_of_bus "$bus"; done | sort -u); do + devs=$(usb_devices | while read -r bus vp kind pr; do [ "$(usb_ctrl_of_bus "$bus")" = "$ctrl" ] && echo "$vp $kind $pr"; done) + kinds=$(echo "$devs" | awk '{print $2}' | sort -u | tr '\n' ' ') + case "$kinds" in + "keyboard "|"mouse "|"keyboard mouse ") + if group_clean "$ctrl"; then echo "pci $ctrl"; continue; fi;; + esac + echo "$devs" | awk '$2=="keyboard"||$2=="mouse"{print "usb", $1}' + done + else + for item in ${spec//,/ }; do + case "$item" in + 0000:*) echo "pci $item";; + *:*) echo "usb $item";; + *) echo "unknown USB spec '$item' - want vid:pid or 0000:bb:dd.f" >&2; exit 1;; + esac + done + fi | sort -u | while read -r kind id; do + if [ "$kind" = usb ]; then + printf " \n \n \n \n \n \n \n" "${id%:*}" "${id#*:}" "${id%:*}" "${id#*:}" + else + IFS=':.' read -r dm bs sl fn <<< "$id" + printf " \n \n
\n \n \n \n" "$dm" "$bs" "$sl" "$fn" "$dm" "$bs" "$sl" "$fn" + fi + done +} +if [ -z "$USBSPEC" ] && [ "$ASSUME" != 1 ]; then + echo + echo "USB devices on this host:" + usb_devices | while read -r bus vp kind pr; do + ctrl=$(usb_ctrl_of_bus "$bus"); printf ' %-9s %-9s %-40s controller %s%s\n' "$vp" "$kind" "$pr" "$ctrl" "$(group_clean "$ctrl" || echo ' (shared IOMMU group)')" + done + echo " auto = keyboard and mouse; a whole controller when only they sit on it" + echo " none = no USB passthrough" + echo " or a list: vid:pid,vid:pid,0000:bb:dd.f (a PCI address passes that whole controller)" + echo "The host loses whatever is passed for as long as the guest runs." + read -rp "USB passthrough [none]: " USBSPEC; USBSPEC=${USBSPEC:-none} +fi +USBSPEC=${USBSPEC:-none} +if [ "$USBSPEC" != none ] && [ "$USBSPEC" != auto ]; then + for item in ${USBSPEC//,/ }; do + [[ $item =~ ^[0-9a-f]{4}:[0-9a-f]{4}$ ]] || [[ $item =~ ^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-7]$ ]] || + { echo "bad -u item '$item': want vid:pid or 0000:bb:dd.f"; exit 1; } + done +fi +[ "$USBSPEC" = none ] || USB_HOSTDEVS=$(usb_plan "$USBSPEC") + +# The patched QEMU refuses a virtio disk, so the full level always converts; the +# native level runs stock QEMU and gets the choice. +CONVERT=0 +if [ "$PROFILE" = full ]; then CONVERT=1 +elif [ "$PROFILE" = native ]; then + echo + echo "Disk and devices. Moving the disks to emulated NVMe and replacing the virtio device set" + echo "(balloon, RNG, agent channels, virtiofs, virtio NIC and inputs) removes every device that" + echo "names the emulator. The guest must have stornvme boot-start first; see docs/GUEST-SETUP.md." + a=$(ask "move disks to NVMe and replace the virtio devices? (y/n)" y); [ "$a" = y ] && CONVERT=1 +fi + +# ------------------------------------------------------------- pin mapping --- +if [ "$RESERVE" = 0 ]; then + # Helper threads go on the far end of what is left, on the guest's own NUMA + # node when the host has more than one. + mapfile -t NEAR < <(for c in "${REST[@]}"; do [ "$(node_of "$c")" = "$GNODE" ] && echo "$c"; done) + [ "${#NEAR[@]}" -ge 2 ] || NEAR=("${REST[@]}") + EMU=${NEAR[$(( ${#NEAR[@]} - 2 ))]}; IOC=${NEAR[$(( ${#NEAR[@]} - 1 ))]} + GUEST=("${GRP0[@]:0:$CORES}") +else + GUEST=("${GRP0[@]:0:$CORES}") + EMU=${GRP0[$CORES]}; IOC=${GRP0[$(( CORES + 1 ))]} +fi +[ "${#GUEST[@]}" -eq "$CORES" ] || { echo "not enough physical cores for $CORES"; exit 1; } + +# The level decides which QEMU the domain runs. "full" needs the patched build; +# the other two must not silently depend on it. +SYSQEMU=$(command -v qemu-system-x86_64 2>/dev/null || echo /usr/bin/qemu-system-x86_64) +PATCHED=/opt/qemu-native/bin/qemu-system-x86_64 +if [ "$PROFILE" = full ] && [ -x "$PATCHED" ]; then + EMULATOR="$PATCHED" +else + EMULATOR="$SYSQEMU" +fi + +# --------------------------------------------------------------- firmware --- +# libvirt's firmware autoselection has no descriptor with enrolled keys on most +# distributions, so a Secure Boot store with real keys is generated once per +# domain (OEM-named PK/KEK, Microsoft db) and named explicitly. Changing the +# template resets the domain's EFI variable store, which BitLocker notices. +LOADER=/usr/share/edk2/x64/OVMF_CODE.secboot.4m.fd +NVRAM_TPL=""; NVRAM_PATH="/var/lib/libvirt/qemu/nvram/${DOM}_VARS.fd"; NVRAM_RESET=0 +if [ -z "$SECBOOT" ] && [ "$PROFILE" != tuned ]; then + echo + echo "Secure Boot. A key store with an OEM-named PK/KEK and the Microsoft db is generated" + echo "for the domain, so the guest sees Secure Boot enforcing, which some software refuses to run without." + echo "Enrolling it resets the domain's EFI variable store; BitLocker will notice." + SECBOOT=$(ask "enable Secure Boot with enrolled keys? (on/off)" on) +fi +SECBOOT=${SECBOOT:-off} +case "$SECBOOT" in on|off) ;; *) echo "-s takes on or off"; exit 1;; esac +if [ "$SECBOOT" = on ] && [ "$PROFILE" != tuned ]; then + if ! "${C[@]}" dumpxml --inactive "$DOM" | grep -q '/dev/null; then + echo "NOTE: virt-fw-vars (python-virt-firmware) not found - Secure Boot keys cannot be" + echo " enrolled, the firmware block is left as it is." + SECBOOT=off + elif [ ! -f "$LOADER" ]; then + echo "NOTE: $LOADER not found - firmware block left as it is."; SECBOOT=off + else + NVRAM_TPL="$DATA/OVMF_VARS.enrolled.fd" + if [ ! -f "$NVRAM_TPL" ]; then + virt-fw-vars -i /usr/share/edk2/x64/OVMF_VARS.4m.fd -o "$NVRAM_TPL" \ + --enroll-generate "$(dmi board_vendor)" --secure-boot >/dev/null 2>&1 || { echo "virt-fw-vars failed"; exit 1; } + fi + OLD_TPL=$("${C[@]}" dumpxml --inactive "$DOM" | grep -o "]*>[^<]*" | head -1 | sed 's/.*>\([^<]*\)<.*/\1/') + # A new template only takes effect on a store that does not exist yet. When + # the domain already has one at the standard path it has to go; a store + # another domain also points at is never touched - that domain keeps it. + if [ "$OLD_TPL" != "$NVRAM_TPL" ] && [ "$OLD_PATH" = "$NVRAM_PATH" ]; then + for other in $("${C[@]}" list --all --name); do + [ "$other" = "$DOM" ] && continue + if "${C[@]}" dumpxml --inactive "$other" 2>/dev/null | grep -q "]*>$NVRAM_PATH<"; then + echo "ERROR: $NVRAM_PATH is also the firmware store of domain '$other'. Give $DOM its" + echo " own nvram path first, or run with -s off."; exit 1 + fi + done + NVRAM_RESET=1 + fi + fi +fi + +THP=$(cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || echo "") + +echo +echo "will configure $DOM:" +echo " level $PROFILE" +echo " vCPUs $VCPUS ($CORES cores x $SMT threads) on host cores ${GUEST[*]}" +echo " emulator core $EMU | iothread core $IOC" +echo " binary $EMULATOR" +echo " memory ${RAMG} GiB" +[ -n "$MODEL" ] && [ "$PROFILE" != tuned ] && echo " declares $MODEL" +if [ "$PROFILE" != tuned ]; then + [ "$CONVERT" = 1 ] && echo " disk emulated NVMe, serial $NVME_SERIAL; virtio devices replaced" + [ "$SECBOOT" = on ] && echo " firmware Secure Boot, keys enrolled from $NVRAM_TPL" + [ "$RANDOMIZE" = 1 ] && echo " identity SMBIOS serials, MAC $MAC and memory module from $DATA/identity.env" + [ "$PROFILE" = full ] && echo " display none - the guest console goes dark, use SSH or RDP" +fi +[ -n "$USB_HOSTDEVS" ] && echo " usb $(echo "$USB_HOSTDEVS" | grep -c ' "$BACKUP/$DOM.before-setup.xml" + +PINS="" +for ((k=0;k\n" +done + +PROFILE="$PROFILE" VENDOR="$VENDOR" VCPUS="$VCPUS" CORES="$CORES" SMT="$SMT" RAMG="$RAMG" \ +MODEL="$MODEL" EMULATOR="$EMULATOR" PINS="$PINS" \ +EMUSET="$(pair "$EMU")" IOCSET="$(pair "$IOC")" NUMANODE="$([ "$NNODES" -gt 1 ] && echo "$GNODE")" \ +SECBOOT="$SECBOOT" LOADER="$LOADER" NVRAM_TPL="$NVRAM_TPL" NVRAM_PATH="$NVRAM_PATH" CONVERT="$CONVERT" \ +RANDOMIZE="$RANDOMIZE" NVME_SERIAL="$NVME_SERIAL" MAC="$MAC" USB_HOSTDEVS="$USB_HOSTDEVS" \ +SYS_SERIAL="$SYS_SERIAL" BOARD_SERIAL="$BOARD_SERIAL" CHASSIS_SERIAL="$CHASSIS_SERIAL" \ +DMI_SYS_VENDOR="$(dmi sys_vendor)" DMI_PRODUCT="$(dmi product_name)" DMI_PRODUCT_VERSION="$(dmi product_version)" \ +DMI_SKU="$(dmi product_sku)" DMI_FAMILY="$(dmi product_family)" DMI_BOARD_VENDOR="$(dmi board_vendor)" \ +DMI_BOARD="$(dmi board_name)" DMI_BOARD_VERSION="$(dmi board_version)" DMI_CHASSIS_VENDOR="$(dmi chassis_vendor)" \ +DMI_CHASSIS_VERSION="$(dmi chassis_version)" DMI_BIOS_VENDOR="$(dmi bios_vendor)" DMI_BIOS_VERSION="$(dmi bios_version)" \ +DMI_BIOS_DATE="$(dmi bios_date)" \ +python3 - "$BACKUP/$DOM.before-setup.xml" "$BACKUP/$DOM.setup.xml" <<'XMLGEN_END' +import io, os, re, sys +from xml.sax.saxutils import escape +src, out = sys.argv[1], sys.argv[2] +E = os.environ +prof, vcpus, cores, smt = E["PROFILE"], int(E["VCPUS"]), int(E["CORES"]), int(E["SMT"]) +s = io.open(src, encoding="utf-8").read() +conformant = prof != "tuned" + +kib = int(E["RAMG"]) * 1024 * 1024 +s = re.sub(r"\d+", "%d" % kib, s, count=1) +s = re.sub(r"\d+", + "%d" % kib, s, count=1) +s = re.sub(r"\d+", + "%d" % vcpus, s, count=1) +s = re.sub(r"", + "" % (cores, smt), + s, count=1) +s = re.sub(r"[^<]*", "%s" % E["EMULATOR"], s, count=1) + +block = (" 1\n \n" + E["PINS"].replace("\\n", "\n") + + " \n" % E["EMUSET"] + + " \n" % E["IOCSET"] + + " \n") +if E["NUMANODE"]: + block += " \n \n \n" % E["NUMANODE"] +s = re.sub(r"\s*\d+", "", s) +s = re.sub(r"\s*.*?", "", s, flags=re.S) +s = re.sub(r"\s*.*?", "", s, flags=re.S) +s = s.replace(" ", block + " ", 1) + +feats = [" "] +if E.get("VENDOR") == "AuthenticAMD": + feats.insert(0, " ") +if conformant: + feats.insert(0, " ") + feats += [" ", + " ", + " "] +s = re.sub(r"\n\s*", "", s) +s = s.replace("threads='%d'/>" % smt, "threads='%d'/>\n" % smt + "\n".join(feats), 1) + +s = re.sub(r"\s*.*?", "", s, flags=re.S) +s = re.sub(r"\s*", "", s, flags=re.S) +s = re.sub(r"\s*", "", s) +if conformant: + s = s.replace(" ", + " \n \n \n ", 1) +else: + hv = (" \n" + " \n \n" + " \n \n" + " \n \n" + " \n \n \n" + " \n \n" + " \n \n" + " \n \n" + " \n \n \n") + s = s.replace(" ", hv + " ", 1) + s = s.replace(" ", + " \n" + " ", 1) + + +# memfd + shared memory backing is only needed for virtiofs, and it blocks +# transparent hugepages outright - shared mappings are not anonymous, so THP +# for anon never applies and the whole guest runs on 4 KiB pages. Measured on +# an 8 GiB guest: 0 MiB AnonHugePages with it, 8110 MiB without. +if ".*?", "", s, flags=re.S) +if "" not in s: + s = s.replace(" ", " \n \n" + " \n \n ", 1) + +if conformant and E["MODEL"]: + s = re.sub(r"version=AMD Ryzen [^,]*,", "version=%s," % E["MODEL"], s) + +if conformant and E["CONVERT"] == "1": + # Disks go to emulated NVMe: the patched QEMU refuses virtio, Windows boots + # NVMe with its inbox driver, and libvirt adds the controller itself. The + # first disk gets the identity's serial, further disks a derived one. + n = [0] + if E["RANDOMIZE"] == "1": + s = re.sub(r"((?:(?!).)*?)\s*[^<]*", r"\1", s, flags=re.S) + s = re.sub(r"(]*>\s*)[^<]*", r"\1%s" % E["NVME_SERIAL"], s) + def to_nvme(m): + d = m.group(0) + if "device='disk'" not in d or ("bus='nvme'" in d and "" in d): + return d + d = re.sub(r"", r"", d) + d = re.sub(r"", + r"", d) + d = re.sub(r"\s*
", "", d) + if "" not in d: + serial = E["NVME_SERIAL"] if n[0] == 0 else E["NVME_SERIAL"][:-1] + "0123456789ABCDEF"[n[0] % 16] + d = d.replace("", " %s\n " % serial) + n[0] += 1 + return d + s = re.sub(r".*?", to_nvme, s, flags=re.S) + # everything virtio or agent-shaped names the emulator; a real machine has none of it + s = re.sub(r"\s*.*?guest_agent.*?", "", s, flags=re.S) + s = re.sub(r"\s*.*?", "", s, flags=re.S) + s = re.sub(r"\s*]*>.*?", "", s, flags=re.S) + s = re.sub(r"\s*]*/>", "", s) + s = re.sub(r"\s*", "", s, flags=re.S) + s = re.sub(r"\s*.*?", "", s, flags=re.S) + s = re.sub(r"\s*.*?", "", s, flags=re.S) + s = re.sub(r"\s*.*?", "", s, flags=re.S) + s = re.sub(r"\s*", "", s) + s = re.sub(r".*?", "", s, flags=re.S) + s = re.sub(r"", "", s) + s = re.sub(r"(\s*)?", "", s) + if prof == "full": + s = re.sub(r"", "", s, flags=re.S) + +if conformant: + if E["RANDOMIZE"] == "1": + s = re.sub(r"", "" % E["MAC"], s, count=1) + def ent(name, key): + v = E.get(key, "") + return " %s\n" % (name, escape(v)) if v else "" + sysinfo = (" \n \n" + ent("vendor", "DMI_BIOS_VENDOR") + + ent("version", "DMI_BIOS_VERSION") + ent("date", "DMI_BIOS_DATE") + " \n \n" + + ent("manufacturer", "DMI_SYS_VENDOR") + ent("product", "DMI_PRODUCT") + ent("version", "DMI_PRODUCT_VERSION") + + ent("serial", "SYS_SERIAL") + ent("sku", "DMI_SKU") + ent("family", "DMI_FAMILY") + + " \n \n" + ent("manufacturer", "DMI_BOARD_VENDOR") + ent("product", "DMI_BOARD") + + ent("version", "DMI_BOARD_VERSION") + ent("serial", "BOARD_SERIAL") + " \n \n" + + ent("manufacturer", "DMI_CHASSIS_VENDOR") + ent("version", "DMI_CHASSIS_VERSION") + ent("serial", "CHASSIS_SERIAL") + + " \n \n") + s = re.sub(r"\s*.*?", "", s, flags=re.S) + s = s.replace(" ", "", s) + if "", " \n ", 1) + + if E["SECBOOT"] == "on": + s = re.sub(r"", "", s, count=1) + s = re.sub(r"\s*.*?", "", s, flags=re.S) + s = re.sub(r"\s*]*>[^<]*", "", s) + s = re.sub(r"\s*]*>[^<]*", "", s) + s = re.sub(r"\s*]*/>", "", s) + fw = ("\n %s" + "\n %s" % + (E["LOADER"], E["NVRAM_TPL"], E["NVRAM_PATH"])) + s = re.sub(r"(hvm)", r"\1" + fw.replace("\\", "\\\\"), s, count=1) + if "" not in s: + s = s.replace(" ", " \n ", 1) + +# USB passthrough: replace what this tool put in before, leave other hostdevs alone +s = re.sub(r"\s*(?:(?!).)*(?:(?!).)*", "", s, flags=re.S) +if E["USB_HOSTDEVS"]: + s = s.replace(" ", E["USB_HOSTDEVS"] + " ", 1) + +io.open(out, "w", encoding="utf-8").write(s) +XMLGEN_END + +# libvirt only copies the template when the domain's own store does not exist +# yet, so enrolling keys means dropping the old store; libvirt does that for us. +if [ "$NVRAM_RESET" = 1 ]; then + if [ "$ASSUME" != 1 ]; then read -rp "reset the firmware variable store of $DOM? [y/N]: " a; [ "$a" = y ] || { echo aborted; exit 1; }; fi + "${C[@]}" undefine "$DOM" --nvram >/dev/null +fi +"${C[@]}" define "$BACKUP/$DOM.setup.xml" >/dev/null + +# ------------------------------------------------ qemu command-line args --- +# libvirt has no XML for the CPUID brand string, raw SMBIOS structures or extra +# ACPI tables, so they ride in as . The generated files live +# in a per-domain directory, so a package upgrade or a moved checkout does not +# change a defined domain underneath it. Only the arguments this script owns +# are replaced; anything else already in the block is left alone. +QARGS=() +if [ "$PROFILE" != tuned ]; then + kb() { local s; s=$(cat "/sys/devices/system/cpu/cpu0/cache/index$1/size"); echo "${s%K}"; } + SPAN=$(( (CORES + MAXC - 1) / MAXC )) + python3 "$SELF/scripts/generate-tables.py" --output-dir "$DATA" \ + --cache-l1 $(( CORES * ($(kb 0) + $(kb 1)) )) --cache-l2 $(( CORES * $(kb 2) )) \ + --cache-l3 $(( SPAN * $(kb 3) )) >/dev/null + for f in "$DATA"/type*.bin; do QARGS+=("-smbios" "file=$f"); done + if [ -d "$SELF/acpi" ]; then + cp "$SELF"/acpi/*.aml "$DATA"/ + for f in "$DATA"/*.aml; do QARGS+=("-acpitable" "file=$f"); done + fi + # A second -cpu REPLACES the first rather than merging, so it is built from + # libvirt's own generated line and never hand-written. It is incompatible + # with , which is why the tuned level skips it. + # libvirt's own -cpu comes first; a previous run's override sits at the + # end, quoted because the brand has spaces, so split like a shell would. + if [ -n "$MODEL" ]; then + python3 - "$BACKUP/$DOM.setup.xml" "$BACKUP/$DOM.nohostdev.xml" <<'STRIP_END' +import io, re, sys +s = io.open(sys.argv[1], encoding="utf-8").read() +io.open(sys.argv[2], "w", encoding="utf-8").write(re.sub(r"\s*", "", s, flags=re.S)) +STRIP_END + GEN=$("${C[@]}" domxml-to-native --format qemu-argv --xml "$BACKUP/$DOM.nohostdev.xml" | python3 -c ' +import shlex, sys +a = shlex.split(sys.stdin.read()) +print(a[a.index("-cpu") + 1] if "-cpu" in a else "")') + if [ -n "$GEN" ] && [[ "$GEN" != *hv-* ]]; then QARGS+=("-cpu" "$GEN,model-id=$MODEL"); fi + # SMBIOS type 4 defaults to manufacturer "QEMU" and the machine name as + # the version; a comma inside a QEMU option value is written twice. + case "$VENDOR:$FAMILY" in + AuthenticAMD:23) SOCK=AM4; CPUVEND="Advanced Micro Devices,, Inc.";; + AuthenticAMD:*) SOCK=AM5; CPUVEND="Advanced Micro Devices,, Inc.";; + GenuineIntel:*) SOCK=$([ "$MODELNO" -ge 151 ] && echo LGA1700 || echo LGA1200); CPUVEND="Intel(R) Corporation";; + *) SOCK=CPU; CPUVEND="$VENDOR";; + esac + MAXMHZ=$(( $(cat /sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq 2>/dev/null || echo 4500000) / 1000 )) + QARGS+=("-smbios" "type=4,sock_pfx=$SOCK,manufacturer=$CPUVEND,version=$MODEL,max-speed=$MAXMHZ,current-speed=$MAXMHZ") + fi + if [ "$RANDOMIZE" = 1 ]; then + QARGS+=("-smbios" "type=17,loc_pfx=DIMM,bank=P0 CHANNEL A,manufacturer=$MEM_MANUFACTURER,part=$MEM_PART,serial=$MEM_SERIAL,speed=$MEM_SPEED") + fi +fi +QARGS_NL="$(printf '%s\n' "${QARGS[@]}")" python3 - "$BACKUP/$DOM.setup.xml" <<'QARGS_END' +import io, os, re, sys +p = sys.argv[1] +args = [a for a in os.environ["QARGS_NL"].split("\n") if a] +s = io.open(p, encoding="utf-8").read() +if "xmlns:qemu=" not in s: + s = s.replace("", + "", 1) +kept, envs = [], [] +m = re.search(r"\n\s*(.*?)", s, re.S) +if m: + envs = re.findall(r"]*/>", m.group(1)) + vals = re.findall(r"", m.group(1)) + i = 0 + while i < len(vals): + owned = vals[i] == "-cpu" or (vals[i] in ("-smbios", "-acpitable") and i + 1 < len(vals) + and vals[i + 1].startswith(("file=", "type="))) + if owned: + i += 2 + else: + kept.append(vals[i]); i += 1 + s = s[:m.start()] + s[m.end():] +vals = kept + args +if vals or envs: + block = (" \n" + + "".join(" \n" % v.replace("'", "'") for v in vals) + + "".join(" %s\n" % e for e in envs) + + " \n") + s = s.replace("", block + "", 1) +io.open(p, "w", encoding="utf-8").write(s) +QARGS_END +"${C[@]}" define "$BACKUP/$DOM.setup.xml" >/dev/null + +echo +if [ "$PROFILE" = full ]; then + ok=1 + [ "$EMULATOR" = "$PATCHED" ] || { echo "MISSING: patched QEMU not found at $PATCHED"; ok=0; } + # DKMS puts them in updates/dkms/, the manual install script in updates/ + ko=$(find "/usr/lib/modules/$(uname -r)/updates" -name 'kvm.ko*' 2>/dev/null | head -1) + if [ -n "$ko" ]; then + built=$(modinfo -F srcversion "$ko" 2>/dev/null) + live=$(cat /sys/module/kvm/srcversion 2>/dev/null) + if [ -z "$built" ] || [ "$built" != "$live" ]; then + echo "MISSING: patched KVM modules installed but not loaded." + echo " With all VMs off: sudo modprobe -r kvm_amd kvm && sudo modprobe kvm_amd" + ok=0 + fi + else + echo "MISSING: no patched KVM modules - install vfio-native-kvm-dkms." + ok=0 + fi + [ "$ok" = 1 ] && echo "patched QEMU and KVM modules both in place." + if [ -n "$MODEL" ] && [ -f /sys/module/kvm_amd/parameters/cpuid_passthrough ]; then + echo + echo "The TIMER check needs CPUID passthrough, which is off by default and must be" + echo "switched on AFTER the guest has booted (it hangs a booting guest). Once the" + echo "guest is up, on the host:" + echo " echo '$MODEL' | sudo tee /sys/module/kvm_amd/parameters/brand_string" + echo " echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough" + echo "Switch it off again (echo N) before rebooting the guest." + fi +fi + +if [ ! -e /usr/lib/udev/rules.d/99-vfio-native-vnet-offload.rules ] && [ ! -e /etc/udev/rules.d/99-vfio-native-vnet-offload.rules ]; then + echo "NOTE: the e1000e offload udev rule is not installed. SSH into the guest will fail with" + echo " 'Corrupted MAC on input' until it is:" + echo " sudo install -Dm644 $SELF/scripts/99-vfio-native-vnet-offload.rules /etc/udev/rules.d/ && sudo udevadm control --reload-rules" +fi +gov=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo unknown) +[ "$gov" = performance ] || echo "host governor is '$gov' - run: sudo cpupower frequency-set -g performance" + +echo +echo "$DOM configured: $PROFILE, $VCPUS vCPU, ${RAMG} GiB." +echo "revert: virsh -c qemu:///system define $BACKUP/$DOM.before-setup.xml" +if command -v vm-native-verify >/dev/null; then echo "verify: vm-native-verify" +else echo "verify: $(dirname "$0")/verify-perf.sh"; fi diff --git a/scripts/verify-perf.sh b/scripts/verify-perf.sh new file mode 100755 index 0000000..9ece376 --- /dev/null +++ b/scripts/verify-perf.sh @@ -0,0 +1,109 @@ +#!/bin/bash +# Check that a corrected guest is actually performing as it should. +# +# @@SELFNAME@@ [user@ip] [ssh-key] +# +# Copies the benchmark in if it is missing, runs it, and grades the numbers that +# matter against the thresholds a correctly configured guest meets. Read-only: +# it changes nothing on the host or in the guest. +set -uo pipefail + +GUEST="${1:-User@192.168.122.222}" +KEY="${2:-}" +[ -n "$KEY" ] || for k in "$HOME/.ssh/win11-native" "$HOME/.ssh/win11"; do [ -f "$k" ] && { KEY="$k"; break; }; done +# Locate the project data (patches, benchmark sources), whether running from a +# checkout or installed as a package. +for d in "$(cd "$(dirname "$0")/.." 2>/dev/null && pwd)" /usr/share/vfio-native; do + if [ -d "$d/bench" ] || [ -d "$d/patches" ]; then SELF="$d"; break; fi +done +SELF="${SELF:-$(cd "$(dirname "$0")/.." && pwd)}" +BENCH="$SELF/bench/vmbench.exe" +[ -w "$SELF/bench" ] 2>/dev/null || BENCH="${XDG_CACHE_HOME:-$HOME/.cache}/vfio-native/vmbench.exe" +SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10) + +command -v ssh >/dev/null || { echo "ssh not found"; exit 1; } +[ -n "$KEY" ] && [ -f "$KEY" ] || { echo "no ssh key for the guest - pass it: $(basename "$0") user@guest-ip ~/.ssh/key"; exit 1; } + +if [ ! -f "$BENCH" ]; then + echo "building vmbench.exe..." + command -v x86_64-w64-mingw32-gcc >/dev/null || { echo "need mingw-w64-gcc to build it"; exit 1; } + mkdir -p "$(dirname "$BENCH")" + x86_64-w64-mingw32-gcc -O2 -o "$BENCH" "$SELF/bench/vmbench.c" || exit 1 +fi + +"${SSH[@]}" "$GUEST" 'exit' 2>/dev/null || { echo "cannot reach $GUEST over ssh"; exit 1; } +scp -q -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=no \ + "$BENCH" "$GUEST:C:/Users/User/vmbench.exe" 2>/dev/null + +# A guest that has just booted is still indexing, patching and starting services, +# and that shows up entirely in the stall counts. Let it settle first. +SETTLE="${SETTLE:-60}" +if [ "$SETTLE" -gt 0 ]; then + echo "letting the guest settle for ${SETTLE}s (SETTLE=0 to skip)..." + sleep "$SETTLE" +fi + +echo "running benchmark in the guest, this takes a few minutes..." +OUT=$("${SSH[@]}" "$GUEST" 'C:\Users\User\vmbench.exe all' 2>/dev/null) +[ -n "$OUT" ] || { echo "benchmark produced no output"; exit 1; } +echo "$OUT" +echo + +val() { echo "$OUT" | sed -n "s/.*$1=\([0-9.]*\).*/\1/p" | head -1; } + +pass=0; fail=0 +check() { # name value op limit explanation + local n="$1" v="$2" op="$3" lim="$4" why="$5" ok + if [ -z "$v" ]; then printf ' ?? %-22s (not measured)\n' "$n"; return; fi + ok=$(awk -v a="$v" -v b="$lim" "BEGIN{print (a $op b)?1:0}") + if [ "$ok" = "1" ]; then printf ' OK %-22s %-10s\n' "$n" "$v"; pass=$((pass+1)) + else printf ' ?? %-22s %-10s <- %s\n' "$n" "$v" "$why"; fail=$((fail+1)); fi +} + +if [ -f /sys/module/kvm_amd/parameters/cpuid_passthrough ]; then + echo "kvm_amd cpuid_passthrough=$(cat /sys/module/kvm_amd/parameters/cpuid_passthrough) brand='$(cat /sys/module/kvm_amd/parameters/brand_string)'" +fi +# A guest can silently run on 4 KiB pages. With one qemu process on the host its +# AnonHugePages should be close to the guest's RAM; with several, name the domain. +pids=$(pgrep -f "[q]emu-system-x86_64" | tr '\n' ' ') +[ -n "${VMDOMAIN:-}" ] && pids=$(pgrep -f "[q]emu-system-x86_64.*guest=$VMDOMAIN" | tr '\n' ' ') +if [ "$(echo $pids | wc -w)" = 1 ] && thp=$(sudo -n awk '/AnonHugePages/ {s+=$2} END {print int(s/1024)}' /proc/$pids/smaps 2>/dev/null); then + ram=$(awk '/VmRSS/ {print int($2/1024)}' /proc/$pids/status 2>/dev/null) + echo "hugepages: ${thp} MiB of ${ram} MiB resident are transparent hugepages$([ "${thp:-0}" -lt $(( ${ram:-1} / 2 )) ] && echo ' <- LOW: check memoryBacking and /sys/kernel/mm/transparent_hugepage/enabled')" +elif [ -n "$pids" ]; then + # another user's smaps needs root; the host-wide counter does not, and with one guest it is close enough + echo "hugepages: host AnonHugePages $(awk '/AnonHugePages/ {print int($2/1024)}' /proc/meminfo) MiB across $(echo $pids | wc -w) guest(s) (sudo for a per-guest figure; THP $(cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null | grep -o '\[.*\]'))" +fi +echo +echo "results:" +check "QPC cost (ns)" "$(val qpc_ns)" "<" 50 "should be ~15. Over ~1000 means Windows lost the boot TSC race - leave the host 4+ free cores, then reboot to re-measure" +check "rdtsc cost (ns)" "$(val rdtsc_ns)" "<" 20 "unexpectedly slow TSC read" +check "1-thread (Mops)" "$(val 'threads=1 aggregate_Mops')" ">" 5000 "single-thread throughput low - check the host governor is 'performance'" +check "L3 latency (ns)" "$(echo "$OUT" | sed -n 's/memlat L3_8M ns=\([0-9.]*\)/\1/p')" "<" 13 "high L3 latency suggests the vCPUs are spread across both CCDs" +check "jitter p99.99 (us)" "$(val 'p99.99')" "<" 20 "scheduling tail is long - check pinning and that the emulator is off the vCPU cores" +check "stalls >100us" "$(echo "$OUT" | sed -n 's/.*over_100us=\([0-9]*\).*/\1/p')" "<" 20 "frequent long stalls. If the guest booted recently, let it idle and re-run" + +echo +if [ "$fail" -eq 0 ]; then + echo "All $pass checks in range - the guest is configured correctly." +else + echo "$pass in range, $fail out of range. See the notes above each one." +fi + +cat <<'EOF' + +For reference, a correctly configured guest on a 7950X measures roughly: + + QPC cost ~15 ns (a 32-vCPU guest measures ~1300 ns instead) + rdtsc cost ~7 ns + 1 thread ~5300 Mops + L1 / L2 / L3 ~0.8 / ~3.4 / ~10 ns + DRAM ~80-90 ns + memory read ~50 GB/s + jitter p99.99 ~3 us + stalls >100us 0-2 over a 10 second run + +The two that matter most are QPC cost and the jitter tail. Software that polls the +clock in a tight loop calls QPC thousands of times a second, and the jitter tail is +what shows up as hitching in an interactive session. Mean throughput shows neither. +EOF