diff --git a/README.md b/README.md index 5a91c9d..741d8be 100644 --- a/README.md +++ b/README.md @@ -165,25 +165,32 @@ vm-native-verify `vm-native-verify` also prints whether CPUID passthrough is on, which is the last step. -### 6. Switch on CPUID passthrough, after every guest boot +### 6. CPUID passthrough, on after boot and off before the next one The `TIMER` check times the world switch on an intercepted `CPUID`, and the only way to stop -paying it is to not exit. `vm-native-setup` prints the two lines for your declared SKU; on this -host they are: +paying it is to not exit. That switch has to be off while a guest cold-boots - a booting Windows +enumerates CPUID bits KVM synthesises and hangs if they vanish half way through - and on once it +is up. Let the libvirt hook handle that around a guest: + +```sh +sudo vm-native-cpuid enable win11 # off for the cold boot, on once the guest is on the network, off on stop +sudo vm-native-cpuid status # what is enabled, and the live switch state +sudo vm-native-cpuid disable win11 # stop automating this guest +sudo vm-native-cpuid revert # remove the hook and reset the switch +``` + +Or drive it by hand. `vm-native-setup` prints the two lines for your declared SKU; run them once +the guest is up, never before, and switch it off (`echo N`) before the next boot: ```sh echo 'AMD Ryzen 7 7700X 8-Core Processor' | sudo tee /sys/module/kvm_amd/parameters/brand_string echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough ``` -Run them once the guest is up, never before: a booting Windows enumerates CPUID bits KVM -synthesises, and hangs if they vanish half way through. Switch it off again (`echo N`) before the -next boot. The module applies it only to vCPU threads pinned to exactly one host CPU, so on an -unpinned guest it does nothing rather than something wrong. - -It is a module parameter, so it is one switch for the whole host: every pinned guest gets it, and -a cold boot of any of them while it is on hits the race. With more than one such guest, switch it -off before any of them boots and on again once they are all up. +The module applies it only to vCPU threads pinned to exactly one host CPU, so on an unpinned guest +it does nothing rather than something wrong. It is one switch and one brand string for the whole +host, so it serves one hardened guest at a time - the hook enforces that and leaves a second guest +alone. For the detection score, run VMAware in the guest **from the console session, not over SSH** - OpenSSH lands you in session 0, which is not where an interactive desktop session runs. See diff --git a/docs/GUEST-SETUP.md b/docs/GUEST-SETUP.md index c86a546..f73b14b 100644 --- a/docs/GUEST-SETUP.md +++ b/docs/GUEST-SETUP.md @@ -165,7 +165,7 @@ echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough vm-native-verify ``` -`vm-native-setup` printed those two lines with your SKU. The switch is host-wide: it applies to +`vm-native-setup` printed those two lines with your SKU, or `sudo vm-native-cpuid enable ` does the same on/off automatically around the guest (off for the cold boot, on once its network is up). The switch is host-wide: it applies to every pinned guest at once, and a cold boot of any of them while it is on hits the enumeration race, so with several guests turn it off before any boots and on when they are all up. Then the scanner, from the console session, as `docs/TESTING.md` describes. The reference guest reads 1/85. diff --git a/packaging/vfio-native/.SRCINFO b/packaging/vfio-native/.SRCINFO index c78dd6d..0a934e5 100644 --- a/packaging/vfio-native/.SRCINFO +++ b/packaging/vfio-native/.SRCINFO @@ -1,6 +1,6 @@ pkgbase = vfio-native pkgdesc = Present a libvirt guest as a self-consistent physical machine, and tune it - pkgver = 1.1.0 + pkgver = 1.1.1 pkgrel = 1 url = https://git.archworks.co/sandwich/vfio-native install = vfio-native.install @@ -17,7 +17,7 @@ pkgbase = vfio-native optdepends = cpupower: set the host CPU governor optdepends = vfio-native-kvm-dkms: patched KVM modules for the full level optdepends = vfio-native-qemu: patched QEMU for the full level - source = git+https://git.archworks.co/sandwich/vfio-native.git#tag=v1.1.0 + source = git+https://git.archworks.co/sandwich/vfio-native.git#tag=v1.1.1 sha256sums = SKIP pkgname = vfio-native diff --git a/packaging/vfio-native/PKGBUILD b/packaging/vfio-native/PKGBUILD index 1443828..4a5c22c 100644 --- a/packaging/vfio-native/PKGBUILD +++ b/packaging/vfio-native/PKGBUILD @@ -8,7 +8,7 @@ # vfio-native-qemu QEMU 11.1.1 with the platform-identity patches, in /opt pkgname=vfio-native -pkgver=1.1.0 +pkgver=1.1.1 pkgrel=1 pkgdesc="Present a libvirt guest as a self-consistent physical machine, and tune it" arch=('any') @@ -29,6 +29,7 @@ package() { install -Dm755 scripts/setup-vm.sh "${pkgdir}/usr/bin/vm-native-setup" install -Dm755 scripts/verify-perf.sh "${pkgdir}/usr/bin/vm-native-verify" install -Dm755 scripts/gpu-passthrough.sh "${pkgdir}/usr/bin/vm-native-gpu" + install -Dm755 scripts/cpuid-passthrough.sh "${pkgdir}/usr/bin/vm-native-cpuid" local share="${pkgdir}/usr/share/vfio-native" install -Dm644 bench/vmbench.c "${share}/bench/vmbench.c" @@ -36,6 +37,8 @@ package() { install -Dm755 scripts/install-modules.sh "${share}/scripts/install-modules.sh" install -Dm755 scripts/restore-stock-kvm.sh "${share}/scripts/restore-stock-kvm.sh" install -Dm755 scripts/generate-tables.py "${share}/scripts/generate-tables.py" + install -Dm755 scripts/libvirt-hook-cpuid-passthrough.sh "${share}/scripts/libvirt-hook-cpuid-passthrough.sh" + install -Dm755 scripts/cpuid-passthrough-watch "${share}/scripts/cpuid-passthrough-watch" install -Dm644 -t "${share}/acpi" acpi/*.aml acpi/*.dsl install -dm755 "${share}/patches" cp -r patches/. "${share}/patches/" diff --git a/scripts/cpuid-passthrough-watch b/scripts/cpuid-passthrough-watch new file mode 100755 index 0000000..7d9301a --- /dev/null +++ b/scripts/cpuid-passthrough-watch @@ -0,0 +1,41 @@ +#!/bin/bash +# Waits for a just-started guest's network to actually come up, then turns +# kvm_amd cpuid_passthrough on. +# +# The readiness signal is the guest's own tap RX counter: it is fresh every boot, +# it needs nothing enabled inside the guest (no SSH, no RDP, no agent), and it only +# moves once the guest's NIC driver has really loaded - which is well past the CPU +# enumeration that the switch must not change under. A DHCP lease left over from a +# previous boot cannot trip it early. +# +# Launched as a transient systemd unit by the cpuid-passthrough hook, so it is free +# to call virsh (the hook itself must not - that deadlocks libvirtd). + +DOMAIN=$1 +BRAND=$2 +PARAM=/sys/module/kvm_amd/parameters +V="virsh -c qemu:///system" + +running() { [ "$($V domstate "$DOMAIN" 2>/dev/null)" = running ]; } + +tap="" +i=0 +for _ in $(seq 1 65); do # ~195 s cap, then flip anyway if still up + i=$((i + 1)) + running || { sleep 3; continue; } # not "running" yet at prepare time - wait + [ -z "$tap" ] && tap=$($V domiflist "$DOMAIN" 2>/dev/null | + awk '$1 ~ /^(vnet|tap|macvtap)/ {print $1; exit}') + rx="/sys/class/net/$tap/statistics/rx_packets" + # the iteration floor keeps a stray pre-OS packet (a UEFI netboot attempt) from + # tripping the flip before the guest is even past its interrupt and timer setup + if [ "$i" -ge 4 ] && [ -n "$tap" ] && [ -r "$rx" ] && + [ "$(cat "$rx" 2>/dev/null || echo 0)" -ge 4 ]; then + break + fi + sleep 3 +done + +running || exit 0 # guest went away before it came up +printf '%s' "$BRAND" > "$PARAM/brand_string" +echo Y > "$PARAM/cpuid_passthrough" +logger -t vfio-cpuid "$DOMAIN network up: cpuid_passthrough=Y" diff --git a/scripts/cpuid-passthrough.sh b/scripts/cpuid-passthrough.sh new file mode 100755 index 0000000..6d7c6de --- /dev/null +++ b/scripts/cpuid-passthrough.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# vm-native-cpuid - manage the automatic cpuid_passthrough toggle for hardened guests. +# +# vm-native-cpuid enable apply the automatic cpuid passthrough to +# vm-native-cpuid disable stop applying it to +# vm-native-cpuid status show the config and the live module state +# vm-native-cpuid revert remove the hook, watcher and config; reset the module +# +# The switch and brand string are one global kvm_amd parameter, so it serves one +# hardened guest at a time. It is off until you enable it here - no domain is +# touched otherwise. Around an enabled guest the libvirt hook forces the switch off +# for the cold boot, a watcher flips it on once the guest's network is up, and it +# goes off again on stop. + +set -uo pipefail + +CONF=/etc/vfio-native/cpuid-passthrough.conf +HOOK=/etc/libvirt/hooks/qemu.d/30-cpuid-passthrough.sh +WATCH=/usr/lib/vfio-native/cpuid-passthrough-watch +PARAM=/sys/module/kvm_amd/parameters +SELF=$(cd "$(dirname "$0")/.." 2>/dev/null && pwd) + +die() { echo "$*" >&2; exit 1; } +need_root() { [ "$(id -u)" = 0 ] || die "run this as root"; } + +load() { ENABLED=yes; DOMAINS=""; [ -r "$CONF" ] && . "$CONF"; } +save() { mkdir -p "$(dirname "$CONF")"; printf 'ENABLED=%s\nDOMAINS="%s"\n' "$ENABLED" "$DOMAINS" > "$CONF"; } + +# copy the hook and watcher into place from the installed tree or this checkout +install_hook() { + local src="" + for d in /usr/share/vfio-native "$SELF"; do + [ -f "$d/scripts/libvirt-hook-cpuid-passthrough.sh" ] && src="$d" + done + [ -n "$src" ] || die "cannot find the hook source (install vfio-native, or run from a checkout)" + install -Dm755 "$src/scripts/libvirt-hook-cpuid-passthrough.sh" "$HOOK" + install -Dm755 "$src/scripts/cpuid-passthrough-watch" "$WATCH" +} + +case "${1:-}" in +enable) + need_root; [ -n "${2:-}" ] || die "usage: vm-native-cpuid enable " + load; install_hook; ENABLED=yes + case " $DOMAINS " in *" $2 "*) ;; *) DOMAINS="${DOMAINS:+$DOMAINS }$2" ;; esac + save + echo "$2 enabled. cpuid passthrough is now automatic for: $DOMAINS" + echo "on start it forces N for the cold boot, flips Y once the guest is on the network, N again on stop." + ;; +disable) + need_root; [ -n "${2:-}" ] || die "usage: vm-native-cpuid disable " + load; DOMAINS=$(printf ' %s ' "$DOMAINS" | sed "s/ $2 / /g" | xargs || true); save + systemctl stop "vfio-cpuid-watch-$2.service" 2>/dev/null || true + echo "$2 disabled. remaining: ${DOMAINS:-none}" + ;; +status) + load + echo "config: $CONF" + echo "enabled: $ENABLED" + echo "domains: ${DOMAINS:-none}" + echo "hook: $([ -f "$HOOK" ] && echo installed || echo 'not installed')" + if [ -r "$PARAM/cpuid_passthrough" ]; then + echo "module: cpuid_passthrough=$(cat "$PARAM/cpuid_passthrough") brand='$(cat "$PARAM/brand_string")'" + else + echo "module: patched kvm_amd not loaded" + fi + ;; +revert) + need_root; load + for d in $DOMAINS; do systemctl stop "vfio-cpuid-watch-$d.service" 2>/dev/null || true; done + rm -f "$HOOK" "$WATCH" "$CONF" /run/vfio-native/cpuid-active + [ -w "$PARAM/cpuid_passthrough" ] && echo N > "$PARAM/cpuid_passthrough" + echo "reverted: hook, watcher and config removed; cpuid_passthrough=N." + ;; +*) + sed -n '2,13p' "$0" | sed 's/^#\( \|$\)//' + ;; +esac diff --git a/scripts/libvirt-hook-cpuid-passthrough.sh b/scripts/libvirt-hook-cpuid-passthrough.sh new file mode 100755 index 0000000..58f9b3b --- /dev/null +++ b/scripts/libvirt-hook-cpuid-passthrough.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# libvirt qemu hook: drive the kvm_amd cpuid_passthrough switch around a guest's life. +# +# The switch must be OFF while a guest cold-boots - raw CPUID changes shape mid +# enumeration and Windows hangs - and ON once the guest is up, where it clears the +# TIMER detection. This applies only to the domains you opt in with +# `vm-native-cpuid enable `; it does nothing to any other guest. +# +# On start it sets the brand string from the guest's declared SKU and forces N, +# then a detached watcher flips Y once the guest's network is up; on stop it sets N. +# cpuid_passthrough and brand_string are one global kvm_amd parameter, so a single +# hardened guest is served at a time; a second is logged and left alone. +# +# Exits 0 on every path. The hook never calls virsh (that deadlocks libvirtd); the +# watcher that does is a transient systemd unit and runs after this returns. + +CONF=/etc/vfio-native/cpuid-passthrough.conf +ENABLED=yes +DOMAINS="" +[ -r "$CONF" ] && . "$CONF" +[ "$ENABLED" = yes ] || exit 0 + +PARAM=/sys/module/kvm_amd/parameters +[ -w "$PARAM/cpuid_passthrough" ] || exit 0 # stock module, or not an AMD host + +DOMAIN=$1 +OPERATION=$2 +case " $DOMAINS " in *" $DOMAIN "*) ;; *) exit 0 ;; esac # not an opted-in domain + +RUN=/run/vfio-native +ACTIVE=$RUN/cpuid-active +WATCH=/usr/lib/vfio-native/cpuid-passthrough-watch + +case "$OPERATION" in +prepare) + # the guest's declared SKU, straight from the -cpu model-id in the XML on stdin + BRAND=$(grep -oE "model-id=[^'\"]+" | head -1 | sed 's/^model-id=//') + [ -n "$BRAND" ] || exit 0 # not a full-fidelity guest + + mkdir -p "$RUN" + held=$( [ -e "$ACTIVE" ] && cut -d: -f1 "$ACTIVE" ) + if [ -n "$held" ] && [ "$held" != "$DOMAIN" ]; then + logger -t vfio-cpuid "cpuid passthrough held by $held; $DOMAIN left unhardened (one guest at a time)" + exit 0 + fi + + printf '%s' "$BRAND" > "$PARAM/brand_string" + echo N > "$PARAM/cpuid_passthrough" # safe for the cold boot + printf '%s:%s' "$DOMAIN" "$BRAND" > "$ACTIVE" + logger -t vfio-cpuid "$DOMAIN start: brand='$BRAND' cpuid_passthrough=N, watcher armed" + + # transient systemd unit, not a bare background job: libvirt reaps anything left + # in the hook's own process tree; systemd-run escapes it + [ -x "$WATCH" ] && systemd-run --collect --quiet \ + --unit="vfio-cpuid-watch-${DOMAIN}" "$WATCH" "$DOMAIN" "$BRAND" + ;; + +release|stopped) + held=$( [ -e "$ACTIVE" ] && cut -d: -f1 "$ACTIVE" ) + [ "$held" = "$DOMAIN" ] || exit 0 + systemctl stop "vfio-cpuid-watch-${DOMAIN}.service" 2>/dev/null + echo N > "$PARAM/cpuid_passthrough" + rm -f "$ACTIVE" + logger -t vfio-cpuid "$DOMAIN $OPERATION: cpuid_passthrough=N" + ;; +esac + +exit 0 diff --git a/scripts/setup-vm.sh b/scripts/setup-vm.sh index f2d97d0..641d46b 100755 --- a/scripts/setup-vm.sh +++ b/scripts/setup-vm.sh @@ -814,12 +814,12 @@ if [ "$PROFILE" = full ]; then [ "$ok" = 1 ] && echo "patched QEMU and KVM modules both in place." if [ -n "$MODEL" ] && [ -f /sys/module/kvm_amd/parameters/cpuid_passthrough ]; then echo - echo "The TIMER check needs CPUID passthrough, which is off by default and must be" - echo "switched on AFTER the guest has booted (it hangs a booting guest). Once the" - echo "guest is up, on the host:" + echo "The TIMER check needs CPUID passthrough, which must be off while the guest cold" + echo "boots and on once it is up. Let the hook handle that around this guest:" + echo " sudo vm-native-cpuid enable $DOM" + echo "Or drive it by hand, after the guest has booted, on the host:" echo " echo '$MODEL' | sudo tee /sys/module/kvm_amd/parameters/brand_string" - echo " echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough" - echo "Switch it off again (echo N) before rebooting the guest." + echo " echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough # N again before the next boot" fi fi