feat(cpuid): automate the passthrough toggle around enabled guests

README.md

@@ -165,25 +165,32 @@ vm-native-verify
 
 `vm-native-verify` also prints whether CPUID passthrough is on, which is the last step.
 
-### 6. Switch on CPUID passthrough, after every guest boot
+### 6. CPUID passthrough, on after boot and off before the next one
 
 The `TIMER` check times the world switch on an intercepted `CPUID`, and the only way to stop
-paying it is to not exit. `vm-native-setup` prints the two lines for your declared SKU; on this
+paying it is to not exit. That switch has to be off while a guest cold-boots - a booting Windows
-host they are:
+enumerates CPUID bits KVM synthesises and hangs if they vanish half way through - and on once it
+is up. Let the libvirt hook handle that around a guest:
+
+```sh
+sudo vm-native-cpuid enable win11     # off for the cold boot, on once the guest is on the network, off on stop
+sudo vm-native-cpuid status           # what is enabled, and the live switch state
+sudo vm-native-cpuid disable win11    # stop automating this guest
+sudo vm-native-cpuid revert           # remove the hook and reset the switch
+```
+
+Or drive it by hand. `vm-native-setup` prints the two lines for your declared SKU; run them once
+the guest is up, never before, and switch it off (`echo N`) before the next boot:
 
 ```sh
 echo 'AMD Ryzen 7 7700X 8-Core Processor' | sudo tee /sys/module/kvm_amd/parameters/brand_string
 echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough
 ```
 
-Run them once the guest is up, never before: a booting Windows enumerates CPUID bits KVM
+The module applies it only to vCPU threads pinned to exactly one host CPU, so on an unpinned guest
-synthesises, and hangs if they vanish half way through. Switch it off again (`echo N`) before the
+it does nothing rather than something wrong. It is one switch and one brand string for the whole
-next boot. The module applies it only to vCPU threads pinned to exactly one host CPU, so on an
+host, so it serves one hardened guest at a time - the hook enforces that and leaves a second guest
-unpinned guest it does nothing rather than something wrong.
+alone.
-
-It is a module parameter, so it is one switch for the whole host: every pinned guest gets it, and
-a cold boot of any of them while it is on hits the race. With more than one such guest, switch it
-off before any of them boots and on again once they are all up.
 
 For the detection score, run VMAware in the guest **from the console session, not over SSH** -
 OpenSSH lands you in session 0, which is not where an interactive desktop session runs. See

docs/GUEST-SETUP.md

@@ -165,7 +165,7 @@ echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough
 vm-native-verify
 ```
 
-`vm-native-setup` printed those two lines with your SKU. The switch is host-wide: it applies to
+`vm-native-setup` printed those two lines with your SKU, or `sudo vm-native-cpuid enable <domain>` does the same on/off automatically around the guest (off for the cold boot, on once its network is up). The switch is host-wide: it applies to
 every pinned guest at once, and a cold boot of any of them while it is on hits the enumeration
 race, so with several guests turn it off before any boots and on when they are all up. Then the
 scanner, from the console session, as `docs/TESTING.md` describes. The reference guest reads 1/85.

packaging/vfio-native/.SRCINFO

@@ -1,6 +1,6 @@
 pkgbase = vfio-native
 	pkgdesc = Present a libvirt guest as a self-consistent physical machine, and tune it
-	pkgver = 1.1.0
+	pkgver = 1.1.1
 	pkgrel = 1
 	url = https://git.archworks.co/sandwich/vfio-native
 	install = vfio-native.install
@@ -17,7 +17,7 @@ pkgbase = vfio-native
 	optdepends = cpupower: set the host CPU governor
 	optdepends = vfio-native-kvm-dkms: patched KVM modules for the full level
 	optdepends = vfio-native-qemu: patched QEMU for the full level
-	source = git+https://git.archworks.co/sandwich/vfio-native.git#tag=v1.1.0
+	source = git+https://git.archworks.co/sandwich/vfio-native.git#tag=v1.1.1
 	sha256sums = SKIP
 
 pkgname = vfio-native

packaging/vfio-native/PKGBUILD

@@ -8,7 +8,7 @@
 #   vfio-native-qemu       QEMU 11.1.1 with the platform-identity patches, in /opt
 
 pkgname=vfio-native
-pkgver=1.1.0
+pkgver=1.1.1
 pkgrel=1
 pkgdesc="Present a libvirt guest as a self-consistent physical machine, and tune it"
 arch=('any')
@@ -29,6 +29,7 @@ package() {
     install -Dm755 scripts/setup-vm.sh        "${pkgdir}/usr/bin/vm-native-setup"
     install -Dm755 scripts/verify-perf.sh     "${pkgdir}/usr/bin/vm-native-verify"
     install -Dm755 scripts/gpu-passthrough.sh "${pkgdir}/usr/bin/vm-native-gpu"
+    install -Dm755 scripts/cpuid-passthrough.sh "${pkgdir}/usr/bin/vm-native-cpuid"
 
     local share="${pkgdir}/usr/share/vfio-native"
     install -Dm644 bench/vmbench.c    "${share}/bench/vmbench.c"
@@ -36,6 +37,8 @@ package() {
     install -Dm755 scripts/install-modules.sh   "${share}/scripts/install-modules.sh"
     install -Dm755 scripts/restore-stock-kvm.sh "${share}/scripts/restore-stock-kvm.sh"
     install -Dm755 scripts/generate-tables.py   "${share}/scripts/generate-tables.py"
+    install -Dm755 scripts/libvirt-hook-cpuid-passthrough.sh "${share}/scripts/libvirt-hook-cpuid-passthrough.sh"
+    install -Dm755 scripts/cpuid-passthrough-watch          "${share}/scripts/cpuid-passthrough-watch"
     install -Dm644 -t "${share}/acpi" acpi/*.aml acpi/*.dsl
     install -dm755 "${share}/patches"
     cp -r patches/. "${share}/patches/"

scripts/cpuid-passthrough-watch

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Waits for a just-started guest's network to actually come up, then turns
+# kvm_amd cpuid_passthrough on.
+#
+# The readiness signal is the guest's own tap RX counter: it is fresh every boot,
+# it needs nothing enabled inside the guest (no SSH, no RDP, no agent), and it only
+# moves once the guest's NIC driver has really loaded - which is well past the CPU
+# enumeration that the switch must not change under. A DHCP lease left over from a
+# previous boot cannot trip it early.
+#
+# Launched as a transient systemd unit by the cpuid-passthrough hook, so it is free
+# to call virsh (the hook itself must not - that deadlocks libvirtd).
+
+DOMAIN=$1
+BRAND=$2
+PARAM=/sys/module/kvm_amd/parameters
+V="virsh -c qemu:///system"
+
+running() { [ "$($V domstate "$DOMAIN" 2>/dev/null)" = running ]; }
+
+tap=""
+i=0
+for _ in $(seq 1 65); do                  # ~195 s cap, then flip anyway if still up
+    i=$((i + 1))
+    running || { sleep 3; continue; }     # not "running" yet at prepare time - wait
+    [ -z "$tap" ] && tap=$($V domiflist "$DOMAIN" 2>/dev/null |
+                           awk '$1 ~ /^(vnet|tap|macvtap)/ {print $1; exit}')
+    rx="/sys/class/net/$tap/statistics/rx_packets"
+    # the iteration floor keeps a stray pre-OS packet (a UEFI netboot attempt) from
+    # tripping the flip before the guest is even past its interrupt and timer setup
+    if [ "$i" -ge 4 ] && [ -n "$tap" ] && [ -r "$rx" ] &&
+       [ "$(cat "$rx" 2>/dev/null || echo 0)" -ge 4 ]; then
+        break
+    fi
+    sleep 3
+done
+
+running || exit 0                         # guest went away before it came up
+printf '%s' "$BRAND" > "$PARAM/brand_string"
+echo Y > "$PARAM/cpuid_passthrough"
+logger -t vfio-cpuid "$DOMAIN network up: cpuid_passthrough=Y"

scripts/cpuid-passthrough.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# vm-native-cpuid - manage the automatic cpuid_passthrough toggle for hardened guests.
+#
+#   vm-native-cpuid enable <domain>    apply the automatic cpuid passthrough to <domain>
+#   vm-native-cpuid disable <domain>   stop applying it to <domain>
+#   vm-native-cpuid status             show the config and the live module state
+#   vm-native-cpuid revert             remove the hook, watcher and config; reset the module
+#
+# The switch and brand string are one global kvm_amd parameter, so it serves one
+# hardened guest at a time. It is off until you enable it here - no domain is
+# touched otherwise. Around an enabled guest the libvirt hook forces the switch off
+# for the cold boot, a watcher flips it on once the guest's network is up, and it
+# goes off again on stop.
+
+set -uo pipefail
+
+CONF=/etc/vfio-native/cpuid-passthrough.conf
+HOOK=/etc/libvirt/hooks/qemu.d/30-cpuid-passthrough.sh
+WATCH=/usr/lib/vfio-native/cpuid-passthrough-watch
+PARAM=/sys/module/kvm_amd/parameters
+SELF=$(cd "$(dirname "$0")/.." 2>/dev/null && pwd)
+
+die() { echo "$*" >&2; exit 1; }
+need_root() { [ "$(id -u)" = 0 ] || die "run this as root"; }
+
+load() { ENABLED=yes; DOMAINS=""; [ -r "$CONF" ] && . "$CONF"; }
+save() { mkdir -p "$(dirname "$CONF")"; printf 'ENABLED=%s\nDOMAINS="%s"\n' "$ENABLED" "$DOMAINS" > "$CONF"; }
+
+# copy the hook and watcher into place from the installed tree or this checkout
+install_hook() {
+    local src=""
+    for d in /usr/share/vfio-native "$SELF"; do
+        [ -f "$d/scripts/libvirt-hook-cpuid-passthrough.sh" ] && src="$d"
+    done
+    [ -n "$src" ] || die "cannot find the hook source (install vfio-native, or run from a checkout)"
+    install -Dm755 "$src/scripts/libvirt-hook-cpuid-passthrough.sh" "$HOOK"
+    install -Dm755 "$src/scripts/cpuid-passthrough-watch"          "$WATCH"
+}
+
+case "${1:-}" in
+enable)
+    need_root; [ -n "${2:-}" ] || die "usage: vm-native-cpuid enable <domain>"
+    load; install_hook; ENABLED=yes
+    case " $DOMAINS " in *" $2 "*) ;; *) DOMAINS="${DOMAINS:+$DOMAINS }$2" ;; esac
+    save
+    echo "$2 enabled. cpuid passthrough is now automatic for: $DOMAINS"
+    echo "on start it forces N for the cold boot, flips Y once the guest is on the network, N again on stop."
+    ;;
+disable)
+    need_root; [ -n "${2:-}" ] || die "usage: vm-native-cpuid disable <domain>"
+    load; DOMAINS=$(printf ' %s ' "$DOMAINS" | sed "s/ $2 / /g" | xargs || true); save
+    systemctl stop "vfio-cpuid-watch-$2.service" 2>/dev/null || true
+    echo "$2 disabled. remaining: ${DOMAINS:-none}"
+    ;;
+status)
+    load
+    echo "config:  $CONF"
+    echo "enabled: $ENABLED"
+    echo "domains: ${DOMAINS:-none}"
+    echo "hook:    $([ -f "$HOOK" ] && echo installed || echo 'not installed')"
+    if [ -r "$PARAM/cpuid_passthrough" ]; then
+        echo "module:  cpuid_passthrough=$(cat "$PARAM/cpuid_passthrough") brand='$(cat "$PARAM/brand_string")'"
+    else
+        echo "module:  patched kvm_amd not loaded"
+    fi
+    ;;
+revert)
+    need_root; load
+    for d in $DOMAINS; do systemctl stop "vfio-cpuid-watch-$d.service" 2>/dev/null || true; done
+    rm -f "$HOOK" "$WATCH" "$CONF" /run/vfio-native/cpuid-active
+    [ -w "$PARAM/cpuid_passthrough" ] && echo N > "$PARAM/cpuid_passthrough"
+    echo "reverted: hook, watcher and config removed; cpuid_passthrough=N."
+    ;;
+*)
+    sed -n '2,13p' "$0" | sed 's/^#\( \|$\)//'
+    ;;
+esac

scripts/libvirt-hook-cpuid-passthrough.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# libvirt qemu hook: drive the kvm_amd cpuid_passthrough switch around a guest's life.
+#
+# The switch must be OFF while a guest cold-boots - raw CPUID changes shape mid
+# enumeration and Windows hangs - and ON once the guest is up, where it clears the
+# TIMER detection. This applies only to the domains you opt in with
+# `vm-native-cpuid enable <domain>`; it does nothing to any other guest.
+#
+# On start it sets the brand string from the guest's declared SKU and forces N,
+# then a detached watcher flips Y once the guest's network is up; on stop it sets N.
+# cpuid_passthrough and brand_string are one global kvm_amd parameter, so a single
+# hardened guest is served at a time; a second is logged and left alone.
+#
+# Exits 0 on every path. The hook never calls virsh (that deadlocks libvirtd); the
+# watcher that does is a transient systemd unit and runs after this returns.
+
+CONF=/etc/vfio-native/cpuid-passthrough.conf
+ENABLED=yes
+DOMAINS=""
+[ -r "$CONF" ] && . "$CONF"
+[ "$ENABLED" = yes ] || exit 0
+
+PARAM=/sys/module/kvm_amd/parameters
+[ -w "$PARAM/cpuid_passthrough" ] || exit 0    # stock module, or not an AMD host
+
+DOMAIN=$1
+OPERATION=$2
+case " $DOMAINS " in *" $DOMAIN "*) ;; *) exit 0 ;; esac    # not an opted-in domain
+
+RUN=/run/vfio-native
+ACTIVE=$RUN/cpuid-active
+WATCH=/usr/lib/vfio-native/cpuid-passthrough-watch
+
+case "$OPERATION" in
+prepare)
+    # the guest's declared SKU, straight from the -cpu model-id in the XML on stdin
+    BRAND=$(grep -oE "model-id=[^'\"]+" | head -1 | sed 's/^model-id=//')
+    [ -n "$BRAND" ] || exit 0                  # not a full-fidelity guest
+
+    mkdir -p "$RUN"
+    held=$( [ -e "$ACTIVE" ] && cut -d: -f1 "$ACTIVE" )
+    if [ -n "$held" ] && [ "$held" != "$DOMAIN" ]; then
+        logger -t vfio-cpuid "cpuid passthrough held by $held; $DOMAIN left unhardened (one guest at a time)"
+        exit 0
+    fi
+
+    printf '%s' "$BRAND" > "$PARAM/brand_string"
+    echo N > "$PARAM/cpuid_passthrough"         # safe for the cold boot
+    printf '%s:%s' "$DOMAIN" "$BRAND" > "$ACTIVE"
+    logger -t vfio-cpuid "$DOMAIN start: brand='$BRAND' cpuid_passthrough=N, watcher armed"
+
+    # transient systemd unit, not a bare background job: libvirt reaps anything left
+    # in the hook's own process tree; systemd-run escapes it
+    [ -x "$WATCH" ] && systemd-run --collect --quiet \
+        --unit="vfio-cpuid-watch-${DOMAIN}" "$WATCH" "$DOMAIN" "$BRAND"
+    ;;
+
+release|stopped)
+    held=$( [ -e "$ACTIVE" ] && cut -d: -f1 "$ACTIVE" )
+    [ "$held" = "$DOMAIN" ] || exit 0
+    systemctl stop "vfio-cpuid-watch-${DOMAIN}.service" 2>/dev/null
+    echo N > "$PARAM/cpuid_passthrough"
+    rm -f "$ACTIVE"
+    logger -t vfio-cpuid "$DOMAIN $OPERATION: cpuid_passthrough=N"
+    ;;
+esac
+
+exit 0

scripts/setup-vm.sh

@@ -814,12 +814,12 @@ if [ "$PROFILE" = full ]; then
     [ "$ok" = 1 ] && echo "patched QEMU and KVM modules both in place."
     if [ -n "$MODEL" ] && [ -f /sys/module/kvm_amd/parameters/cpuid_passthrough ]; then
         echo
-        echo "The TIMER check needs CPUID passthrough, which is off by default and must be"
+        echo "The TIMER check needs CPUID passthrough, which must be off while the guest cold"
-        echo "switched on AFTER the guest has booted (it hangs a booting guest). Once the"
+        echo "boots and on once it is up. Let the hook handle that around this guest:"
-        echo "guest is up, on the host:"
+        echo "  sudo vm-native-cpuid enable $DOM"
+        echo "Or drive it by hand, after the guest has booted, on the host:"
         echo "  echo '$MODEL' | sudo tee /sys/module/kvm_amd/parameters/brand_string"
-        echo "  echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough"
+        echo "  echo Y | sudo tee /sys/module/kvm_amd/parameters/cpuid_passthrough   # N again before the next boot"
-        echo "Switch it off again (echo N) before rebooting the guest."
     fi
 fi