fix(cleanup): wait for the guest shutdown before forcing the VM off

roles/cleanup/defaults/main.yml

@@ -3,3 +3,7 @@
 cleanup_verify_boot: true
 cleanup_boot_timeout: 300
 cleanup_remove_on_failure: true
+
+# How long to let the guest finish the shutdown started in shutdown.yml before forcing it off
+cleanup_poweroff_retries: 24
+cleanup_poweroff_delay: 5

roles/cleanup/tasks/proxmox.yml

@@ -6,7 +6,34 @@
   module_defaults:
     community.proxmox.proxmox_disk: "{{ _proxmox_auth }}"
     community.proxmox.proxmox_kvm: "{{ _proxmox_auth_node }}"
+    community.proxmox.proxmox_vm_info: "{{ _proxmox_auth_node }}"
   block:
+    # shutdown.yml halts the guest without waiting for it, so stopping a VM Proxmox has
+    # already stopped errors out. Wait the shutdown out, and force off only a guest that
+    # ignored it. The disks come off afterwards, so they are never detached from a
+    # running VM.
+    - name: Wait for the installer environment to power off
+      community.proxmox.proxmox_vm_info:
+        vmid: "{{ system_cfg.id }}"
+      register: cleanup_proxmox_power
+      until: >-
+        ((cleanup_proxmox_power.proxmox_vms | default([]) | first | default({})).status
+         | default('')) == 'stopped'
+      retries: "{{ cleanup_poweroff_retries }}"
+      delay: "{{ cleanup_poweroff_delay }}"
+      failed_when: false
+      no_log: true
+
+    - name: Force the installer environment off when it did not shut down
+      when: >-
+        ((cleanup_proxmox_power.proxmox_vms | default([]) | first | default({})).status
+         | default('')) != 'stopped'
+      community.proxmox.proxmox_kvm:
+        vmid: "{{ system_cfg.id }}"
+        state: stopped
+        force: true
+      no_log: true
+
     - name: Cleanup Setup Disks
       community.proxmox.proxmox_disk:
         name: "{{ hostname }}"
@@ -21,13 +48,6 @@
       failed_when: false
       no_log: true
 
-    - name: Ensure the installer environment is powered off
-      community.proxmox.proxmox_kvm:
-        vmid: "{{ system_cfg.id }}"
-        state: stopped
-        force: true
-      no_log: true
-
     - name: Boot the installed OS
       community.proxmox.proxmox_kvm:
         vmid: "{{ system_cfg.id }}"

roles/cleanup/tasks/vmware.yml

@@ -5,13 +5,28 @@
   become: false
   module_defaults:
     community.vmware.vmware_guest: "{{ _vmware_auth }}"
+    community.vmware.vmware_guest_info: "{{ _vmware_auth }}"
     vmware.vmware.vm_powerstate: "{{ _vmware_auth }}"
   no_log: true
   block:
-    - name: Ensure the installer environment is powered off
+    # shutdown.yml halts the guest without waiting for it, so powering off a VM that
+    # vCenter has already stopped raises InvalidPowerState. Wait the shutdown out, and
+    # force off only a guest that ignored it.
+    - name: Wait for the installer environment to power off
+      community.vmware.vmware_guest_info:
+        name: "{{ hostname }}"
+      register: cleanup_vmware_power
+      until: (cleanup_vmware_power.instance.hw_power_status | default('') | lower) == 'poweredoff'
+      retries: "{{ cleanup_poweroff_retries }}"
+      delay: "{{ cleanup_poweroff_delay }}"
+      failed_when: false
+
+    - name: Force the installer environment off when it did not shut down
+      when: (cleanup_vmware_power.instance.hw_power_status | default('') | lower) != 'poweredoff'
       vmware.vmware.vm_powerstate:
         name: "{{ hostname }}"
         state: powered-off
+        force: true
 
     - name: Remove CD-ROM from VM in vCenter
       community.vmware.vmware_guest: