Skip to content

Commit 0c55772

Browse files
dhsrivasdhsrivas
committed
Add guest boot sanity tests with passthrough device support
Add test and configurations to validate the following scenarios: 1. Launch guests in different guest interrupt modes: a. AVIC b. APIC c. x2APIC d. x2AVIC 2. Boot the above guests with vCPU counts: 512, 288, 256, 254, 128, 64 3. Boot guests with PCI passthrough devices. 4. Boot guests with emulated AMD IOMMU and emulated Intel IOMMU. Notes: 1. AVIC and APIC support fewer than 255 vCPUs. 2. x2APIC and x2AVIC modes require extended-apicid=on, or the presence of emulated AMD IOMMU / emulated Intel IOMMU, to support booting with >255 vCPUs. Signed-off-by: Dheeraj Kumar Srivastava <[email protected]>
1 parent 5997bad commit 0c55772

File tree

5 files changed

+382
-0
lines changed

5 files changed

+382
-0
lines changed

‎qemu/tests/cfg/emulated_amd_iommu.cfg‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
- emulated_amd_iommu:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
machine_type_extra_params = "kernel-irqchip=split"
14+
extra_params = " -global kvm-pit.lost_tick_policy=discard"
15+
vcpu_sockets = 1
16+
vcpu_dies = 1
17+
vcpu_threads = 1
18+
mem = 4G
19+
mode = "x2apic"
20+
variants:
21+
- no_passthrough:
22+
required_qemu = [8.2.90,)
23+
# TODO Add passthrough variant
24+
variants:
25+
- vcpu_512:
26+
smp_fixed = 512
27+
vcpu_maxcpus = 512
28+
vcpu_cores = 512
29+
- vcpu_288:
30+
smp_fixed = 288
31+
vcpu_maxcpus = 288
32+
vcpu_cores = 288
33+
- vcpu_256:
34+
smp_fixed = 256
35+
vcpu_maxcpus = 256
36+
vcpu_cores = 256
37+
- vcpu_254:
38+
smp_fixed = 254
39+
vcpu_maxcpus = 254
40+
vcpu_cores = 254
41+
- vcpu_128:
42+
smp_fixed = 128
43+
vcpu_maxcpus = 128
44+
vcpu_cores = 128
45+
- vcpu_64:
46+
smp_fixed = 64
47+
vcpu_maxcpus = 64
48+
vcpu_cores = 64
49+
variants:
50+
- x2apic_emul_amd_iommu:
51+
cpu_model_flags += ",+x2apic"
52+
extra_params += " -device amd-iommu,intremap=on,xtsup=on"
53+
kvm_probe_module_parameters = "avic=0"
54+
- x2avic_emul_amd_iommu:
55+
cpu_model_flags += ",+x2apic"
56+
extra_params += " -device amd-iommu,intremap=on,xtsup=on"
57+
kvm_probe_module_parameters = "avic=1"

‎qemu/tests/cfg/iommu_accelerated_guest_mode.cfg‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
- iommu_accelerated_guest_mode:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
machine_type_extra_params = "kernel-irqchip=split"
14+
extra_params = " -global kvm-pit.lost_tick_policy=discard"
15+
vcpu_sockets = 1
16+
vcpu_dies = 1
17+
vcpu_threads = 1
18+
mem = 4G
19+
mode = "x2apic"
20+
kvm_probe_module_parameters = "avic=1"
21+
# Update pci_device with host PCI device/s for passthrough to the guest
22+
# enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0"
23+
# Note: Without a PCI device input, the guest will still boot, but the
24+
# test won't exercise the host IOMMU.
25+
#pci_device = ""
26+
variants:
27+
- vcpu_512:
28+
smp_fixed = 512
29+
vcpu_maxcpus = 512
30+
vcpu_cores = 512
31+
- vcpu_288:
32+
smp_fixed = 288
33+
vcpu_maxcpus = 288
34+
vcpu_cores = 288
35+
- vcpu_256:
36+
smp_fixed = 256
37+
vcpu_maxcpus = 256
38+
vcpu_cores = 256
39+
- vcpu_254:
40+
smp_fixed = 254
41+
vcpu_maxcpus = 254
42+
vcpu_cores = 254
43+
- vcpu_128:
44+
smp_fixed = 128
45+
vcpu_maxcpus = 128
46+
vcpu_cores = 128
47+
- vcpu_64:
48+
smp_fixed = 64
49+
vcpu_maxcpus = 64
50+
vcpu_cores = 64
51+
variants:
52+
- avic:
53+
mode = "apic"
54+
cpu_model_flags += ",-x2apic"
55+
no vcpu_512, vcpu_288, vcpu_256
56+
- x2avic_ext_apic_id:
57+
cpu_model_flags += ",+x2apic"
58+
cpu_model_flags += ",kvm-msi-ext-dest-id=on"
59+
- x2avic_emul_intel_iommu:
60+
intel_iommu = yes
61+
iommu_eim = on
62+
iommu_intremap = on
63+
guest_iommu_option = pt
64+
iommu_caching_mode = on
65+
cpu_model_flags += ",+x2apic"

‎qemu/tests/cfg/iommu_guest_mode.cfg‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
- iommu_guest_mode:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
machine_type_extra_params = "kernel-irqchip=split"
14+
extra_params = ""
15+
vcpu_sockets = 1
16+
vcpu_dies = 1
17+
vcpu_threads = 1
18+
mem = 4G
19+
mode = "x2apic"
20+
kvm_probe_module_parameters = "avic=0"
21+
# Update pci_device with host PCI device/s for passthrough to the guest
22+
# enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0"
23+
# Note: Without a PCI device input, the guest will still boot, but the
24+
# test won't exercise the host IOMMU.
25+
#pci_device = ""
26+
variants:
27+
- vcpu_1024:
28+
smp_fixed = 1024
29+
vcpu_maxcpus = 1024
30+
vcpu_cores = 1024
31+
- vcpu_512:
32+
smp_fixed = 512
33+
vcpu_maxcpus = 512
34+
vcpu_cores = 512
35+
- vcpu_288:
36+
smp_fixed = 288
37+
vcpu_maxcpus = 288
38+
vcpu_cores = 288
39+
- vcpu_256:
40+
smp_fixed = 256
41+
vcpu_maxcpus = 256
42+
vcpu_cores = 256
43+
- vcpu_254:
44+
smp_fixed = 254
45+
vcpu_maxcpus = 254
46+
vcpu_cores = 254
47+
- vcpu_128:
48+
smp_fixed = 128
49+
vcpu_maxcpus = 128
50+
vcpu_cores = 128
51+
- vcpu_64:
52+
smp_fixed = 64
53+
vcpu_maxcpus = 64
54+
vcpu_cores = 64
55+
variants:
56+
- apic:
57+
mode = "apic"
58+
cpu_model_flags += ",-x2apic"
59+
no vcpu_1024, vcpu_512, vcpu_288, vcpu_256
60+
- x2apic_ext_apic_id:
61+
cpu_model_flags += ",+x2apic"
62+
cpu_model_flags += ",kvm-msi-ext-dest-id=on"
63+
- x2apic_emul_intel_iommu:
64+
intel_iommu = yes
65+
iommu_eim = on
66+
iommu_intremap = on
67+
guest_iommu_option = pt
68+
iommu_caching_mode = on
69+
cpu_model_flags += ",+x2apic"

‎qemu/tests/cfg/vfio_pci_passthrough.cfg‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
- vfio_pci_passthrough:
2+
type = qemu_pci_passthrough
3+
start_vm = no
4+
only Linux
5+
only x86_64
6+
only HostCpuVendor.amd
7+
pcie_extra_root_port = 0
8+
vt_ulimit_nofile = 65536
9+
usb_devices = ""
10+
auto_cpu_model = "no"
11+
cpu_model = "host"
12+
machine_type = "q35"
13+
extra_params = ""
14+
mem = 4G
15+
# Update pci_device with host PCI device/s for passthrough to the guest
16+
# enabling IOMMU functionality validation. eg. pci_device = "0000:01:00.0"
17+
# Note: Without a PCI device input, the guest will still boot, but the
18+
# test won't exercise the host IOMMU.
19+
#pci_device = ""

‎qemu/tests/qemu_pci_passthrough.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
"""
2+
Guest boot sanity test with passthrough device in different mode
3+
- Apic, X2apic, Avic, X2avic
4+
"""
5+
6+
import os
7+
from avocado.utils import process, dmesg, pci, linux_modules, genio
8+
from virttest import env_process
9+
10+
11+
def run(test, params, env): # pylint: disable=R0915
12+
"""
13+
Guest boot sanity test with passthrough device
14+
Steps:
15+
1. Launch a guest with a vfio pci passthrough device
16+
2. Verify if vfio pci passthrough of device to guest is successful
17+
and guest boots in expected mode - apic, avic, x2apic and x2avic.
18+
19+
:param test: QEMU test object.
20+
:param params: Dictionary with test parameters:
21+
- pci_device: Pci device to passthrough. Default: ""
22+
- mode: Defines guest modes - [apic, x2apic]. Default: x2apic.
23+
- kvm_probe_module_parameters: To enable/disable avic on host.
24+
Possible values ["avic=1", "avic=0"]
25+
- login_timeout: VM login timeout in seconds. Default: 240).
26+
:param env: Dictionary with test environment.
27+
:raises: cancel if
28+
1. At start of test, KVM module is not loaded.
29+
2. KVM, msr, vfio_pci is not built or loadable.
30+
3. pci_device != ""
31+
a. vfio_pci is not built or loadable.
32+
b. Interrupt remapping is not enabled on host. Required for
33+
passthrough.
34+
c. Pci device/s not found on host system.
35+
4. kvm_probe_module_parameters = "avic=1"
36+
a. Host doesnot support expected mode - avic or x2avic.
37+
b. Host doesnot have expected mode enabled for guest - avic
38+
or x2avic.
39+
fails if
40+
1. Pci device cannot be bind to vfio_pci module for passthrough.
41+
(pci_device != "")
42+
2. Unable to login to guest within login timeout.
43+
"""
44+
kvm_probe_module_parameters = params.get("kvm_probe_module_parameters", "")
45+
login_timeout = int(params.get("login_timeout", 240))
46+
pci_device = params.get("pci_device", "")
47+
mode = params.get("mode", "x2apic")
48+
driver_list = []
49+
session = None
50+
vm = None
51+
52+
try:
53+
54+
def check_avic_support():
55+
"""
56+
Check if system supports avic.
57+
"""
58+
cmd = "rdmsr -p 0 0xc00110dd --bitfield 13:13"
59+
out = process.run(cmd, sudo=True, shell=True).stdout_text.strip()
60+
if out == "0":
61+
test.cancel("System doesnot support avic")
62+
63+
def check_x2avic_support():
64+
"""
65+
check if system supports x2avic.
66+
"""
67+
cmd = "rdmsr -p 0 0xc00110dd --bitfield 18:18"
68+
out = process.run(cmd, sudo=True, shell=True).stdout_text.strip()
69+
if out == "0":
70+
test.cancel("System doesnot support x2avic")
71+
72+
def verify_avic_enablement(mode):
73+
"""
74+
Check AVIC and x2AVIC status in dmesg logs diff.
75+
76+
:param mode: Whether apic or x2apic.
77+
"""
78+
79+
# Check for the "avic enabled" in dmesg
80+
if not dmesg.check_kernel_logs("AVIC enabled"):
81+
test.cancel("AVIC not enabled after loading kvm_amd with avic=1")
82+
83+
# Check for the "x2avic enabled" only if the test is "x2apic"
84+
if (not dmesg.check_kernel_logs("x2AVIC enabled")) and (mode == "x2apic"):
85+
test.cancel("x2AVIC not enabled after loading kvm_amd with avic=1.")
86+
87+
def prepare_pci_passthrough():
88+
"""
89+
Validate IOMMU, Interrupt Remapping, and vfio-pci module availability
90+
"""
91+
92+
# Check if interrupt remapping is enabled on system
93+
if not dmesg.check_kernel_logs("AMD-Vi: Interrupt remapping enabled"):
94+
test.cancel("IOMMU interrupt remaping is not enabled")
95+
96+
# Check and load vfio-pci module
97+
linux_modules.configure_module("vfio-pci", "CONFIG_VFIO_PCI")
98+
99+
def guest_system_details(session):
100+
"""
101+
Collect guest system details
102+
103+
:param session: active guest login session
104+
"""
105+
test.log.debug("Debug: %s", session.cmd_output("cat /etc/os-release"))
106+
test.log.debug("Debug: %s", session.cmd_output("uname -a"))
107+
test.log.debug("Debug: %s", session.cmd_output("ls /boot/"))
108+
test.log.debug("Debug: %s", session.cmd_output("lspci -k"))
109+
test.log.debug("Debug: %s", session.cmd_output("lscpu"))
110+
test.log.debug("Debug: %s", session.cmd_output("lsblk"))
111+
test.log.debug("Debug: %s", session.cmd_output("df -h"))
112+
test.log.debug("Debug: %s", session.cmd_output("dmesg"))
113+
114+
# Check system support for avic or x2avic
115+
if kvm_probe_module_parameters == "avic=1":
116+
linux_modules.configure_module("msr", "CONFIG_X86_MSR")
117+
if mode == "apic":
118+
check_avic_support()
119+
if mode == "x2apic":
120+
check_x2avic_support()
121+
# Validate dmesg for avic and x2avic enablement
122+
verify_avic_enablement(mode)
123+
124+
# Passthrough device/s and validate if passthrough is successful
125+
if pci_device != "":
126+
# Perform pre-checks and prereq enablements before pci passthrough
127+
prepare_pci_passthrough()
128+
129+
# Prepare for pci passthrough
130+
for i in range(len(pci_device.split(" "))):
131+
# Check if device input is valid
132+
if pci_device.split(" ")[i] not in pci.get_pci_addresses():
133+
test.cancel(
134+
"Please provide full pci address of a valid pci device"
135+
)
136+
137+
driver_list.append(pci.get_driver(pci_device.split(" ")[i]))
138+
pci.attach_driver(pci_device.split(" ")[i], "vfio-pci")
139+
params[
140+
"extra_params"
141+
] += f" -device vfio-pci,host={pci_device.split(' ')[i]}"
142+
143+
params["start_vm"] = "yes"
144+
vm = env.get_vm(params["main_vm"])
145+
try:
146+
env_process.preprocess_vm(test, params, env, params.get("main_vm"))
147+
vm.verify_alive()
148+
except Exception as e:
149+
test.fail(f"Failed to create VM: {str(e)}")
150+
try:
151+
session = vm.wait_for_login(timeout=login_timeout)
152+
except Exception as e:
153+
test.fail(f"Failed to login VM: {str(e)}")
154+
vm.verify_kernel_crash()
155+
156+
# Collect guest system details
157+
guest_system_details(session)
158+
except ValueError as e:
159+
test.fail(f"{e}")
160+
finally:
161+
if session:
162+
session.close()
163+
if vm:
164+
vm.destroy()
165+
try:
166+
if pci_device != "":
167+
for i in range(len(pci_device.split(" "))):
168+
if pci_device.split(" ")[i] not in pci.get_pci_addresses():
169+
break
170+
pci.attach_driver(pci_device.split(" ")[i], driver_list[i])
171+
except ValueError as e:
172+
test.fail(f"Failed to reset devices after test: Reason {e}")

0 commit comments

Comments
 (0)