1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 from __future__ import absolute_import
25 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.network_services.utils import get_nsb_option
28 from yardstick.network_services.utils import provision_tool
29 from yardstick.benchmark.contexts.standalone import StandaloneContext
31 log = logging.getLogger(__name__)
36 <uuid>{random_uuid}</uuid>
37 <memory unit="KiB">102400</memory>
38 <currentMemory unit="KiB">102400</currentMemory>
42 <vcpu placement="static">20</vcpu>
44 <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
52 <cpu match="exact" mode="custom">
53 <model fallback="allow">SandyBridge</model>
54 <topology cores="10" sockets="1" threads="2" />
57 <timer name="rtc" tickpolicy="catchup" />
58 <timer name="pit" tickpolicy="delay" />
59 <timer name="hpet" present="no" />
61 <on_poweroff>destroy</on_poweroff>
62 <on_reboot>restart</on_reboot>
63 <on_crash>restart</on_crash>
65 <emulator>/usr/bin/kvm-spice</emulator>
66 <disk device="disk" type="file">
67 <driver name="qemu" type="qcow2" />
68 <source file="{vm_image}"/>
69 <target bus="virtio" dev="vda" />
70 <address bus="0x00" domain="0x0000"
71 function="0x0" slot="0x04" type="pci" />
73 <controller index="0" model="ich9-ehci1" type="usb">
74 <address bus="0x00" domain="0x0000"
75 function="0x7" slot="0x05" type="pci" />
77 <controller index="0" model="ich9-uhci1" type="usb">
78 <master startport="0" />
79 <address bus="0x00" domain="0x0000" function="0x0"
80 multifunction="on" slot="0x05" type="pci" />
82 <controller index="0" model="ich9-uhci2" type="usb">
83 <master startport="2" />
84 <address bus="0x00" domain="0x0000"
85 function="0x1" slot="0x05" type="pci" />
87 <controller index="0" model="ich9-uhci3" type="usb">
88 <master startport="4" />
89 <address bus="0x00" domain="0x0000"
90 function="0x2" slot="0x05" type="pci" />
92 <controller index="0" model="pci-root" type="pci" />
97 <target port="0" type="serial" />
99 <input bus="usb" type="tablet" />
100 <input bus="ps2" type="mouse" />
101 <input bus="ps2" type="keyboard" />
102 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
104 <model heads="1" type="cirrus" vram="16384" />
105 <address bus="0x00" domain="0x0000"
106 function="0x0" slot="0x02" type="pci" />
108 <memballoon model="virtio">
109 <address bus="0x00" domain="0x0000"
110 function="0x0" slot="0x06" type="pci" />
112 <interface type="bridge">
113 <mac address="{mac_addr}" />
114 <source bridge="virbr0" />
121 class Sriov(StandaloneContext):
124 self.file_path = None
126 self.vm_deploy = False
128 self.first_run = True
129 self.dpdk_nic_bind = ""
137 self.parse_pod_and_get_data(self.file_path)
139 def parse_pod_and_get_data(self, file_path):
140 self.file_path = file_path
141 log.debug("parsing pod file: {0}".format(self.file_path))
143 with open(self.file_path) as stream:
144 cfg = yaml.load(stream)
146 log.error("File {0} does not exist".format(self.file_path))
149 self.nodes.extend([node for node in cfg["nodes"]
150 if node["role"] != "Sriov"])
151 self.sriov.extend([node for node in cfg["nodes"]
152 if node["role"] == "Sriov"])
153 self.user = self.sriov[0]['user']
154 self.ssh_ip = self.sriov[0]['ip']
155 if self.sriov[0]['auth_type'] == "password":
156 self.passwd = self.sriov[0]['password']
158 self.ssh_port = self.sriov[0]['ssh_port']
159 self.key_filename = self.sriov[0]['key_filename']
161 def ssh_remote_machine(self):
162 if self.sriov[0]['auth_type'] == "password":
163 self.connection = ssh.SSH(
166 password=self.passwd)
167 self.connection.wait()
169 if self.ssh_port is not None:
170 ssh_port = self.ssh_port
172 ssh_port = ssh.DEFAULT_PORT
173 self.connection = ssh.SSH(
177 key_filename=self.key_filename)
178 self.connection.wait()
179 self.dpdk_nic_bind = provision_tool(
181 os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py"))
183 def get_nic_details(self):
187 'pci': self.sriov[0]['phy_ports'],
188 'phy_driver': self.sriov[0]['phy_driver'],
189 'vf_macs': self.sriov[0]['vf_macs']
191 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
192 for i, _ in enumerate(nic_details['pci']):
193 err, out, _ = self.connection.execute(
194 "{dpdk_nic_bind} --force -b {driver} {port}".format(
195 dpdk_nic_bind=self.dpdk_nic_bind,
196 driver=self.sriov[0]['phy_driver'],
197 port=self.sriov[0]['phy_ports'][i]))
198 err, out, _ = self.connection.execute(
199 "lshw -c network -businfo | grep '{port}'".format(
200 port=self.sriov[0]['phy_ports'][i]))
202 err, out, _ = self.connection.execute(
203 "ip -s link show {interface}".format(
204 interface=out.split()[1]))
205 nic_details['interface'][i] = str(a)
206 log.info("{0}".format(nic_details))
209 def install_req_libs(self):
211 log.info("Installing required libraries...")
212 err, out, _ = self.connection.execute("apt-get update")
213 log.debug("{0}".format(out))
214 err, out, _ = self.connection.execute(
215 "apt-get -y install qemu-kvm libvirt-bin")
216 log.debug("{0}".format(out))
217 err, out, _ = self.connection.execute(
218 "apt-get -y install libvirt-dev bridge-utils numactl")
219 log.debug("{0}".format(out))
220 self.first_run = False
222 def configure_nics_for_sriov(self, host_driver, nic_details):
224 self.connection.execute(
225 "rmmod {0}".format(host_driver))[1].splitlines()
226 self.connection.execute(
227 "modprobe {0} num_vfs=1".format(host_driver))[1].splitlines()
228 nic_details['vf_pci'] = {}
229 for i in range(len(nic_details['pci'])):
230 self.connection.execute(
231 "echo 1 > /sys/bus/pci/devices/{0}/sriov_numvfs".format(
232 nic_details['pci'][i]))
233 err, out, _ = self.connection.execute(
234 "ip link set {interface} vf 0 mac {mac}".format(
235 interface=nic_details['interface'][i],
236 mac=nic_details['vf_macs'][i]))
238 vf_pci[i] = self.get_vf_datas(
240 nic_details['pci'][i],
241 nic_details['vf_macs'][i])
242 nic_details['vf_pci'][i] = vf_pci[i]
243 log.debug("NIC DETAILS : {0}".format(nic_details))
246 def setup_sriov_context(self, pcis, nic_details, host_driver):
247 blacklist = "/etc/modprobe.d/blacklist.conf"
249 # 1 : Blacklist the vf driver in /etc/modprobe.d/blacklist.conf
250 vfnic = "{0}vf".format(host_driver)
251 lines = self.read_from_file(blacklist)
252 if vfnic not in lines:
253 vfblacklist = "blacklist {vfnic}".format(vfnic=vfnic)
254 self.connection.execute(
255 "echo {vfblacklist} >> {blacklist}".format(
256 vfblacklist=vfblacklist,
257 blacklist=blacklist))
259 # 2 : modprobe host_driver with num_vfs
260 nic_details = self.configure_nics_for_sriov(host_driver, nic_details)
262 # 3: Setup vm_sriov.xml to launch VM
263 cfg_sriov = '/tmp/vm_sriov.xml'
264 mac = [0x00, 0x24, 0x81,
265 random.randint(0x00, 0x7f),
266 random.randint(0x00, 0xff),
267 random.randint(0x00, 0xff)]
268 mac_address = ':'.join(map(lambda x: "%02x" % x, mac))
269 vm_sriov_xml = VM_TEMPLATE.format(
270 random_uuid=uuid.uuid4(),
271 mac_addr=mac_address,
272 vm_image=self.sriov[0]["images"])
273 with open(cfg_sriov, 'w') as f:
274 f.write(vm_sriov_xml)
276 vf = nic_details['vf_pci']
277 for index in range(len(nic_details['vf_pci'])):
278 self.add_sriov_interface(
283 self.connection.execute(
284 "ifconfig {interface} up".format(
285 interface=nic_details['interface'][index]))
287 # 4: Create and start the VM
288 self.connection.put(cfg_sriov, cfg_sriov)
290 err, out = self.check_output("virsh list --name | grep -i vm1")
293 log.info("VM is already present")
295 # FIXME: launch through libvirt
296 log.info("virsh create ...")
297 err, out, _ = self.connection.execute(
298 "virsh create /tmp/vm_sriov.xml")
300 log.error("err : {0}".format(err))
301 log.error("{0}".format(_))
302 log.debug("out : {0}".format(out))
306 # 5: Tunning for better performace
308 self.connection.execute(
309 "echo 1 > /sys/module/kvm/parameters/"
310 "allow_unsafe_assigned_interrupts")
311 self.connection.execute(
312 "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
314 def add_sriov_interface(self, index, vf_pci, vfmac, xml):
316 pattern = "0000:(\d+):(\d+).(\d+)"
317 m = re.search(pattern, vf_pci, re.MULTILINE)
318 device = root.find('devices')
320 interface = ET.SubElement(device, 'interface')
321 interface.set('managed', 'yes')
322 interface.set('type', 'hostdev')
324 mac = ET.SubElement(interface, 'mac')
325 mac.set('address', vfmac)
326 source = ET.SubElement(interface, 'source')
328 addr = ET.SubElement(source, "address")
329 addr.set('domain', "0x0")
330 addr.set('bus', "{0}".format(m.group(1)))
331 addr.set('function', "{0}".format(m.group(3)))
332 addr.set('slot', "{0}".format(m.group(2)))
333 addr.set('type', "pci")
335 vf_pci = ET.SubElement(interface, 'address')
336 vf_pci.set('type', 'pci')
337 vf_pci.set('domain', '0x0000')
338 vf_pci.set('bus', '0x00')
339 vf_pci.set('slot', '0x0{0}'.format(index + 7))
340 vf_pci.set('function', '0x00')
344 # This is roughly compatible with check_output function in subprocess
345 # module which is only available in python 2.7
346 def check_output(self, cmd, stderr=None):
347 # Run a command and capture its output
348 err, out, _ = self.connection.execute(cmd)
351 def get_virtual_devices(self, pci):
353 err, extra_info = self.check_output(
354 "cat /sys/bus/pci/devices/{0}/virtfn0/uevent".format(pci))
355 pattern = "PCI_SLOT_NAME=(?P<name>[0-9:.\s.]+)"
356 m = re.search(pattern, extra_info, re.MULTILINE)
359 pf_vfs.update({pci: str(m.group(1).rstrip())})
360 log.info("pf_vfs : {0}".format(pf_vfs))
363 def get_vf_datas(self, key, value, vfmac):
365 pattern = "0000:(\d+):(\d+).(\d+)"
368 vfs = self.get_virtual_devices(value)
369 log.info("vfs: {0}".format(vfs))
370 for k, v in vfs.items():
371 m = re.search(pattern, k, re.MULTILINE)
372 m1 = re.search(pattern, value, re.MULTILINE)
373 if m.group(1) == m1.group(1):
374 vfret["vf_pci"] = str(v)
379 def read_from_file(self, filename):
381 with open(filename, 'r') as the_file:
382 data = the_file.read()
385 def write_to_file(self, filename, content):
386 with open(filename, 'w') as the_file:
387 the_file.write(content)
389 def pin_vcpu(self, pcis):
390 nodes = self.get_numa_nodes()
391 log.info("{0}".format(nodes))
392 num_nodes = len(nodes)
393 for i in range(0, 10):
394 self.connection.execute(
395 "virsh vcpupin vm1 {0} {1}".format(
396 i, nodes[str(num_nodes - 1)][i]))
398 def get_numa_nodes(self):
399 nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
401 for node_sysfs in nodes_sysfs:
402 num = os.path.basename(node_sysfs).replace("node", "")
403 with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
404 cpulist = cpulist_file.read().strip()
405 nodes[num] = self.split_cpu_list(cpulist)
406 log.info("nodes: {0}".format(nodes))
409 def split_cpu_list(self, cpu_list):
411 ranges = cpu_list.split(',')
412 bounds = ([int(b) for b in r.split('-')] for r in ranges)
414 (range(bound[0], bound[1] + 1 if len(bound) == 2
415 else bound[0] + 1) for bound in bounds)
417 return sorted(itertools.chain.from_iterable(range_objects))
421 def destroy_vm(self):
422 host_driver = self.sriov[0]["phy_driver"]
423 err, out = self.check_output("virsh list --name | grep -i vm1")
424 log.info("{0}".format(out))
426 self.connection.execute("virsh shutdown vm1")
427 self.connection.execute("virsh destroy vm1")
428 self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
429 self.check_output("modprobe {0}".format(host_driver))[
432 log.error("error : {0}".format(err))