1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 from __future__ import absolute_import
25 import xml.etree.ElementTree as ET
26 from yardstick import ssh
27 from yardstick.network_services.utils import get_nsb_option
28 from yardstick.network_services.utils import provision_tool
29 from yardstick.benchmark.contexts.standalone import StandaloneContext
31 log = logging.getLogger(__name__)
36 <uuid>{random_uuid}</uuid>
37 <memory unit="KiB">102400</memory>
38 <currentMemory unit="KiB">102400</currentMemory>
42 <vcpu placement="static">20</vcpu>
44 <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type>
52 <cpu match="exact" mode="custom">
53 <model fallback="allow">SandyBridge</model>
54 <topology cores="10" sockets="1" threads="2" />
57 <timer name="rtc" tickpolicy="catchup" />
58 <timer name="pit" tickpolicy="delay" />
59 <timer name="hpet" present="no" />
61 <on_poweroff>destroy</on_poweroff>
62 <on_reboot>restart</on_reboot>
63 <on_crash>restart</on_crash>
65 <emulator>/usr/bin/kvm-spice</emulator>
66 <disk device="disk" type="file">
67 <driver name="qemu" type="qcow2" />
68 <source file="{vm_image}"/>
69 <target bus="virtio" dev="vda" />
70 <address bus="0x00" domain="0x0000"
71 function="0x0" slot="0x04" type="pci" />
73 <controller index="0" model="ich9-ehci1" type="usb">
74 <address bus="0x00" domain="0x0000"
75 function="0x7" slot="0x05" type="pci" />
77 <controller index="0" model="ich9-uhci1" type="usb">
78 <master startport="0" />
79 <address bus="0x00" domain="0x0000" function="0x0"
80 multifunction="on" slot="0x05" type="pci" />
82 <controller index="0" model="ich9-uhci2" type="usb">
83 <master startport="2" />
84 <address bus="0x00" domain="0x0000"
85 function="0x1" slot="0x05" type="pci" />
87 <controller index="0" model="ich9-uhci3" type="usb">
88 <master startport="4" />
89 <address bus="0x00" domain="0x0000"
90 function="0x2" slot="0x05" type="pci" />
92 <controller index="0" model="pci-root" type="pci" />
97 <target port="0" type="serial" />
99 <input bus="usb" type="tablet" />
100 <input bus="ps2" type="mouse" />
101 <input bus="ps2" type="keyboard" />
102 <graphics autoport="yes" listen="0.0.0.0" port="-1" type="vnc" />
104 <model heads="1" type="cirrus" vram="16384" />
105 <address bus="0x00" domain="0x0000"
106 function="0x0" slot="0x02" type="pci" />
108 <memballoon model="virtio">
109 <address bus="0x00" domain="0x0000"
110 function="0x0" slot="0x06" type="pci" />
112 <interface type="bridge">
113 <mac address="{mac_addr}" />
114 <source bridge="virbr0" />
121 class Sriov(StandaloneContext):
124 self.file_path = None
126 self.vm_deploy = False
128 self.first_run = True
129 self.dpdk_nic_bind = ""
138 self.parse_pod_and_get_data(self.file_path)
140 def parse_pod_and_get_data(self, file_path):
141 self.file_path = file_path
142 log.debug("parsing pod file: {0}".format(self.file_path))
144 with open(self.file_path) as stream:
145 cfg = yaml.load(stream)
147 log.error("File {0} does not exist".format(self.file_path))
150 self.sriov.extend([node for node in cfg["nodes"]
151 if node["role"] == "Sriov"])
152 self.user = self.sriov[0]['user']
153 self.ssh_ip = self.sriov[0]['ip']
154 if self.sriov[0]['auth_type'] == "password":
155 self.passwd = self.sriov[0]['password']
157 self.ssh_port = self.sriov[0]['ssh_port']
158 self.key_filename = self.sriov[0]['key_filename']
160 def ssh_remote_machine(self):
161 if self.sriov[0]['auth_type'] == "password":
162 self.connection = ssh.SSH(
165 password=self.passwd)
166 self.connection.wait()
168 if self.ssh_port is not None:
169 ssh_port = self.ssh_port
171 ssh_port = ssh.DEFAULT_PORT
172 self.connection = ssh.SSH(
176 key_filename=self.key_filename)
177 self.connection.wait()
178 self.dpdk_nic_bind = provision_tool(
180 os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py"))
182 def get_nic_details(self):
186 'pci': self.sriov[0]['phy_ports'],
187 'phy_driver': self.sriov[0]['phy_driver'],
188 'vf_macs': self.sriov[0]['vf_macs']
190 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
191 for i, _ in enumerate(nic_details['pci']):
192 err, out, _ = self.connection.execute(
193 "{dpdk_nic_bind} --force -b {driver} {port}".format(
194 dpdk_nic_bind=self.dpdk_nic_bind,
195 driver=self.sriov[0]['phy_driver'],
196 port=self.sriov[0]['phy_ports'][i]))
197 err, out, _ = self.connection.execute(
198 "lshw -c network -businfo | grep '{port}'".format(
199 port=self.sriov[0]['phy_ports'][i]))
201 err, out, _ = self.connection.execute(
202 "ip -s link show {interface}".format(
203 interface=out.split()[1]))
204 nic_details['interface'][i] = str(a)
205 log.info("{0}".format(nic_details))
208 def install_req_libs(self):
210 log.info("Installing required libraries...")
211 err, out, _ = self.connection.execute("apt-get update")
212 log.debug("{0}".format(out))
213 err, out, _ = self.connection.execute(
214 "apt-get -y install qemu-kvm libvirt-bin")
215 log.debug("{0}".format(out))
216 err, out, _ = self.connection.execute(
217 "apt-get -y install libvirt-dev bridge-utils numactl")
218 log.debug("{0}".format(out))
219 self.first_run = False
221 def configure_nics_for_sriov(self, host_driver, nic_details):
223 self.connection.execute(
224 "rmmod {0}".format(host_driver))[1].splitlines()
225 self.connection.execute(
226 "modprobe {0} num_vfs=1".format(host_driver))[1].splitlines()
227 nic_details['vf_pci'] = {}
228 for i in range(len(nic_details['pci'])):
229 self.connection.execute(
230 "echo 1 > /sys/bus/pci/devices/{0}/sriov_numvfs".format(
231 nic_details['pci'][i]))
232 err, out, _ = self.connection.execute(
233 "ip link set {interface} vf 0 mac {mac}".format(
234 interface=nic_details['interface'][i],
235 mac=nic_details['vf_macs'][i]))
237 vf_pci[i] = self.get_vf_datas(
239 nic_details['pci'][i],
240 nic_details['vf_macs'][i])
241 nic_details['vf_pci'][i] = vf_pci[i]
242 log.debug("NIC DETAILS : {0}".format(nic_details))
245 def setup_sriov_context(self, pcis, nic_details, host_driver):
246 blacklist = "/etc/modprobe.d/blacklist.conf"
248 # 1 : Blacklist the vf driver in /etc/modprobe.d/blacklist.conf
249 vfnic = "{0}vf".format(host_driver)
250 lines = self.read_from_file(blacklist)
251 if vfnic not in lines:
252 vfblacklist = "blacklist {vfnic}".format(vfnic=vfnic)
253 self.connection.execute(
254 "echo {vfblacklist} >> {blacklist}".format(
255 vfblacklist=vfblacklist,
256 blacklist=blacklist))
258 # 2 : modprobe host_driver with num_vfs
259 nic_details = self.configure_nics_for_sriov(host_driver, nic_details)
261 # 3: Setup vm_sriov.xml to launch VM
262 cfg_sriov = '/tmp/vm_sriov.xml'
263 mac = [0x00, 0x24, 0x81,
264 random.randint(0x00, 0x7f),
265 random.randint(0x00, 0xff),
266 random.randint(0x00, 0xff)]
267 mac_address = ':'.join(map(lambda x: "%02x" % x, mac))
268 vm_sriov_xml = VM_TEMPLATE.format(
269 random_uuid=uuid.uuid4(),
270 mac_addr=mac_address,
271 vm_image=self.sriov[0]["images"])
272 with open(cfg_sriov, 'w') as f:
273 f.write(vm_sriov_xml)
275 vf = nic_details['vf_pci']
276 for index in range(len(nic_details['vf_pci'])):
277 self.add_sriov_interface(
282 self.connection.execute(
283 "ifconfig {interface} up".format(
284 interface=nic_details['interface'][index]))
286 # 4: Create and start the VM
287 self.connection.put(cfg_sriov, cfg_sriov)
289 err, out = self.check_output("virsh list --name | grep -i vm1")
292 log.info("VM is already present")
294 # FIXME: launch through libvirt
295 log.info("virsh create ...")
296 err, out, _ = self.connection.execute(
297 "virsh create /tmp/vm_sriov.xml")
299 log.error("err : {0}".format(err))
300 log.error("{0}".format(_))
301 log.debug("out : {0}".format(out))
305 # 5: Tunning for better performace
307 self.connection.execute(
308 "echo 1 > /sys/module/kvm/parameters/"
309 "allow_unsafe_assigned_interrupts")
310 self.connection.execute(
311 "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
313 def add_sriov_interface(self, index, vf_pci, vfmac, xml):
315 pattern = "0000:(\d+):(\d+).(\d+)"
316 m = re.search(pattern, vf_pci, re.MULTILINE)
317 device = root.find('devices')
319 interface = ET.SubElement(device, 'interface')
320 interface.set('managed', 'yes')
321 interface.set('type', 'hostdev')
323 mac = ET.SubElement(interface, 'mac')
324 mac.set('address', vfmac)
325 source = ET.SubElement(interface, 'source')
327 addr = ET.SubElement(source, "address")
328 addr.set('domain', "0x0")
329 addr.set('bus', "{0}".format(m.group(1)))
330 addr.set('function', "{0}".format(m.group(3)))
331 addr.set('slot', "{0}".format(m.group(2)))
332 addr.set('type', "pci")
334 vf_pci = ET.SubElement(interface, 'address')
335 vf_pci.set('type', 'pci')
336 vf_pci.set('domain', '0x0000')
337 vf_pci.set('bus', '0x00')
338 vf_pci.set('slot', '0x0{0}'.format(index + 7))
339 vf_pci.set('function', '0x00')
343 # This is roughly compatible with check_output function in subprocess
344 # module which is only available in python 2.7
345 def check_output(self, cmd, stderr=None):
346 # Run a command and capture its output
347 err, out, _ = self.connection.execute(cmd)
350 def get_virtual_devices(self, pci):
352 err, extra_info = self.check_output(
353 "cat /sys/bus/pci/devices/{0}/virtfn0/uevent".format(pci))
354 pattern = "PCI_SLOT_NAME=(?P<name>[0-9:.\s.]+)"
355 m = re.search(pattern, extra_info, re.MULTILINE)
358 pf_vfs.update({pci: str(m.group(1).rstrip())})
359 log.info("pf_vfs : {0}".format(pf_vfs))
362 def get_vf_datas(self, key, value, vfmac):
364 pattern = "0000:(\d+):(\d+).(\d+)"
367 vfs = self.get_virtual_devices(value)
368 log.info("vfs: {0}".format(vfs))
369 for k, v in vfs.items():
370 m = re.search(pattern, k, re.MULTILINE)
371 m1 = re.search(pattern, value, re.MULTILINE)
372 if m.group(1) == m1.group(1):
373 vfret["vf_pci"] = str(v)
378 def read_from_file(self, filename):
380 with open(filename, 'r') as the_file:
381 data = the_file.read()
384 def write_to_file(self, filename, content):
385 with open(filename, 'w') as the_file:
386 the_file.write(content)
388 def pin_vcpu(self, pcis):
389 nodes = self.get_numa_nodes()
390 log.info("{0}".format(nodes))
391 num_nodes = len(nodes)
392 for i in range(0, 10):
393 self.connection.execute(
394 "virsh vcpupin vm1 {0} {1}".format(
395 i, nodes[str(num_nodes - 1)][i]))
397 def get_numa_nodes(self):
398 nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
400 for node_sysfs in nodes_sysfs:
401 num = os.path.basename(node_sysfs).replace("node", "")
402 with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
403 cpulist = cpulist_file.read().strip()
404 nodes[num] = self.split_cpu_list(cpulist)
405 log.info("nodes: {0}".format(nodes))
408 def split_cpu_list(self, cpu_list):
410 ranges = cpu_list.split(',')
411 bounds = ([int(b) for b in r.split('-')] for r in ranges)
413 (range(bound[0], bound[1] + 1 if len(bound) == 2
414 else bound[0] + 1) for bound in bounds)
416 return sorted(itertools.chain.from_iterable(range_objects))
420 def destroy_vm(self):
421 host_driver = self.sriov[0]["phy_driver"]
422 err, out = self.check_output("virsh list --name | grep -i vm1")
423 log.info("{0}".format(out))
425 self.connection.execute("virsh shutdown vm1")
426 self.connection.execute("virsh destroy vm1")
427 self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
428 self.check_output("modprobe {0}".format(host_driver))[
431 log.error("error : {0}".format(err))