1 # Copyright (c) 2016-2017 Intel Corporation
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
15 from __future__ import absolute_import
22 from yardstick import ssh
23 from yardstick.benchmark.contexts.standalone import StandaloneContext
25 BIN_PATH = "/opt/isb_bin/"
26 DPDK_NIC_BIND = "dpdk_nic_bind.py"
28 log = logging.getLogger(__name__)
33 <uuid>18230c0c-635d-4c50-b2dc-a213d30acb34</uuid>
34 <memory unit='KiB'>20971520</memory>
35 <currentMemory unit="KiB">20971520</currentMemory>
39 <vcpu placement='static'>20</vcpu>
41 <type arch='x86_64' machine='pc'>hvm</type>
48 <cpu match="exact" mode='host-model'>
49 <model fallback='allow'/>
50 <topology sockets='1' cores='10' threads='2'/>
52 <on_poweroff>destroy</on_poweroff>
53 <on_reboot>restart</on_reboot>
54 <on_crash>destroy</on_crash>
56 <emulator>/usr/bin/qemu-system-x86_64</emulator>
57 <disk type='file' device='disk'>
58 <driver name='qemu' type='qcow2' cache='none'/>
59 <source file="{vm_image}"/>
60 <target dev='vda' bus='virtio'/>
61 <address bus="0x00" domain="0x0000"
62 function="0x0" slot="0x04" type="pci" />
64 <!--disk type='dir' device='disk'>
65 <driver name='qemu' type='fat'/>
66 <source dir='/opt/isb_bin/dpdk'/>
67 <target dev='vdb' bus='virtio'/>
70 <interface type="bridge">
71 <mac address="00:00:00:ab:cd:ef" />
72 <source bridge="br-int" />
74 <interface type='vhostuser'>
75 <mac address='00:00:00:00:00:01'/>
76 <source type='unix' path='/usr/local/var/run/openvswitch/dpdkvhostuser0' mode='client'/>
77 <model type='virtio'/>
79 <host mrg_rxbuf='off'/>
82 <interface type='vhostuser'>
83 <mac address='00:00:00:00:00:02'/>
84 <source type='unix' path='/usr/local/var/run/openvswitch/dpdkvhostuser1' mode='client'/>
85 <model type='virtio'/>
87 <host mrg_rxbuf='off'/>
94 <target type='serial' port='0'/>
96 <graphics autoport="yes" listen="0.0.0.0" port="1" type="vnc" />
102 class Ovsdpdk(StandaloneContext):
105 self.file_path = None
107 self.vm_deploy = False
109 self.first_run = True
110 self.dpdk_nic_bind = BIN_PATH + DPDK_NIC_BIND
118 '''initializes itself'''
120 self.parse_pod_and_get_data()
122 def parse_pod_and_get_data(self, file_path):
123 self.file_path = file_path
124 print("parsing pod file: {0}".format(self.file_path))
126 with open(self.file_path) as stream:
127 cfg = yaml.load(stream)
129 print("File {0} does not exist".format(self.file_path))
132 self.ovs.extend([node for node in cfg["nodes"]
133 if node["role"] == "Ovsdpdk"])
134 self.user = self.ovs[0]['user']
135 self.ssh_ip = self.ovs[0]['ip']
136 if self.ovs[0]['auth_type'] == "password":
137 self.passwd = self.ovs[0]['password']
139 self.ssh_port = self.ovs[0]['ssh_port']
140 self.key_filename = self.ovs[0]['key_filename']
142 def ssh_remote_machine(self):
143 if self.ovs[0]['auth_type'] == "password":
144 self.connection = ssh.SSH(
147 password=self.passwd)
148 self.connection.wait()
150 if self.ssh_port is not None:
151 ssh_port = self.ssh_port
153 ssh_port = ssh.DEFAULT_PORT
154 self.connection = ssh.SSH(
158 key_filename=self.key_filename)
159 self.connection.wait()
161 def get_nic_details(self):
163 nic_details['interface'] = {}
164 nic_details['pci'] = self.ovs[0]['phy_ports']
165 nic_details['phy_driver'] = self.ovs[0]['phy_driver']
166 nic_details['vports_mac'] = self.ovs[0]['vports_mac']
167 # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
168 for i, _ in enumerate(nic_details['pci']):
169 err, out, _ = self.connection.execute(
170 "{dpdk_nic_bind} --force -b {driver} {port}".format(
171 dpdk_nic_bind=self.dpdk_nic_bind,
172 driver=self.ovs[0]['phy_driver'],
173 port=self.ovs[0]['phy_ports'][i]))
174 err, out, _ = self.connection.execute(
175 "lshw -c network -businfo | grep '{port}'".format(
176 port=self.ovs[0]['phy_ports'][i]))
178 err, out, _ = self.connection.execute(
179 "ip -s link show {interface}".format(
180 interface=out.split()[1]))
181 nic_details['interface'][i] = str(a)
182 print("{0}".format(nic_details))
185 def install_req_libs(self):
187 err, out, _ = self.connection.execute("apt-get update")
188 print("{0}".format(out))
189 err, out, _ = self.connection.execute(
190 "apt-get -y install qemu-kvm libvirt-bin")
191 print("{0}".format(out))
192 err, out, _ = self.connection.execute(
193 "apt-get -y install libvirt-dev bridge-utils numactl")
194 print("{0}".format(out))
195 self.first_run = False
197 def setup_ovs(self, vpcis):
198 self.connection.execute("/usr/bin/chmod 0666 /dev/vfio/*")
199 self.connection.execute("/usr/bin/chmod a+x /dev/vfio")
200 self.connection.execute("pkill -9 ovs")
201 self.connection.execute("ps -ef | grep ovs | grep -v grep | "
202 "awk '{print $2}' | xargs -r kill -9")
203 self.connection.execute("killall -r 'ovs*'")
204 self.connection.execute(
205 "mkdir -p {0}/etc/openvswitch".format(self.ovs[0]["vpath"]))
206 self.connection.execute(
207 "mkdir -p {0}/var/run/openvswitch".format(self.ovs[0]["vpath"]))
208 self.connection.execute(
209 "rm {0}/etc/openvswitch/conf.db".format(self.ovs[0]["vpath"]))
210 self.connection.execute(
211 "ovsdb-tool create {0}/etc/openvswitch/conf.db "
212 "{0}/share/openvswitch/"
213 "vswitch.ovsschema".format(self.ovs[0]["vpath"]))
214 self.connection.execute("modprobe vfio-pci")
215 self.connection.execute("chmod a+x /dev/vfio")
216 self.connection.execute("chmod 0666 /dev/vfio/*")
218 self.connection.execute(
219 "/opt/isb_bin/dpdk_nic_bind.py "
220 "--bind=vfio-pci {0}".format(vpci))
222 def start_ovs_serverswitch(self):
223 self.connection.execute("mkdir -p /usr/local/var/run/openvswitch")
224 self.connection.execute(
225 "ovsdb-server --remote=punix:"
226 "/usr/local/var/run/openvswitch/db.sock --pidfile --detach")
227 self.connection.execute(
228 "ovs-vsctl --no-wait set "
229 "Open_vSwitch . other_config:dpdk-init=true")
230 self.connection.execute(
231 "ovs-vsctl --no-wait set "
232 "Open_vSwitch . other_config:dpdk-lcore-mask=0x3")
233 self.connection.execute(
234 "ovs-vsctl --no-wait set "
235 "Open_vSwitch . other_config:dpdk-socket-mem='2048,0'")
236 self.connection.execute(
237 "ovs-vswitchd unix:{0}/"
238 "var/run/openvswitch/db.sock --pidfile --detach "
239 "--log-file=/var/log/openvswitch/"
240 "ovs-vswitchd.log".format(
241 self.ovs[0]["vpath"]))
242 self.connection.execute(
243 "ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=2C")
245 def setup_ovs_bridge(self):
246 self.connection.execute("ovs-vsctl del-br br0")
247 self.connection.execute(
248 "rm -rf /usr/local/var/run/openvswitch/dpdkvhostuser*")
249 self.connection.execute(
250 "ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev")
251 self.connection.execute(
252 "ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk")
253 self.connection.execute(
254 "ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk")
255 self.connection.execute(
256 "ovs-vsctl add-port br0 dpdkvhostuser0 -- set Interface "
257 "dpdkvhostuser0 type=dpdkvhostuser")
258 self.connection.execute("ovs-vsctl add-port br0 dpdkvhostuser1 "
259 "-- set Interface dpdkvhostuser1 "
260 "type=dpdkvhostuser")
261 self.connection.execute(
262 "chmod 0777 {0}/var/run/"
263 "openvswitch/dpdkvhostuser*".format(self.ovs[0]["vpath"]))
265 def add_oflows(self):
266 self.connection.execute("ovs-ofctl del-flows br0")
267 for flow in self.ovs[0]["flow"]:
268 self.connection.execute(flow)
269 self.connection.execute("ovs-ofctl dump-flows br0")
270 self.connection.execute(
271 "ovs-vsctl set Interface dpdk0 options:n_rxq=4")
272 self.connection.execute(
273 "ovs-vsctl set Interface dpdk1 options:n_rxq=4")
275 def setup_ovs_context(self, pcis, nic_details, host_driver):
277 ''' 1: Setup vm_ovs.xml to launch VM.'''
278 cfg_ovs = '/tmp/vm_ovs.xml'
279 vm_ovs_xml = VM_TEMPLATE.format(vm_image=self.ovs[0]["images"])
280 with open(cfg_ovs, 'w') as f:
283 ''' 2: Create and start the VM'''
284 self.connection.put(cfg_ovs, cfg_ovs)
286 err, out = self.check_output("virsh list --name | grep -i vm1")
288 print("VM is already present")
290 ''' FIXME: launch through libvirt'''
291 print("virsh create ...")
292 err, out, _ = self.connection.execute(
293 "virsh create /tmp/vm_ovs.xml")
295 print("err : {0}".format(err))
296 print("{0}".format(_))
297 print("out : {0}".format(out))
299 ''' 3: Tuning for better performace.'''
301 self.connection.execute(
302 "echo 1 > /sys/module/kvm/parameters/"
303 "allow_unsafe_assigned_interrupts")
304 self.connection.execute(
305 "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
306 print("After tuning performance ...")
308 ''' This is roughly compatible with check_output function in subprocess
309 module which is only available in python 2.7.'''
310 def check_output(self, cmd, stderr=None):
311 '''Run a command and capture its output'''
312 err, out, _ = self.connection.execute(cmd)
315 def read_from_file(self, filename):
317 with open(filename, 'r') as the_file:
318 data = the_file.read()
321 def write_to_file(self, filename, content):
322 with open(filename, 'w') as the_file:
323 the_file.write(content)
325 def pin_vcpu(self, pcis):
326 nodes = self.get_numa_nodes()
327 print("{0}".format(nodes))
328 num_nodes = len(nodes)
329 for i in range(0, 10):
330 self.connection.execute(
331 "virsh vcpupin vm1 {0} {1}".format(
332 i, nodes[str(num_nodes - 1)][i % len(nodes[str(num_nodes - 1)])]))
334 def get_numa_nodes(self):
335 nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
337 for node_sysfs in nodes_sysfs:
338 num = os.path.basename(node_sysfs).replace("node", "")
339 with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
340 cpulist = cpulist_file.read().strip()
341 print("cpulist: {0}".format(cpulist))
342 nodes[num] = self.split_cpu_list(cpulist)
343 print("nodes: {0}".format(nodes))
346 def split_cpu_list(self, cpu_list):
348 ranges = cpu_list.split(',')
349 bounds = ([int(b) for b in r.split('-')] for r in ranges)
351 (range(bound[0], bound[1] + 1 if len(bound) == 2
352 else bound[0] + 1) for bound in bounds)
354 return sorted(itertools.chain.from_iterable(range_objects))
358 def destroy_vm(self):
359 host_driver = self.ovs[0]['phy_driver']
360 err, out = self.check_output("virsh list --name | grep -i vm1")
361 print("{0}".format(out))
363 self.connection.execute("virsh shutdown vm1")
364 self.connection.execute("virsh destroy vm1")
365 self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
366 self.check_output("modprobe {0}".format(host_driver))[
369 print("error : ", err)