vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50         self._logger = logging.getLogger(__name__)
  51         self._logfile = os.path.join(
  52             S.getValue('LOG_DIR'),
  53             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  54         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  55         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  56         self._net1 = get_test_param('guest_nic1_name', None)
  57         if self._net1 == None:
  58             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  59         else:
  60             self._net1 = self._net1.split(',')[self._number]
  61         self._net2 = get_test_param('guest_nic2_name', None)
  62         if self._net2 == None:
  63             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  64         else:
  65             self._net2 = self._net2.split(',')[self._number]
  66
  67         # set guest loopback application based on VNF configuration
  68         # cli option take precedence to config file values
  69         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  70
  71         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
  72         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  73         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  74            not S.getValue('VNF').endswith('PciPassthrough'):
  75
  76             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  77                               self._testpmd_fwd_mode, 'io')
  78             self._testpmd_fwd_mode = 'io'
  79
  80         name = 'Client%d' % self._number
  81         vnc = ':%d' % self._number
  82         # don't use taskset to affinize main qemu process; It causes hangup
  83         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  84         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  85                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  86                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  87                      '-cpu', 'host,migratable=off',
  88                      '-drive', 'if=scsi,file=' +
  89                      S.getValue('GUEST_IMAGE')[self._number],
  90                      '-boot', 'c', '--enable-kvm',
  91                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  92                      '-object',
  93                      'memory-backend-file,id=mem,size=' +
  94                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  95                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  96                      '-numa', 'node,memdev=mem -mem-prealloc',
  97                      '-nographic', '-vnc', str(vnc), '-name', name,
  98                      '-snapshot', '-net none', '-no-reboot',
  99                      '-drive',
 100                      'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
 101                      S.getValue('GUEST_SHARE_DIR')[self._number],
 102                     ]
 103         self._configure_logging()
 104
 105     def _configure_logging(self):
 106         """
 107         Configure logging.
 108         """
 109         self.GuestCommandFilter.prefix = self._log_prefix
 110
 111         logger = logging.getLogger()
 112         cmd_logger = logging.FileHandler(
 113             filename=os.path.join(S.getValue('LOG_DIR'),
 114                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 115             str(self._number))
 116         cmd_logger.setLevel(logging.DEBUG)
 117         cmd_logger.addFilter(self.GuestCommandFilter())
 118         logger.addHandler(cmd_logger)
 119
 120     # startup/Shutdown
 121
 122     def start(self):
 123         """
 124         Start QEMU instance, login and prepare for commands.
 125         """
 126         super(IVnfQemu, self).start()
 127         if S.getValue('VNF_AFFINITIZATION_ON'):
 128             self._affinitize()
 129
 130         if self._timeout:
 131             self._config_guest_loopback()
 132
 133     def stop(self):
 134         """
 135         Stops VNF instance gracefully first.
 136         """
 137         try:
 138             # exit testpmd if needed
 139             if self._guest_loopback == 'testpmd':
 140                 self.execute_and_wait('stop', 120, "Done")
 141                 self.execute_and_wait('quit', 120, "[bB]ye")
 142
 143             # turn off VM
 144             self.execute_and_wait('poweroff', 120, "Power down")
 145
 146         except pexpect.TIMEOUT:
 147             self.kill()
 148
 149         # wait until qemu shutdowns
 150         self._logger.debug('Wait for QEMU to terminate')
 151         for dummy in range(30):
 152             time.sleep(1)
 153             if not self.is_running():
 154                 break
 155
 156         # just for case that graceful shutdown failed
 157         super(IVnfQemu, self).stop()
 158
 159     # helper functions
 160
 161     def _login(self, timeout=120):
 162         """
 163         Login to QEMU instance.
 164
 165         This can be used immediately after booting the machine, provided a
 166         sufficiently long ``timeout`` is given.
 167
 168         :param timeout: Timeout to wait for login to complete.
 169
 170         :returns: None
 171         """
 172         # if no timeout was set, we likely started QEMU without waiting for it
 173         # to boot. This being the case, we best check that it has finished
 174         # first.
 175         if not self._timeout:
 176             self._expect_process(timeout=timeout)
 177
 178         self._child.sendline(S.getValue('GUEST_USERNAME'))
 179         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 180         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 181
 182         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 183
 184     def send_and_pass(self, cmd, timeout=30):
 185         """
 186         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 187
 188         :param cmd: Command to send to guest.
 189         :param timeout: Time to wait for prompt before checking return code.
 190
 191         :returns: None
 192         """
 193         self.execute(cmd)
 194         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 195         self.execute('echo $?')
 196         self._child.expect('^0$', timeout=1)  # expect a 0
 197         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 198
 199     def _affinitize(self):
 200         """
 201         Affinitize the SMP cores of a QEMU instance.
 202
 203         This is a bit of a hack. The 'socat' utility is used to
 204         interact with the QEMU HMP. This is necessary due to the lack
 205         of QMP in older versions of QEMU, like v1.6.2. In future
 206         releases, this should be replaced with calls to libvirt or
 207         another Python-QEMU wrapper library.
 208
 209         :returns: None
 210         """
 211         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 212
 213         self._logger.info('Affinitizing guest...')
 214
 215         cur_locale = locale.getdefaultlocale()[1]
 216         proc = subprocess.Popen(
 217             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 218         output = subprocess.check_output(
 219             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 220             stdin=proc.stdout)
 221         proc.wait()
 222
 223         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 224             match = None
 225             for line in output.decode(cur_locale).split('\n'):
 226                 match = re.search(thread_id % cpu, line)
 227                 if match:
 228                     self._affinitize_pid(
 229                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 230                         match.group(1))
 231                     break
 232
 233             if not match:
 234                 self._logger.error('Failed to affinitize guest core #%d. Could'
 235                                    ' not parse tid.', cpu)
 236
 237     def _config_guest_loopback(self):
 238         """
 239         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 240         """
 241         if self._guest_loopback == 'testpmd':
 242             self._login()
 243             self._configure_testpmd()
 244         elif self._guest_loopback == 'l2fwd':
 245             self._login()
 246             self._configure_l2fwd()
 247         elif self._guest_loopback == 'linux_bridge':
 248             self._login()
 249             self._configure_linux_bridge()
 250         elif self._guest_loopback != 'buildin':
 251             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 252                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 253
 254     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 255         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 256
 257     def execute_and_wait(self, cmd, timeout=30,
 258                          prompt=S.getValue('GUEST_PROMPT')):
 259         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 260                                                prompt=prompt)
 261
 262     def _modify_dpdk_makefile(self):
 263         """
 264         Modifies DPDK makefile in Guest before compilation if needed
 265         """
 266         pass
 267
 268     def _configure_copy_sources(self, dirname):
 269         """
 270         Mount shared directory and copy DPDK and l2fwd sources
 271         """
 272         # mount shared directory
 273         self.execute_and_wait('umount /dev/sdb1')
 274         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 275         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 276         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 277                               S.getValue('OVS_DPDK_SHARE'))
 278         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 279         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 280                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 281         self.execute_and_wait('umount /dev/sdb1')
 282
 283     def _configure_disable_firewall(self):
 284         """
 285         Disable firewall in VM
 286         """
 287         for iptables in ['iptables', 'ip6tables']:
 288             # filter table
 289             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 290                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 291             # mangle table
 292             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 293                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 294             # nat table
 295             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 296                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 297
 298             # flush rules and delete chains created by user
 299             for table in ['filter', 'mangle', 'nat']:
 300                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 301                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 302
 303
 304     def _configure_testpmd(self):
 305         """
 306         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 307         """
 308         self._configure_copy_sources('DPDK')
 309         self._configure_disable_firewall()
 310
 311         # Guest images _should_ have 1024 hugepages by default,
 312         # but just in case:'''
 313         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 314
 315         # Mount hugepages
 316         self.execute_and_wait('mkdir -p /dev/hugepages')
 317         self.execute_and_wait(
 318             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 319
 320         # build and configure system for dpdk
 321         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 322                               '/DPDK')
 323         self.execute_and_wait('export CC=gcc')
 324         self.execute_and_wait('export RTE_SDK=' +
 325                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 326         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 327
 328         # modify makefile if needed
 329         self._modify_dpdk_makefile()
 330
 331         # disable network interfaces, so DPDK can take care of them
 332         self.execute_and_wait('ifdown ' + self._net1)
 333         self.execute_and_wait('ifdown ' + self._net2)
 334
 335         # build and insert igb_uio and rebind interfaces to it
 336         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 337                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 338         self.execute_and_wait('modprobe uio')
 339         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 340                               S.getValue('RTE_TARGET'))
 341         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 342         self.execute_and_wait(
 343             './tools/dpdk_nic_bind.py -u' ' ' +
 344             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 345             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 346         self.execute_and_wait(
 347             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 348             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 349             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 350         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 351
 352         # build and run 'test-pmd'
 353         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 354                               '/DPDK/app/test-pmd')
 355         self.execute_and_wait('make clean')
 356         self.execute_and_wait('make')
 357         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 358                               ' --burst=64 -i --txqflags=0xf00 ' +
 359                               '--disable-hw-vlan', 60, "Done")
 360         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 361         self.execute_and_wait('start', 20,
 362                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 363
 364     def _configure_l2fwd(self):
 365         """
 366         Configure VM to perform L2 forwarding between NICs by l2fwd module
 367         """
 368         self._configure_copy_sources('l2fwd')
 369         self._configure_disable_firewall()
 370
 371         # build and configure system for l2fwd
 372         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 373                               '/l2fwd')
 374         self.execute_and_wait('export CC=gcc')
 375
 376         self.execute_and_wait('make')
 377         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 378                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 379                               ' net2=' + self._net2)
 380
 381     def _configure_linux_bridge(self):
 382         """
 383         Configure VM to perform L2 forwarding between NICs by linux bridge
 384         """
 385         self._configure_disable_firewall()
 386
 387         self.execute('ip addr add ' +
 388                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
 389                      ' dev ' + self._net1)
 390         self.execute('ip link set dev ' + self._net1 + ' up')
 391
 392         self.execute('ip addr add ' +
 393                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
 394                      ' dev ' + self._net2)
 395         self.execute('ip link set dev ' + self._net2 + ' up')
 396
 397         # configure linux bridge
 398         self.execute('brctl addbr br0')
 399         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 400         self.execute('ip addr add ' +
 401                      S.getValue('VANILLA_BRIDGE_IP')[self._number] +
 402                      ' dev br0')
 403         self.execute('ip link set dev br0 up')
 404
 405         # Add the arp entries for the IXIA ports and the bridge you are using.
 406         # Use command line values if provided.
 407         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 408                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 409         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 410                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 411
 412         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 413
 414         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 415                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 416         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 417                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 418
 419         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 420
 421         # Enable forwarding
 422         self.execute('sysctl -w net.ipv4.ip_forward=1')
 423
 424         # Controls source route verification
 425         # 0 means no source validation
 426         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 427         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 428         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')