vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23
  24 from conf import settings as S
  25 from conf import get_test_param
  26 from vnfs.vnf.vnf import IVnf
  27
  28 class IVnfQemu(IVnf):
  29     """
  30     Abstract class for controling an instance of QEMU
  31     """
  32     _cmd = None
  33     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  34     _proc_name = 'qemu'
  35
  36     class GuestCommandFilter(logging.Filter):
  37         """
  38         Filter out strings beginning with 'guestcmd :'.
  39         """
  40         def filter(self, record):
  41             return record.getMessage().startswith(self.prefix)
  42
  43     def __init__(self):
  44         """
  45         Initialisation function.
  46         """
  47         super(IVnfQemu, self).__init__()
  48         self._logger = logging.getLogger(__name__)
  49         self._logfile = os.path.join(
  50             S.getValue('LOG_DIR'),
  51             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  52         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  53         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  54         self._net1 = get_test_param('guest_nic1_name', None)
  55         if self._net1 == None:
  56             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  57         else:
  58             self._net1 = self._net1.split(',')[self._number]
  59         self._net2 = get_test_param('guest_nic2_name', None)
  60         if self._net2 == None:
  61             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  62         else:
  63             self._net2 = self._net2.split(',')[self._number]
  64
  65         # set guest loopback application based on VNF configuration
  66         # cli option take precedence to config file values
  67         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  68
  69         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
  70         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  71         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  72            not S.getValue('VNF').endswith('PciPassthrough'):
  73
  74             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  75                               self._testpmd_fwd_mode, 'io')
  76             self._testpmd_fwd_mode = 'io'
  77
  78         name = 'Client%d' % self._number
  79         vnc = ':%d' % self._number
  80         # don't use taskset to affinize main qemu process; It causes hangup
  81         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  82         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  83                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  84                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  85                      '-cpu', 'host,migratable=off',
  86                      '-drive', 'if=scsi,file=' +
  87                      S.getValue('GUEST_IMAGE')[self._number],
  88                      '-boot', 'c', '--enable-kvm',
  89                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  90                      '-object',
  91                      'memory-backend-file,id=mem,size=' +
  92                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  93                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  94                      '-numa', 'node,memdev=mem -mem-prealloc',
  95                      '-nographic', '-vnc', str(vnc), '-name', name,
  96                      '-snapshot', '-net none', '-no-reboot',
  97                      '-drive',
  98                      'if=scsi,format=raw,file=fat:rw:%s,snapshot=off' %
  99                      S.getValue('GUEST_SHARE_DIR')[self._number],
 100                     ]
 101         self._configure_logging()
 102
 103     def _configure_logging(self):
 104         """
 105         Configure logging.
 106         """
 107         self.GuestCommandFilter.prefix = self._log_prefix
 108
 109         logger = logging.getLogger()
 110         cmd_logger = logging.FileHandler(
 111             filename=os.path.join(S.getValue('LOG_DIR'),
 112                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 113             str(self._number))
 114         cmd_logger.setLevel(logging.DEBUG)
 115         cmd_logger.addFilter(self.GuestCommandFilter())
 116         logger.addHandler(cmd_logger)
 117
 118     # startup/Shutdown
 119
 120     def start(self):
 121         """
 122         Start QEMU instance, login and prepare for commands.
 123         """
 124         super(IVnfQemu, self).start()
 125         if S.getValue('VNF_AFFINITIZATION_ON'):
 126             self._affinitize()
 127
 128         if self._timeout:
 129             self._config_guest_loopback()
 130
 131     def stop(self):
 132         """
 133         Stops VNF instance gracefully first.
 134         """
 135         # exit testpmd if needed
 136         if self._guest_loopback == 'testpmd':
 137             self.execute_and_wait('stop', 120, "Done")
 138             self.execute_and_wait('quit', 120, "bye")
 139
 140         # turn off VM
 141         self.execute_and_wait('poweroff', 120, "Power down")
 142
 143         # just for case that graceful shutdown failed
 144         super(IVnfQemu, self).stop()
 145
 146     # helper functions
 147
 148     def _login(self, timeout=120):
 149         """
 150         Login to QEMU instance.
 151
 152         This can be used immediately after booting the machine, provided a
 153         sufficiently long ``timeout`` is given.
 154
 155         :param timeout: Timeout to wait for login to complete.
 156
 157         :returns: None
 158         """
 159         # if no timeout was set, we likely started QEMU without waiting for it
 160         # to boot. This being the case, we best check that it has finished
 161         # first.
 162         if not self._timeout:
 163             self._expect_process(timeout=timeout)
 164
 165         self._child.sendline(S.getValue('GUEST_USERNAME'))
 166         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 167         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 168
 169         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 170
 171     def send_and_pass(self, cmd, timeout=30):
 172         """
 173         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 174
 175         :param cmd: Command to send to guest.
 176         :param timeout: Time to wait for prompt before checking return code.
 177
 178         :returns: None
 179         """
 180         self.execute(cmd)
 181         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 182         self.execute('echo $?')
 183         self._child.expect('^0$', timeout=1)  # expect a 0
 184         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 185
 186     def _affinitize(self):
 187         """
 188         Affinitize the SMP cores of a QEMU instance.
 189
 190         This is a bit of a hack. The 'socat' utility is used to
 191         interact with the QEMU HMP. This is necessary due to the lack
 192         of QMP in older versions of QEMU, like v1.6.2. In future
 193         releases, this should be replaced with calls to libvirt or
 194         another Python-QEMU wrapper library.
 195
 196         :returns: None
 197         """
 198         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 199
 200         self._logger.info('Affinitizing guest...')
 201
 202         cur_locale = locale.getdefaultlocale()[1]
 203         proc = subprocess.Popen(
 204             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 205         output = subprocess.check_output(
 206             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 207             stdin=proc.stdout)
 208         proc.wait()
 209
 210         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 211             match = None
 212             for line in output.decode(cur_locale).split('\n'):
 213                 match = re.search(thread_id % cpu, line)
 214                 if match:
 215                     self._affinitize_pid(
 216                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 217                         match.group(1))
 218                     break
 219
 220             if not match:
 221                 self._logger.error('Failed to affinitize guest core #%d. Could'
 222                                    ' not parse tid.', cpu)
 223
 224     def _config_guest_loopback(self):
 225         """
 226         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 227         """
 228         if self._guest_loopback == 'testpmd':
 229             self._login()
 230             self._configure_testpmd()
 231         elif self._guest_loopback == 'l2fwd':
 232             self._login()
 233             self._configure_l2fwd()
 234         elif self._guest_loopback == 'linux_bridge':
 235             self._login()
 236             self._configure_linux_bridge()
 237         elif self._guest_loopback != 'buildin':
 238             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 239                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 240
 241     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 242         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 243
 244     def execute_and_wait(self, cmd, timeout=30,
 245                          prompt=S.getValue('GUEST_PROMPT')):
 246         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 247                                                prompt=prompt)
 248
 249     def _modify_dpdk_makefile(self):
 250         """
 251         Modifies DPDK makefile in Guest before compilation
 252         """
 253         pass
 254
 255     def _configure_copy_sources(self, dirname):
 256         """
 257         Mount shared directory and copy DPDK and l2fwd sources
 258         """
 259         # mount shared directory
 260         self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
 261         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 262         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 263         self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
 264                               S.getValue('OVS_DPDK_SHARE'))
 265         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 266         self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 267                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 268
 269     def _configure_disable_firewall(self):
 270         """
 271         Disable firewall in VM
 272         """
 273         for iptables in ['iptables', 'ip6tables']:
 274             # filter table
 275             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 276                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 277             # mangle table
 278             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 279                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 280             # nat table
 281             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 282                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 283
 284             # flush rules and delete chains created by user
 285             for table in ['filter', 'mangle', 'nat']:
 286                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 287                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 288
 289
 290     def _configure_testpmd(self):
 291         """
 292         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 293         """
 294         self._configure_copy_sources('DPDK')
 295         self._configure_disable_firewall()
 296
 297         # Guest images _should_ have 1024 hugepages by default,
 298         # but just in case:'''
 299         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 300
 301         # Mount hugepages
 302         self.execute_and_wait('mkdir -p /dev/hugepages')
 303         self.execute_and_wait(
 304             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 305
 306         # build and configure system for dpdk
 307         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 308                               '/DPDK')
 309         self.execute_and_wait('export CC=gcc')
 310         self.execute_and_wait('export RTE_SDK=' +
 311                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 312         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 313
 314         # modify makefile if needed
 315         self._modify_dpdk_makefile()
 316
 317         # disable network interfaces, so DPDK can take care of them
 318         self.execute_and_wait('ifdown ' + self._net1)
 319         self.execute_and_wait('ifdown ' + self._net2)
 320
 321         # build and insert igb_uio and rebind interfaces to it
 322         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 323                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 324         self.execute_and_wait('modprobe uio')
 325         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 326                               S.getValue('RTE_TARGET'))
 327         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 328         self.execute_and_wait(
 329             './tools/dpdk_nic_bind.py -u' ' ' +
 330             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 331             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 332         self.execute_and_wait(
 333             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 334             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 335             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 336         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 337
 338         # build and run 'test-pmd'
 339         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 340                               '/DPDK/app/test-pmd')
 341         self.execute_and_wait('make clean')
 342         self.execute_and_wait('make')
 343         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 344                               ' --burst=64 -i --txqflags=0xf00 ' +
 345                               '--disable-hw-vlan', 60, "Done")
 346         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 347         self.execute_and_wait('start', 20,
 348                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 349
 350     def _configure_l2fwd(self):
 351         """
 352         Configure VM to perform L2 forwarding between NICs by l2fwd module
 353         """
 354         self._configure_copy_sources('l2fwd')
 355         self._configure_disable_firewall()
 356
 357         # build and configure system for l2fwd
 358         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 359                               '/l2fwd')
 360         self.execute_and_wait('export CC=gcc')
 361
 362         self.execute_and_wait('make')
 363         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 364                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 365                               ' net2=' + self._net2)
 366
 367     def _configure_linux_bridge(self):
 368         """
 369         Configure VM to perform L2 forwarding between NICs by linux bridge
 370         """
 371         self._configure_disable_firewall()
 372
 373         self.execute('ifconfig ' + self._net1 + ' ' +
 374                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
 375
 376         self.execute('ifconfig ' + self._net2 + ' ' +
 377                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
 378
 379         # configure linux bridge
 380         self.execute('brctl addbr br0')
 381         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 382         self.execute('ifconfig br0 ' +
 383                      S.getValue('VANILLA_BRIDGE_IP')[self._number])
 384
 385         # Add the arp entries for the IXIA ports and the bridge you are using.
 386         # Use command line values if provided.
 387         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 388                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 389         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 390                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 391
 392         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 393
 394         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 395                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 396         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 397                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 398
 399         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 400
 401         # Enable forwarding
 402         self.execute('sysctl -w net.ipv4.ip_forward=1')
 403
 404         # Controls source route verification
 405         # 0 means no source validation
 406         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 407         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 408         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')