vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23 import time
  24 import pexpect
  25
  26 from conf import settings as S
  27 from conf import get_test_param
  28 from vnfs.vnf.vnf import IVnf
  29
  30 class IVnfQemu(IVnf):
  31     """
  32     Abstract class for controling an instance of QEMU
  33     """
  34     _cmd = None
  35     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  36     _proc_name = 'qemu'
  37
  38     class GuestCommandFilter(logging.Filter):
  39         """
  40         Filter out strings beginning with 'guestcmd :'.
  41         """
  42         def filter(self, record):
  43             return record.getMessage().startswith(self.prefix)
  44
  45     def __init__(self):
  46         """
  47         Initialisation function.
  48         """
  49         super(IVnfQemu, self).__init__()
  50         self._logger = logging.getLogger(__name__)
  51         self._logfile = os.path.join(
  52             S.getValue('LOG_DIR'),
  53             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  54         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  55         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  56         self._net1 = get_test_param('guest_nic1_name', None)
  57         if self._net1 == None:
  58             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  59         else:
  60             self._net1 = self._net1.split(',')[self._number]
  61         self._net2 = get_test_param('guest_nic2_name', None)
  62         if self._net2 == None:
  63             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  64         else:
  65             self._net2 = self._net2.split(',')[self._number]
  66
  67         # set guest loopback application based on VNF configuration
  68         # cli option take precedence to config file values
  69         self._guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
  70
  71         self._testpmd_fwd_mode = S.getValue('GUEST_TESTPMD_FWD_MODE')
  72         # in case of SRIOV we must ensure, that MAC addresses are not swapped
  73         if S.getValue('SRIOV_ENABLED') and self._testpmd_fwd_mode.startswith('mac') and \
  74            not S.getValue('VNF').endswith('PciPassthrough'):
  75
  76             self._logger.info("SRIOV detected, forwarding mode of testpmd was changed from '%s' to '%s'",
  77                               self._testpmd_fwd_mode, 'io')
  78             self._testpmd_fwd_mode = 'io'
  79
  80         name = 'Client%d' % self._number
  81         vnc = ':%d' % self._number
  82         # don't use taskset to affinize main qemu process; It causes hangup
  83         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  84         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  85                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  86                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  87                      '-cpu', 'host,migratable=off',
  88                      '-drive', 'if={},file='.format(S.getValue(
  89                         'GUEST_BOOT_DRIVE_TYPE')) +
  90                      S.getValue('GUEST_IMAGE')[self._number],
  91                      '-boot', 'c', '--enable-kvm',
  92                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  93                      '-object',
  94                      'memory-backend-file,id=mem,size=' +
  95                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  96                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  97                      '-numa', 'node,memdev=mem -mem-prealloc',
  98                      '-nographic', '-vnc', str(vnc), '-name', name,
  99                      '-snapshot', '-net none', '-no-reboot',
 100                      '-drive',
 101                      'if=%s,format=raw,file=fat:rw:%s,snapshot=off' %
 102                      (S.getValue('GUEST_SHARED_DRIVE_TYPE'),
 103                       S.getValue('GUEST_SHARE_DIR')[self._number]),
 104                     ]
 105         self._configure_logging()
 106
 107     def _configure_logging(self):
 108         """
 109         Configure logging.
 110         """
 111         self.GuestCommandFilter.prefix = self._log_prefix
 112
 113         logger = logging.getLogger()
 114         cmd_logger = logging.FileHandler(
 115             filename=os.path.join(S.getValue('LOG_DIR'),
 116                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 117             str(self._number))
 118         cmd_logger.setLevel(logging.DEBUG)
 119         cmd_logger.addFilter(self.GuestCommandFilter())
 120         logger.addHandler(cmd_logger)
 121
 122     # startup/Shutdown
 123
 124     def start(self):
 125         """
 126         Start QEMU instance, login and prepare for commands.
 127         """
 128         super(IVnfQemu, self).start()
 129         if S.getValue('VNF_AFFINITIZATION_ON'):
 130             self._affinitize()
 131
 132         if self._timeout:
 133             self._config_guest_loopback()
 134
 135     def stop(self):
 136         """
 137         Stops VNF instance gracefully first.
 138         """
 139         try:
 140             # exit testpmd if needed
 141             if self._guest_loopback == 'testpmd':
 142                 self.execute_and_wait('stop', 120, "Done")
 143                 self.execute_and_wait('quit', 120, "[bB]ye")
 144
 145             # turn off VM
 146             self.execute_and_wait('poweroff', 120, "Power down")
 147
 148         except pexpect.TIMEOUT:
 149             self.kill()
 150
 151         # wait until qemu shutdowns
 152         self._logger.debug('Wait for QEMU to terminate')
 153         for dummy in range(30):
 154             time.sleep(1)
 155             if not self.is_running():
 156                 break
 157
 158         # just for case that graceful shutdown failed
 159         super(IVnfQemu, self).stop()
 160
 161     # helper functions
 162
 163     def _login(self, timeout=120):
 164         """
 165         Login to QEMU instance.
 166
 167         This can be used immediately after booting the machine, provided a
 168         sufficiently long ``timeout`` is given.
 169
 170         :param timeout: Timeout to wait for login to complete.
 171
 172         :returns: None
 173         """
 174         # if no timeout was set, we likely started QEMU without waiting for it
 175         # to boot. This being the case, we best check that it has finished
 176         # first.
 177         if not self._timeout:
 178             self._expect_process(timeout=timeout)
 179
 180         self._child.sendline(S.getValue('GUEST_USERNAME'))
 181         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 182         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 183
 184         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 185
 186     def send_and_pass(self, cmd, timeout=30):
 187         """
 188         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 189
 190         :param cmd: Command to send to guest.
 191         :param timeout: Time to wait for prompt before checking return code.
 192
 193         :returns: None
 194         """
 195         self.execute(cmd)
 196         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 197         self.execute('echo $?')
 198         self._child.expect('^0$', timeout=1)  # expect a 0
 199         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 200
 201     def _affinitize(self):
 202         """
 203         Affinitize the SMP cores of a QEMU instance.
 204
 205         This is a bit of a hack. The 'socat' utility is used to
 206         interact with the QEMU HMP. This is necessary due to the lack
 207         of QMP in older versions of QEMU, like v1.6.2. In future
 208         releases, this should be replaced with calls to libvirt or
 209         another Python-QEMU wrapper library.
 210
 211         :returns: None
 212         """
 213         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 214
 215         self._logger.info('Affinitizing guest...')
 216
 217         cur_locale = locale.getdefaultlocale()[1]
 218         proc = subprocess.Popen(
 219             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 220         output = subprocess.check_output(
 221             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 222             stdin=proc.stdout)
 223         proc.wait()
 224
 225         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 226             match = None
 227             for line in output.decode(cur_locale).split('\n'):
 228                 match = re.search(thread_id % cpu, line)
 229                 if match:
 230                     self._affinitize_pid(
 231                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 232                         match.group(1))
 233                     break
 234
 235             if not match:
 236                 self._logger.error('Failed to affinitize guest core #%d. Could'
 237                                    ' not parse tid.', cpu)
 238
 239     def _config_guest_loopback(self):
 240         """
 241         Configure VM to run VNF, e.g. port forwarding application based on the configuration
 242         """
 243         if self._guest_loopback == 'testpmd':
 244             self._login()
 245             self._configure_testpmd()
 246         elif self._guest_loopback == 'l2fwd':
 247             self._login()
 248             self._configure_l2fwd()
 249         elif self._guest_loopback == 'linux_bridge':
 250             self._login()
 251             self._configure_linux_bridge()
 252         elif self._guest_loopback != 'buildin':
 253             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 254                                ' "buildin" will be used as a fallback.', self._guest_loopback)
 255
 256     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 257         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 258
 259     def execute_and_wait(self, cmd, timeout=30,
 260                          prompt=S.getValue('GUEST_PROMPT')):
 261         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 262                                                prompt=prompt)
 263
 264     def _modify_dpdk_makefile(self):
 265         """
 266         Modifies DPDK makefile in Guest before compilation if needed
 267         """
 268         pass
 269
 270     def _configure_copy_sources(self, dirname):
 271         """
 272         Mount shared directory and copy DPDK and l2fwd sources
 273         """
 274         # mount shared directory
 275         self.execute_and_wait('umount /dev/sdb1')
 276         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 277         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 278         self.execute_and_wait('mount -o ro,iocharset=utf8 /dev/sdb1 ' +
 279                               S.getValue('OVS_DPDK_SHARE'))
 280         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 281         self.execute_and_wait('cp -r ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 282                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 283         self.execute_and_wait('umount /dev/sdb1')
 284
 285     def _configure_disable_firewall(self):
 286         """
 287         Disable firewall in VM
 288         """
 289         for iptables in ['iptables', 'ip6tables']:
 290             # filter table
 291             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 292                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 293             # mangle table
 294             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 295                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 296             # nat table
 297             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 298                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 299
 300             # flush rules and delete chains created by user
 301             for table in ['filter', 'mangle', 'nat']:
 302                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 303                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 304
 305
 306     def _configure_testpmd(self):
 307         """
 308         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 309         """
 310         self._configure_copy_sources('DPDK')
 311         self._configure_disable_firewall()
 312
 313         # Guest images _should_ have 1024 hugepages by default,
 314         # but just in case:'''
 315         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 316
 317         # Mount hugepages
 318         self.execute_and_wait('mkdir -p /dev/hugepages')
 319         self.execute_and_wait(
 320             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 321
 322         # build and configure system for dpdk
 323         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 324                               '/DPDK')
 325         self.execute_and_wait('export CC=gcc')
 326         self.execute_and_wait('export RTE_SDK=' +
 327                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 328         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 329
 330         # modify makefile if needed
 331         self._modify_dpdk_makefile()
 332
 333         # disable network interfaces, so DPDK can take care of them
 334         self.execute_and_wait('ifdown ' + self._net1)
 335         self.execute_and_wait('ifdown ' + self._net2)
 336
 337         # build and insert igb_uio and rebind interfaces to it
 338         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 339                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 340         self.execute_and_wait('modprobe uio')
 341         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 342                               S.getValue('RTE_TARGET'))
 343         self.execute_and_wait('./tools/dpdk*bind.py --status')
 344         self.execute_and_wait(
 345             './tools/dpdk*bind.py -u' ' ' +
 346             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 347             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 348         self.execute_and_wait(
 349             './tools/dpdk*bind.py -b igb_uio' ' ' +
 350             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 351             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 352         self.execute_and_wait('./tools/dpdk*bind.py --status')
 353
 354         # build and run 'test-pmd'
 355         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 356                               '/DPDK/app/test-pmd')
 357         self.execute_and_wait('make clean')
 358         self.execute_and_wait('make')
 359         if int(S.getValue('GUEST_NIC_QUEUES')):
 360             self.execute_and_wait(
 361                 './testpmd {} -n4 --socket-mem 512 --'.format(
 362                     S.getValue('GUEST_TESTPMD_CPU_MASK')) +
 363                 ' --burst=64 -i --txqflags=0xf00 ' +
 364                 '--nb-cores={} --rxq={} --txq={} '.format(
 365                     S.getValue('GUEST_TESTPMD_NB_CORES'),
 366                     S.getValue('GUEST_TESTPMD_TXQ'),
 367                     S.getValue('GUEST_TESTPMD_RXQ')) +
 368                 '--disable-hw-vlan', 60, "Done")
 369         else:
 370             self.execute_and_wait(
 371                 './testpmd {} -n 4 --socket-mem 512 --'.format(
 372                     S.getValue('GUEST_TESTPMD_CPU_MASK')) +
 373                 ' --burst=64 -i --txqflags=0xf00 ' +
 374                 '--disable-hw-vlan', 60, "Done")
 375         self.execute('set fwd ' + self._testpmd_fwd_mode, 1)
 376         self.execute_and_wait('start', 20,
 377                               'TX RS bit threshold=.+ - TXQ flags=0xf00')
 378
 379     def _configure_l2fwd(self):
 380         """
 381         Configure VM to perform L2 forwarding between NICs by l2fwd module
 382         """
 383         self._configure_copy_sources('l2fwd')
 384         self._configure_disable_firewall()
 385
 386         # build and configure system for l2fwd
 387         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 388                               '/l2fwd')
 389         self.execute_and_wait('export CC=gcc')
 390
 391         self.execute_and_wait('make')
 392         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 393                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 394                               ' net2=' + self._net2)
 395
 396     def _configure_linux_bridge(self):
 397         """
 398         Configure VM to perform L2 forwarding between NICs by linux bridge
 399         """
 400         self._configure_disable_firewall()
 401
 402         self.execute('ip addr add ' +
 403                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number] +
 404                      ' dev ' + self._net1)
 405         self.execute('ip link set dev ' + self._net1 + ' up')
 406
 407         self.execute('ip addr add ' +
 408                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number] +
 409                      ' dev ' + self._net2)
 410         self.execute('ip link set dev ' + self._net2 + ' up')
 411
 412         # configure linux bridge
 413         self.execute('brctl addbr br0')
 414         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 415         self.execute('ip addr add ' +
 416                      S.getValue('VANILLA_BRIDGE_IP')[self._number] +
 417                      ' dev br0')
 418         self.execute('ip link set dev br0 up')
 419
 420         # Add the arp entries for the IXIA ports and the bridge you are using.
 421         # Use command line values if provided.
 422         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 423                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 424         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 425                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 426
 427         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 428
 429         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 430                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 431         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 432                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 433
 434         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 435
 436         # Enable forwarding
 437         self.execute('sysctl -w net.ipv4.ip_forward=1')
 438
 439         # Controls source route verification
 440         # 0 means no source validation
 441         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 442         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 443         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')