vnfs/qemu/qemu.py

   1 # Copyright 2015-2016 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23
  24 from conf import settings as S
  25 from conf import get_test_param
  26 from vnfs.vnf.vnf import IVnf
  27
  28 class IVnfQemu(IVnf):
  29     """
  30     Abstract class for controling an instance of QEMU
  31     """
  32     _cmd = None
  33     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  34     _proc_name = 'qemu'
  35
  36     class GuestCommandFilter(logging.Filter):
  37         """
  38         Filter out strings beginning with 'guestcmd :'.
  39         """
  40         def filter(self, record):
  41             return record.getMessage().startswith(self.prefix)
  42
  43     def __init__(self):
  44         """
  45         Initialisation function.
  46         """
  47         super(IVnfQemu, self).__init__()
  48         self._logger = logging.getLogger(__name__)
  49         self._logfile = os.path.join(
  50             S.getValue('LOG_DIR'),
  51             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  52         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  53         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  54         self._net1 = get_test_param('guest_nic1_name', None)
  55         if self._net1 == None:
  56             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  57         else:
  58             self._net1 = self._net1.split(',')[self._number]
  59         self._net2 = get_test_param('guest_nic2_name', None)
  60         if self._net2 == None:
  61             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  62         else:
  63             self._net2 = self._net2.split(',')[self._number]
  64
  65
  66         name = 'Client%d' % self._number
  67         vnc = ':%d' % self._number
  68         # don't use taskset to affinize main qemu process; It causes hangup
  69         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  70         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  71                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  72                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  73                      '-cpu', 'host',
  74                      '-drive', 'if=scsi,file=' +
  75                      S.getValue('GUEST_IMAGE')[self._number],
  76                      '-boot', 'c', '--enable-kvm',
  77                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  78                      '-object',
  79                      'memory-backend-file,id=mem,size=' +
  80                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  81                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  82                      '-numa', 'node,memdev=mem -mem-prealloc',
  83                      '-nographic', '-vnc', str(vnc), '-name', name,
  84                      '-snapshot', '-net none', '-no-reboot',
  85                      '-drive',
  86                      'if=scsi,file=fat:rw:%s,snapshot=off' %
  87                      S.getValue('GUEST_SHARE_DIR')[self._number],
  88                     ]
  89         self._configure_logging()
  90
  91     def _configure_logging(self):
  92         """
  93         Configure logging.
  94         """
  95         self.GuestCommandFilter.prefix = self._log_prefix
  96
  97         logger = logging.getLogger()
  98         cmd_logger = logging.FileHandler(
  99             filename=os.path.join(S.getValue('LOG_DIR'),
 100                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 101             str(self._number))
 102         cmd_logger.setLevel(logging.DEBUG)
 103         cmd_logger.addFilter(self.GuestCommandFilter())
 104         logger.addHandler(cmd_logger)
 105
 106     # startup/Shutdown
 107
 108     def start(self):
 109         """
 110         Start QEMU instance, login and prepare for commands.
 111         """
 112         super(IVnfQemu, self).start()
 113         if S.getValue('VNF_AFFINITIZATION_ON'):
 114             self._affinitize()
 115
 116         if self._timeout:
 117             self._config_guest_loopback()
 118
 119     # helper functions
 120
 121     def _login(self, timeout=120):
 122         """
 123         Login to QEMU instance.
 124
 125         This can be used immediately after booting the machine, provided a
 126         sufficiently long ``timeout`` is given.
 127
 128         :param timeout: Timeout to wait for login to complete.
 129
 130         :returns: None
 131         """
 132         # if no timeout was set, we likely started QEMU without waiting for it
 133         # to boot. This being the case, we best check that it has finished
 134         # first.
 135         if not self._timeout:
 136             self._expect_process(timeout=timeout)
 137
 138         self._child.sendline(S.getValue('GUEST_USERNAME'))
 139         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 140         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 141
 142         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 143
 144     def send_and_pass(self, cmd, timeout=30):
 145         """
 146         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 147
 148         :param cmd: Command to send to guest.
 149         :param timeout: Time to wait for prompt before checking return code.
 150
 151         :returns: None
 152         """
 153         self.execute(cmd)
 154         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 155         self.execute('echo $?')
 156         self._child.expect('^0$', timeout=1)  # expect a 0
 157         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 158
 159     def _affinitize(self):
 160         """
 161         Affinitize the SMP cores of a QEMU instance.
 162
 163         This is a bit of a hack. The 'socat' utility is used to
 164         interact with the QEMU HMP. This is necessary due to the lack
 165         of QMP in older versions of QEMU, like v1.6.2. In future
 166         releases, this should be replaced with calls to libvirt or
 167         another Python-QEMU wrapper library.
 168
 169         :returns: None
 170         """
 171         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 172
 173         self._logger.info('Affinitizing guest...')
 174
 175         cur_locale = locale.getdefaultlocale()[1]
 176         proc = subprocess.Popen(
 177             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 178         output = subprocess.check_output(
 179             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 180             stdin=proc.stdout)
 181         proc.wait()
 182
 183         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 184             match = None
 185             for line in output.decode(cur_locale).split('\n'):
 186                 match = re.search(thread_id % cpu, line)
 187                 if match:
 188                     self._affinitize_pid(
 189                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 190                         match.group(1))
 191                     break
 192
 193             if not match:
 194                 self._logger.error('Failed to affinitize guest core #%d. Could'
 195                                    ' not parse tid.', cpu)
 196
 197     def _config_guest_loopback(self):
 198         """
 199         Configure VM to run VNF (e.g. port forwarding application)
 200         """
 201         # set guest loopback application based on VNF configuration
 202         # cli option take precedence to config file values
 203         guest_loopback = S.getValue('GUEST_LOOPBACK')[self._number]
 204
 205         if guest_loopback == 'testpmd':
 206             self._login()
 207             self._configure_testpmd()
 208         elif guest_loopback == 'l2fwd':
 209             self._login()
 210             self._configure_l2fwd()
 211         elif guest_loopback == 'linux_bridge':
 212             self._login()
 213             self._configure_linux_bridge()
 214         elif guest_loopback != 'buildin':
 215             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 216                                ' "buildin" will be used as a fallback.', guest_loopback)
 217
 218     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 219         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 220
 221     def execute_and_wait(self, cmd, timeout=30,
 222                          prompt=S.getValue('GUEST_PROMPT')):
 223         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 224                                                prompt=prompt)
 225
 226     def _modify_dpdk_makefile(self):
 227         """
 228         Modifies DPDK makefile in Guest before compilation
 229         """
 230         pass
 231
 232     def _configure_copy_sources(self, dirname):
 233         """
 234         Mount shared directory and copy DPDK and l2fwd sources
 235         """
 236         # mount shared directory
 237         self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
 238         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 239         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 240         self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
 241                               S.getValue('OVS_DPDK_SHARE'))
 242         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 243         self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 244                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 245
 246     def _configure_disable_firewall(self):
 247         """
 248         Disable firewall in VM
 249         """
 250         for iptables in ['iptables', 'ip6tables']:
 251             # filter table
 252             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 253                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 254             # mangle table
 255             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 256                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 257             # nat table
 258             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 259                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 260
 261             # flush rules and delete chains created by user
 262             for table in ['filter', 'mangle', 'nat']:
 263                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 264                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 265
 266
 267     def _configure_testpmd(self):
 268         """
 269         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 270         """
 271         self._configure_copy_sources('DPDK')
 272         self._configure_disable_firewall()
 273
 274         # Guest images _should_ have 1024 hugepages by default,
 275         # but just in case:'''
 276         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 277
 278         # Mount hugepages
 279         self.execute_and_wait('mkdir -p /dev/hugepages')
 280         self.execute_and_wait(
 281             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 282
 283         # build and configure system for dpdk
 284         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 285                               '/DPDK')
 286         self.execute_and_wait('export CC=gcc')
 287         self.execute_and_wait('export RTE_SDK=' +
 288                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 289         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 290
 291         # modify makefile if needed
 292         self._modify_dpdk_makefile()
 293
 294         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 295                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 296         self.execute_and_wait('modprobe uio')
 297         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 298                               S.getValue('RTE_TARGET'))
 299         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 300         self.execute_and_wait(
 301             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 302             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 303             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 304
 305         # build and run 'test-pmd'
 306         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 307                               '/DPDK/app/test-pmd')
 308         self.execute_and_wait('make clean')
 309         self.execute_and_wait('make')
 310         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 311                               ' --burst=64 -i --txqflags=0xf00 ' +
 312                               '--disable-hw-vlan', 60, "Done")
 313         self.execute('set fwd mac_retry', 1)
 314         self.execute_and_wait('start', 20,
 315                               'TX RS bit threshold=0 - TXQ flags=0xf00')
 316
 317     def _configure_l2fwd(self):
 318         """
 319         Configure VM to perform L2 forwarding between NICs by l2fwd module
 320         """
 321         self._configure_copy_sources('l2fwd')
 322         self._configure_disable_firewall()
 323
 324         # build and configure system for l2fwd
 325         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 326                               '/l2fwd')
 327         self.execute_and_wait('export CC=gcc')
 328
 329         self.execute_and_wait('make')
 330         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 331                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 332                               ' net2=' + self._net2)
 333
 334     def _configure_linux_bridge(self):
 335         """
 336         Configure VM to perform L2 forwarding between NICs by linux bridge
 337         """
 338         self._configure_disable_firewall()
 339
 340         self.execute('ifconfig ' + self._net1 + ' ' +
 341                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
 342
 343         self.execute('ifconfig ' + self._net2 + ' ' +
 344                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
 345
 346         # configure linux bridge
 347         self.execute('brctl addbr br0')
 348         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 349         self.execute('ifconfig br0 ' +
 350                      S.getValue('VANILLA_BRIDGE_IP')[self._number])
 351
 352         # Add the arp entries for the IXIA ports and the bridge you are using.
 353         # Use command line values if provided.
 354         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 355                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 356         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 357                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 358
 359         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 360
 361         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 362                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 363         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 364                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 365
 366         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 367
 368         # Enable forwarding
 369         self.execute('sysctl -w net.ipv4.ip_forward=1')
 370
 371         # Controls source route verification
 372         # 0 means no source validation
 373         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 374         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 375         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')