vnfs/qemu/qemu.py

   1 # Copyright 2015 Intel Corporation.
   2 #
   3 # Licensed under the Apache License, Version 2.0 (the "License");
   4 # you may not use this file except in compliance with the License.
   5 # You may obtain a copy of the License at
   6 #
   7 #   http://www.apache.org/licenses/LICENSE-2.0
   8 #
   9 # Unless required by applicable law or agreed to in writing, software
  10 # distributed under the License is distributed on an "AS IS" BASIS,
  11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12 # See the License for the specific language governing permissions and
  13 # limitations under the License.
  14
  15 """Automation of QEMU hypervisor for launching vhost-cuse enabled guests.
  16 """
  17
  18 import os
  19 import logging
  20 import locale
  21 import re
  22 import subprocess
  23
  24 from conf import settings as S
  25 from conf import get_test_param
  26 from vnfs.vnf.vnf import IVnf
  27
  28 class IVnfQemu(IVnf):
  29     """
  30     Abstract class for controling an instance of QEMU
  31     """
  32     _cmd = None
  33     _expect = S.getValue('GUEST_PROMPT_LOGIN')
  34     _proc_name = 'qemu'
  35
  36     class GuestCommandFilter(logging.Filter):
  37         """
  38         Filter out strings beginning with 'guestcmd :'.
  39         """
  40         def filter(self, record):
  41             return record.getMessage().startswith(self.prefix)
  42
  43     def __init__(self):
  44         """
  45         Initialisation function.
  46         """
  47         super(IVnfQemu, self).__init__()
  48         self._logger = logging.getLogger(__name__)
  49         self._logfile = os.path.join(
  50             S.getValue('LOG_DIR'),
  51             S.getValue('LOG_FILE_QEMU')) + str(self._number)
  52         self._timeout = S.getValue('GUEST_TIMEOUT')[self._number]
  53         self._monitor = '%s/vm%dmonitor' % ('/tmp', self._number)
  54         self._net1 = get_test_param('guest_nic1_name', None)
  55         if self._net1 == None:
  56             self._net1 = S.getValue('GUEST_NIC1_NAME')[self._number]
  57         else:
  58             self._net1 = self._net1.split(',')[self._number]
  59         self._net2 = get_test_param('guest_nic2_name', None)
  60         if self._net2 == None:
  61             self._net2 = S.getValue('GUEST_NIC2_NAME')[self._number]
  62         else:
  63             self._net2 = self._net2.split(',')[self._number]
  64
  65
  66         name = 'Client%d' % self._number
  67         vnc = ':%d' % self._number
  68         # don't use taskset to affinize main qemu process; It causes hangup
  69         # of 2nd VM in case of DPDK. It also slows down VM responsivnes.
  70         self._cmd = ['sudo', '-E', S.getValue('QEMU_BIN'),
  71                      '-m', S.getValue('GUEST_MEMORY')[self._number],
  72                      '-smp', str(S.getValue('GUEST_SMP')[self._number]),
  73                      '-cpu', 'host',
  74                      '-drive', 'if=scsi,file=' +
  75                      S.getValue('GUEST_IMAGE')[self._number],
  76                      '-boot', 'c', '--enable-kvm',
  77                      '-monitor', 'unix:%s,server,nowait' % self._monitor,
  78                      '-object',
  79                      'memory-backend-file,id=mem,size=' +
  80                      str(S.getValue('GUEST_MEMORY')[self._number]) + 'M,' +
  81                      'mem-path=' + S.getValue('HUGEPAGE_DIR') + ',share=on',
  82                      '-numa', 'node,memdev=mem -mem-prealloc',
  83                      '-nographic', '-vnc', str(vnc), '-name', name,
  84                      '-snapshot', '-net none', '-no-reboot',
  85                      '-drive',
  86                      'if=scsi,file=fat:rw:%s,snapshot=off' %
  87                      S.getValue('GUEST_SHARE_DIR')[self._number],
  88                     ]
  89         self._configure_logging()
  90
  91     def _configure_logging(self):
  92         """
  93         Configure logging.
  94         """
  95         self.GuestCommandFilter.prefix = self._log_prefix
  96
  97         logger = logging.getLogger()
  98         cmd_logger = logging.FileHandler(
  99             filename=os.path.join(S.getValue('LOG_DIR'),
 100                                   S.getValue('LOG_FILE_GUEST_CMDS')) +
 101             str(self._number))
 102         cmd_logger.setLevel(logging.DEBUG)
 103         cmd_logger.addFilter(self.GuestCommandFilter())
 104         logger.addHandler(cmd_logger)
 105
 106     # startup/Shutdown
 107
 108     def start(self):
 109         """
 110         Start QEMU instance, login and prepare for commands.
 111         """
 112         super(IVnfQemu, self).start()
 113         if S.getValue('VNF_AFFINITIZATION_ON'):
 114             self._affinitize()
 115
 116         if self._timeout:
 117             self._config_guest_loopback()
 118
 119     # helper functions
 120
 121     def _login(self, timeout=120):
 122         """
 123         Login to QEMU instance.
 124
 125         This can be used immediately after booting the machine, provided a
 126         sufficiently long ``timeout`` is given.
 127
 128         :param timeout: Timeout to wait for login to complete.
 129
 130         :returns: None
 131         """
 132         # if no timeout was set, we likely started QEMU without waiting for it
 133         # to boot. This being the case, we best check that it has finished
 134         # first.
 135         if not self._timeout:
 136             self._expect_process(timeout=timeout)
 137
 138         self._child.sendline(S.getValue('GUEST_USERNAME'))
 139         self._child.expect(S.getValue('GUEST_PROMPT_PASSWORD'), timeout=5)
 140         self._child.sendline(S.getValue('GUEST_PASSWORD'))
 141
 142         self._expect_process(S.getValue('GUEST_PROMPT'), timeout=5)
 143
 144     def send_and_pass(self, cmd, timeout=30):
 145         """
 146         Send ``cmd`` and wait ``timeout`` seconds for it to pass.
 147
 148         :param cmd: Command to send to guest.
 149         :param timeout: Time to wait for prompt before checking return code.
 150
 151         :returns: None
 152         """
 153         self.execute(cmd)
 154         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 155         self.execute('echo $?')
 156         self._child.expect('^0$', timeout=1)  # expect a 0
 157         self.wait(S.getValue('GUEST_PROMPT'), timeout=timeout)
 158
 159     def _affinitize(self):
 160         """
 161         Affinitize the SMP cores of a QEMU instance.
 162
 163         This is a bit of a hack. The 'socat' utility is used to
 164         interact with the QEMU HMP. This is necessary due to the lack
 165         of QMP in older versions of QEMU, like v1.6.2. In future
 166         releases, this should be replaced with calls to libvirt or
 167         another Python-QEMU wrapper library.
 168
 169         :returns: None
 170         """
 171         thread_id = (r'.* CPU #%d: .* thread_id=(\d+)')
 172
 173         self._logger.info('Affinitizing guest...')
 174
 175         cur_locale = locale.getdefaultlocale()[1]
 176         proc = subprocess.Popen(
 177             ('echo', 'info cpus'), stdout=subprocess.PIPE)
 178         output = subprocess.check_output(
 179             ('sudo', 'socat', '-', 'UNIX-CONNECT:%s' % self._monitor),
 180             stdin=proc.stdout)
 181         proc.wait()
 182
 183         for cpu in range(0, int(S.getValue('GUEST_SMP')[self._number])):
 184             match = None
 185             for line in output.decode(cur_locale).split('\n'):
 186                 match = re.search(thread_id % cpu, line)
 187                 if match:
 188                     self._affinitize_pid(
 189                         S.getValue('GUEST_CORE_BINDING')[self._number][cpu],
 190                         match.group(1))
 191                     break
 192
 193             if not match:
 194                 self._logger.error('Failed to affinitize guest core #%d. Could'
 195                                    ' not parse tid.', cpu)
 196
 197     def _config_guest_loopback(self):
 198         """
 199         Configure VM to run VNF (e.g. port forwarding application)
 200         """
 201         # set guest loopback application based on VNF configuration
 202         # cli option take precedence to config file values
 203         guest_loopback = get_test_param('guest_loopback', S.getValue('GUEST_LOOPBACK')[self._number])
 204         if guest_loopback == 'testpmd':
 205             self._login()
 206             self._configure_testpmd()
 207         elif guest_loopback == 'l2fwd':
 208             self._login()
 209             self._configure_l2fwd()
 210         elif guest_loopback == 'linux_bridge':
 211             self._login()
 212             self._configure_linux_bridge()
 213         elif guest_loopback != 'buildin':
 214             self._logger.error('Unsupported guest loopback method "%s" was specified. Option'
 215                                ' "buildin" will be used as a fallback.', guest_loopback)
 216
 217     def wait(self, prompt=S.getValue('GUEST_PROMPT'), timeout=30):
 218         super(IVnfQemu, self).wait(prompt=prompt, timeout=timeout)
 219
 220     def execute_and_wait(self, cmd, timeout=30,
 221                          prompt=S.getValue('GUEST_PROMPT')):
 222         super(IVnfQemu, self).execute_and_wait(cmd, timeout=timeout,
 223                                                prompt=prompt)
 224
 225     def _modify_dpdk_makefile(self):
 226         """
 227         Modifies DPDK makefile in Guest before compilation
 228         """
 229         pass
 230
 231     def _configure_copy_sources(self, dirname):
 232         """
 233         Mount shared directory and copy DPDK and l2fwd sources
 234         """
 235         # mount shared directory
 236         self.execute_and_wait('umount ' + S.getValue('OVS_DPDK_SHARE'))
 237         self.execute_and_wait('rm -rf ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 238         self.execute_and_wait('mkdir -p ' + S.getValue('OVS_DPDK_SHARE'))
 239         self.execute_and_wait('mount -o iocharset=utf8 /dev/sdb1 ' +
 240                               S.getValue('OVS_DPDK_SHARE'))
 241         self.execute_and_wait('mkdir -p ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 242         self.execute_and_wait('cp -ra ' + os.path.join(S.getValue('OVS_DPDK_SHARE'), dirname) +
 243                               ' ' + S.getValue('GUEST_OVS_DPDK_DIR'))
 244
 245     def _configure_disable_firewall(self):
 246         """
 247         Disable firewall in VM
 248         """
 249         for iptables in ['iptables', 'ip6tables']:
 250             # filter table
 251             for chain in ['INPUT', 'FORWARD', 'OUTPUT']:
 252                 self.execute_and_wait("{} -t filter -P {} ACCEPT".format(iptables, chain))
 253             # mangle table
 254             for chain in ['PREROUTING', 'INPUT', 'FORWARD', 'OUTPUT', 'POSTROUTING']:
 255                 self.execute_and_wait("{} -t mangle -P {} ACCEPT".format(iptables, chain))
 256             # nat table
 257             for chain in ['PREROUTING', 'INPUT', 'OUTPUT', 'POSTROUTING']:
 258                 self.execute_and_wait("{} -t nat -P {} ACCEPT".format(iptables, chain))
 259
 260             # flush rules and delete chains created by user
 261             for table in ['filter', 'mangle', 'nat']:
 262                 self.execute_and_wait("{} -t {} -F".format(iptables, table))
 263                 self.execute_and_wait("{} -t {} -X".format(iptables, table))
 264
 265
 266     def _configure_testpmd(self):
 267         """
 268         Configure VM to perform L2 forwarding between NICs by DPDK's testpmd
 269         """
 270         self._configure_copy_sources('DPDK')
 271         self._configure_disable_firewall()
 272
 273         # Guest images _should_ have 1024 hugepages by default,
 274         # but just in case:'''
 275         self.execute_and_wait('sysctl vm.nr_hugepages=1024')
 276
 277         # Mount hugepages
 278         self.execute_and_wait('mkdir -p /dev/hugepages')
 279         self.execute_and_wait(
 280             'mount -t hugetlbfs hugetlbfs /dev/hugepages')
 281
 282         # build and configure system for dpdk
 283         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 284                               '/DPDK')
 285         self.execute_and_wait('export CC=gcc')
 286         self.execute_and_wait('export RTE_SDK=' +
 287                               S.getValue('GUEST_OVS_DPDK_DIR') + '/DPDK')
 288         self.execute_and_wait('export RTE_TARGET=%s' % S.getValue('RTE_TARGET'))
 289
 290         # modify makefile if needed
 291         self._modify_dpdk_makefile()
 292
 293         self.execute_and_wait('make RTE_OUTPUT=$RTE_SDK/$RTE_TARGET -C '
 294                               '$RTE_SDK/lib/librte_eal/linuxapp/igb_uio')
 295         self.execute_and_wait('modprobe uio')
 296         self.execute_and_wait('insmod %s/kmod/igb_uio.ko' %
 297                               S.getValue('RTE_TARGET'))
 298         self.execute_and_wait('./tools/dpdk_nic_bind.py --status')
 299         self.execute_and_wait(
 300             './tools/dpdk_nic_bind.py -b igb_uio' ' ' +
 301             S.getValue('GUEST_NET1_PCI_ADDRESS')[self._number] + ' ' +
 302             S.getValue('GUEST_NET2_PCI_ADDRESS')[self._number])
 303
 304         # build and run 'test-pmd'
 305         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 306                               '/DPDK/app/test-pmd')
 307         self.execute_and_wait('make clean')
 308         self.execute_and_wait('make')
 309         self.execute_and_wait('./testpmd -c 0x3 -n 4 --socket-mem 512 --'
 310                               ' --burst=64 -i --txqflags=0xf00 ' +
 311                               '--disable-hw-vlan', 60, "Done")
 312         self.execute('set fwd mac_retry', 1)
 313         self.execute_and_wait('start', 20,
 314                               'TX RS bit threshold=0 - TXQ flags=0xf00')
 315
 316     def _configure_l2fwd(self):
 317         """
 318         Configure VM to perform L2 forwarding between NICs by l2fwd module
 319         """
 320         self._configure_copy_sources('l2fwd')
 321         self._configure_disable_firewall()
 322
 323         # build and configure system for l2fwd
 324         self.execute_and_wait('cd ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 325                               '/l2fwd')
 326         self.execute_and_wait('export CC=gcc')
 327
 328         self.execute_and_wait('make')
 329         self.execute_and_wait('insmod ' + S.getValue('GUEST_OVS_DPDK_DIR') +
 330                               '/l2fwd' + '/l2fwd.ko net1=' + self._net1 +
 331                               ' net2=' + self._net2)
 332
 333     def _configure_linux_bridge(self):
 334         """
 335         Configure VM to perform L2 forwarding between NICs by linux bridge
 336         """
 337         self._configure_disable_firewall()
 338
 339         self.execute('ifconfig ' + self._net1 + ' ' +
 340                      S.getValue('VANILLA_NIC1_IP_CIDR')[self._number])
 341
 342         self.execute('ifconfig ' + self._net2 + ' ' +
 343                      S.getValue('VANILLA_NIC2_IP_CIDR')[self._number])
 344
 345         # configure linux bridge
 346         self.execute('brctl addbr br0')
 347         self.execute('brctl addif br0 ' + self._net1 + ' ' + self._net2)
 348         self.execute('ifconfig br0 ' +
 349                      S.getValue('VANILLA_BRIDGE_IP')[self._number])
 350
 351         # Add the arp entries for the IXIA ports and the bridge you are using.
 352         # Use command line values if provided.
 353         trafficgen_mac = get_test_param('vanilla_tgen_port1_mac',
 354                                         S.getValue('VANILLA_TGEN_PORT1_MAC'))
 355         trafficgen_ip = get_test_param('vanilla_tgen_port1_ip',
 356                                        S.getValue('VANILLA_TGEN_PORT1_IP'))
 357
 358         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 359
 360         trafficgen_mac = get_test_param('vanilla_tgen_port2_mac',
 361                                         S.getValue('VANILLA_TGEN_PORT2_MAC'))
 362         trafficgen_ip = get_test_param('vanilla_tgen_port2_ip',
 363                                        S.getValue('VANILLA_TGEN_PORT2_IP'))
 364
 365         self.execute('arp -s ' + trafficgen_ip + ' ' + trafficgen_mac)
 366
 367         # Enable forwarding
 368         self.execute('sysctl -w net.ipv4.ip_forward=1')
 369
 370         # Controls source route verification
 371         # 0 means no source validation
 372         self.execute('sysctl -w net.ipv4.conf.all.rp_filter=0')
 373         self.execute('sysctl -w net.ipv4.conf.' + self._net1 + '.rp_filter=0')
 374         self.execute('sysctl -w net.ipv4.conf.' + self._net2 + '.rp_filter=0')