1 # Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
2 # Copyright 2010 United States Government as represented by the
3 # Administrator of the National Aeronautics and Space Administration.
6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
7 # not use this file except in compliance with the License. You may obtain
8 # a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15 # License for the specific language governing permissions and limitations
18 """Implements vlans, bridges, and iptables rules using linux utilities."""
27 from oslo_concurrency import processutils
28 from oslo_config import cfg
29 from oslo_log import log as logging
30 from oslo_serialization import jsonutils
31 from oslo_utils import excutils
32 from oslo_utils import importutils
33 from oslo_utils import timeutils
36 from nova import exception
37 from nova.i18n import _, _LE, _LW
38 from nova import objects
39 from nova.openstack.common import fileutils
40 from nova import paths
41 from nova.pci import utils as pci_utils
42 from nova import utils
44 LOG = logging.getLogger(__name__)
48 cfg.MultiStrOpt('dhcpbridge_flagfile',
49 default=['/etc/nova/nova-dhcpbridge.conf'],
50 help='Location of flagfiles for dhcpbridge'),
51 cfg.StrOpt('networks_path',
52 default=paths.state_path_def('networks'),
53 help='Location to keep network config files'),
54 cfg.StrOpt('public_interface',
56 help='Interface for public IP addresses'),
57 cfg.StrOpt('dhcpbridge',
58 default=paths.bindir_def('nova-dhcpbridge'),
59 help='Location of nova-dhcpbridge'),
60 cfg.StrOpt('routing_source_ip',
62 help='Public IP of network host'),
63 cfg.IntOpt('dhcp_lease_time',
65 help='Lifetime of a DHCP lease in seconds'),
66 cfg.MultiStrOpt('dns_server',
68 help='If set, uses specific DNS server for dnsmasq. Can'
69 ' be specified multiple times.'),
70 cfg.BoolOpt('use_network_dns_servers',
72 help='If set, uses the dns1 and dns2 from the network ref.'
74 cfg.ListOpt('dmz_cidr',
76 help='A list of dmz ranges that should be accepted'),
77 cfg.MultiStrOpt('force_snat_range',
79 help='Traffic to this range will always be snatted to the '
80 'fallback ip, even if it would normally be bridged out '
81 'of the node. Can be specified multiple times.'),
82 cfg.StrOpt('dnsmasq_config_file',
84 help='Override the default dnsmasq settings with this file'),
85 cfg.StrOpt('linuxnet_interface_driver',
86 default='nova.network.linux_net.LinuxBridgeInterfaceDriver',
87 help='Driver used to create ethernet devices.'),
88 cfg.StrOpt('linuxnet_ovs_integration_bridge',
90 help='Name of Open vSwitch bridge used with linuxnet'),
91 cfg.BoolOpt('send_arp_for_ha',
93 help='Send gratuitous ARPs for HA setup'),
94 cfg.IntOpt('send_arp_for_ha_count',
96 help='Send this many gratuitous ARPs for HA setup'),
97 cfg.BoolOpt('use_single_default_gateway',
99 help='Use single default gateway. Only first nic of vm will '
100 'get default gateway from dhcp server'),
101 cfg.MultiStrOpt('forward_bridge_interface',
103 help='An interface that bridges can forward to. If this '
104 'is set to all then all traffic will be forwarded. '
105 'Can be specified multiple times.'),
106 cfg.StrOpt('metadata_host',
108 help='The IP address for the metadata API server'),
109 cfg.IntOpt('metadata_port',
111 help='The port for the metadata API port'),
112 cfg.StrOpt('iptables_top_regex',
114 help='Regular expression to match the iptables rule that '
115 'should always be on the top.'),
116 cfg.StrOpt('iptables_bottom_regex',
118 help='Regular expression to match the iptables rule that '
119 'should always be on the bottom.'),
120 cfg.StrOpt('iptables_drop_action',
122 help='The table that iptables to jump to when a packet is '
124 cfg.IntOpt('ovs_vsctl_timeout',
126 help='Amount of time, in seconds, that ovs_vsctl should wait '
127 'for a response from the database. 0 is to wait forever.'),
128 cfg.BoolOpt('fake_network',
130 help='If passed, use fake network devices and addresses'),
131 cfg.IntOpt('ebtables_exec_attempts',
133 help='Number of times to retry ebtables commands on failure.'),
134 cfg.FloatOpt('ebtables_retry_interval',
136 help='Number of seconds to wait between ebtables retries.'),
140 CONF.register_opts(linux_net_opts)
141 CONF.import_opt('host', 'nova.netconf')
142 CONF.import_opt('use_ipv6', 'nova.netconf')
143 CONF.import_opt('my_ip', 'nova.netconf')
144 CONF.import_opt('network_device_mtu', 'nova.objects.network')
147 # NOTE(vish): Iptables supports chain names of up to 28 characters, and we
148 # add up to 12 characters to binary_name which is used as a prefix,
149 # so we limit it to 16 characters.
150 # (max_chain_name_length - len('-POSTROUTING') == 16)
151 def get_binary_name():
152 """Grab the name of the binary we're running in."""
153 return os.path.basename(inspect.stack()[-1][1])[:16]
155 binary_name = get_binary_name()
158 class IptablesRule(object):
161 You shouldn't need to use this class directly, it's only used by
166 def __init__(self, chain, rule, wrap=True, top=False):
172 def __eq__(self, other):
173 return ((self.chain == other.chain) and
174 (self.rule == other.rule) and
175 (self.top == other.top) and
176 (self.wrap == other.wrap))
178 def __ne__(self, other):
179 return not self == other
183 chain = '%s-%s' % (binary_name, self.chain)
186 # new rules should have a zero [packet: byte] count
187 return '[0:0] -A %s %s' % (chain, self.rule)
190 class IptablesTable(object):
191 """An iptables table."""
195 self.remove_rules = []
197 self.unwrapped_chains = set()
198 self.remove_chains = set()
201 def has_chain(self, name, wrap=True):
203 return name in self.chains
205 return name in self.unwrapped_chains
207 def add_chain(self, name, wrap=True):
208 """Adds a named chain to the table.
210 The chain name is wrapped to be unique for the component creating
211 it, so different components of Nova can safely create identically
212 named chains without interfering with one another.
214 At the moment, its wrapped name is <binary name>-<chain name>,
215 so if nova-compute creates a chain named 'OUTPUT', it'll actually
216 end up named 'nova-compute-OUTPUT'.
220 self.chains.add(name)
222 self.unwrapped_chains.add(name)
225 def remove_chain(self, name, wrap=True):
226 """Remove named chain.
228 This removal "cascades". All rule in the chain are removed, as are
229 all rules in other chains that jump to it.
231 If the chain is not found, this is merely logged.
235 chain_set = self.chains
237 chain_set = self.unwrapped_chains
239 if name not in chain_set:
240 LOG.warning(_LW('Attempted to remove chain %s which does not '
245 # non-wrapped chains and rules need to be dealt with specially,
246 # so we keep a list of them to be iterated over in apply()
248 self.remove_chains.add(name)
249 chain_set.remove(name)
251 self.remove_rules += filter(lambda r: r.chain == name, self.rules)
252 self.rules = filter(lambda r: r.chain != name, self.rules)
255 jump_snippet = '-j %s-%s' % (binary_name, name)
257 jump_snippet = '-j %s' % (name,)
260 self.remove_rules += filter(lambda r: jump_snippet in r.rule,
262 self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules)
264 def add_rule(self, chain, rule, wrap=True, top=False):
265 """Add a rule to the table.
267 This is just like what you'd feed to iptables, just without
268 the '-A <chain name>' bit at the start.
270 However, if you need to jump to one of your wrapped chains,
271 prepend its name with a '$' which will ensure the wrapping
272 is applied correctly.
275 if wrap and chain not in self.chains:
276 raise ValueError(_('Unknown chain: %r') % chain)
279 rule = ' '.join(map(self._wrap_target_chain, rule.split(' ')))
281 rule_obj = IptablesRule(chain, rule, wrap, top)
282 if rule_obj in self.rules:
283 LOG.debug("Skipping duplicate iptables rule addition. "
284 "%(rule)r already in %(rules)r",
285 {'rule': rule_obj, 'rules': self.rules})
287 self.rules.append(IptablesRule(chain, rule, wrap, top))
290 def _wrap_target_chain(self, s):
291 if s.startswith('$'):
292 return '%s-%s' % (binary_name, s[1:])
295 def remove_rule(self, chain, rule, wrap=True, top=False):
296 """Remove a rule from a chain.
298 Note: The rule must be exactly identical to the one that was added.
299 You cannot switch arguments around like you can with the iptables
304 self.rules.remove(IptablesRule(chain, rule, wrap, top))
306 self.remove_rules.append(IptablesRule(chain, rule, wrap, top))
309 LOG.warning(_LW('Tried to remove rule that was not there:'
310 ' %(chain)r %(rule)r %(wrap)r %(top)r'),
311 {'chain': chain, 'rule': rule,
312 'top': top, 'wrap': wrap})
314 def remove_rules_regex(self, regex):
315 """Remove all rules matching regex."""
316 if isinstance(regex, six.string_types):
317 regex = re.compile(regex)
318 num_rules = len(self.rules)
319 self.rules = filter(lambda r: not regex.match(str(r)), self.rules)
320 removed = num_rules - len(self.rules)
325 def empty_chain(self, chain, wrap=True):
326 """Remove all rules from a chain."""
327 chained_rules = [rule for rule in self.rules
328 if rule.chain == chain and rule.wrap == wrap]
331 for rule in chained_rules:
332 self.rules.remove(rule)
335 class IptablesManager(object):
336 """Wrapper for iptables.
338 See IptablesTable for some usage docs
340 A number of chains are set up to begin with.
342 First, nova-filter-top. It's added at the top of FORWARD and OUTPUT. Its
343 name is not wrapped, so it's shared between the various nova workers. It's
344 intended for rules that need to live at the top of the FORWARD and OUTPUT
345 chains. It's in both the ipv4 and ipv6 set of tables.
347 For ipv4 and ipv6, the built-in INPUT, OUTPUT, and FORWARD filter chains
348 are wrapped, meaning that the "real" INPUT chain has a rule that jumps to
349 the wrapped INPUT chain, etc. Additionally, there's a wrapped chain named
350 "local" which is jumped to from nova-filter-top.
352 For ipv4, the built-in PREROUTING, OUTPUT, and POSTROUTING nat chains are
353 wrapped in the same was as the built-in filter chains. Additionally,
354 there's a snat chain that is applied after the POSTROUTING chain.
358 def __init__(self, execute=None):
360 self.execute = _execute
362 self.execute = execute
364 self.ipv4 = {'filter': IptablesTable(),
365 'nat': IptablesTable(),
366 'mangle': IptablesTable()}
367 self.ipv6 = {'filter': IptablesTable()}
369 self.iptables_apply_deferred = False
371 # Add a nova-filter-top chain. It's intended to be shared
372 # among the various nova components. It sits at the very top
373 # of FORWARD and OUTPUT.
374 for tables in [self.ipv4, self.ipv6]:
375 tables['filter'].add_chain('nova-filter-top', wrap=False)
376 tables['filter'].add_rule('FORWARD', '-j nova-filter-top',
377 wrap=False, top=True)
378 tables['filter'].add_rule('OUTPUT', '-j nova-filter-top',
379 wrap=False, top=True)
381 tables['filter'].add_chain('local')
382 tables['filter'].add_rule('nova-filter-top', '-j $local',
385 # Wrap the built-in chains
386 builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'],
387 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING'],
388 'mangle': ['POSTROUTING']},
389 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}}
391 for ip_version in builtin_chains:
394 elif ip_version == 6:
397 for table, chains in builtin_chains[ip_version].iteritems():
399 tables[table].add_chain(chain)
400 tables[table].add_rule(chain, '-j $%s' % (chain,),
403 # Add a nova-postrouting-bottom chain. It's intended to be shared
404 # among the various nova components. We set it as the last chain
405 # of POSTROUTING chain.
406 self.ipv4['nat'].add_chain('nova-postrouting-bottom', wrap=False)
407 self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom',
410 # We add a snat chain to the shared nova-postrouting-bottom chain
411 # so that it's applied last.
412 self.ipv4['nat'].add_chain('snat')
413 self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $snat',
416 # And then we add a float-snat chain and jump to first thing in
418 self.ipv4['nat'].add_chain('float-snat')
419 self.ipv4['nat'].add_rule('snat', '-j $float-snat')
421 def defer_apply_on(self):
422 self.iptables_apply_deferred = True
424 def defer_apply_off(self):
425 self.iptables_apply_deferred = False
429 for table in self.ipv4.itervalues():
433 for table in self.ipv6.itervalues():
439 if self.iptables_apply_deferred:
444 LOG.debug("Skipping apply due to lack of new rules")
446 @utils.synchronized('iptables', external=True)
448 """Apply the current in-memory set of iptables rules.
450 This will blow away any rules left over from previous runs of the
451 same component of Nova, and replace them with our current set of
452 rules. This happens atomically, thanks to iptables-restore.
455 s = [('iptables', self.ipv4)]
457 s += [('ip6tables', self.ipv6)]
459 for cmd, tables in s:
460 all_tables, _err = self.execute('%s-save' % (cmd,), '-c',
463 all_lines = all_tables.split('\n')
464 for table_name, table in tables.iteritems():
465 start, end = self._find_table(all_lines, table_name)
466 all_lines[start:end] = self._modify_rules(
467 all_lines[start:end], table, table_name)
469 self.execute('%s-restore' % (cmd,), '-c', run_as_root=True,
470 process_input='\n'.join(all_lines),
472 LOG.debug("IPTablesManager.apply completed with success")
474 def _find_table(self, lines, table_name):
476 # length only <2 when fake iptables
479 start = lines.index('*%s' % table_name) - 1
481 # Couldn't find table_name
483 end = lines[start:].index('COMMIT') + start + 2
486 def _modify_rules(self, current_lines, table, table_name):
487 unwrapped_chains = table.unwrapped_chains
488 chains = table.chains
489 remove_chains = table.remove_chains
491 remove_rules = table.remove_rules
493 if not current_lines:
494 fake_table = ['#Generated by nova',
495 '*' + table_name, 'COMMIT',
496 '#Completed by nova']
497 current_lines = fake_table
499 # Remove any trace of our rules
500 new_filter = filter(lambda line: binary_name not in line,
506 if CONF.iptables_top_regex:
507 regex = re.compile(CONF.iptables_top_regex)
508 temp_filter = filter(lambda line: regex.search(line), new_filter)
509 for rule_str in temp_filter:
510 new_filter = filter(lambda s: s.strip() != rule_str.strip(),
512 top_rules = temp_filter
514 if CONF.iptables_bottom_regex:
515 regex = re.compile(CONF.iptables_bottom_regex)
516 temp_filter = filter(lambda line: regex.search(line), new_filter)
517 for rule_str in temp_filter:
518 new_filter = filter(lambda s: s.strip() != rule_str.strip(),
520 bottom_rules = temp_filter
524 for rules_index, rule in enumerate(new_filter):
526 if rule.startswith(':'):
529 if not rule.startswith(':'):
535 our_rules = top_rules
540 # rule.top == True means we want this rule to be at the top.
541 # Further down, we weed out duplicates from the bottom of the
542 # list, so here we remove the dupes ahead of time.
544 # We don't want to remove an entry if it has non-zero
545 # [packet:byte] counts and replace it with [0:0], so let's
546 # go look for a duplicate, and over-ride our table rule if
549 # ignore [packet:byte] counts at beginning of line
550 if rule_str.startswith('['):
551 rule_str = rule_str.split(']', 1)[1]
552 dup_filter = filter(lambda s: rule_str.strip() in s.strip(),
555 new_filter = filter(lambda s:
556 rule_str.strip() not in s.strip(),
558 # if no duplicates, use original rule
560 # grab the last entry, if there is one
567 our_rules += [rule_str]
569 bot_rules += [rule_str]
571 our_rules += bot_rules
573 new_filter[rules_index:rules_index] = our_rules
575 new_filter[rules_index:rules_index] = [':%s - [0:0]' % (name,)
576 for name in unwrapped_chains]
577 new_filter[rules_index:rules_index] = [':%s-%s - [0:0]' %
581 commit_index = new_filter.index('COMMIT')
582 new_filter[commit_index:commit_index] = bottom_rules
585 def _weed_out_duplicates(line):
586 # ignore [packet:byte] counts at beginning of lines
587 if line.startswith('['):
588 line = line.split(']', 1)[1]
590 if line in seen_lines:
596 def _weed_out_removes(line):
597 # We need to find exact matches here
598 if line.startswith(':'):
599 # it's a chain, for example, ":nova-billing - [0:0]"
600 # strip off everything except the chain name
601 line = line.split(':')[1]
602 line = line.split('- [')[0]
604 for chain in remove_chains:
606 remove_chains.remove(chain)
608 elif line.startswith('['):
610 # ignore [packet:byte] counts at beginning of lines
611 line = line.split(']', 1)[1]
613 for rule in remove_rules:
614 # ignore [packet:byte] counts at beginning of rules
616 rule_str = rule_str.split(' ', 1)[1]
617 rule_str = rule_str.strip()
619 remove_rules.remove(rule)
625 # We filter duplicates, letting the *last* occurrence take
626 # precedence. We also filter out anything in the "remove"
629 new_filter = filter(_weed_out_duplicates, new_filter)
630 new_filter = filter(_weed_out_removes, new_filter)
633 # flush lists, just in case we didn't find something
634 remove_chains.clear()
635 for rule in remove_rules:
636 remove_rules.remove(rule)
641 # NOTE(jkoelker) This is just a nice little stub point since mocking
642 # builtins with mox is a nightmare
643 def write_to_file(file, data, mode='w'):
644 with open(file, mode) as f:
648 def is_pid_cmdline_correct(pid, match):
649 """Ensure that the cmdline for a pid seems sane
651 Because pids are recycled, blindly killing by pid is something to
652 avoid. This provides the ability to include a substring that is
653 expected in the cmdline as a safety check.
656 with open('/proc/%d/cmdline' % pid) as f:
658 return match in cmdline
659 except EnvironmentError:
663 def metadata_forward():
664 """Create forwarding rule for metadata."""
665 if CONF.metadata_host != '127.0.0.1':
666 iptables_manager.ipv4['nat'].add_rule('PREROUTING',
667 '-s 0.0.0.0/0 -d 169.254.169.254/32 '
668 '-p tcp -m tcp --dport 80 -j DNAT '
669 '--to-destination %s:%s' %
673 iptables_manager.ipv4['nat'].add_rule('PREROUTING',
674 '-s 0.0.0.0/0 -d 169.254.169.254/32 '
675 '-p tcp -m tcp --dport 80 '
676 '-j REDIRECT --to-ports %s' %
678 iptables_manager.apply()
681 def _iptables_dest(ip):
682 if ((netaddr.IPAddress(ip).version == 4 and ip == '127.0.0.1')
684 return '-m addrtype --dst-type LOCAL'
689 def metadata_accept():
690 """Create the filter accept rule for metadata."""
692 rule = ('-p tcp -m tcp --dport %s %s -j ACCEPT' %
693 (CONF.metadata_port, _iptables_dest(CONF.metadata_host)))
695 if netaddr.IPAddress(CONF.metadata_host).version == 4:
696 iptables_manager.ipv4['filter'].add_rule('INPUT', rule)
698 iptables_manager.ipv6['filter'].add_rule('INPUT', rule)
700 iptables_manager.apply()
703 def add_snat_rule(ip_range, is_external=False):
704 if CONF.routing_source_ip:
706 if CONF.force_snat_range:
707 snat_range = CONF.force_snat_range
711 snat_range = ['0.0.0.0/0']
712 for dest_range in snat_range:
713 rule = ('-s %s -d %s -j SNAT --to-source %s'
714 % (ip_range, dest_range, CONF.routing_source_ip))
715 if not is_external and CONF.public_interface:
716 rule += ' -o %s' % CONF.public_interface
717 iptables_manager.ipv4['nat'].add_rule('snat', rule)
718 iptables_manager.apply()
721 def init_host(ip_range, is_external=False):
722 """Basic networking setup goes here."""
723 # NOTE(devcamcar): Cloud public SNAT entries and the default
724 # SNAT rule for outbound traffic.
726 add_snat_rule(ip_range, is_external)
730 for snat_range in CONF.force_snat_range:
731 rules.append('PREROUTING -p ipv4 --ip-src %s --ip-dst %s '
732 '-j redirect --redirect-target ACCEPT' %
733 (ip_range, snat_range))
735 ensure_ebtables_rules(rules, 'nat')
737 iptables_manager.ipv4['nat'].add_rule('POSTROUTING',
738 '-s %s -d %s/32 -j ACCEPT' %
739 (ip_range, CONF.metadata_host))
741 for dmz in CONF.dmz_cidr:
742 iptables_manager.ipv4['nat'].add_rule('POSTROUTING',
743 '-s %s -d %s -j ACCEPT' %
746 iptables_manager.ipv4['nat'].add_rule('POSTROUTING',
747 '-s %(range)s -d %(range)s '
748 '-m conntrack ! --ctstate DNAT '
751 iptables_manager.apply()
754 def send_arp_for_ip(ip, device, count):
755 out, err = _execute('arping', '-U', ip,
758 run_as_root=True, check_exit_code=False)
761 LOG.debug('arping error for ip %s', ip)
764 def bind_floating_ip(floating_ip, device):
765 """Bind ip to public interface."""
766 _execute('ip', 'addr', 'add', str(floating_ip) + '/32',
768 run_as_root=True, check_exit_code=[0, 2, 254])
770 if CONF.send_arp_for_ha and CONF.send_arp_for_ha_count > 0:
771 send_arp_for_ip(floating_ip, device, CONF.send_arp_for_ha_count)
774 def unbind_floating_ip(floating_ip, device):
775 """Unbind a public ip from public interface."""
776 _execute('ip', 'addr', 'del', str(floating_ip) + '/32',
778 run_as_root=True, check_exit_code=[0, 2, 254])
781 def ensure_metadata_ip():
782 """Sets up local metadata ip."""
783 _execute('ip', 'addr', 'add', '169.254.169.254/32',
784 'scope', 'link', 'dev', 'lo',
785 run_as_root=True, check_exit_code=[0, 2, 254])
788 def ensure_vpn_forward(public_ip, port, private_ip):
789 """Sets up forwarding rules for vlan."""
790 iptables_manager.ipv4['filter'].add_rule('FORWARD',
793 '-j ACCEPT' % private_ip)
794 iptables_manager.ipv4['nat'].add_rule('PREROUTING',
796 '--dport %s -j DNAT --to %s:1194' %
797 (public_ip, port, private_ip))
798 iptables_manager.ipv4['nat'].add_rule('OUTPUT',
800 '--dport %s -j DNAT --to %s:1194' %
801 (public_ip, port, private_ip))
802 iptables_manager.apply()
805 def ensure_floating_forward(floating_ip, fixed_ip, device, network):
806 """Ensure floating ip forwarding rule."""
807 # NOTE(vish): Make sure we never have duplicate rules for the same ip
808 regex = '.*\s+%s(/32|\s+|$)' % floating_ip
809 num_rules = iptables_manager.ipv4['nat'].remove_rules_regex(regex)
811 msg = _LW('Removed %(num)d duplicate rules for floating ip %(float)s')
812 LOG.warn(msg, {'num': num_rules, 'float': floating_ip})
813 for chain, rule in floating_forward_rules(floating_ip, fixed_ip, device):
814 iptables_manager.ipv4['nat'].add_rule(chain, rule)
815 iptables_manager.apply()
816 if device != network['bridge']:
817 ensure_ebtables_rules(*floating_ebtables_rules(fixed_ip, network))
820 def remove_floating_forward(floating_ip, fixed_ip, device, network):
821 """Remove forwarding for floating ip."""
822 for chain, rule in floating_forward_rules(floating_ip, fixed_ip, device):
823 iptables_manager.ipv4['nat'].remove_rule(chain, rule)
824 iptables_manager.apply()
825 if device != network['bridge']:
826 remove_ebtables_rules(*floating_ebtables_rules(fixed_ip, network))
829 def floating_ebtables_rules(fixed_ip, network):
830 """Makes sure only in-network traffic is bridged."""
831 return (['PREROUTING --logical-in %s -p ipv4 --ip-src %s '
832 '! --ip-dst %s -j redirect --redirect-target ACCEPT' %
833 (network['bridge'], fixed_ip, network['cidr'])], 'nat')
836 def floating_forward_rules(floating_ip, fixed_ip, device):
838 rule = '-s %s -j SNAT --to %s' % (fixed_ip, floating_ip)
840 rules.append(('float-snat', rule + ' -d %s' % fixed_ip))
841 rules.append(('float-snat', rule + ' -o %s' % device))
843 rules.append(('float-snat', rule))
845 ('PREROUTING', '-d %s -j DNAT --to %s' % (floating_ip, fixed_ip)))
847 ('OUTPUT', '-d %s -j DNAT --to %s' % (floating_ip, fixed_ip)))
848 rules.append(('POSTROUTING', '-s %s -m conntrack --ctstate DNAT -j SNAT '
850 (fixed_ip, floating_ip)))
854 def clean_conntrack(fixed_ip):
856 _execute('conntrack', '-D', '-r', fixed_ip, run_as_root=True,
857 check_exit_code=[0, 1])
858 except processutils.ProcessExecutionError:
859 LOG.exception(_LE('Error deleting conntrack entries for %s'), fixed_ip)
862 def _enable_ipv4_forwarding():
863 sysctl_key = 'net.ipv4.ip_forward'
864 stdout, stderr = _execute('sysctl', '-n', sysctl_key)
865 if stdout.strip() is not '1':
866 _execute('sysctl', '-w', '%s=1' % sysctl_key, run_as_root=True)
869 @utils.synchronized('lock_gateway', external=True)
870 def initialize_gateway_device(dev, network_ref):
874 _enable_ipv4_forwarding()
876 # NOTE(vish): The ip for dnsmasq has to be the first address on the
877 # bridge for it to respond to requests properly
879 prefix = network_ref.cidr.prefixlen
880 except AttributeError:
881 prefix = network_ref['cidr'].rpartition('/')[2]
883 full_ip = '%s/%s' % (network_ref['dhcp_server'], prefix)
884 new_ip_params = [[full_ip, 'brd', network_ref['broadcast']]]
886 out, err = _execute('ip', 'addr', 'show', 'dev', dev,
888 for line in out.split('\n'):
889 fields = line.split()
890 if fields and fields[0] == 'inet':
891 if fields[-2] in ('secondary', 'dynamic'):
892 ip_params = fields[1:-2]
894 ip_params = fields[1:-1]
895 old_ip_params.append(ip_params)
896 if ip_params[0] != full_ip:
897 new_ip_params.append(ip_params)
898 if not old_ip_params or old_ip_params[0][0] != full_ip:
900 result = _execute('ip', 'route', 'show', 'dev', dev)
903 for line in out.split('\n'):
904 fields = line.split()
905 if fields and 'via' in fields:
906 old_routes.append(fields)
907 _execute('ip', 'route', 'del', fields[0],
908 'dev', dev, run_as_root=True)
909 for ip_params in old_ip_params:
910 _execute(*_ip_bridge_cmd('del', ip_params, dev),
911 run_as_root=True, check_exit_code=[0, 2, 254])
912 for ip_params in new_ip_params:
913 _execute(*_ip_bridge_cmd('add', ip_params, dev),
914 run_as_root=True, check_exit_code=[0, 2, 254])
916 for fields in old_routes:
917 _execute('ip', 'route', 'add', *fields,
919 if CONF.send_arp_for_ha and CONF.send_arp_for_ha_count > 0:
920 send_arp_for_ip(network_ref['dhcp_server'], dev,
921 CONF.send_arp_for_ha_count)
923 _execute('ip', '-f', 'inet6', 'addr',
924 'change', network_ref['cidr_v6'],
925 'dev', dev, run_as_root=True)
928 def get_dhcp_leases(context, network_ref):
929 """Return a network's hosts config in dnsmasq leasefile format."""
932 if network_ref['multi_host']:
934 for fixedip in objects.FixedIPList.get_by_network(context,
937 # NOTE(cfb): Don't return a lease entry if the IP isn't
940 hosts.append(_host_lease(fixedip))
942 return '\n'.join(hosts)
945 def get_dhcp_hosts(context, network_ref, fixedips):
946 """Get network's hosts config in dhcp-host format."""
949 for fixedip in fixedips:
950 if fixedip.allocated:
951 if fixedip.virtual_interface.address not in macs:
952 hosts.append(_host_dhcp(fixedip))
953 macs.add(fixedip.virtual_interface.address)
954 return '\n'.join(hosts)
957 def get_dns_hosts(context, network_ref):
958 """Get network's DNS hosts in hosts format."""
960 for fixedip in objects.FixedIPList.get_by_network(context, network_ref):
961 if fixedip.allocated:
962 hosts.append(_host_dns(fixedip))
963 return '\n'.join(hosts)
966 def _add_dnsmasq_accept_rules(dev):
967 """Allow DHCP and DNS traffic through to dnsmasq."""
968 table = iptables_manager.ipv4['filter']
969 for port in [67, 53]:
970 for proto in ['udp', 'tcp']:
971 args = {'dev': dev, 'port': port, 'proto': proto}
972 table.add_rule('INPUT',
973 '-i %(dev)s -p %(proto)s -m %(proto)s '
974 '--dport %(port)s -j ACCEPT' % args)
975 iptables_manager.apply()
978 def _remove_dnsmasq_accept_rules(dev):
979 """Remove DHCP and DNS traffic allowed through to dnsmasq."""
980 table = iptables_manager.ipv4['filter']
981 for port in [67, 53]:
982 for proto in ['udp', 'tcp']:
983 args = {'dev': dev, 'port': port, 'proto': proto}
984 table.remove_rule('INPUT',
985 '-i %(dev)s -p %(proto)s -m %(proto)s '
986 '--dport %(port)s -j ACCEPT' % args)
987 iptables_manager.apply()
990 # NOTE(russellb) Curious why this is needed? Check out this explanation from
991 # markmc: https://bugzilla.redhat.com/show_bug.cgi?id=910619#c6
992 def _add_dhcp_mangle_rule(dev):
993 table = iptables_manager.ipv4['mangle']
994 table.add_rule('POSTROUTING',
995 '-o %s -p udp -m udp --dport 68 -j CHECKSUM '
996 '--checksum-fill' % dev)
997 iptables_manager.apply()
1000 def _remove_dhcp_mangle_rule(dev):
1001 table = iptables_manager.ipv4['mangle']
1002 table.remove_rule('POSTROUTING',
1003 '-o %s -p udp -m udp --dport 68 -j CHECKSUM '
1004 '--checksum-fill' % dev)
1005 iptables_manager.apply()
1008 def get_dhcp_opts(context, network_ref, fixedips):
1009 """Get network's hosts config in dhcp-opts format."""
1010 gateway = network_ref['gateway']
1011 # NOTE(vish): if we are in multi-host mode and we are not sharing
1012 # addresses, then we actually need to hand out the
1013 # dhcp server address as the gateway.
1014 if network_ref['multi_host'] and not (network_ref['share_address'] or
1015 CONF.share_dhcp_address):
1016 gateway = network_ref['dhcp_server']
1018 if CONF.use_single_default_gateway:
1019 for fixedip in fixedips:
1020 if fixedip.allocated:
1021 vif_id = fixedip.virtual_interface_id
1022 if fixedip.default_route:
1023 hosts.append(_host_dhcp_opts(vif_id, gateway))
1025 hosts.append(_host_dhcp_opts(vif_id))
1027 hosts.append(_host_dhcp_opts(None, gateway))
1028 return '\n'.join(hosts)
1031 def release_dhcp(dev, address, mac_address):
1032 if device_exists(dev):
1034 utils.execute('dhcp_release', dev, address, mac_address,
1036 except processutils.ProcessExecutionError:
1037 raise exception.NetworkDhcpReleaseFailed(address=address,
1038 mac_address=mac_address)
1041 def update_dhcp(context, dev, network_ref):
1042 conffile = _dhcp_file(dev, 'conf')
1044 if network_ref['multi_host']:
1046 fixedips = objects.FixedIPList.get_by_network(context,
1049 write_to_file(conffile, get_dhcp_hosts(context, network_ref, fixedips))
1050 restart_dhcp(context, dev, network_ref, fixedips)
1053 def update_dns(context, dev, network_ref):
1054 hostsfile = _dhcp_file(dev, 'hosts')
1056 if network_ref['multi_host']:
1058 fixedips = objects.FixedIPList.get_by_network(context,
1061 write_to_file(hostsfile, get_dns_hosts(context, network_ref))
1062 restart_dhcp(context, dev, network_ref, fixedips)
1065 def update_dhcp_hostfile_with_text(dev, hosts_text):
1066 conffile = _dhcp_file(dev, 'conf')
1067 write_to_file(conffile, hosts_text)
1071 pid = _dnsmasq_pid_for(dev)
1073 # Check that the process exists and looks like a dnsmasq process
1074 conffile = _dhcp_file(dev, 'conf')
1075 if is_pid_cmdline_correct(pid, conffile.split('/')[-1]):
1076 _execute('kill', '-9', pid, run_as_root=True)
1078 LOG.debug('Pid %d is stale, skip killing dnsmasq', pid)
1079 _remove_dnsmasq_accept_rules(dev)
1080 _remove_dhcp_mangle_rule(dev)
1083 # NOTE(ja): Sending a HUP only reloads the hostfile, so any
1084 # configuration options (like dchp-range, vlan, ...)
1086 @utils.synchronized('dnsmasq_start')
1087 def restart_dhcp(context, dev, network_ref, fixedips):
1088 """(Re)starts a dnsmasq server for a given network.
1090 If a dnsmasq instance is already running then send a HUP
1091 signal causing it to reload, otherwise spawn a new instance.
1094 conffile = _dhcp_file(dev, 'conf')
1096 optsfile = _dhcp_file(dev, 'opts')
1097 write_to_file(optsfile, get_dhcp_opts(context, network_ref, fixedips))
1098 os.chmod(optsfile, 0o644)
1100 _add_dhcp_mangle_rule(dev)
1102 # Make sure dnsmasq can actually read it (it setuid()s to "nobody")
1103 os.chmod(conffile, 0o644)
1105 pid = _dnsmasq_pid_for(dev)
1107 # if dnsmasq is already running, then tell it to reload
1109 if is_pid_cmdline_correct(pid, conffile.split('/')[-1]):
1111 _execute('kill', '-HUP', pid, run_as_root=True)
1112 _add_dnsmasq_accept_rules(dev)
1114 except Exception as exc:
1115 LOG.error(_LE('kill -HUP dnsmasq threw %s'), exc)
1117 LOG.debug('Pid %d is stale, relaunching dnsmasq', pid)
1120 'CONFIG_FILE=%s' % jsonutils.dumps(CONF.dhcpbridge_flagfile),
1121 'NETWORK_ID=%s' % str(network_ref['id']),
1124 '--bind-interfaces',
1125 '--conf-file=%s' % CONF.dnsmasq_config_file,
1126 '--pid-file=%s' % _dhcp_file(dev, 'pid'),
1127 '--dhcp-optsfile=%s' % _dhcp_file(dev, 'opts'),
1128 '--listen-address=%s' % network_ref['dhcp_server'],
1129 '--except-interface=lo',
1130 '--dhcp-range=set:%s,%s,static,%s,%ss' %
1131 (network_ref['label'],
1132 network_ref['dhcp_start'],
1133 network_ref['netmask'],
1134 CONF.dhcp_lease_time),
1135 '--dhcp-lease-max=%s' % len(netaddr.IPNetwork(network_ref['cidr'])),
1136 '--dhcp-hostsfile=%s' % _dhcp_file(dev, 'conf'),
1137 '--dhcp-script=%s' % CONF.dhcpbridge,
1141 # dnsmasq currently gives an error for an empty domain,
1142 # rather than ignoring. So only specify it if defined.
1143 if CONF.dhcp_domain:
1144 cmd.append('--domain=%s' % CONF.dhcp_domain)
1146 dns_servers = CONF.dns_server
1147 if CONF.use_network_dns_servers:
1148 if network_ref.get('dns1'):
1149 dns_servers.append(network_ref.get('dns1'))
1150 if network_ref.get('dns2'):
1151 dns_servers.append(network_ref.get('dns2'))
1152 if network_ref['multi_host']:
1153 cmd.append('--addn-hosts=%s' % _dhcp_file(dev, 'hosts'))
1155 cmd.append('--no-resolv')
1156 for dns_server in dns_servers:
1157 cmd.append('--server=%s' % dns_server)
1159 _execute(*cmd, run_as_root=True)
1161 _add_dnsmasq_accept_rules(dev)
1164 @utils.synchronized('radvd_start')
1165 def update_ra(context, dev, network_ref):
1166 conffile = _ra_file(dev, 'conf')
1171 MinRtrAdvInterval 3;
1172 MaxRtrAdvInterval 10;
1179 """ % (dev, network_ref['cidr_v6'])
1180 write_to_file(conffile, conf_str)
1182 # Make sure radvd can actually read it (it setuid()s to "nobody")
1183 os.chmod(conffile, 0o644)
1185 pid = _ra_pid_for(dev)
1187 # if radvd is already running, then tell it to reload
1189 if is_pid_cmdline_correct(pid, conffile):
1191 _execute('kill', pid, run_as_root=True)
1192 except Exception as exc:
1193 LOG.error(_LE('killing radvd threw %s'), exc)
1195 LOG.debug('Pid %d is stale, relaunching radvd', pid)
1198 '-C', '%s' % _ra_file(dev, 'conf'),
1199 '-p', '%s' % _ra_file(dev, 'pid')]
1201 _execute(*cmd, run_as_root=True)
1204 def _host_lease(fixedip):
1205 """Return a host string for an address in leasefile format."""
1206 timestamp = timeutils.utcnow()
1207 seconds_since_epoch = calendar.timegm(timestamp.utctimetuple())
1208 return '%d %s %s %s *' % (seconds_since_epoch + CONF.dhcp_lease_time,
1209 fixedip.virtual_interface.address,
1211 fixedip.instance.hostname or '*')
1214 def _host_dhcp_network(vif_id):
1215 return 'NW-%s' % vif_id
1218 def _host_dhcp(fixedip):
1219 """Return a host string for an address in dhcp-host format."""
1220 # NOTE(cfb): dnsmasq on linux only supports 64 characters in the hostname
1221 # field (LP #1238910). Since the . counts as a character we need
1222 # to truncate the hostname to only 63 characters.
1223 hostname = fixedip.instance.hostname
1224 if len(hostname) > 63:
1225 LOG.warning(_LW('hostname %s too long, truncating.') % (hostname))
1226 hostname = fixedip.instance.hostname[:2] + '-' +\
1227 fixedip.instance.hostname[-60:]
1228 if CONF.use_single_default_gateway:
1229 net = _host_dhcp_network(fixedip.virtual_interface_id)
1230 return '%s,%s.%s,%s,net:%s' % (fixedip.virtual_interface.address,
1236 return '%s,%s.%s,%s' % (fixedip.virtual_interface.address,
1242 def _host_dns(fixedip):
1243 return '%s\t%s.%s' % (fixedip.address,
1244 fixedip.instance.hostname,
1248 def _host_dhcp_opts(vif_id=None, gateway=None):
1249 """Return an empty gateway option."""
1251 if vif_id is not None:
1252 values.append(_host_dhcp_network(vif_id))
1253 # NOTE(vish): 3 is the dhcp option for gateway.
1256 values.append('%s' % gateway)
1257 return ','.join(values)
1260 def _execute(*cmd, **kwargs):
1261 """Wrapper around utils._execute for fake_network."""
1262 if CONF.fake_network:
1263 LOG.debug('FAKE NET: %s', ' '.join(map(str, cmd)))
1266 return utils.execute(*cmd, **kwargs)
1269 def device_exists(device):
1270 """Check if ethernet device exists."""
1271 return os.path.exists('/sys/class/net/%s' % device)
1274 def _dhcp_file(dev, kind):
1275 """Return path to a pid, leases, hosts or conf file for a bridge/device."""
1276 fileutils.ensure_tree(CONF.networks_path)
1277 return os.path.abspath('%s/nova-%s.%s' % (CONF.networks_path,
1282 def _ra_file(dev, kind):
1283 """Return path to a pid or conf file for a bridge/device."""
1284 fileutils.ensure_tree(CONF.networks_path)
1285 return os.path.abspath('%s/nova-ra-%s.%s' % (CONF.networks_path,
1290 def _dnsmasq_pid_for(dev):
1291 """Returns the pid for prior dnsmasq instance for a bridge/device.
1293 Returns None if no pid file exists.
1295 If machine has rebooted pid might be incorrect (caller should check).
1298 pid_file = _dhcp_file(dev, 'pid')
1300 if os.path.exists(pid_file):
1302 with open(pid_file, 'r') as f:
1303 return int(f.read())
1304 except (ValueError, IOError):
1308 def _ra_pid_for(dev):
1309 """Returns the pid for prior radvd instance for a bridge/device.
1311 Returns None if no pid file exists.
1313 If machine has rebooted pid might be incorrect (caller should check).
1316 pid_file = _ra_file(dev, 'pid')
1318 if os.path.exists(pid_file):
1319 with open(pid_file, 'r') as f:
1320 return int(f.read())
1323 def _ip_bridge_cmd(action, params, device):
1324 """Build commands to add/del ips to bridges/devices."""
1325 cmd = ['ip', 'addr', action]
1327 cmd.extend(['dev', device])
1331 def _set_device_mtu(dev, mtu=None):
1332 """Set the device MTU."""
1335 mtu = CONF.network_device_mtu
1337 utils.execute('ip', 'link', 'set', dev, 'mtu',
1338 mtu, run_as_root=True,
1339 check_exit_code=[0, 1, 2, 254])
1342 def _create_veth_pair(dev1_name, dev2_name):
1343 """Create a pair of veth devices with the specified names,
1344 deleting any previous devices with those names.
1346 for dev in [dev1_name, dev2_name]:
1349 utils.execute('ip', 'link', 'add', dev1_name, 'type', 'veth', 'peer',
1350 'name', dev2_name, run_as_root=True)
1351 for dev in [dev1_name, dev2_name]:
1352 utils.execute('ip', 'link', 'set', dev, 'up', run_as_root=True)
1353 utils.execute('ip', 'link', 'set', dev, 'promisc', 'on',
1355 _set_device_mtu(dev)
1358 def _ovs_vsctl(args):
1359 full_args = ['ovs-vsctl', '--timeout=%s' % CONF.ovs_vsctl_timeout] + args
1361 return utils.execute(*full_args, run_as_root=True)
1362 except Exception as e:
1363 LOG.error(_LE("Unable to execute %(cmd)s. Exception: %(exception)s"),
1364 {'cmd': full_args, 'exception': e})
1365 raise exception.AgentError(method=full_args)
1368 def create_ovs_vif_port(bridge, dev, iface_id, mac, instance_id):
1369 _ovs_vsctl(['--', '--if-exists', 'del-port', dev, '--',
1370 'add-port', bridge, dev,
1371 '--', 'set', 'Interface', dev,
1372 'external-ids:iface-id=%s' % iface_id,
1373 'external-ids:iface-status=active',
1374 'external-ids:attached-mac=%s' % mac,
1375 'external-ids:vm-uuid=%s' % instance_id])
1376 _set_device_mtu(dev)
1379 def delete_ovs_vif_port(bridge, dev):
1380 _ovs_vsctl(['--', '--if-exists', 'del-port', bridge, dev])
1384 def ovs_set_vhostuser_port_type(dev):
1385 _ovs_vsctl(['--', 'set', 'Interface', dev, 'type=dpdkvhostuser'])
1388 def create_ivs_vif_port(dev, iface_id, mac, instance_id):
1389 utils.execute('ivs-ctl', 'add-port',
1390 dev, run_as_root=True)
1393 def delete_ivs_vif_port(dev):
1394 utils.execute('ivs-ctl', 'del-port', dev,
1396 utils.execute('ip', 'link', 'delete', dev,
1400 def create_tap_dev(dev, mac_address=None):
1401 if not device_exists(dev):
1403 # First, try with 'ip'
1404 utils.execute('ip', 'tuntap', 'add', dev, 'mode', 'tap',
1405 run_as_root=True, check_exit_code=[0, 2, 254])
1406 except processutils.ProcessExecutionError:
1407 # Second option: tunctl
1408 utils.execute('tunctl', '-b', '-t', dev, run_as_root=True)
1410 utils.execute('ip', 'link', 'set', dev, 'address', mac_address,
1411 run_as_root=True, check_exit_code=[0, 2, 254])
1412 utils.execute('ip', 'link', 'set', dev, 'up', run_as_root=True,
1413 check_exit_code=[0, 2, 254])
1416 def delete_net_dev(dev):
1417 """Delete a network device only if it exists."""
1418 if device_exists(dev):
1420 utils.execute('ip', 'link', 'delete', dev, run_as_root=True,
1421 check_exit_code=[0, 2, 254])
1422 LOG.debug("Net device removed: '%s'", dev)
1423 except processutils.ProcessExecutionError:
1424 with excutils.save_and_reraise_exception():
1425 LOG.error(_LE("Failed removing net device: '%s'"), dev)
1428 def delete_bridge_dev(dev):
1429 """Delete a network bridge."""
1430 if device_exists(dev):
1432 utils.execute('ip', 'link', 'set', dev, 'down', run_as_root=True)
1433 utils.execute('brctl', 'delbr', dev, run_as_root=True)
1434 except processutils.ProcessExecutionError:
1435 with excutils.save_and_reraise_exception():
1436 LOG.error(_LE("Failed removing bridge device: '%s'"), dev)
1439 # Similar to compute virt layers, the Linux network node
1440 # code uses a flexible driver model to support different ways
1441 # of creating ethernet interfaces and attaching them to the network.
1442 # In the case of a network host, these interfaces
1443 # act as gateway/dhcp/vpn/etc. endpoints not VM interfaces.
1444 interface_driver = None
1447 def _get_interface_driver():
1448 global interface_driver
1449 if not interface_driver:
1450 interface_driver = importutils.import_object(
1451 CONF.linuxnet_interface_driver)
1452 return interface_driver
1455 def plug(network, mac_address, gateway=True):
1456 return _get_interface_driver().plug(network, mac_address, gateway)
1459 def unplug(network):
1460 return _get_interface_driver().unplug(network)
1463 def get_dev(network):
1464 return _get_interface_driver().get_dev(network)
1467 class LinuxNetInterfaceDriver(object):
1468 """Abstract class that defines generic network host API
1469 for all Linux interface drivers.
1472 def plug(self, network, mac_address):
1473 """Create Linux device, return device name."""
1474 raise NotImplementedError()
1476 def unplug(self, network):
1477 """Destroy Linux device, return device name."""
1478 raise NotImplementedError()
1480 def get_dev(self, network):
1481 """Get device name."""
1482 raise NotImplementedError()
1485 # plugs interfaces using Linux Bridge
1486 class LinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver):
1488 def plug(self, network, mac_address, gateway=True):
1489 vlan = network.get('vlan')
1490 if vlan is not None:
1491 iface = CONF.vlan_interface or network['bridge_interface']
1492 LinuxBridgeInterfaceDriver.ensure_vlan_bridge(
1499 iface = 'vlan%s' % vlan
1501 iface = CONF.flat_interface or network['bridge_interface']
1502 LinuxBridgeInterfaceDriver.ensure_bridge(
1507 if network['share_address'] or CONF.share_dhcp_address:
1508 isolate_dhcp_address(iface, network['dhcp_server'])
1509 # NOTE(vish): applying here so we don't get a lock conflict
1510 iptables_manager.apply()
1511 return network['bridge']
1513 def unplug(self, network, gateway=True):
1514 vlan = network.get('vlan')
1515 if vlan is not None:
1516 iface = 'vlan%s' % vlan
1517 LinuxBridgeInterfaceDriver.remove_vlan_bridge(vlan,
1520 iface = CONF.flat_interface or network['bridge_interface']
1521 LinuxBridgeInterfaceDriver.remove_bridge(network['bridge'],
1524 if network['share_address'] or CONF.share_dhcp_address:
1525 remove_isolate_dhcp_address(iface, network['dhcp_server'])
1527 iptables_manager.apply()
1528 return self.get_dev(network)
1530 def get_dev(self, network):
1531 return network['bridge']
1534 def ensure_vlan_bridge(vlan_num, bridge, bridge_interface,
1535 net_attrs=None, mac_address=None,
1537 """Create a vlan and bridge unless they already exist."""
1538 interface = LinuxBridgeInterfaceDriver.ensure_vlan(vlan_num,
1539 bridge_interface, mac_address,
1541 LinuxBridgeInterfaceDriver.ensure_bridge(bridge, interface, net_attrs)
1545 def remove_vlan_bridge(vlan_num, bridge):
1546 """Delete a bridge and vlan."""
1547 LinuxBridgeInterfaceDriver.remove_bridge(bridge)
1548 LinuxBridgeInterfaceDriver.remove_vlan(vlan_num)
1551 @utils.synchronized('lock_vlan', external=True)
1552 def ensure_vlan(vlan_num, bridge_interface, mac_address=None, mtu=None):
1553 """Create a vlan unless it already exists."""
1554 interface = 'vlan%s' % vlan_num
1555 if not device_exists(interface):
1556 LOG.debug('Starting VLAN interface %s', interface)
1557 _execute('ip', 'link', 'add', 'link', bridge_interface,
1558 'name', interface, 'type', 'vlan',
1559 'id', vlan_num, run_as_root=True,
1560 check_exit_code=[0, 2, 254])
1561 # (danwent) the bridge will inherit this address, so we want to
1562 # make sure it is the value set from the NetworkManager
1564 _execute('ip', 'link', 'set', interface, 'address',
1565 mac_address, run_as_root=True,
1566 check_exit_code=[0, 2, 254])
1567 _execute('ip', 'link', 'set', interface, 'up', run_as_root=True,
1568 check_exit_code=[0, 2, 254])
1569 # NOTE(vish): set mtu every time to ensure that changes to mtu get
1571 _set_device_mtu(interface, mtu)
1575 @utils.synchronized('lock_vlan', external=True)
1576 def remove_vlan(vlan_num):
1577 """Delete a vlan."""
1578 vlan_interface = 'vlan%s' % vlan_num
1579 delete_net_dev(vlan_interface)
1582 @utils.synchronized('lock_bridge', external=True)
1583 def ensure_bridge(bridge, interface, net_attrs=None, gateway=True,
1585 """Create a bridge unless it already exists.
1587 :param interface: the interface to create the bridge on.
1588 :param net_attrs: dictionary with attributes used to create bridge.
1589 :param gateway: whether or not the bridge is a gateway.
1590 :param filtering: whether or not to create filters on the bridge.
1592 If net_attrs is set, it will add the net_attrs['gateway'] to the bridge
1593 using net_attrs['broadcast'] and net_attrs['cidr']. It will also add
1594 the ip_v6 address specified in net_attrs['cidr_v6'] if use_ipv6 is set.
1596 The code will attempt to move any ips that already exist on the
1597 interface onto the bridge and reset the default gateway if necessary.
1600 if not device_exists(bridge):
1601 LOG.debug('Starting Bridge %s', bridge)
1602 _execute('brctl', 'addbr', bridge, run_as_root=True)
1603 _execute('brctl', 'setfd', bridge, 0, run_as_root=True)
1604 # _execute('brctl setageing %s 10' % bridge, run_as_root=True)
1605 _execute('brctl', 'stp', bridge, 'off', run_as_root=True)
1606 # (danwent) bridge device MAC address can't be set directly.
1607 # instead it inherits the MAC address of the first device on the
1608 # bridge, which will either be the vlan interface, or a
1610 _execute('ip', 'link', 'set', bridge, 'up', run_as_root=True)
1613 LOG.debug('Adding interface %(interface)s to bridge %(bridge)s',
1614 {'interface': interface, 'bridge': bridge})
1615 out, err = _execute('brctl', 'addif', bridge, interface,
1616 check_exit_code=False, run_as_root=True)
1617 if (err and err != "device %s is already a member of a bridge; "
1618 "can't enslave it to bridge %s.\n" % (interface, bridge)):
1619 msg = _('Failed to add interface: %s') % err
1620 raise exception.NovaException(msg)
1622 out, err = _execute('ip', 'link', 'set', interface, 'up',
1623 check_exit_code=False, run_as_root=True)
1625 # NOTE(vish): This will break if there is already an ip on the
1626 # interface, so we move any ips to the bridge
1627 # NOTE(danms): We also need to copy routes to the bridge so as
1628 # not to break existing connectivity on the interface
1630 out, err = _execute('ip', 'route', 'show', 'dev', interface)
1631 for line in out.split('\n'):
1632 fields = line.split()
1633 if fields and 'via' in fields:
1634 old_routes.append(fields)
1635 _execute('ip', 'route', 'del', *fields,
1637 out, err = _execute('ip', 'addr', 'show', 'dev', interface,
1639 for line in out.split('\n'):
1640 fields = line.split()
1641 if fields and fields[0] == 'inet':
1642 if fields[-2] in ('secondary', 'dynamic', ):
1643 params = fields[1:-2]
1645 params = fields[1:-1]
1646 _execute(*_ip_bridge_cmd('del', params, fields[-1]),
1647 run_as_root=True, check_exit_code=[0, 2, 254])
1648 _execute(*_ip_bridge_cmd('add', params, bridge),
1649 run_as_root=True, check_exit_code=[0, 2, 254])
1650 for fields in old_routes:
1651 _execute('ip', 'route', 'add', *fields,
1655 # Don't forward traffic unless we were told to be a gateway
1656 ipv4_filter = iptables_manager.ipv4['filter']
1658 for rule in get_gateway_rules(bridge):
1659 ipv4_filter.add_rule(*rule)
1661 ipv4_filter.add_rule('FORWARD',
1662 ('--in-interface %s -j %s'
1663 % (bridge, CONF.iptables_drop_action)))
1664 ipv4_filter.add_rule('FORWARD',
1665 ('--out-interface %s -j %s'
1666 % (bridge, CONF.iptables_drop_action)))
1669 @utils.synchronized('lock_bridge', external=True)
1670 def remove_bridge(bridge, gateway=True, filtering=True):
1671 """Delete a bridge."""
1672 if not device_exists(bridge):
1676 ipv4_filter = iptables_manager.ipv4['filter']
1678 for rule in get_gateway_rules(bridge):
1679 ipv4_filter.remove_rule(*rule)
1681 drop_actions = ['DROP']
1682 if CONF.iptables_drop_action != 'DROP':
1683 drop_actions.append(CONF.iptables_drop_action)
1685 for drop_action in drop_actions:
1686 ipv4_filter.remove_rule('FORWARD',
1687 ('--in-interface %s -j %s'
1688 % (bridge, drop_action)))
1689 ipv4_filter.remove_rule('FORWARD',
1690 ('--out-interface %s -j %s'
1691 % (bridge, drop_action)))
1692 delete_bridge_dev(bridge)
1695 # NOTE(cfb): This is a temporary fix to LP #1316621. We really want to call
1696 # ebtables with --concurrent. In order to do that though we need
1697 # libvirt to support this. Additionally since ebtables --concurrent
1698 # will hang indefinitely waiting on the lock we need to teach
1699 # oslo_concurrency.processutils how to timeout a long running
1700 # process first. Once those are complete we can replace all of this
1701 # with calls to ebtables --concurrent and a reasonable timeout.
1702 def _exec_ebtables(*cmd, **kwargs):
1703 check_exit_code = kwargs.pop('check_exit_code', True)
1705 # List of error strings to re-try.
1707 'Multiple ebtables programs',
1710 # We always try at least once
1711 attempts = CONF.ebtables_exec_attempts
1715 while count <= attempts:
1716 # Updated our counters if needed
1717 sleep = CONF.ebtables_retry_interval * count
1719 # NOTE(cfb): ebtables reports all errors with a return code of 255.
1720 # As such we can't know if we hit a locking error, or some
1721 # other error (like a rule doesn't exist) so we have to
1724 _execute(*cmd, check_exit_code=[0], **kwargs)
1725 except processutils.ProcessExecutionError as exc:
1726 # See if we can retry the error.
1727 if any(error in exc.stderr for error in retry_strings):
1728 if count > attempts and check_exit_code:
1729 LOG.warning(_LW('%s failed. Not Retrying.'), ' '.join(cmd))
1732 # We need to sleep a bit before retrying
1733 LOG.warning(_LW("%(cmd)s failed. Sleeping %(time)s "
1734 "seconds before retry."),
1735 {'cmd': ' '.join(cmd), 'time': sleep})
1738 # Not eligible for retry
1740 LOG.warning(_LW('%s failed. Not Retrying.'), ' '.join(cmd))
1749 @utils.synchronized('ebtables', external=True)
1750 def ensure_ebtables_rules(rules, table='filter'):
1752 cmd = ['ebtables', '-t', table, '-D'] + rule.split()
1753 _exec_ebtables(*cmd, check_exit_code=False, run_as_root=True)
1755 _exec_ebtables(*cmd, run_as_root=True)
1758 @utils.synchronized('ebtables', external=True)
1759 def remove_ebtables_rules(rules, table='filter'):
1761 cmd = ['ebtables', '-t', table, '-D'] + rule.split()
1762 _exec_ebtables(*cmd, check_exit_code=False, run_as_root=True)
1765 def isolate_dhcp_address(interface, address):
1766 # block arp traffic to address across the interface
1768 rules.append('INPUT -p ARP -i %s --arp-ip-dst %s -j DROP'
1769 % (interface, address))
1770 rules.append('OUTPUT -p ARP -o %s --arp-ip-src %s -j DROP'
1771 % (interface, address))
1772 rules.append('FORWARD -p IPv4 -i %s --ip-protocol udp '
1773 '--ip-destination-port 67:68 -j DROP'
1775 rules.append('FORWARD -p IPv4 -o %s --ip-protocol udp '
1776 '--ip-destination-port 67:68 -j DROP'
1778 # NOTE(vish): the above is not possible with iptables/arptables
1779 ensure_ebtables_rules(rules)
1782 def remove_isolate_dhcp_address(interface, address):
1783 # block arp traffic to address across the interface
1785 rules.append('INPUT -p ARP -i %s --arp-ip-dst %s -j DROP'
1786 % (interface, address))
1787 rules.append('OUTPUT -p ARP -o %s --arp-ip-src %s -j DROP'
1788 % (interface, address))
1789 rules.append('FORWARD -p IPv4 -i %s --ip-protocol udp '
1790 '--ip-destination-port 67:68 -j DROP'
1792 rules.append('FORWARD -p IPv4 -o %s --ip-protocol udp '
1793 '--ip-destination-port 67:68 -j DROP'
1795 remove_ebtables_rules(rules)
1796 # NOTE(vish): the above is not possible with iptables/arptables
1799 def get_gateway_rules(bridge):
1800 interfaces = CONF.forward_bridge_interface
1801 if 'all' in interfaces:
1802 return [('FORWARD', '-i %s -j ACCEPT' % bridge),
1803 ('FORWARD', '-o %s -j ACCEPT' % bridge)]
1805 for iface in CONF.forward_bridge_interface:
1807 rules.append(('FORWARD', '-i %s -o %s -j ACCEPT' % (bridge,
1809 rules.append(('FORWARD', '-i %s -o %s -j ACCEPT' % (iface,
1811 rules.append(('FORWARD', '-i %s -o %s -j ACCEPT' % (bridge, bridge)))
1812 rules.append(('FORWARD', '-i %s -j %s' % (bridge,
1813 CONF.iptables_drop_action)))
1814 rules.append(('FORWARD', '-o %s -j %s' % (bridge,
1815 CONF.iptables_drop_action)))
1819 # plugs interfaces using Open vSwitch
1820 class LinuxOVSInterfaceDriver(LinuxNetInterfaceDriver):
1822 def plug(self, network, mac_address, gateway=True):
1823 dev = self.get_dev(network)
1824 if not device_exists(dev):
1825 bridge = CONF.linuxnet_ovs_integration_bridge
1826 _ovs_vsctl(['--', '--may-exist', 'add-port', bridge, dev,
1827 '--', 'set', 'Interface', dev, 'type=internal',
1828 '--', 'set', 'Interface', dev,
1829 'external-ids:iface-id=%s' % dev,
1830 '--', 'set', 'Interface', dev,
1831 'external-ids:iface-status=active',
1832 '--', 'set', 'Interface', dev,
1833 'external-ids:attached-mac=%s' % mac_address])
1834 _execute('ip', 'link', 'set', dev, 'address', mac_address,
1836 _set_device_mtu(dev, network.get('mtu'))
1837 _execute('ip', 'link', 'set', dev, 'up', run_as_root=True)
1839 # If we weren't instructed to act as a gateway then add the
1840 # appropriate flows to block all non-dhcp traffic.
1841 _execute('ovs-ofctl',
1842 'add-flow', bridge, 'priority=1,actions=drop',
1844 _execute('ovs-ofctl', 'add-flow', bridge,
1845 'udp,tp_dst=67,dl_dst=%s,priority=2,actions=normal' %
1846 mac_address, run_as_root=True)
1847 # .. and make sure iptbles won't forward it as well.
1848 iptables_manager.ipv4['filter'].add_rule('FORWARD',
1849 '--in-interface %s -j %s' % (bridge,
1850 CONF.iptables_drop_action))
1851 iptables_manager.ipv4['filter'].add_rule('FORWARD',
1852 '--out-interface %s -j %s' % (bridge,
1853 CONF.iptables_drop_action))
1855 for rule in get_gateway_rules(bridge):
1856 iptables_manager.ipv4['filter'].add_rule(*rule)
1860 def unplug(self, network):
1861 dev = self.get_dev(network)
1862 bridge = CONF.linuxnet_ovs_integration_bridge
1863 _ovs_vsctl(['--', '--if-exists', 'del-port', bridge, dev])
1866 def get_dev(self, network):
1867 dev = 'gw-' + str(network['uuid'][0:11])
1871 # plugs interfaces using Linux Bridge when using NeutronManager
1872 class NeutronLinuxBridgeInterfaceDriver(LinuxNetInterfaceDriver):
1874 BRIDGE_NAME_PREFIX = 'brq'
1875 GATEWAY_INTERFACE_PREFIX = 'gw-'
1877 def plug(self, network, mac_address, gateway=True):
1878 dev = self.get_dev(network)
1879 bridge = self.get_bridge(network)
1881 # If we weren't instructed to act as a gateway then add the
1882 # appropriate flows to block all non-dhcp traffic.
1883 # .. and make sure iptbles won't forward it as well.
1884 iptables_manager.ipv4['filter'].add_rule('FORWARD',
1885 ('--in-interface %s -j %s'
1886 % (bridge, CONF.iptables_drop_action)))
1887 iptables_manager.ipv4['filter'].add_rule('FORWARD',
1888 ('--out-interface %s -j %s'
1889 % (bridge, CONF.iptables_drop_action)))
1892 for rule in get_gateway_rules(bridge):
1893 iptables_manager.ipv4['filter'].add_rule(*rule)
1895 create_tap_dev(dev, mac_address)
1897 if not device_exists(bridge):
1898 LOG.debug("Starting bridge %s ", bridge)
1899 utils.execute('brctl', 'addbr', bridge, run_as_root=True)
1900 utils.execute('brctl', 'setfd', bridge, str(0), run_as_root=True)
1901 utils.execute('brctl', 'stp', bridge, 'off', run_as_root=True)
1902 utils.execute('ip', 'link', 'set', bridge, 'address', mac_address,
1903 run_as_root=True, check_exit_code=[0, 2, 254])
1904 utils.execute('ip', 'link', 'set', bridge, 'up', run_as_root=True,
1905 check_exit_code=[0, 2, 254])
1906 LOG.debug("Done starting bridge %s", bridge)
1908 full_ip = '%s/%s' % (network['dhcp_server'],
1909 network['cidr'].rpartition('/')[2])
1910 utils.execute('ip', 'address', 'add', full_ip, 'dev', bridge,
1911 run_as_root=True, check_exit_code=[0, 2, 254])
1915 def unplug(self, network):
1916 dev = self.get_dev(network)
1917 if not device_exists(dev):
1923 def get_dev(self, network):
1924 dev = self.GATEWAY_INTERFACE_PREFIX + str(network['uuid'][0:11])
1927 def get_bridge(self, network):
1928 bridge = self.BRIDGE_NAME_PREFIX + str(network['uuid'][0:11])
1931 # provide compatibility with existing configs
1932 QuantumLinuxBridgeInterfaceDriver = NeutronLinuxBridgeInterfaceDriver
1934 iptables_manager = IptablesManager()
1937 def set_vf_interface_vlan(pci_addr, mac_addr, vlan=0):
1938 pf_ifname = pci_utils.get_ifname_by_pci_address(pci_addr,
1940 vf_ifname = pci_utils.get_ifname_by_pci_address(pci_addr)
1941 vf_num = pci_utils.get_vf_num_by_pci_address(pci_addr)
1943 # Set the VF's mac address and vlan
1944 exit_code = [0, 2, 254]
1945 port_state = 'up' if vlan > 0 else 'down'
1946 utils.execute('ip', 'link', 'set', pf_ifname,
1951 check_exit_code=exit_code)
1952 # Bring up/down the VF's interface
1953 utils.execute('ip', 'link', 'set', vf_ifname,
1956 check_exit_code=exit_code)