1 # Copyright (c) 2017 Cable Television Laboratories, Inc. ("CableLabs")
2 # and others. All rights reserved.
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at:
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
17 from neutronclient.common.exceptions import NotFound
18 from neutronclient.neutron.client import Client
20 from snaps.domain.network import (
21 Port, SecurityGroup, SecurityGroupRule, Router, InterfaceRouter, Subnet,
23 from snaps.domain.project import NetworkQuotas
24 from snaps.domain.vm_inst import FloatingIp
25 from snaps.openstack.utils import keystone_utils
27 __author__ = 'spisarski'
29 logger = logging.getLogger('neutron_utils')
32 Utilities for basic neutron API calls
36 def neutron_client(os_creds):
38 Instantiates and returns a client for communications with OpenStack's
40 :param os_creds: the credentials for connecting to the OpenStack remote API
41 :return: the client object
43 return Client(api_version=os_creds.network_api_version,
44 session=keystone_utils.keystone_session(os_creds),
45 region_name=os_creds.region_name)
48 def create_network(neutron, os_creds, network_settings):
50 Creates a network for OpenStack
51 :param neutron: the client
52 :param os_creds: the OpenStack credentials
53 :param network_settings: A dictionary containing the network configuration
54 and is responsible for creating the network
56 :return: a SNAPS-OO Network domain object if found else None
58 logger.info('Creating network with name ' + network_settings.name)
59 json_body = network_settings.dict_for_neutron(os_creds)
60 os_network = neutron.create_network(body=json_body)
63 network = get_network_by_id(neutron, os_network['network']['id'])
66 for subnet_settings in network_settings.subnet_settings:
69 create_subnet(neutron, subnet_settings, os_creds, network))
72 'Unexpected error creating subnet [%s] for network [%s]',
73 subnet_settings.name, network.name)
75 for subnet in subnets:
76 delete_subnet(neutron, subnet)
78 delete_network(neutron, network)
82 return get_network_by_id(neutron, network.id)
85 def delete_network(neutron, network):
87 Deletes a network for OpenStack
88 :param neutron: the client
89 :param network: a SNAPS-OO Network domain object
91 if neutron and network:
93 for subnet in network.subnets:
94 logger.info('Deleting subnet with name ' + subnet.name)
96 delete_subnet(neutron, subnet)
100 logger.info('Deleting network with name ' + network.name)
101 neutron.delete_network(network.id)
104 def get_network(neutron, network_settings=None, network_name=None,
107 Returns Network SNAPS-OO domain object the first network found with
108 either the given attributes from the network_settings object if not None,
109 else the query will use just the name from the network_name parameter.
110 When the project_id is included, that will be added to the query filter.
111 :param neutron: the client
112 :param network_settings: the NetworkConfig object used to create filter
113 :param network_name: the name of the network to retrieve
114 :param project_id: the id of the network's project
115 :return: a SNAPS-OO Network domain object
119 net_filter['name'] = network_settings.name
121 net_filter['name'] = network_name
124 net_filter['project_id'] = project_id
126 networks = neutron.list_networks(**net_filter)
127 for network, netInsts in networks.items():
128 for inst in netInsts:
129 return __map_network(neutron, inst)
132 def __get_os_network_by_id(neutron, network_id):
134 Returns the OpenStack network object (dictionary) with the given ID else
136 :param neutron: the client
137 :param network_id: the id of the network to retrieve
138 :return: a SNAPS-OO Network domain object
140 networks = neutron.list_networks(**{'id': network_id})
141 for network in networks['networks']:
142 if network['id'] == network_id:
146 def get_network_by_id(neutron, network_id):
148 Returns the SNAPS Network domain object for the given ID else None
149 :param neutron: the client
150 :param network_id: the id of the network to retrieve
151 :return: a SNAPS-OO Network domain object
153 os_network = __get_os_network_by_id(neutron, network_id)
155 return __map_network(neutron, os_network)
158 def __map_network(neutron, os_network):
160 Returns the network object (dictionary) with the given ID else None
161 :param neutron: the client
162 :param os_network: the OpenStack Network dict
163 :return: a SNAPS-OO Network domain object
165 subnets = get_subnets_by_network_id(neutron, os_network['id'])
166 os_network['subnets'] = subnets
167 return Network(**os_network)
170 def create_subnet(neutron, subnet_settings, os_creds, network):
172 Creates a network subnet for OpenStack
173 :param neutron: the client
174 :param subnet_settings: A dictionary containing the subnet configuration
175 and is responsible for creating the subnet request
177 :param os_creds: the OpenStack credentials
178 :param network: the network object
179 :return: a SNAPS-OO Subnet domain object
181 if neutron and network and subnet_settings:
182 json_body = {'subnets': [subnet_settings.dict_for_neutron(
183 os_creds, network=network)]}
184 logger.info('Creating subnet with name ' + subnet_settings.name)
185 subnets = neutron.create_subnet(body=json_body)
186 return Subnet(**subnets['subnets'][0])
188 raise NeutronException('Failed to create subnet')
191 def delete_subnet(neutron, subnet):
193 Deletes a network subnet for OpenStack
194 :param neutron: the client
195 :param subnet: a SNAPS-OO Subnet domain object
197 if neutron and subnet:
198 logger.info('Deleting subnet with name ' + subnet.name)
199 neutron.delete_subnet(subnet.id)
202 def get_subnet(neutron, subnet_settings=None, subnet_name=None):
204 Returns the first subnet object that fits the query else None including
205 if subnet_settings or subnet_name parameters are None.
206 :param neutron: the client
207 :param subnet_settings: the subnet settings of the object to retrieve
208 :param subnet_name: the name of the subnet to retrieve
209 :return: a SNAPS-OO Subnet domain object or None
213 sub_filter['name'] = subnet_settings.name
214 sub_filter['cidr'] = subnet_settings.cidr
215 if subnet_settings.gateway_ip:
216 sub_filter['gateway_ip'] = subnet_settings.gateway_ip
218 if subnet_settings.enable_dhcp is not None:
219 sub_filter['enable_dhcp'] = subnet_settings.enable_dhcp
221 if subnet_settings.destination:
222 sub_filter['destination'] = subnet_settings.destination
224 if subnet_settings.nexthop:
225 sub_filter['nexthop'] = subnet_settings.nexthop
227 if subnet_settings.ipv6_ra_mode:
228 sub_filter['ipv6_ra_mode'] = subnet_settings.ipv6_ra_mode
230 if subnet_settings.ipv6_address_mode:
231 sub_filter['ipv6_address_mode'] = subnet_settings.ipv6_address_mode
233 sub_filter['name'] = subnet_name
237 subnets = neutron.list_subnets(**sub_filter)
238 for subnet in subnets['subnets']:
239 return Subnet(**subnet)
242 def get_subnet_by_id(neutron, subnet_id):
244 Returns a SNAPS-OO Subnet domain object for a given ID
245 :param neutron: the OpenStack neutron client
246 :param subnet_id: the subnet ID
247 :return: a Subnet object
249 os_subnet = neutron.show_subnet(subnet_id)
250 if os_subnet and 'subnet' in os_subnet:
251 return Subnet(**os_subnet['subnet'])
254 def get_subnets_by_network(neutron, network):
256 Returns a list of SNAPS-OO Subnet domain objects
257 :param neutron: the OpenStack neutron client
258 :param network: the SNAPS-OO Network domain object
259 :return: a list of Subnet objects
261 return get_subnets_by_network_id(neutron, network.id)
264 def get_subnets_by_network_id(neutron, network_id):
266 Returns a list of SNAPS-OO Subnet domain objects
267 :param neutron: the OpenStack neutron client
268 :param network_id: the subnet's ID
269 :return: a list of Subnet objects
273 os_subnets = neutron.list_subnets(network_id=network_id)
275 for os_subnet in os_subnets['subnets']:
276 out.append(Subnet(**os_subnet))
281 def create_router(neutron, os_creds, router_settings):
283 Creates a router for OpenStack
284 :param neutron: the client
285 :param os_creds: the OpenStack credentials
286 :param router_settings: A dictionary containing the router configuration
287 and is responsible for creating the subnet request
289 :return: a SNAPS-OO Router domain object
292 json_body = router_settings.dict_for_neutron(neutron, os_creds)
293 logger.info('Creating router with name - ' + router_settings.name)
294 os_router = neutron.create_router(json_body)
295 return __map_router(neutron, os_router['router'])
297 logger.error("Failed to create router.")
298 raise NeutronException('Failed to create router')
301 def delete_router(neutron, router):
303 Deletes a router for OpenStack
304 :param neutron: the client
305 :param router: a SNAPS-OO Router domain object
307 if neutron and router:
308 logger.info('Deleting router with name - ' + router.name)
309 neutron.delete_router(router=router.id)
312 def get_router_by_id(neutron, router_id):
314 Returns a router with a given ID, else None if not found
315 :param neutron: the client
316 :param router_id: the Router ID
317 :return: a SNAPS-OO Router domain object
319 router = neutron.show_router(router_id)
321 return __map_router(neutron, router['router'])
324 def get_router(neutron, router_settings=None, router_name=None):
326 Returns the first router object (dictionary) found the given the settings
327 values if not None, else finds the first with the value of the router_name
329 :param neutron: the client
330 :param router_settings: the RouterConfig object
331 :param router_name: the name of the network to retrieve
332 :return: a SNAPS-OO Router domain object
334 router_filter = dict()
336 router_filter['name'] = router_settings.name
337 if router_settings.admin_state_up is not None:
338 router_filter['admin_state_up'] = router_settings.admin_state_up
340 router_filter['name'] = router_name
344 routers = neutron.list_routers(**router_filter)
346 for routerInst in routers['routers']:
347 return __map_router(neutron, routerInst)
352 def __map_router(neutron, os_router):
354 Takes an OpenStack router instance and maps it to a SNAPS Router domain
356 :param neutron: the neutron client
357 :param os_router: the OpenStack Router object
360 device_ports = neutron.list_ports(
361 **{'device_id': os_router['id']})['ports']
362 port_subnets = list()
364 # Order by create date
365 sorted_ports = sorted(
366 device_ports, key=lambda dev_port: dev_port['created_at'])
368 for port in sorted_ports:
370 for fixed_ip in port['fixed_ips']:
371 subnet = get_subnet_by_id(neutron, fixed_ip['subnet_id'])
372 if subnet and subnet.network_id == port['network_id']:
373 subnets.append(subnet)
374 port_subnets.append((Port(**port), subnets))
376 os_router['port_subnets'] = port_subnets
377 return Router(**os_router)
380 def add_interface_router(neutron, router, subnet=None, port=None):
382 Adds an interface router for OpenStack for either a subnet or port.
383 Exception will be raised if requesting for both.
384 :param neutron: the client
385 :param router: the router object
386 :param subnet: the subnet object
387 :param port: the port object
388 :return: the interface router object
391 raise NeutronException(
392 'Cannot add interface to the router. Both subnet and '
393 'port were sent in. Either or please.')
395 if neutron and router and (router or subnet):
396 logger.info('Adding interface to router with name ' + router.name)
397 os_intf_router = neutron.add_interface_router(
398 router=router.id, body=__create_port_json_body(subnet, port))
399 return InterfaceRouter(**os_intf_router)
401 raise NeutronException(
402 'Unable to create interface router as neutron client,'
403 ' router or subnet were not created')
406 def remove_interface_router(neutron, router, subnet=None, port=None):
408 Removes an interface router for OpenStack
409 :param neutron: the client
410 :param router: the SNAPS-OO Router domain object
411 :param subnet: the subnet object (either subnet or port, not both)
412 :param port: the port object
416 logger.info('Removing router interface from router named ' +
418 neutron.remove_interface_router(
420 body=__create_port_json_body(subnet, port))
421 except NotFound as e:
422 logger.warning('Could not remove router interface. NotFound - %s',
426 logger.warning('Could not remove router interface, No router object')
429 def __create_port_json_body(subnet=None, port=None):
431 Returns the dictionary required for creating and deleting router
432 interfaces. Will only work on a subnet or port object. Will throw and
433 exception if parameters contain both or neither
434 :param subnet: the subnet object
435 :param port: the port object
439 raise NeutronException(
440 'Cannot create JSON body with both subnet and port')
441 if not subnet and not port:
442 raise NeutronException(
443 'Cannot create JSON body without subnet or port')
446 return {"subnet_id": subnet.id}
448 return {"port_id": port.id}
451 def create_port(neutron, os_creds, port_settings):
453 Creates a port for OpenStack
454 :param neutron: the client
455 :param os_creds: the OpenStack credentials
456 :param port_settings: the settings object for port configuration
457 :return: the SNAPS-OO Port domain object
459 json_body = port_settings.dict_for_neutron(neutron, os_creds)
460 logger.info('Creating port for network with name - %s',
461 port_settings.network_name)
462 os_port = neutron.create_port(body=json_body)['port']
463 return Port(name=os_port['name'], id=os_port['id'],
464 ips=os_port['fixed_ips'],
465 mac_address=os_port['mac_address'],
466 allowed_address_pairs=os_port['allowed_address_pairs'])
469 def delete_port(neutron, port):
471 Removes an OpenStack port
472 :param neutron: the client
473 :param port: the SNAPS-OO Port domain object
475 logger.info('Deleting port with name ' + port.name)
476 neutron.delete_port(port.id)
479 def get_port(neutron, port_settings=None, port_name=None):
481 Returns the first port object (dictionary) found for the given query
482 :param neutron: the client
483 :param port_settings: the PortConfig object used for generating the query
484 :param port_name: if port_settings is None, this name is the value to place
486 :return: a SNAPS-OO Port domain object
491 if port_settings.name and len(port_settings.name) > 0:
492 port_filter['name'] = port_settings.name
493 if port_settings.admin_state_up:
494 port_filter['admin_state_up'] = port_settings.admin_state_up
495 if port_settings.device_id:
496 port_filter['device_id'] = port_settings.device_id
497 if port_settings.mac_address:
498 port_filter['mac_address'] = port_settings.mac_address
499 if port_settings.network_name:
500 network = get_network(neutron,
501 network_name=port_settings.network_name)
502 port_filter['network_id'] = network.id
504 port_filter['name'] = port_name
506 ports = neutron.list_ports(**port_filter)
507 for port in ports['ports']:
512 def get_port_by_id(neutron, port_id):
514 Returns a SNAPS-OO Port domain object for the given ID or none if not found
515 :param neutron: the client
516 :param port_id: the to query
517 :return: a SNAPS-OO Port domain object or None
519 port = neutron.show_port(port_id)
521 return Port(**port['port'])
525 def get_ports(neutron, network, ips=None):
527 Returns a list of SNAPS-OO Port objects for all OpenStack Port objects that
528 are associated with the 'network' parameter
529 :param neutron: the client
530 :param network: SNAPS-OO Network domain object
531 :param ips: the IPs to lookup if not None
532 :return: a SNAPS-OO Port domain object or None if not found
535 ports = neutron.list_ports(**{'network_id': network.id})
536 for port in ports['ports']:
538 for fixed_ips in port['fixed_ips']:
539 if ('ip_address' in fixed_ips and
540 fixed_ips['ip_address'] in ips) or ips is None:
541 out.append(Port(**port))
544 out.append(Port(**port))
549 def create_security_group(neutron, keystone, sec_grp_settings):
551 Creates a security group object in OpenStack
552 :param neutron: the Neutron client
553 :param keystone: the Keystone client
554 :param sec_grp_settings: the security group settings
555 :return: a SNAPS-OO SecurityGroup domain object
557 logger.info('Creating security group with name - %s',
558 sec_grp_settings.name)
559 os_group = neutron.create_security_group(
560 sec_grp_settings.dict_for_neutron(keystone))
561 return __map_os_security_group(neutron, os_group['security_group'])
564 def delete_security_group(neutron, sec_grp):
566 Deletes a security group object from OpenStack
567 :param neutron: the client
568 :param sec_grp: the SNAPS SecurityGroup object to delete
570 logger.info('Deleting security group with name - %s', sec_grp.name)
571 neutron.delete_security_group(sec_grp.id)
574 def get_security_group(neutron, sec_grp_settings=None, sec_grp_name=None,
577 Returns the first security group for a given query. The query gets built
578 from the sec_grp_settings parameter if not None, else only the name of
579 the security group will be used, else if the query parameters are None then
580 None will be returned
581 :param neutron: the client
582 :param sec_grp_settings: an instance of SecurityGroupConfig object
583 :param sec_grp_name: the name of security group object to retrieve
584 :param project_id: the ID of the project/tentant object that owns the
585 secuity group to retrieve
586 :return: a SNAPS-OO SecurityGroup domain object or None if not found
589 sec_grp_filter = dict()
591 sec_grp_filter['tenant_id'] = project_id
594 sec_grp_filter['name'] = sec_grp_settings.name
596 if sec_grp_settings.description:
597 sec_grp_filter['description'] = sec_grp_settings.description
599 sec_grp_filter['name'] = sec_grp_name
603 groups = neutron.list_security_groups(**sec_grp_filter)
604 for group in groups['security_groups']:
605 return __map_os_security_group(neutron, group)
608 def __map_os_security_group(neutron, os_sec_grp):
610 Creates a SecurityGroup SNAPS domain object from an OpenStack Security
612 :param neutron: the neutron client for performing rule lookups
613 :param os_sec_grp: the OpenStack Security Group dict object
614 :return: a SecurityGroup object
616 os_sec_grp['rules'] = get_rules_by_security_group_id(
617 neutron, os_sec_grp['id'])
618 return SecurityGroup(**os_sec_grp)
621 def get_security_group_by_id(neutron, sec_grp_id):
623 Returns the first security group object of the given name else None
624 :param neutron: the client
625 :param sec_grp_id: the id of the security group to retrieve
626 :return: a SNAPS-OO SecurityGroup domain object or None if not found
628 logger.info('Retrieving security group with ID - ' + sec_grp_id)
630 groups = neutron.list_security_groups(**{'id': sec_grp_id})
631 for group in groups['security_groups']:
632 if group['id'] == sec_grp_id:
633 return __map_os_security_group(neutron, group)
637 def create_security_group_rule(neutron, sec_grp_rule_settings):
639 Creates a security group object in OpenStack
640 :param neutron: the client
641 :param sec_grp_rule_settings: the security group rule settings
642 :return: a SNAPS-OO SecurityGroupRule domain object
644 logger.info('Creating security group to security group - %s',
645 sec_grp_rule_settings.sec_grp_name)
646 os_rule = neutron.create_security_group_rule(
647 sec_grp_rule_settings.dict_for_neutron(neutron))
648 return SecurityGroupRule(**os_rule['security_group_rule'])
651 def delete_security_group_rule(neutron, sec_grp_rule):
653 Deletes a security group object from OpenStack
654 :param neutron: the client
655 :param sec_grp_rule: the SNAPS SecurityGroupRule object to delete
657 logger.info('Deleting security group rule with ID - %s',
659 neutron.delete_security_group_rule(sec_grp_rule.id)
662 def get_rules_by_security_group(neutron, sec_grp):
664 Retrieves all of the rules for a given security group
665 :param neutron: the client
666 :param sec_grp: a list of SNAPS SecurityGroupRule domain objects
668 return get_rules_by_security_group_id(neutron, sec_grp.id)
671 def get_rules_by_security_group_id(neutron, sec_grp_id):
673 Retrieves all of the rules for a given security group
674 :param neutron: the client
675 :param sec_grp_id: the ID of the associated security group
677 logger.info('Retrieving security group rules associate with the '
678 'security group with ID - %s', sec_grp_id)
680 rules = neutron.list_security_group_rules(
681 **{'security_group_id': sec_grp_id})
682 for rule in rules['security_group_rules']:
683 if rule['security_group_id'] == sec_grp_id:
684 out.append(SecurityGroupRule(**rule))
688 def get_rule_by_id(neutron, sec_grp, rule_id):
690 Deletes a security group object from OpenStack
691 :param neutron: the client
692 :param sec_grp: the SNAPS SecurityGroup domain object
693 :param rule_id: the rule's ID
694 :param sec_grp: a SNAPS SecurityGroupRule domain object
696 rules = neutron.list_security_group_rules(
697 **{'security_group_id': sec_grp.id})
698 for rule in rules['security_group_rules']:
699 if rule['id'] == rule_id:
700 return SecurityGroupRule(**rule)
704 def get_external_networks(neutron):
706 Returns a list of external OpenStack network object/dict for all external
708 :param neutron: the client
709 :return: a list of external networks of Type SNAPS-OO domain class Network
712 for network in neutron.list_networks(
713 **{'router:external': True})['networks']:
714 out.append(__map_network(neutron, network))
718 def get_floating_ips(neutron, ports=None):
720 Returns all of the floating IPs
721 When ports is not None, FIPs returned must be associated with one of the
722 ports in the list and a tuple 2 where the first element being the port's
723 ID and the second being the FloatingIp SNAPS-OO domain object.
724 When ports is None, all known FloatingIp SNAPS-OO domain objects will be
726 :param neutron: the Neutron client
727 :param ports: a list of tuple 2 where index 0 is the port name and index 1
728 is the SNAPS-OO Port object
729 :return: a list of tuple 2 (port_id, SNAPS FloatingIp) objects when ports
730 is not None else a list of Port objects
733 fips = neutron.list_floatingips()
734 for fip in fips['floatingips']:
736 for port_name, port in ports:
737 if port and port.id == fip['port_id']:
738 out.append((port.id, FloatingIp(**fip)))
741 out.append(FloatingIp(**fip))
746 def create_floating_ip(neutron, ext_net_name):
748 Returns the floating IP object that was created with this call
749 :param neutron: the Neutron client
750 :param ext_net_name: the name of the external network on which to apply the
752 :return: the SNAPS FloatingIp object
754 logger.info('Creating floating ip to external network - ' + ext_net_name)
755 ext_net = get_network(neutron, network_name=ext_net_name)
757 fip = neutron.create_floatingip(
759 {'floating_network_id': ext_net.id}})
761 return FloatingIp(id=fip['floatingip']['id'],
762 ip=fip['floatingip']['floating_ip_address'])
764 raise NeutronException(
765 'Cannot create floating IP, external network not found')
768 def get_floating_ip(neutron, floating_ip):
770 Returns a floating IP object that should be identical to the floating_ip
772 :param neutron: the Neutron client
773 :param floating_ip: the SNAPS FloatingIp object
774 :return: hopefully the same floating IP object input
776 logger.debug('Attempting to retrieve existing floating ip with IP - %s',
778 os_fip = __get_os_floating_ip(neutron, floating_ip)
780 return FloatingIp(id=os_fip['id'], ip=os_fip['floating_ip_address'])
783 def __get_os_floating_ip(neutron, floating_ip):
785 Returns an OpenStack floating IP object
787 :param neutron: the Neutron client
788 :param floating_ip: the SNAPS FloatingIp object
789 :return: hopefully the same floating IP object input
791 logger.debug('Attempting to retrieve existing floating ip with IP - %s',
793 fips = neutron.list_floatingips(ip=floating_ip.id)
795 for fip in fips['floatingips']:
796 if fip['id'] == floating_ip.id:
800 def delete_floating_ip(neutron, floating_ip):
802 Responsible for deleting a floating IP
803 :param neutron: the Neutron client
804 :param floating_ip: the SNAPS FloatingIp object
807 logger.debug('Attempting to delete existing floating ip with IP - %s',
809 return neutron.delete_floatingip(floating_ip.id)
812 def get_network_quotas(neutron, project_id):
814 Returns a list of all available keypairs
815 :param neutron: the neutron client
816 :param project_id: the project's ID of the quotas to lookup
817 :return: an object of type NetworkQuotas or None if not found
819 quota = neutron.show_quota(project_id)
821 return NetworkQuotas(**quota['quota'])
824 def update_quotas(neutron, project_id, network_quotas):
826 Updates the networking quotas for a given project
827 :param neutron: the Neutron client
828 :param project_id: the project's ID that requires quota updates
829 :param network_quotas: an object of type NetworkQuotas containing the
834 update_body['security_group'] = network_quotas.security_group
835 update_body['security_group_rule'] = network_quotas.security_group_rule
836 update_body['floatingip'] = network_quotas.floatingip
837 update_body['network'] = network_quotas.network
838 update_body['port'] = network_quotas.port
839 update_body['router'] = network_quotas.router
840 update_body['subnet'] = network_quotas.subnet
842 return neutron.update_quota(project_id, {'quota': update_body})
845 class NeutronException(Exception):
847 Exception when calls to the Keystone client cannot be served properly