Merge "Render isolated network templates using jinja2"
authorJenkins <jenkins@review.openstack.org>
Thu, 3 Aug 2017 04:30:48 +0000 (04:30 +0000)
committerGerrit Code Review <review@openstack.org>
Thu, 3 Aug 2017 04:30:48 +0000 (04:30 +0000)
j2_excludes.yaml
network/network.network.j2.yaml [new file with mode: 0644]
network/networks.j2.yaml
network/ports/port.network.j2.yaml [new file with mode: 0644]
network/ports/port_from_pool.network.j2.yaml [new file with mode: 0644]
network_data.yaml
tools/process-templates.py

index 063e63d..356068f 100644 (file)
@@ -8,3 +8,39 @@ name:
   - puppet/blockstorage-role.yaml
   - puppet/objectstorage-role.yaml
   - puppet/cephstorage-role.yaml
+  - network/internal_api.yaml
+  - network/external.yaml
+  - network/storage.yaml
+  - network/storage_mgmt.yaml
+  - network/tenant.yaml
+  - network/management.yaml
+  - network/internal_api_v6.yaml
+  - network/external_v6.yaml
+  - network/storage_v6.yaml
+  - network/storage_mgmt_v6.yaml
+  - network/tenant_v6.yaml
+  - network/management_v6.yaml
+  - network/ports/internal_api.yaml
+  - network/ports/external.yaml
+  - network/ports/storage.yaml
+  - network/ports/storage_mgmt.yaml
+  - network/ports/tenant.yaml
+  - network/ports/management.yaml
+  - network/ports/internal_api_v6.yaml
+  - network/ports/external_v6.yaml
+  - network/ports/storage_v6.yaml
+  - network/ports/storage_mgmt_v6.yaml
+  - network/ports/tenant_v6.yaml
+  - network/ports/management_v6.yaml
+  - network/ports/internal_api_from_pool.yaml
+  - network/ports/external_from_pool.yaml
+  - network/ports/storage_from_pool.yaml
+  - network/ports/storage_mgmt_from_pool.yaml
+  - network/ports/tenant_from_pool.yaml
+  - network/ports/management_from_pool.yaml
+  - network/ports/internal_api_from_pool_v6.yaml
+  - network/ports/external_from_pool_v6.yaml
+  - network/ports/storage_from_pool_v6.yaml
+  - network/ports/storage_mgmt_from_pool_v6.yaml
+  - network/ports/tenant_from_pool_v6.yaml
+  - network/ports/management_from_pool_v6.yaml
diff --git a/network/network.network.j2.yaml b/network/network.network.j2.yaml
new file mode 100644 (file)
index 0000000..2c223c1
--- /dev/null
@@ -0,0 +1,92 @@
+heat_template_version: pike
+
+description: >
+  {{network.name}} network definition (automatically generated).
+
+parameters:
+  # the defaults here work for static IP assignment (IPAM) only
+  {{network.name}}NetCidr:
+    default: {{network.ip_subnet|default("")}}
+    description: Cidr for the {{network.name_lower}} network.
+    type: string
+  {{network.name}}NetValueSpecs:
+    default: {'provider:physical_network': '{{network.name_lower}}', 'provider:network_type': 'flat'}
+    description: Value specs for the {{network.name_lower}} network.
+    type: json
+  {{network.name}}NetAdminStateUp:
+    default: false
+    description: This admin state of the network.
+    type: boolean
+  {{network.name}}NetEnableDHCP:
+    default: false
+    description: Whether to enable DHCP on the associated subnet.
+    type: boolean
+  {{network.name}}NetShared:
+    default: false
+    description: Whether this network is shared across all tenants.
+    type: boolean
+  {{network.name}}NetName:
+    default: {{network.name_lower}}
+    description: The name of the  {{network.name_lower}} network.
+    type: string
+  {{network.name}}SubnetName:
+    default: {{network.name_lower}}_subnet
+    description: The name of the {{network.name_lower}} subnet in Neutron.
+    type: string
+  {{network.name}}AllocationPools:
+    default: {{network.allocation_pools|default([])}}
+    description: Ip allocation pool range for the {{network.name_lower}} network.
+    type: json
+  {{network.name}}InterfaceDefaultRoute:
+    default: {{network.gateway_ip|default("not_defined")}}
+    description: default route for the {{network.name_lower}} network
+    type: string
+{%- if network.vlan %}
+  {{network.name}}NetworkVlanID:
+    default: {{network.vlan}}
+    description: Vlan ID for the {{network.name}} network traffic.
+    type: number
+{%- endif %}
+{%- if network.ipv6 %}
+  IPv6AddressMode:
+    default: dhcpv6-stateful
+    description: Neutron subnet IPv6 address mode
+    type: string
+  IPv6RAMode:
+    default: dhcpv6-stateful
+    description: Neutron subnet IPv6 router advertisement mode
+    type: string
+{%- endif %}
+
+resources:
+  {{network.name}}Network:
+    type: OS::Neutron::Net
+    properties:
+      admin_state_up: {get_param: {{network.name}}NetAdminStateUp}
+      name: {get_param: {{network.name}}NetName}
+      shared: {get_param: {{network.name}}NetShared}
+      value_specs: {get_param: {{network.name}}NetValueSpecs}
+
+  {{network.name}}Subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      cidr: {get_param: {{network.name}}NetCidr}
+      name: {get_param: {{network.name}}SubnetName}
+      network: {get_resource: {{network.name}}Network}
+      allocation_pools: {get_param: {{network.name}}AllocationPools}
+      gateway_ip: {get_param: {{network.name}}InterfaceDefaultRoute}
+{%-  if network.ipv6 %}
+      ip_version: 6
+      ipv6_address_mode: {get_param: IPv6AddressMode}
+      ipv6_ra_mode: {get_param: IPv6RAMode}
+{%- else %}
+      enable_dhcp: {get_param: {{network.name}}NetEnableDHCP}
+{%- endif %}
+
+outputs:
+  OS::stack_id:
+    description: {{network.name_lower}} network
+    value: {get_resource: {{network.name}}Network}
+  subnet_cidr:
+    value: {get_attr: {{network.name}}Subnet, cidr}
+
index 5aec597..c790d37 100644 (file)
@@ -5,11 +5,7 @@ description: Create networks to split out Overcloud traffic
 resources:
 
   {%- for network in networks %}
-    {%- if network.name != 'InternalApi' %}
   {{network.name}}Network:
-    {%- else  %}
-  InternalNetwork:
-    {%- endif %}
     type: OS::TripleO::Network::{{network.name}}
   {%- endfor %}
 
@@ -23,15 +19,8 @@ outputs:
       # NOTE(gfidente): we need to replace the null value with a
       # string to work around https://bugs.launchpad.net/heat/+bug/1700025
       {%- for network in networks %}
-        {%- if network.name != 'InternalApi' %}
       {{network.name_lower}}:
         yaql:
           data: {get_attr: [{{network.name}}Network, subnet_cidr]}
           expression: str($.data).replace('null', 'disabled')
-        {%- else  %}
-      {{network.name_lower}}:
-        yaql:
-          data: {get_attr: [InternalNetwork, subnet_cidr]}
-          expression: str($.data).replace('null', 'disabled')
-        {%- endif %}
       {%- endfor %}
diff --git a/network/ports/port.network.j2.yaml b/network/ports/port.network.j2.yaml
new file mode 100644 (file)
index 0000000..ded3e79
--- /dev/null
@@ -0,0 +1,72 @@
+heat_template_version: pike
+
+description: >
+  Creates a port on the {{network.name}} network. The IP address will be chosen
+  automatically if FixedIPs is empty.
+
+parameters:
+  {{network.name}}NetName:
+    description: Name of the {{network.name_lower}} neutron network
+    default: {{network.name_lower|default(network.name|lower)}}
+    type: string
+  PortName:
+    description: Name of the port
+    default: ''
+    type: string
+  ControlPlaneIP: # Here for compatibility with noop.yaml
+    description: IP address on the control plane
+    default: ''
+    type: string
+  ControlPlaneNetwork: # Here for compatibility with ctlplane_vip.yaml
+    description: The name of the undercloud Neutron control plane
+    default: ctlplane
+    type: string
+  FixedIPs:
+    description: >
+        Control the IP allocation for the VIP port. E.g.
+        [{'ip_address':'1.2.3.4'}]
+    default: []
+    type: json
+  IPPool: # Here for compatibility with from_pool.yaml
+    default: {}
+    type: json
+  NodeIndex: # Here for compatibility with from_pool.yaml
+    default: 0
+    type: number
+
+resources:
+
+  {{network.name}}Port:
+    type: OS::Neutron::Port
+    properties:
+      network: {get_param: {{network.name}}NetName}
+      name: {get_param: PortName}
+      fixed_ips: {get_param: FixedIPs}
+      replacement_policy: AUTO
+
+outputs:
+  ip_address:
+    description: {{network.name}} network IP
+    value: {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]}
+  ip_address_uri:
+{%- if network.ipv6  %}
+    description: {{network.name}} network IP (with brackets for IPv6 URLs)
+    value:
+          list_join:
+          - ''
+          - - '['
+            - {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]}
+            - ']'
+{%- else %}
+    description: {{network.name}} network IP (for compatibility with IPv6 URLs)
+    value: {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]}
+{%-  endif %}
+  ip_subnet:
+    description: IP/Subnet CIDR for the {{network.name}} network IP
+    value:
+          list_join:
+            - ''
+            - - {get_attr: [{{network.name}}Port, fixed_ips, 0, ip_address]}
+              - '/'
+              - {str_split: ['/', {get_attr: [{{network.name}}Port, subnets, 0, cidr]}, 1]}
+
diff --git a/network/ports/port_from_pool.network.j2.yaml b/network/ports/port_from_pool.network.j2.yaml
new file mode 100644 (file)
index 0000000..9c08ec7
--- /dev/null
@@ -0,0 +1,65 @@
+heat_template_version: pike
+
+description: >
+  Creates a port on the {{network.name}} network, using a map of IPs per role.
+  Each role has a map of IPs in <Role>IPs parameters, with a list of IPs by
+  network (lower_name or lower case). For example:
+  ControllerIPs:
+    external:
+    - 1.2.3.4 # First controller
+    - 1.2.3.5 # Second controller
+
+parameters:
+  {{network.name}}NetName:
+    description: Name of the {{network.name}} neutron network
+    default: {{network.name_lower}}
+    type: string
+  PortName:
+    description: Name of the port
+    default: ''
+    type: string
+  ControlPlaneIP: # Here for compatibility with noop.yaml
+    description: IP address on the control plane
+    default: ''
+    type: string
+  ControlPlaneNetwork: # Here for compatibility with ctlplane_vip.yaml
+    description: The name of the undercloud Neutron control plane
+    default: ctlplane
+    type: string
+  IPPool: # Set in <Role>IPs map, see environments/ips-from-pool-all.yaml
+    default: {}
+    type: json
+  NodeIndex: # First node in the role will get first IP, and so on...
+    default: 0
+    type: number
+  {{network.name}}NetCidr:
+    default: {{network.ip_subnet}}
+    description: Cidr for the {{network.name_lower}} network.
+    type: string
+
+outputs:
+  ip_address:
+    description: {{network.name}} network IP
+    value: {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]}
+  ip_address_uri:
+{%- if network.ipv6 %}
+    description: {{network.name}} network IP (with brackets for IPv6 URLs)
+    value:
+          list_join:
+          - ''
+          - - '['
+            - {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]}
+            - ']'
+{%- else %}
+    description: {{network.name}} network IP (for compatibility with {{network.name_lower}}_v6.yaml)
+    value: {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]}
+{%- endif %}
+  ip_subnet:
+    description: IP/Subnet CIDR for the {{network.name}} network IP
+    value:
+      list_join:
+      - ''
+      - - {get_param: [IPPool, {get_param: {{network.name}}NetName}, {get_param: NodeIndex}]}
+        - '/'
+        - {str_split: ['/', {get_param: {{network.name}}NetCidr}, 1]}
+
index 23c231f..947769a 100644 (file)
@@ -5,30 +5,59 @@
 # name: Name of the network (mandatory)
 # name_lower: lowercase version of name used for filenames
 #             (optional, defaults to name.lower())
-# vlan: vlan for the network (optional)
-# gateway: gateway for the network (optional)
 # enabled: Is the network enabled (optional, defaults to true)
+# ipv6: Does this network use IPv6 IPs? (optional, defaults to false)
+#                   (optional, may use parameter defaults in environment to set)
+# vlan: vlan for the network (optional)
 # vip: Enable creation of a virtual IP on this network
-# [TODO] (dsneddon@redhat.com) - Enable dynamic creation of VIP ports, to support
-# VIPs on non-default networks. See https://bugs.launchpad.net/tripleo/+bug/1667104
+#      [TODO] (dsneddon@redhat.com) - Enable dynamic creation of VIP ports,
+#      to support VIPs on non-default networks.
+#      See https://bugs.launchpad.net/tripleo/+bug/1667104
+# ip_subnet: IP/CIDR, e.g. '192.168.24.0/24' (optional, may use parameter defaults)
+# allocation_pools: IP range list e.g. [{'start':'10.0.0.4', 'end':'10.0.0.250}]
+# gateway_ip: gateway for the network (optional, may use parameter defaults)
+# NOTE: IP-related values set parameter defaults in templates, may be overridden.
+#
+# Example:
+# - name Example
+#   vip: false
+#   ip_subnet: '10.0.2.0/24'
+#   allocation_pools: [{'start': '10.0.2.4', 'end': '10.0.2.250'}]
+#   gateway_ip: '10.0.2.254'
 #
+# TODO (dsneddon) remove existing templates from j2_excludes.yaml
+#                 and generate all templates dynamically.
+
 - name: External
   vip: true
   name_lower: external
+  ip_subnet: '10.0.0.0/24'
+  allocation_pools: [{'start': '10.0.0.4', 'end': '10.0.0.250'}]
+  gateway_ip: '10.0.0.1'
 - name: InternalApi
   name_lower: internal_api
   vip: true
+  ip_subnet: '172.16.2.0/24'
+  allocation_pools: [{'start': '172.16.2.4', 'end': '172.16.2.250'}]
 - name: Storage
   vip: true
   name_lower: storage
+  ip_subnet: '172.16.1.0/24'
+  allocation_pools: [{'start': '172.16.1.4', 'end': '172.16.1.250'}]
 - name: StorageMgmt
   name_lower: storage_mgmt
   vip: true
+  ip_subnet: '172.16.3.0/24'
+  allocation_pools: [{'start': '172.16.3.4', 'end': '172.16.3.250'}]
 - name: Tenant
   vip: false  # Tenant network does not use VIPs
   name_lower: tenant
+  ip_subnet: '172.16.0.0/24'
+  allocation_pools: [{'start': '172.16.0.4', 'end': '172.16.0.250'}]
 - name: Management
   # Management network is disabled by default
   enabled: false
   vip: false  # Management network does not use VIPs
   name_lower: management
+  ip_subnet: '10.0.1.0/24'
+  allocation_pools: [{'start': '10.0.1.4', 'end': '10.0.1.250'}]
index badc142..07c27ba 100755 (executable)
@@ -96,6 +96,16 @@ def process_templates(template_path, role_data_path, output_dir,
     r_map = {}
     for r in role_data:
         r_map[r.get('name')] = r
+
+    n_map = {}
+    for n in network_data:
+        if (n.get('enabled') is not False):
+            n_map[n.get('name')] = n
+            if not n.get('name_lower'):
+                n_map[n.get('name')]['name_lower'] = n.get('name').lower()
+        else:
+            print("skipping %s network: network is disabled" % n.get('name'))
+
     excl_templates = ['%s/%s' % (template_path, e)
                       for e in j2_excludes.get('name')]
 
@@ -126,10 +136,13 @@ def process_templates(template_path, role_data_path, output_dir,
 
             for f in files:
                 file_path = os.path.join(subdir, f)
-                # We do two templating passes here:
+                # We do three templating passes here:
                 # 1. *.role.j2.yaml - we template just the role name
                 #    and create multiple files (one per role)
-                # 2. *.j2.yaml - we template with all roles_data,
+                # 2  *.network.j2.yaml - we template the network name and
+                #    data and create multiple files for networks and
+                #    network ports (one per network)
+                # 3. *.j2.yaml - we template with all roles_data,
                 #    and create one file common to all roles
                 if f.endswith('.role.j2.yaml'):
                     print("jinja2 rendering role template %s" % f)
@@ -167,6 +180,30 @@ def process_templates(template_path, role_data_path, output_dir,
 
                             else:
                                 print('skipping rendering of %s' % out_f_path)
+
+                elif f.endswith('.network.j2.yaml'):
+                    print("jinja2 rendering network template %s" % f)
+                    with open(file_path) as j2_template:
+                        template_data = j2_template.read()
+                    print("jinja2 rendering networks %s" % ",".join(n_map))
+                    for network in n_map:
+                        j2_data = {'network': n_map[network]}
+                        # Output file names in "<name>.yaml" format
+                        out_f = os.path.basename(f).replace('.network.j2.yaml',
+                                                            '.yaml')
+                        if os.path.dirname(file_path).endswith('ports'):
+                            out_f = out_f.replace('port',
+                                                  n_map[network]['name_lower'])
+                        else:
+                            out_f = out_f.replace('network',
+                                                  n_map[network]['name_lower'])
+                        out_f_path = os.path.join(out_dir, out_f)
+                        if not (out_f_path in excl_templates):
+                            _j2_render_to_file(template_data, j2_data,
+                                               out_f_path)
+                        else:
+                            print('skipping rendering of %s' % out_f_path)
+
                 elif f.endswith('.j2.yaml'):
                     print("jinja2 rendering normal template %s" % f)
                     with open(file_path) as j2_template: