when:
- stor4nfv is defined and stor4nfv == "Enable"
+- name: copy sriov playbook to kargo
+ copy:
+ src: "{{ run_dir }}/roles/sriov"
+ dest: /opt/kargo_k8s/roles/network_plugin
+
+- name: copy sriov-apps playbook to kargo
+ copy:
+ src: "{{ run_dir }}/roles/sriov-apps/"
+ dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/sriov
+
+- name: append sriov to network plugin
+ blockinfile:
+ path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml
+ block: " - role: network_plugin/sriov\n when: kube_network_plugin == 'sriov'\n \
+ tags: sriov\n"
+
+- name: append sriov apps to network plugin
+ blockinfile:
+ path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml
+ block: " - role: kubernetes-apps/network_plugin/sriov\n \
+ when: kube_network_plugin == 'sriov'\n tags: sriov\n"
+
+- name: append sriov to valid kube_network_plugin list
+ replace:
+ path: "{{ item.path }}"
+ regexp: "{{ item.regexp }}"
+ replace: "{{ item.replace }}"
+ with_items:
+ - {path: "/opt/kargo_k8s/roles/kubernetes/master/templates/manifests/\
+kube-controller-manager.manifest.j2",
+ regexp: '"cloud", "flannel"',
+ replace: '"cloud", "flannel", "sriov"'}
+ - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2',
+ regexp: '"calico", "canal", "flannel", "weave"',
+ replace: '"calico", "canal", "flannel", "weave", "sriov"'}
+ - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.standard.env.j2',
+ regexp: '"calico", "canal", "flannel", "weave"',
+ replace: '"calico", "canal", "flannel", "weave", "sriov"'}
+ - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.rkt.service.j2',
+ regexp: '"calico", "weave", "canal", "flannel"',
+ replace: '"calico", "weave", "canal", "flannel", "sriov"'}
+ - {path: '/opt/kargo_k8s/roles/kubernetes/preinstall/tasks/main.yml',
+ regexp: '"calico", "weave", "canal", "flannel"',
+ replace: '"calico", "weave", "canal", "flannel", "sriov"'}
+
- name: run kargo playbook
shell: |
cd /opt/kargo_k8s
---
aptpackages:
-- bridge-utils
-- debootstrap
-- ifenslave
-- ifenslave-2.6
-- lsof
-- lvm2
-- ntp
-- ntpdate
-- sudo
-- vlan
-- tcpdump
+ - bridge-utils
+ - debootstrap
+ - ifenslave
+ - ifenslave-2.6
+ - lsof
+ - lvm2
+ - ntp
+ - ntpdate
+ - sudo
+ - vlan
+ - tcpdump
yumpackages:
-- bridge-utils
-- iputils
-- lvm2
-- ntp
-- tcpdump
-- vim
+ - bridge-utils
+ - iputils
+ - lvm2
+ - ntp
+ - tcpdump
+ - vim
src: ifcfg-eth.j2
dest: /etc/sysconfig/network-scripts/ifcfg-{{sys_intf_mappings["external"]["interface"]}}
+- name: generate ifcfg-sriov
+ template:
+ src: ifcfg-sriov.j2
+ dest: /etc/sysconfig/network-scripts/ifcfg-{{ intf_sriov }}
+ when: intf_sriov|length > 0
+
+- name: remove ifcfg-br-sriov script
+ file:
+ path: /etc/sysconfig/network-scripts/ifcfg-br-sriov
+ state: absent
+
- name: remove defualt gw
lineinfile:
dest: /etc/sysconfig/network
--- /dev/null
+DEVICE={{ intf_sriov }}
+BOOTPROTO=none
+ONBOOT=yes
+IPADDR={{ ip_settings[inventory_hostname]["tenant"]["ip"] }}
+NETMASK=255.255.255.0
+DEFROUTE="no"
+{% if sys_intf_mappings["tenant"]["vlan_tag"] | int %}
+{% set intf_vlan = "yes" %}
+{% else %}
+{% set intf_vlan = "no" %}
+{% endif %}
+VLAN={{ intf_vlan }}
--- /dev/null
+---
+intf_sriov: |-
+ {%- set intf_sriov = sys_intf_mappings["tenant"]["interface"] %}
+ {%- if sys_intf_mappings["tenant"]["vlan_tag"] | int %}
+ {%- set intf_sriov = intf_sriov + '.' + sys_intf_mappings["tenant"]["vlan_tag"]|string %}
+ {%- endif %}
+ {{- intf_sriov }}
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: "Sriov | Create ServiceAccount ClusterRole and ClusterRoleBinding"
+ command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-sriov-rbac.yml"
+ run_once: true
+ when: rbac_enabled and sriov_rbac_manifest.changed
+
+- name: Sriov | Create Network Resources
+ kube:
+ name: "kube-sriov"
+ kubectl: "{{ bin_dir }}/kubectl"
+ filename: "{{ kube_config_dir }}/cni-sriov.yml"
+ namespace: "{{system_namespace}}"
+ state: "{{ item | ternary('latest','present') }}"
+ with_items: "{{ sriov_manifest.changed }}"
+ when: inventory_hostname == groups['kube-master'][0]
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+# Limits for apps
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | delete default docker bridge
+ command: ip link delete docker0
+ failed_when: false
+ notify: Sriov | restart docker
+
+# special cases for atomic because it defaults to live-restore: true
+# So we disable live-restore to pickup the new flannel IP. After
+# we enable it, we have to restart docker again to pickup the new
+# setting and restore the original behavior
+- name: Sriov | restart docker
+ command: /bin/true
+ notify:
+ - Sriov | reload systemd
+ - Sriov | reload docker.socket
+ - Sriov | configure docker live-restore true (atomic)
+ - Sriov | reload docker
+ - Sriov | pause while Docker restarts
+ - Sriov | wait for docker
+
+- name: Sriov | reload systemd
+ shell: systemctl daemon-reload
+
+- name: Sriov | reload docker.socket
+ service:
+ name: docker.socket
+ state: restarted
+ when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Sriov | configure docker live-restore true (atomic)
+ replace:
+ name: /etc/docker/daemon.json
+ regexp: '"live-restore":.*true'
+ replace: '"live-restore": false'
+ when: is_atomic
+
+- name: Sriov | reload docker
+ service:
+ name: docker
+ state: restarted
+
+- name: Sriov | pause while Docker restarts
+ pause:
+ seconds: 10
+ prompt: "Waiting for docker restart"
+
+- name: Sriov | wait for docker
+ command: "{{ docker_bin_dir }}/docker images"
+ register: docker_ready
+ retries: 10
+ delay: 5
+ until: docker_ready.rc == 0
+
+- name: Sriov | reload kubelet
+ service:
+ name: kubelet
+ state: restarted
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | Verify if br_netfilter module exists
+ shell: "modinfo br_netfilter"
+ register: modinfo_br_netfilter
+ failed_when: modinfo_br_netfilter.rc not in [0, 1]
+ changed_when: false
+
+- name: Sriov | Enable br_netfilter module
+ modprobe:
+ name: br_netfilter
+ state: present
+ when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled
+# when found if br_netfilter is not a module
+- name: Sriov | Check if bridge-nf-call-iptables key exists
+ command: "sysctl net.bridge.bridge-nf-call-iptables"
+ failed_when: false
+ changed_when: false
+ register: sysctl_bridge_nf_call_iptables
+
+- name: Sriov | Enable bridge-nf-call tables
+ sysctl:
+ name: "{{ item }}"
+ state: present
+ value: 1
+ reload: "yes"
+ when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0
+ with_items:
+ - net.bridge.bridge-nf-call-iptables
+ - net.bridge.bridge-nf-call-arptables
+ - net.bridge.bridge-nf-call-ip6tables
+
+- name: Sriov | Install Multus CNI
+ shell: |-
+ /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+ bash -c "git clone https://github.com/Intel-Corp/multus-cni && cd multus-cni \
+ && ./build && cp bin/multus /opt/cni/bin"
+
+- name: Sriov | Install Sriov CNI
+ shell: |-
+ /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+ bash -c "git clone https://github.com/hustcat/sriov-cni && cd sriov-cni \
+ && ./build && cp bin/sriov /opt/cni/bin"
+
+- name: Sriov | Install Flannel CNI
+ shell: |-
+ /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/host/opt/cni/bin/ \
+ {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} \
+ sh -c "cp /opt/cni/bin/* /host/opt/cni/bin/"
+
+- name: Sriov | Remove all file in /etc/cni/net.d
+ shell: |-
+ rm -rf /etc/cni/net.d/
+ mkdir -p /etc/cni/net.d/
+
+- name: Sriov | Generate Sriov CNI Conf
+ copy:
+ content: |
+ {
+ "name": "minion-cni-network",
+ "type": "multus",
+ "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+ "delegates": [
+ {
+ "type": "flannel",
+ "masterplugin": true,
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+ ]
+ }
+ dest: "/etc/cni/net.d/multus-cni.conf"
+ owner: root
+ group: root
+ mode: 0644
+
+- name: Sriov | Enable DHCP CNI
+ shell: /opt/cni/bin/dhcp daemon &
+
+
+- name: Sriov | Create cni-sriov-rbac manifest
+ template:
+ src: cni-sriov-rbac.yml.j2
+ dest: "{{ kube_config_dir }}/cni-sriov-rbac.yml"
+ register: sriov_rbac_manifest
+ when: inventory_hostname == groups['kube-master'][0] and rbac_enabled
+
+- name: Sriov | Create cni-sriov manifest
+ template:
+ src: cni-sriov.yml.j2
+ dest: "{{ kube_config_dir }}/cni-sriov.yml"
+ register: sriov_manifest
+ when: inventory_hostname == groups['kube-master'][0]
+
+- name: Sriov | Sriov tests manifest
+ template:
+ src: sriov-test-pod.yml
+ dest: "{{ kube_config_dir }}/sriov-test-pod.yml"
+ when: inventory_hostname == groups['kube-master'][0]
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: sriov
+ namespace: "{{system_namespace}}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: sriov
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - get
+ - apiGroups:
+ - ""
+ resources:
+ - nodes
+ verbs:
+ - list
+ - watch
+ - apiGroups:
+ - ""
+ resources:
+ - nodes/status
+ verbs:
+ - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: sriov
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: sriov
+subjects:
+- kind: ServiceAccount
+ name: sriov
+ namespace: "{{system_namespace}}"
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: extensions/v1beta1
+kind: ThirdPartyResource
+metadata:
+ name: network.kubernetes.com
+description: "A specification of a Network obj in the kubernetes"
+versions:
+- name: v1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: flannel
+ namespace: {{system_namespace}}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: kube-flannel-cfg
+ namespace: {{system_namespace}}
+ labels:
+ tier: node
+ app: flannel
+data:
+ cni-conf.json: |
+ {
+ "name": "cbr0",
+ "type": "flannel",
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+ net-conf.json: |
+ {
+ "Network": "10.244.0.0/16",
+ "Backend": {
+ "Type": "udp"
+ }
+ }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-flannel-ds
+ namespace: {{system_namespace}}
+ labels:
+ tier: node
+ app: flannel
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: flannel
+ spec:
+ hostNetwork: true
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ serviceAccountName: flannel
+ containers:
+ - name: kube-flannel
+ image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
+ securityContext:
+ privileged: true
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - name: run
+ mountPath: /run
+ - name: flannel-cfg
+ mountPath: /etc/kube-flannel/
+ volumes:
+ - name: run
+ hostPath:
+ path: /run
+ - name: flannel-cfg
+ configMap:
+ name: kube-flannel-cfg
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: flannel-conf
+ namespace: default
+plugin: flannel
+args: '[
+ {
+ "masterplugin": true,
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: sriov-conf1
+ namespace: default
+plugin: sriov
+args: '[
+ {
+ "master": "eth1.101",
+ "pfOnly": true,
+ "ipam": {
+ "type": "host-local",
+ "subnet": "192.168.123.0/24",
+ "rangeStart": "192.168.123.11",
+ "rangeEnd": "192.168.123.21",
+ "routes": [
+ {
+ "dst": "0.0.0.0/0"
+ }
+ ],
+ "gateway": "192.168.123.1"
+ }
+ }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: sriov-conf2
+ namespace: default
+plugin: sriov
+args: '[
+ {
+ "master": "eth1.101",
+ "pfOnly": true,
+ "ipam": {
+ "type": "host-local",
+ "subnet": "192.168.123.0/24",
+ "rangeStart": "192.168.123.31",
+ "rangeEnd": "192.168.123.41",
+ "routes": [
+ {
+ "dst": "0.0.0.0/0"
+ }
+ ],
+ "gateway": "192.168.123.1"
+ }
+ }
+]'
--- /dev/null
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: multus-test1
+ annotations:
+ networks: '[
+ { "name": "flannel-conf" },
+ { "name": "sriov-conf1" }
+ ]'
+spec:
+ containers:
+ - name: multus-test
+ image: "busybox"
+ command: ["top"]
+ stdin: true
+ tty: true
+ nodeSelector:
+ kubernetes.io/hostname: "host1"
+ tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Exists"
+ effect: "NoSchedule"
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: multus-test2
+ annotations:
+ networks: '[
+ { "name": "flannel-conf" },
+ { "name": "sriov-conf2" }
+ ]'
+spec:
+ containers:
+ - name: multus-test
+ image: "busybox"
+ command: ["top"]
+ stdin: true
+ tty: true
+ nodeSelector:
+ kubernetes.io/hostname: "host2"
+ tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Exists"
+ effect: "NoSchedule"
--- /dev/null
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+---
+nic_mappings: []
+bond_mappings: []
+
+provider_net_mappings:
+ - name: br-provider
+ network: physnet
+ interface: eth10
+ type: ovs
+ role:
+ - controller
+
+sys_intf_mappings:
+ - name: mgmt
+ interface: eth0
+ type: normal
+ vlan_tag: None
+ role:
+ - controller
+ - compute
+
+ - name: tenant
+ interface: eth1
+ type: normal
+ vlan_tag: 101
+ role:
+ - controller
+ - compute
+
+ - name: external
+ interface: eth1
+ type: normal
+ vlan_tag: None
+ role:
+ - controller
+ - compute
+
+ip_settings:
+ - name: mgmt
+ ip_ranges:
+ - - "10.1.0.50"
+ - "10.1.0.100"
+ dhcp_ranges:
+ - - "10.1.0.2"
+ - "10.1.0.49"
+ cidr: "10.1.0.0/24"
+ gw: "10.1.0.1"
+ role:
+ - controller
+ - compute
+
+ - name: tenant
+ ip_ranges:
+ - - "172.16.1.2"
+ - "172.16.1.250"
+ cidr: "172.16.1.0/24"
+ role:
+ - controller
+ - compute
+
+ - name: external
+ ip_ranges:
+ - - "192.16.1.210"
+ - "192.16.1.220"
+ cidr: "192.16.1.0/24"
+ gw: "192.16.1.1"
+ role:
+ - controller
+ - compute
+
+internal_vip:
+ ip: 10.1.0.222
+ netmask: "24"
+ interface: mgmt
+
+public_vip:
+ ip: 192.16.1.222
+ netmask: "24"
+ interface: external
+
+onos_nic: eth2
+tenant_net_info:
+ type: vxlan
+ range: "1:1000"
+ provider_network: None
+
+public_net_info:
+ enable: "True"
+ network: ext-net
+ type: flat
+ segment_id: 1000
+ subnet: ext-subnet
+ provider_network: physnet
+ router: router-ext
+ enable_dhcp: "False"
+ no_gateway: "False"
+ external_gw: "192.16.1.1"
+ floating_ip_cidr: "192.16.1.0/24"
+ floating_ip_start: "192.16.1.101"
+ floating_ip_end: "192.16.1.199"
Code Review / compass4nfv.git / commitdiff