Deploy Multus, SRIOV, Vhostuser(VPP+DPDK) with CRD.
Change-Id: I787d5fd61c75f17d50fbaf8f86c08bdc44c557b8
Signed-off-by: Yibo Cai <yibo.cai@arm.com>
--- /dev/null
+---
+- name: Fixup default flannel
+ hosts: kube-master
+ gather_facts: "no"
+ vars_files:
+ - "vars/global"
+ roles:
+ - {role: flannel, tags: [flannel]}
+
+- name: Deploy Multus CNI
+ hosts: all
+ gather_facts: "no"
+ vars_files:
+ - "vars/global"
+ roles:
+ - {role: multus, tags: [multus]}
+
+- name: Deploy SRIOV CNI
+ hosts: all
+ gather_facts: "no"
+ vars_files:
+ - "vars/global"
+ roles:
+ - {role: sriov, tags: [sriov]}
+
+- name: Deploy Vhostuser CNI and VPP
+ hosts: all
+ gather_facts: "yes"
+ vars_files:
+ - "vars/global"
+ roles:
+ - {role: vhost-vpp, tags: [vhost-vpp]}
--- /dev/null
+# compass-tasks: /opt/kargo_k8s/inventory/inventory.cfg
+
+[all]
+host2 ansible_ssh_host=10.1.0.51 ansible_ssh_pass=root ansible_user=root
+host1 ansible_ssh_host=10.1.0.50 ansible_ssh_pass=root ansible_user=root
+
+[kube-master]
+host1
+
+[etcd]
+host1
+
+[kube-node]
+host2
+
+[k8s-cluster:children]
+kube-node
+kube-master
--- /dev/null
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-flannel
+ namespace: "kube-system"
+ labels:
+ tier: node
+ k8s-app: flannel
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ k8s-app: flannel
+ spec:
+ serviceAccountName: flannel
+ containers:
+ - name: kube-flannel
+ image: quay.io/coreos/flannel:v0.9.1-arm64
+ imagePullPolicy: IfNotPresent
+ resources:
+ limits:
+ cpu: 300m
+ memory: 500M
+ requests:
+ cpu: 150m
+ memory: 64M
+ command: ["/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"]
+ securityContext:
+ privileged: true
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - name: run
+ mountPath: /run
+ - name: cni
+ mountPath: /etc/cni/net.d
+ - name: flannel-cfg
+ mountPath: /etc/kube-flannel/
+ # - name: install-cni
+ # image: linaro/flannel-cni-arm64:v0.3.0
+ # command: ["/install-cni.sh"]
+ # env:
+ # # The CNI network config to install on each node.
+ # - name: CNI_NETWORK_CONFIG
+ # valueFrom:
+ # configMapKeyRef:
+ # name: kube-flannel-cfg
+ # key: cni-conf.json
+ # - name: CNI_CONF_NAME
+ # value: "10-flannel.conflist"
+ # volumeMounts:
+ # - name: cni
+ # mountPath: /host/etc/cni/net.d
+ # - name: host-cni-bin
+ # mountPath: /host/opt/cni/bin/
+ hostNetwork: true
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ volumes:
+ - name: run
+ hostPath:
+ path: /run
+ - name: cni
+ hostPath:
+ path: /etc/cni/net.d
+ - name: flannel-cfg
+ configMap:
+ name: kube-flannel-cfg
+ # - name: host-cni-bin
+ # hostPath:
+ # path: /opt/cni/bin
+ updateStrategy:
+ rollingUpdate:
+ maxUnavailable: 20%
+ type: RollingUpdate
--- /dev/null
+---
+- name: Copy flannel daemonset file
+ copy:
+ src: cni-flannel-ds.yml
+ dest: /tmp/cni-flannel-ds.yml
+
+- name: Apply flannel daemonset
+ shell: kubectl apply -f /tmp/cni-flannel-ds.yml
+ ignore_errors: "yes"
+
+- name: Sleep 10 seconds
+ wait_for: timeout=10
--- /dev/null
+{
+ "name": "multus-cni-network",
+ "type": "multus",
+ "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+ "delegates": [{
+ "type": "flannel",
+ "masterplugin": true,
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }]
+}
+
--- /dev/null
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: multus-crd-overpowered
+rules:
+ - apiGroups:
+ - '*'
+ resources:
+ - '*'
+ verbs:
+ - '*'
+ - nonResourceURLs:
+ - '*'
+ verbs:
+ - '*'
--- /dev/null
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: networks.kubernetes.com
+spec:
+ group: kubernetes.com
+ version: v1
+ scope: Namespaced
+ names:
+ plural: networks
+ singular: network
+ kind: Network
+ shortNames:
+ - net
--- /dev/null
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: flannel-networkobj
+plugin: flannel
+args: '[
+ {
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+]'
--- /dev/null
+- name: Restart kubelet
+ service:
+ name: kubelet
+ state: restarted
--- /dev/null
+---
+- name: Copy yaml files
+ copy:
+ src: "{{ item }}"
+ dest: "/tmp/{{ item }}"
+ with_items:
+ - clusterrole.yml
+ - crdnetwork.yml
+ - flannel-obj.yml
+
+- name: Copy macvlan template
+ template:
+ src: macvlan-obj.yml.j2
+ dest: /tmp/macvlan-obj.yml
+
+- name: Copy Multus testpod template
+ template:
+ src: multus-testpod.yml.j2
+ dest: /root/multus-testpod.yml
+
+- name: Create cluster role
+ shell: kubectl apply -f /tmp/clusterrole.yml
+
+- name: Check if role binding is created
+ shell: kubectl get clusterrolebinding multus-node-{{ item }}
+ register: check_rb
+ ignore_errors: "yes"
+ with_items: "{{ groups['all'] }}"
+
+- name: Create role binding
+ shell: >
+ kubectl create clusterrolebinding multus-node-{{ item }}
+ --clusterrole=multus-crd-overpowered
+ --user=system:node:{{ item }}
+ when: check_rb is failed
+ with_items: "{{ groups['all'] }}"
+
+- name: Create network CRD
+ shell: kubectl apply -f /tmp/crdnetwork.yml
+
+- name: Create flannel and macvlan network objects
+ shell: >
+ kubectl apply -f /tmp/flannel-obj.yml &&
+ kubectl apply -f /tmp/macvlan-obj.yml
--- /dev/null
+---
+- name: Build Multus CNI
+ shell: >
+ docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+ bash -c "git clone {{ multus_repo }} multus_cni && cd multus_cni &&
+ git checkout {{ multus_commit }} && ./build && cp bin/multus /opt/cni/bin/"
+ args:
+ creates: /opt/cni/bin/multus
+
+- name: Remove default CNI configuration
+ shell: rm -f /etc/cni/net.d/*
+ args:
+ warn: "no"
+
+- name: Set Multus as default CNI
+ copy:
+ src: 10-multus.conf
+ dest: /etc/cni/net.d/
+ notify:
+ - Restart kubelet
+
+- name: Import CRD task
+ import_tasks: crd.yml
+ when: inventory_hostname == groups["kube-master"][0]
--- /dev/null
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: macvlan-networkobj
+plugin: macvlan
+args: '[
+ {
+ "master": "{{ macvlan_master }}",
+ "mode": "vepa",
+ "ipam": {
+ "type": "host-local",
+ "subnet": "{{ macvlan_subnet }}",
+ "rangeStart": "{{ macvlan_range_start }}",
+ "rangeEnd": "{{ macvlan_range_end }}",
+ "routes": [
+ { "dst": "0.0.0.0/0" }
+ ],
+ "gateway": "{{ macvlan_gateway }}"
+ }
+ }
+]'
--- /dev/null
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: multus-test
+ annotations:
+ networks: '[
+ { "name": "flannel-networkobj" },
+ { "name": "macvlan-networkobj" }
+ ]'
+spec:
+ containers:
+ - name: multus-test
+ image: "busybox"
+ command: ["sleep", "100d"]
+ stdin: true
+ tty: true
+ nodeSelector:
+ kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
--- /dev/null
+---
+- name: Copy SRIOV template
+ template:
+ src: sriov-obj.yml.j2
+ dest: /tmp/sriov-obj.yml
+
+- name: Copy SRIOV testpod template
+ template:
+ src: sriov-testpod.yml.j2
+ dest: /root/sriov-testpod.yml
+
+- name: Create SRIOV network object
+ shell: kubectl apply -f /tmp/sriov-obj.yml
--- /dev/null
+---
+- name: Build SRIOV CNI
+ shell: >
+ docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+ bash -c "git clone {{ sriov_repo }} sriov_cni && cd sriov_cni &&
+ git checkout {{ sriov_commit }} && ./build && cp bin/sriov /opt/cni/bin/"
+ args:
+ creates: /opt/cni/bin/sriov
+
+- name: Import CRD task
+ import_tasks: crd.yml
+ when: inventory_hostname == groups["kube-master"][0]
--- /dev/null
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: sriov-networkobj
+plugin: sriov
+args: '[
+ {
+ "master": "{{ sriov_master }}",
+ "pfOnly": true,
+ "if0name": "net0",
+ "ipam": {
+ "type": "host-local",
+ "subnet": "{{ sriov_subnet }}",
+ "rangeStart": "{{ sriov_range_start }}",
+ "rangeEnd": "{{ sriov_range_end }}",
+ "routes": [
+ {
+ "dst": "0.0.0.0/0"
+ }
+ ],
+ "gateway": "{{ sriov_gateway }}"
+ }
+ }
+]'
--- /dev/null
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: sriov-test
+ annotations:
+ networks: '[
+ { "name": "flannel-networkobj" },
+ { "name": "sriov-networkobj" }
+ ]'
+spec:
+ containers:
+ - name: sriov-test
+ image: "busybox"
+ command: ["sleep", "100d"]
+ stdin: true
+ tty: true
+ nodeSelector:
+ kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
--- /dev/null
+diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
+index e320811..c1b1640 100644
+--- a/drivers/net/virtio/virtio_ethdev.c
++++ b/drivers/net/virtio/virtio_ethdev.c
+@@ -1754,6 +1754,11 @@ virtio_dev_start(struct rte_eth_dev *dev)
+ virtqueue_notify(rxvq->vq);
+ }
+
++ for (i = 0; i < dev->data->nb_tx_queues; i++) {
++ txvq = dev->data->tx_queues[i];
++ virtqueue_notify(txvq->vq);
++ }
++
+ PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
+
+ for (i = 0; i < dev->data->nb_rx_queues; i++) {
--- /dev/null
+FROM ubuntu:xenial
+
+RUN apt-get update && \
+ apt-get install -y git make openssl libcrypto++-dev libnuma-dev && \
+ apt-get autoclean
+
+RUN git clone https://gerrit.fd.io/r/vpp -b stable/1710 /root/vpp-1710
+
+WORKDIR /root/vpp-1710
+COPY ./0001-net-virtio-ethdev.patch dpdk/dpdk-17.08_patches/0001-net-virtio-ethdev.patch
+RUN sed -i "s/sudo -E //g" Makefile
+RUN make UNATTENDED=yes install-dep
+
+WORKDIR /root/vpp-1710/build-root
+RUN ./bootstrap.sh
+RUN make PLATFORM=vpp TAG=vpp_debug vpp-install
+RUN mkdir -p /etc/vpp && \
+ cp /root/vpp-1710/src/vpp/conf/startup.conf /etc/vpp/startup.conf && \
+ cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin && \
+ ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins
+RUN groupadd vpp
+
+ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/dpdk/bin"
+ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin"
--- /dev/null
+#!/bin/bash
+
+set -x
+
+cid=`sed -ne '/hostname/p' /proc/1/task/1/mountinfo | awk -F '/' '{print $6}'`
+cid_s=${cid:0:12}
+filename=${cid_s}-net1.json
+ifstring=`cat /vhost-user-net-plugin/${cid}/${cid_s}-net1.json | awk -F ',' '{print $4}'`
+ifmac=`echo ${ifstring} | awk -F '\"' '{print $4}'`
+
+ipstr=$(cat /vhost-user-net-plugin/${cid}/${cid_s}-net1-ip4.conf |grep "ipAddr")
+ipaddr=$(echo $ipstr | awk -F '\"' '{print $4}')
+ipaddr1=$(echo $ipaddr | cut -d / -f 1)
+
+vdev_str="vdev virtio_user0,path=/vhost-user-net-plugin/$cid/$cid_s-net1,mac=$ifmac"
+
+sed -i.bak '/# dpdk/a\dpdk \{' /etc/vpp/startup.conf
+sed -i.bak "/# vdev eth_bond1,mode=1/a\\$vdev_str" /etc/vpp/startup.conf
+sed -i.bak '/# socket-mem/a\\}' /etc/vpp/startup.conf
+
+vpp -c /etc/vpp/startup.conf &
+
+sleep 40
+
+vppctl set int state VirtioUser0/0/0 up
+vppctl set int ip address VirtioUser0/0/0 ${ipaddr1}/24
+vppctl show int
+vppctl show int address
+
+echo ${ipaddr1} > /vhost-user-net-plugin/$(hostname)
--- /dev/null
+unix {
+ nodaemon
+ log /tmp/vpp.log
+ full-coredump
+ cli-listen /run/vpp/cli.sock
+ gid vpp
+}
+api-trace {
+ on
+}
+api-segment {
+ gid vpp
+}
+cpu {
+ main-core 1
+ corelist-workers 2-3
+ workers 2
+}
+dpdk {
+ uio-driver vfio-pci
+}
--- /dev/null
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: vhostuser-networkobj
+plugin: vhostuser
+args: '[
+ {
+ "type": "vhostuser",
+ "name": "vhostuser-network",
+ "if0name": "net1",
+ "vhost": {
+ "vhost_tool": "/opt/cni/bin/vpp-config.py"
+ },
+ "ipam": {
+ "type": "host-local",
+ "subnet": "10.56.217.0/24",
+ "rangeStart": "10.56.217.131",
+ "rangeEnd": "10.56.217.190",
+ "routes": [
+ {
+ "dst": "0.0.0.0/0"
+ }
+ ],
+ "gateway": "10.56.217.1"
+ }
+ }
+]'
--- /dev/null
+---
+- name: Copy Vhostuser yaml
+ copy:
+ src: vhostuser-obj.yml
+ dest: /tmp/vhostuser-obj.yml
+
+- name: Copy VPP testpod template
+ template:
+ src: vpp-testpod.yml.j2
+ dest: /root/vpp-testpod.yml
+
+- name: Create Vhostuser network object
+ shell: kubectl apply -f /tmp/vhostuser-obj.yml
--- /dev/null
+---
+- name: Build Vhostuser CNI
+ shell: >
+ docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+ bash -c "git clone {{ vhostuser_repo }} vhostuser_cni && cd vhostuser_cni
+ && git checkout {{ vhostuser_commit }} && ./build
+ && cp bin/vhostuser /opt/cni/bin/
+ && cp tests/vpp-config-debug.py /opt/cni/bin/vpp-config.py"
+ args:
+ creates: /opt/cni/bin/vhostuser
+
+- name: Import CRD task
+ import_tasks: crd.yml
+ when: inventory_hostname == groups["kube-master"][0]
+
+- name: Import VPP task
+ import_tasks: vpp.yml
+ when: inventory_hostname in groups["kube-node"]
--- /dev/null
+---
+- name: Create dest directories
+ file:
+ path: "{{ item }}"
+ state: directory
+ with_items:
+ - /tmp/vpp1710/
+ - /var/lib/cni/vhostuser/
+ - /etc/vpp/
+
+- name: Copy VPP files
+ copy:
+ src: "{{ item.src }}"
+ dest: "{{ item.dest }}"
+ with_items:
+ - {src: "Dockerfile.vpp1710-dpdk1708", dest: "/tmp/vpp1710/Dockerfile"}
+ - {src: "0001-net-virtio-ethdev.patch", dest: "/tmp/vpp1710/0001-net-virtio-ethdev.patch"}
+ - {src: "setvpp.sh", dest: "/var/lib/cni/vhostuser/setvpp.sh"}
+ - {src: "startup.conf", dest: "/etc/vpp/startup.conf"}
+
+- name: Check if VPP image exists
+ shell: docker inspect --type=image vpp-1710:virtio-patched > /dev/null 2>&1
+ ignore_errors: "yes"
+ register: check_vpp
+
+- name: Building VPP container. Be patient...
+ shell: docker build -t vpp-1710:virtio-patched --network host .
+ args:
+ chdir: /tmp/vpp1710/
+ when: check_vpp is failed
+
+- name: Copy VPP binaries to host
+ shell: >
+ docker run --rm -v /root/vpp-1710/build-root:/root/vpp-host vpp-1710:virtio-patched
+ /bin/cp -a /root/vpp-1710/build-root/install-vpp_debug-native /root/vpp-host
+ && /bin/cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin
+ && /bin/rm -rf /usr/lib/vpp_plugins
+ && ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins
+ && (groupadd vpp || true)
+
+- name: Copy libcrypto.so.1.0.0 for CentOS
+ shell: >
+ docker run --rm -v /usr/lib64:/root/lib64-centos vpp-1710:virtio-patched
+ /bin/cp /lib/aarch64-linux-gnu/libcrypto.so.1.0.0 /root/lib64-centos/
+ args:
+ creates: /usr/lib64/libcrypto.so.1.0.0
+ when: ansible_os_family == "RedHat"
--- /dev/null
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: vpp-test1
+ annotations:
+ networks: '[
+ { "name": "flannel-networkobj" },
+ { "name": "vhostuser-networkobj" }
+ ]'
+spec:
+ containers:
+ - name: vpp-test1
+ image: vpp-1710:virtio-patched
+ imagePullPolicy: "Never"
+ stdin: true
+ terminationMessagePath: /dev/termination-log
+ tty: true
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /vhost-user-net-plugin
+ name: vhost-user-net-plugin
+ - mountPath: /mnt/huge
+ name: huge
+ nodeSelector:
+ kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
+ volumes:
+ - name: vhost-user-net-plugin
+ hostPath:
+ path: /var/lib/cni/vhostuser
+ - name: huge
+ hostPath:
+ path: /mnt/huge
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: vpp-test2
+ annotations:
+ networks: '[
+ { "name": "flannel-networkobj" },
+ { "name": "vhostuser-networkobj" }
+ ]'
+spec:
+ containers:
+ - name: vpp-test2
+ image: vpp-1710:virtio-patched
+ imagePullPolicy: "Never"
+ stdin: true
+ terminationMessagePath: /dev/termination-log
+ tty: true
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /vhost-user-net-plugin
+ name: vhost-user-net-plugin
+ - mountPath: /mnt/huge
+ name: huge
+ nodeSelector:
+ kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
+ volumes:
+ - name: vhost-user-net-plugin
+ hostPath:
+ path: /var/lib/cni/vhostuser
+ - name: huge
+ hostPath:
+ path: /mnt/huge
--- /dev/null
+multus_repo: https://github.com/Intel-Corp/multus-cni
+multus_commit: 61959e04
+
+sriov_repo: https://github.com/hustcat/sriov-cni
+sriov_commit: 8b7ed984
+
+vhostuser_repo: https://github.com/yibo-cai/vhost-user-net-plugin
+vhostuser_commit: e8dc9d8e
+
+macvlan_master: eth2
+macvlan_subnet: 192.168.166.0/24
+macvlan_range_start: 192.168.166.11
+macvlan_range_end: 192.168.166.30
+macvlan_gateway: 192.168.166.1
+
+sriov_master: eth2
+sriov_subnet: 192.168.166.0/24
+sriov_range_start: 192.168.166.31
+sriov_range_end: 192.168.166.50
+sriov_gateway: 192.168.166.1
Code Review / container4nfv.git / commitdiff