Adding cnishim and cniserver 77/69977/9
authorKuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Tue, 21 Apr 2020 17:19:34 +0000 (17:19 +0000)
committerKuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Thu, 17 Sep 2020 23:25:10 +0000 (16:25 -0700)
- inspired from ovn-kubernetes and sdn openshift - cniserver & cnishim concepts
- removed cni binary to depend on the host ovs binary installation
- encapsulated all the binaries within the ovn and ovs containers
- ovn4nfv-k8s cni server run along with nfn-agent
- cnishim act as the httpclient and cniserver as httpservers
- cnishim wrap all the cni commands to cniserver
- cniserver do the actual network pumping work and send result back to cnishim
- cnishim print the results as per the cni spec requirement
- currently supports only debian installation for ovn daemon
- support for debian kernel installation
- Consolidated all yaml into single ovn4nfv-k8s-plugin

Signed-off-by: Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Change-Id: I1e2b114d90f717baa2ee94ff379c849d73b2754e

15 files changed:
build/Dockerfile
cmd/nfn-agent/nfn-agent.go
cmd/ovn4nfvk8s-cni/app/helper_linux.go
cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
deploy/ovn4nfv-k8s-plugin.yaml [moved from deploy/ovn4nfv-k8s-plugin-daemonset.yml with 90% similarity]
go.mod
go.sum
internal/pkg/cniserver/cni.go [new file with mode: 0644]
internal/pkg/cniserver/cniserver.go [new file with mode: 0644]
internal/pkg/cnishim/cnishim.go [new file with mode: 0644]
internal/pkg/config/config.go
internal/pkg/ovn/utils.go
pkg/controller/pod/pod_controller.go
utilities/docker/debian/Dockerfile
utilities/kernel/debian/install_kernel_modules.sh [new file with mode: 0755]

index 9530772..af88d13 100644 (file)
@@ -8,13 +8,21 @@ ENV https_proxy $HTTPS_PROXY
 ENV no_proxy $NO_PROXY
 
 RUN apt-get update && apt-get install -y -qq apt-transport-https make curl net-tools iproute2 \
- && echo "deb https://packages.wand.net.nz bionic ovs-2.10" >  /etc/apt/sources.list.d/ovs.list \
- && curl https://packages.wand.net.nz/keyring.gpg -o /etc/apt/trusted.gpg.d/wand.gpg \
- && apt-get update && apt install -y -qq ovn-common openvswitch-common openvswitch-switch
+    wget software-properties-common setpriv dpkg-dev netcat jq
+
+RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
+RUN bash -xc "\
+pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \
+wget -q -nv -O- https://api.github.com/repos/akraino-icn/ovs/releases/tags/v2.12.0 2>/dev/null | jq -r '.assets[] | select(.browser_download_url | contains("\""deb"\"")) | .browser_download_url' | wget -i -; \
+dpkg-scanpackages . | gzip -c9  > Packages.gz; \
+popd; \
+"
+RUN echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null
+RUN apt-get update && apt-get install -y -qq ovn-common=2.12.0-1 openvswitch-common=2.12.0-1 openvswitch-switch=2.12.0-1
 
 ENV GOLANG_VERSION 1.14.1
 RUN curl -sSL https://storage.googleapis.com/golang/go$GOLANG_VERSION.linux-amd64.tar.gz \
-       | tar -v -C /usr/local -xz
+        | tar -v -C /usr/local -xz
 
 ENV PATH /usr/local/go/bin:$PATH
 RUN mkdir -p /go/src /go/bin && chmod -R 777 /go
@@ -32,5 +40,4 @@ ENV OPERATOR=/usr/local/bin/nfn-operator \
     USER_NAME=nfn-operator
 
 RUN cp -r build/bin/* /usr/local/bin/
-
 ENTRYPOINT ["entrypoint"]
index 3c85dbd..8d33778 100644 (file)
@@ -9,8 +9,11 @@ import (
        "os"
        "os/signal"
        pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
+        cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
        "ovn4nfv-k8s-plugin/internal/pkg/ovn"
        logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+        "k8s.io/client-go/kubernetes"
+        "k8s.io/client-go/rest"
        "strings"
        "syscall"
        "time"
@@ -298,6 +301,26 @@ func main() {
        client := pb.NewNfnNotifyClient(conn)
        errorChannel = make(chan string)
 
+        // creates the in-cluster config
+       config, err := rest.InClusterConfig()
+       if err != nil {
+                log.Error(err, "Unable to create in-cluster config")
+               return
+       }
+
+       // creates the clientset
+       clientset, err := kubernetes.NewForConfig(config)
+       if err != nil {
+               log.Error(err, "Unable to create clientset for in-cluster config")
+                return
+       }
+
+        cniserver := cs.NewCNIServer("",clientset)
+        err = cniserver.Start(cs.HandleCNIcommandRequest)
+        if err != nil {
+                log.Error(err, "Unable to start cni server")
+                return
+        }
        // Run client in background
        go subscribeNotif(client)
        shutdownHandler(errorChannel)
index 2dba628..1702597 100644 (file)
@@ -10,8 +10,6 @@ import (
        "strings"
 
        "github.com/sirupsen/logrus"
-
-       "github.com/containernetworking/cni/pkg/skel"
        "github.com/containernetworking/cni/pkg/types/current"
        "github.com/containernetworking/plugins/pkg/ip"
        "github.com/containernetworking/plugins/pkg/ns"
@@ -106,10 +104,10 @@ func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress,
 }
 
 // ConfigureInterface sets up the container interface
-var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) {
-       netns, err := ns.GetNS(args.Netns)
+var ConfigureInterface = func(containerNetns, containerID, ifName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) {
+       netns, err := ns.GetNS(containerNetns)
        if err != nil {
-               return nil, fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+               return nil, fmt.Errorf("failed to open netns %q: %v", containerNetns, err)
        }
        defer netns.Close()
 
@@ -118,10 +116,10 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress
                ifaceID = fmt.Sprintf("%s_%s_%s", namespace, podName, interfaceName)
        } else {
                ifaceID = fmt.Sprintf("%s_%s", namespace, podName)
-               interfaceName = args.IfName
+               interfaceName = ifName
                defaultGateway = "true"
        }
-       hostIface, contIface, err := setupInterface(netns, args.ContainerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu)
+       hostIface, contIface, err := setupInterface(netns, containerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu)
        if err != nil {
                return nil, err
        }
@@ -132,7 +130,7 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress
                fmt.Sprintf("external_ids:attached_mac=%s", macAddress),
                fmt.Sprintf("external_ids:iface-id=%s", ifaceID),
                fmt.Sprintf("external_ids:ip_address=%s", ipAddress),
-               fmt.Sprintf("external_ids:sandbox=%s", args.ContainerID),
+               fmt.Sprintf("external_ids:sandbox=%s", containerID),
        }
 
        var out []byte
@@ -160,10 +158,10 @@ func setupRoute(netns ns.NetNS, dst, gw, dev string) error {
 }
 
 // ConfigureRoute sets up the container routes
-var ConfigureRoute = func(args *skel.CmdArgs, dst, gw, dev string) error {
-       netns, err := ns.GetNS(args.Netns)
+var ConfigureRoute = func(containerNetns, dst, gw, dev string) error {
+       netns, err := ns.GetNS(containerNetns)
        if err != nil {
-               return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+               return fmt.Errorf("failed to open netns %q: %v", containerNetns, err)
        }
        defer netns.Close()
        err = setupRoute(netns, dst, gw, dev)
index c176700..cb75ecd 100644 (file)
-// +build linux
-
 package main
 
 import (
-       "encoding/json"
-       "fmt"
-       "net"
        "os"
-       "strconv"
-       "strings"
-       "time"
 
        "github.com/sirupsen/logrus"
        "github.com/urfave/cli"
 
+       cni "ovn4nfv-k8s-plugin/internal/pkg/cnishim"
+       "ovn4nfv-k8s-plugin/internal/pkg/config"
+
        "github.com/containernetworking/cni/pkg/skel"
        "github.com/containernetworking/cni/pkg/types"
-       "github.com/containernetworking/cni/pkg/types/current"
        "github.com/containernetworking/cni/pkg/version"
-       "k8s.io/apimachinery/pkg/util/wait"
-
-       "ovn4nfv-k8s-plugin/internal/pkg/kube"
-
-       "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app"
-       "ovn4nfv-k8s-plugin/internal/pkg/config"
+       "github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
-const (
-       ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
-)
-
-func argString2Map(args string) (map[string]string, error) {
-       argsMap := make(map[string]string)
-
-       pairs := strings.Split(args, ";")
-       for _, pair := range pairs {
-               kv := strings.Split(pair, "=")
-               if len(kv) != 2 {
-                       return nil, fmt.Errorf("ARGS: invalid pair %q", pair)
-               }
-               keyString := kv[0]
-               valueString := kv[1]
-               argsMap[keyString] = valueString
-       }
-
-       return argsMap, nil
-}
-
-func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) {
-       var ovnNet []map[string]string
-
-       if ovnnetwork == "" {
-               return nil, fmt.Errorf("parseOvnNetworkObject:error")
-       }
-
-       if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil {
-               return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork)
-       }
-
-       return ovnNet, nil
-}
-
-func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) {
-
-       if dstObj == nil {
-               return srcObj, nil
-       }
-       src, err := current.NewResultFromResult(srcObj)
-       if err != nil {
-               return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
-       }
-       dst, err := current.NewResultFromResult(dstObj)
-       if err != nil {
-               return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
-       }
-
-       ifacesLength := len(dst.Interfaces)
-
-       for _, iface := range src.Interfaces {
-               dst.Interfaces = append(dst.Interfaces, iface)
-       }
-       for _, ip := range src.IPs {
-               if ip.Interface != nil && *(ip.Interface) != -1 {
-                       ip.Interface = current.Int(*(ip.Interface) + ifacesLength)
-               }
-               dst.IPs = append(dst.IPs, ip)
-       }
-       for _, route := range src.Routes {
-               dst.Routes = append(dst.Routes, route)
-       }
-
-       for _, ns := range src.DNS.Nameservers {
-               dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns)
-       }
-       for _, s := range src.DNS.Search {
-               dst.DNS.Search = append(dst.DNS.Search, s)
-       }
-       for _, opt := range src.DNS.Options {
-               dst.DNS.Options = append(dst.DNS.Options, opt)
-       }
-       // TODO: what about DNS.domain?
-       return dst, nil
-}
-
-func prettyPrint(i interface{}) string {
-       s, _ := json.MarshalIndent(i, "", "\t")
-       return string(s)
-}
-
-func addMultipleInterfaces(args *skel.CmdArgs, ovnAnnotation, namespace, podName string) types.Result {
-       logrus.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ")
-
-       var ovnAnnotatedMap []map[string]string
-       ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
-       if err != nil {
-               logrus.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err)
-               return nil
-       }
-       if namespace == "" || podName == "" {
-               logrus.Errorf("required CNI variable missing")
-               return nil
-       }
-       var interfacesArray []*current.Interface
-       var index int
-       var result *current.Result
-       var dstResult types.Result
-       for _, ovnNet := range ovnAnnotatedMap {
-               ipAddress := ovnNet["ip_address"]
-               macAddress := ovnNet["mac_address"]
-               gatewayIP := ovnNet["gateway_ip"]
-               defaultGateway := ovnNet["defaultGateway"]
-
-               if ipAddress == "" || macAddress == "" {
-                       logrus.Errorf("failed in pod annotation key extract")
-                       return nil
-               }
-
-               index++
-               interfaceName := ovnNet["interface"]
-               if interfaceName == "" {
-                       logrus.Errorf("addMultipleInterfaces: interface can't be null")
-                       return nil
-               }
-               interfacesArray, err = app.ConfigureInterface(args, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU)
-               if err != nil {
-                       logrus.Errorf("Failed to configure interface in pod: %v", err)
-                       return nil
-               }
-               addr, addrNet, err := net.ParseCIDR(ipAddress)
-               if err != nil {
-                       logrus.Errorf("failed to parse IP address %q: %v", ipAddress, err)
-                       return nil
-               }
-               ipVersion := "6"
-               if addr.To4() != nil {
-                       ipVersion = "4"
-               }
-               var routes types.Route
-               if defaultGateway == "true" {
-                       defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0")
-                       routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)}
-
-                       result = &current.Result{
-                               Interfaces: interfacesArray,
-                               IPs: []*current.IPConfig{
-                                       {
-                                               Version:   ipVersion,
-                                               Interface: current.Int(1),
-                                               Address:   net.IPNet{IP: addr, Mask: addrNet.Mask},
-                                               Gateway:   net.ParseIP(gatewayIP),
-                                       },
-                               },
-                               Routes: []*types.Route{&routes},
-                       }
-               } else {
-                       result = &current.Result{
-                               Interfaces: interfacesArray,
-                               IPs: []*current.IPConfig{
-                                       {
-                                               Version:   ipVersion,
-                                               Interface: current.Int(1),
-                                               Address:   net.IPNet{IP: addr, Mask: addrNet.Mask},
-                                               Gateway:   net.ParseIP(gatewayIP),
-                                       },
-                               },
-                       }
-
-               }
-               // Build the result structure to pass back to the runtime
-               dstResult, err = mergeWithResult(types.Result(result), dstResult)
-               if err != nil {
-                       logrus.Errorf("Failed to merge results: %v", err)
-                       return nil
-               }
-       }
-       logrus.Infof("addMultipleInterfaces:  %s", prettyPrint(dstResult))
-       return dstResult
-}
-
-func addRoutes(args *skel.CmdArgs, ovnAnnotation string, dstResult types.Result) types.Result {
-       logrus.Infof("ovn4nfvk8s-cni: addRoutes ")
-
-       var ovnAnnotatedMap []map[string]string
-       ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
-       if err != nil {
-               logrus.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err)
-               return nil
-       }
-
-       var result types.Result
-       var routes []*types.Route
-       for _, ovnNet := range ovnAnnotatedMap {
-               dst := ovnNet["dst"]
-               gw := ovnNet["gw"]
-               dev := ovnNet["dev"]
-               if dst == "" || gw == "" || dev == "" {
-                       logrus.Errorf("failed in pod annotation key extract")
-                       return nil
-               }
-               err = app.ConfigureRoute(args, dst, gw, dev)
-               if err != nil {
-                       logrus.Errorf("Failed to configure interface in pod: %v", err)
-                       return nil
-               }
-               dstAddr, dstAddrNet, _ := net.ParseCIDR(dst)
-               routes = append(routes, &types.Route{
-                       Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask},
-                       GW:  net.ParseIP(gw),
-               })
-       }
-
-       result = &current.Result{
-               Routes: routes,
-       }
-       // Build the result structure to pass back to the runtime
-       dstResult, err = mergeWithResult(result, dstResult)
-       if err != nil {
-               logrus.Errorf("Failed to merge results: %v", err)
-               return nil
-       }
-       logrus.Infof("addRoutes:  %s", prettyPrint(dstResult))
-       return dstResult
-
-}
-
-func cmdAdd(args *skel.CmdArgs) error {
-       logrus.Infof("ovn4nfvk8s-cni: cmdAdd ")
-       conf := &types.NetConf{}
-       if err := json.Unmarshal(args.StdinData, conf); err != nil {
-               return fmt.Errorf("failed to load netconf: %v", err)
-       }
-
-       argsMap, err := argString2Map(args.Args)
-       if err != nil {
-               return err
-       }
-
-       namespace := argsMap["K8S_POD_NAMESPACE"]
-       podName := argsMap["K8S_POD_NAME"]
-       if namespace == "" || podName == "" {
-               return fmt.Errorf("required CNI variable missing")
-       }
-
-       clientset, err := config.NewClientset(&config.Kubernetes)
-       if err != nil {
-               return fmt.Errorf("Could not create clientset for kubernetes: %v", err)
-       }
-       kubecli := &kube.Kube{KClient: clientset}
-
-       // Get the IP address and MAC address from the API server.
-       var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1}
-       var annotation map[string]string
-       if err := wait.ExponentialBackoff(annotationBackoff, func() (bool, error) {
-               annotation, err = kubecli.GetAnnotationsOnPod(namespace, podName)
-               if err != nil {
-                       // TODO: check if err is non recoverable
-                       logrus.Warningf("Error while obtaining pod annotations - %v", err)
-                       return false, nil
-               }
-               if _, ok := annotation[ovn4nfvAnnotationTag]; ok {
-                       return true, nil
-               }
-               return false, nil
-       }); err != nil {
-               return fmt.Errorf("failed to get pod annotation - %v", err)
-       }
-       logrus.Infof("ovn4nfvk8s-cni: Annotation Found ")
-       ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag]
-       if !ok {
-               return fmt.Errorf("Error while obtaining pod annotations")
-       }
-       result := addMultipleInterfaces(args, ovnAnnotation, namespace, podName)
-       // Add Routes to the pod if annotation found for routes
-       ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"]
-       if ok {
-               logrus.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation)
-               result = addRoutes(args, ovnRouteAnnotation, result)
-       }
-
-       return result.Print()
-}
-
-func cmdDel(args *skel.CmdArgs) error {
-       logrus.Infof("ovn4nfvk8s-cni: cmdDel ")
-       for i := 0; i < 10; i++ {
-               ifaceName := args.ContainerID[:14] + strconv.Itoa(i)
-               done, err := app.PlatformSpecificCleanup(ifaceName)
-               if err != nil {
-                       logrus.Errorf("Teardown error: %v", err)
-               }
-               if done {
-                       break
-               }
-       }
-       return nil
-}
-
 func main() {
-       logrus.Infof("ovn4nfvk8s-cni invoked")
+       logrus.Infof("ovn4nfvk8s-cni shim cni")
        c := cli.NewApp()
        c.Name = "ovn4nfvk8s-cni"
        c.Usage = "a CNI plugin to set up or tear down a additional interfaces with OVN"
-       c.Version = "0.0.2"
+       c.Version = "0.1.0"
        c.Flags = config.Flags
 
+       ep := cni.CNIEndpoint("")
        c.Action = func(ctx *cli.Context) error {
                if _, err := config.InitConfig(ctx); err != nil {
                        return err
                }
+               skel.PluginMain(
+                       ep.CmdAdd,
+                       ep.CmdCheck,
+                       ep.CmdDel,
+                       version.All,
+                       buildversion.BuildString("ovn4nfv-k8s shim cni"))
 
-               skel.PluginMain(cmdAdd, nil, cmdDel, version.All, "")
                return nil
        }
 
similarity index 90%
rename from deploy/ovn4nfv-k8s-plugin-daemonset.yml
rename to deploy/ovn4nfv-k8s-plugin.yaml
index 13e749f..5d5017e 100644 (file)
@@ -284,10 +284,9 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: k8s-nfn-sa
-  namespace: operator
+  namespace: kube-system
 
 ---
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
@@ -298,6 +297,7 @@ rules:
   - ""
   resources:
   - pods
+  - pods/status
   - services
   - endpoints
   - persistentvolumeclaims
@@ -361,7 +361,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nfn-operator
-  namespace: operator
+  namespace: kube-system
 spec:
   type: NodePort
   ports:
@@ -378,7 +378,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: ovn-controller-network
-  namespace: operator
+  namespace: kube-system
 data:
   OVN_SUBNET: "10.244.64.0/18"
   OVN_GATEWAYIP: "10.244.64.20/18"
@@ -391,7 +391,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: nfn-operator
-  namespace: operator
+  namespace: kube-system
 spec:
   replicas: 1
   selector:
@@ -408,10 +408,10 @@ spec:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
             - matchExpressions:
-              - key: nfnType
+              - key: ovn4nfv-k8s-plugin
                 operator: In
                 values:
-                - operator
+                - ovn-control-plane
       tolerations:
        - key: "node-role.kubernetes.io/master"
          effect: "NoSchedule"
@@ -429,10 +429,6 @@ spec:
           - containerPort: 50000
             protocol: TCP
           env:
-            - name: HOST_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.hostIP
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -445,7 +441,7 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: ovn4nfv-cni-config
-  namespace: operator
+  namespace: kube-system
   labels:
     app: ovn4nfv
 data:
@@ -459,14 +455,20 @@ data:
           plugin=ovn4nfvk8s-cni
 
           [kubernetes]
-          kubeconfig=/etc/kubernetes/admin.conf
+          kubeconfig=/etc/cni/net.d/ovn4nfv-k8s.d/ovn4nfv-k8s.kubeconfig
+  00-network.conf: |
+          {
+            "name": "ovn4nfv-k8s-plugin",
+            "type": "ovn4nfvk8s-cni",
+            "cniVersion": "0.3.1"
+          }
 
 ---
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
   name: ovn4nfv-cni
-  namespace: operator
+  namespace: kube-system
   labels:
     app: ovn4nfv
 spec:
@@ -483,6 +485,7 @@ spec:
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      serviceAccountName: k8s-nfn-sa
       containers:
       - name: ovn4nfv
         image: integratedcloudnative/ovn4nfv-k8s-plugin:master
@@ -497,13 +500,20 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
         - name: cnibin
           mountPath: /host/opt/cni/bin
         - name: cniconf
           mountPath: /host/etc/openvswitch
         - name: ovn4nfv-cfg
           mountPath: /tmp/ovn4nfv-conf
+        - name: ovn4nfv-cni-net-conf
+          mountPath: /tmp/ovn4nfv-cni
       volumes:
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
         - name: cnibin
           hostPath:
             path: /opt/cni/bin
@@ -516,13 +526,18 @@ spec:
             items:
             - key: ovn4nfv_k8s.conf
               path: ovn4nfv_k8s.conf
-
+        - name: ovn4nfv-cni-net-conf
+          configMap:
+            name: ovn4nfv-cni-config
+            items:
+            - key: 00-network.conf
+              path: 00-network.conf
 ---
 apiVersion: extensions/v1beta1
 kind: DaemonSet
 metadata:
   name: nfn-agent
-  namespace: operator
+  namespace: kube-system
   labels:
     app: nfn-agent
 spec:
@@ -534,11 +549,13 @@ spec:
         app: nfn-agent
     spec:
       hostNetwork: true
+      hostPID: true
       nodeSelector:
         beta.kubernetes.io/arch: amd64
       tolerations:
       - operator: Exists
         effect: NoSchedule
+      serviceAccountName: k8s-nfn-sa
       containers:
       - name: nfn-agent
         image: integratedcloudnative/ovn4nfv-k8s-plugin:master
@@ -556,12 +573,20 @@ spec:
               fieldRef:
                 fieldPath: spec.nodeName
         securityContext:
+          runAsUser: 0
+          capabilities:
+            add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"]
           privileged: true
         volumeMounts:
+        - mountPath: /var/run/dbus/
+          name: host-var-run-dbus
+          readOnly: true
         - mountPath: /run/openvswitch
           name: host-run-ovs
         - mountPath: /var/run/openvswitch
           name: host-var-run-ovs
+        - mountPath: /var/run/ovn4nfv-k8s-plugin
+          name: host-var-cniserver-socket-dir
       volumes:
       - name: host-run-ovs
         hostPath:
@@ -569,3 +594,9 @@ spec:
       - name: host-var-run-ovs
         hostPath:
           path: /var/run/openvswitch
+      - name: host-var-run-dbus
+        hostPath:
+          path: /var/run/dbus
+      - name: host-var-cniserver-socket-dir
+        hostPath:
+          path: /var/run/ovn4nfv-k8s-plugin
diff --git a/go.mod b/go.mod
index d98f1e5..985c693 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
        github.com/google/btree v1.0.0 // indirect
        github.com/google/go-cmp v0.3.1 // indirect
        github.com/gophercloud/gophercloud v0.2.0 // indirect
+       github.com/gorilla/mux v1.7.4
        github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
        github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect
        github.com/hashicorp/golang-lru v0.5.3 // indirect
@@ -67,6 +68,7 @@ require (
        k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
        k8s.io/code-generator v0.17.0 // indirect
        k8s.io/gengo v0.0.0-20191120174120-e74f70b9b27e // indirect
+       k8s.io/klog v1.0.0
        k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
        k8s.io/utils v0.0.0-20190801114015-581e00157fb1
        sigs.k8s.io/controller-runtime v0.2.0-beta.4
diff --git a/go.sum b/go.sum
index fd474b6..4963c29 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -319,6 +319,8 @@ github.com/gophercloud/gophercloud v0.2.0 h1:lD2Bce2xBAMNNcFZ0dObTpXkGLlVIb33RPV
 github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
diff --git a/internal/pkg/cniserver/cni.go b/internal/pkg/cniserver/cni.go
new file mode 100644 (file)
index 0000000..2c91f04
--- /dev/null
@@ -0,0 +1,295 @@
+package cniserver
+
+import (
+        "encoding/json"
+        "k8s.io/apimachinery/pkg/util/wait"
+        "fmt"
+        "net"
+        "strconv"
+        "net/http"
+        "time"
+        "k8s.io/klog"
+
+        "k8s.io/client-go/kubernetes"
+        "github.com/containernetworking/cni/pkg/types"
+        "github.com/containernetworking/cni/pkg/types/current"
+        "ovn4nfv-k8s-plugin/internal/pkg/kube"
+        "k8s.io/apimachinery/pkg/api/errors"
+        "ovn4nfv-k8s-plugin/internal/pkg/config"
+        "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app"
+)
+
+const (
+        ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
+)
+
+func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) {
+       var ovnNet []map[string]string
+
+       if ovnnetwork == "" {
+               return nil, fmt.Errorf("parseOvnNetworkObject:error")
+       }
+
+       if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil {
+               return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork)
+       }
+
+       return ovnNet, nil
+}
+
+func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) {
+
+       if dstObj == nil {
+               return srcObj, nil
+       }
+       src, err := current.NewResultFromResult(srcObj)
+       if err != nil {
+               return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
+       }
+       dst, err := current.NewResultFromResult(dstObj)
+       if err != nil {
+               return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
+       }
+
+       ifacesLength := len(dst.Interfaces)
+
+       for _, iface := range src.Interfaces {
+               dst.Interfaces = append(dst.Interfaces, iface)
+       }
+       for _, ip := range src.IPs {
+               if ip.Interface != nil && *(ip.Interface) != -1 {
+                       ip.Interface = current.Int(*(ip.Interface) + ifacesLength)
+               }
+               dst.IPs = append(dst.IPs, ip)
+       }
+       for _, route := range src.Routes {
+               dst.Routes = append(dst.Routes, route)
+       }
+
+       for _, ns := range src.DNS.Nameservers {
+               dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns)
+       }
+       for _, s := range src.DNS.Search {
+               dst.DNS.Search = append(dst.DNS.Search, s)
+       }
+       for _, opt := range src.DNS.Options {
+               dst.DNS.Options = append(dst.DNS.Options, opt)
+       }
+       // TODO: what about DNS.domain?
+       return dst, nil
+}
+
+func prettyPrint(i interface{}) string {
+       s, _ := json.MarshalIndent(i, "", "\t")
+       return string(s)
+}
+
+func isNotFoundError(err error) bool {
+        statusErr, ok := err.(*errors.StatusError)
+        return ok && statusErr.Status().Code == http.StatusNotFound
+}
+
+func (cr *CNIServerRequest) addMultipleInterfaces(ovnAnnotation, namespace, podName string) types.Result {
+        klog.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ")
+       var ovnAnnotatedMap []map[string]string
+       ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
+       if err != nil {
+               klog.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err)
+               return nil
+       }
+       if namespace == "" || podName == "" {
+               klog.Errorf("required CNI variable missing")
+               return nil
+       }
+       var interfacesArray []*current.Interface
+       var index int
+       var result *current.Result
+       var dstResult types.Result
+       for _, ovnNet := range ovnAnnotatedMap {
+               ipAddress := ovnNet["ip_address"]
+               macAddress := ovnNet["mac_address"]
+               gatewayIP := ovnNet["gateway_ip"]
+               defaultGateway := ovnNet["defaultGateway"]
+
+               if ipAddress == "" || macAddress == "" {
+                       klog.Errorf("failed in pod annotation key extract")
+                       return nil
+               }
+
+               index++
+               interfaceName := ovnNet["interface"]
+               if interfaceName == "" {
+                       klog.Errorf("addMultipleInterfaces: interface can't be null")
+                       return nil
+               }
+               klog.Infof("addMultipleInterfaces: ipAddress %v %v", ipAddress, interfaceName)
+               interfacesArray, err = app.ConfigureInterface(cr.Netns, cr.SandboxID, cr.IfName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU)
+               if err != nil {
+                       klog.Errorf("Failed to configure interface in pod: %v", err)
+                       return nil
+               }
+               addr, addrNet, err := net.ParseCIDR(ipAddress)
+               if err != nil {
+                       klog.Errorf("failed to parse IP address %q: %v", ipAddress, err)
+                       return nil
+               }
+               ipVersion := "6"
+               if addr.To4() != nil {
+                       ipVersion = "4"
+               }
+               var routes types.Route
+               if defaultGateway == "true" {
+                       defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0")
+                       routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)}
+
+                       result = &current.Result{
+                               Interfaces: interfacesArray,
+                               IPs: []*current.IPConfig{
+                                       {
+                                               Version:   ipVersion,
+                                               Interface: current.Int(1),
+                                               Address:   net.IPNet{IP: addr, Mask: addrNet.Mask},
+                                               Gateway:   net.ParseIP(gatewayIP),
+                                       },
+                               },
+                               Routes: []*types.Route{&routes},
+                       }
+               } else {
+                       result = &current.Result{
+                               Interfaces: interfacesArray,
+                               IPs: []*current.IPConfig{
+                                       {
+                                               Version:   ipVersion,
+                                               Interface: current.Int(1),
+                                               Address:   net.IPNet{IP: addr, Mask: addrNet.Mask},
+                                               Gateway:   net.ParseIP(gatewayIP),
+                                       },
+                               },
+                       }
+
+               }
+               // Build the result structure to pass back to the runtime
+               dstResult, err = mergeWithResult(types.Result(result), dstResult)
+               if err != nil {
+                       klog.Errorf("Failed to merge results: %v", err)
+                       return nil
+               }
+       }
+       klog.Infof("addMultipleInterfaces: results %s", prettyPrint(dstResult))
+       return dstResult
+}
+
+func (cr *CNIServerRequest) addRoutes(ovnAnnotation string, dstResult types.Result) types.Result {
+       klog.Infof("ovn4nfvk8s-cni: addRoutes ")
+       var ovnAnnotatedMap []map[string]string
+       ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
+       if err != nil {
+               klog.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err)
+               return nil
+       }
+
+       var result types.Result
+       var routes []*types.Route
+       for _, ovnNet := range ovnAnnotatedMap {
+               dst := ovnNet["dst"]
+               gw := ovnNet["gw"]
+               dev := ovnNet["dev"]
+               if dst == "" || gw == "" || dev == "" {
+                       klog.Errorf("failed in pod annotation key extract")
+                       return nil
+               }
+               err = app.ConfigureRoute(cr.Netns, dst, gw, dev)
+               if err != nil {
+                       klog.Errorf("Failed to configure interface in pod: %v", err)
+                       return nil
+               }
+               dstAddr, dstAddrNet, _ := net.ParseCIDR(dst)
+               routes = append(routes, &types.Route{
+                       Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask},
+                       GW:  net.ParseIP(gw),
+               })
+       }
+
+       result = &current.Result{
+               Routes: routes,
+       }
+       // Build the result structure to pass back to the runtime
+       dstResult, err = mergeWithResult(result, dstResult)
+       if err != nil {
+               klog.Errorf("Failed to merge results: %v", err)
+               return nil
+       }
+       klog.Infof("addRoutes: results %s", prettyPrint(dstResult))
+       return dstResult
+}
+
+func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) {
+        klog.Infof("ovn4nfvk8s-cni: cmdAdd")
+       namespace := cr.PodNamespace
+        podname := cr.PodName
+       if namespace == "" || podname == "" {
+               return nil, fmt.Errorf("required CNI variable missing")
+       }
+        klog.Infof("ovn4nfvk8s-cni: cmdAdd for pod podname:%s and namespace:%s", podname, namespace)
+       kubecli := &kube.Kube{KClient: kclient}
+       // Get the IP address and MAC address from the API server.
+       var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1}
+       var annotation map[string]string
+        var err error
+       if err = wait.ExponentialBackoff(annotationBackoff, func() (bool, error) {
+               annotation, err = kubecli.GetAnnotationsOnPod(namespace, podname)
+               if err != nil {
+                        if isNotFoundError(err) {
+                               return false, fmt.Errorf("Error - pod not found - %v", err)
+                        }
+                        klog.Infof("ovn4nfvk8s-cni: cmdAdd Warning - Error while obtaining pod annotations - %v", err)
+                        return false,nil
+               }
+               if _, ok := annotation[ovn4nfvAnnotationTag]; ok {
+                       return true, nil
+               }
+               return false, nil
+       }); err != nil {
+               return nil, fmt.Errorf("failed to get pod annotation - %v", err)
+       }
+
+       klog.Infof("ovn4nfvk8s-cni: cmdAdd Annotation Found ")
+       ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag]
+       if !ok {
+               return nil, fmt.Errorf("Error while obtaining pod annotations")
+       }
+        result := cr.addMultipleInterfaces(ovnAnnotation, namespace, podname)
+       //Add Routes to the pod if annotation found for routes
+       ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"]
+       if ok {
+               klog.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation)
+               result = cr.addRoutes(ovnRouteAnnotation, result)
+       }
+
+        if result == nil {
+                klog.Errorf("result struct the ovn4nfv-k8s-plugin cniserver")
+                return nil, fmt.Errorf("result is nil from cni server response")
+        }
+
+        responseBytes, err := json.Marshal(result)
+       if err != nil {
+               return nil, fmt.Errorf("failed to marshal pod request response: %v", err)
+       }
+
+       return responseBytes, nil
+}
+
+func (cr *CNIServerRequest) cmdDel() ([]byte, error) {
+        klog.Infof("cmdDel ")
+        for i := 0; i < 10; i++ {
+               ifaceName := cr.SandboxID[:14] + strconv.Itoa(i)
+               done, err := app.PlatformSpecificCleanup(ifaceName)
+               if err != nil {
+                       klog.Errorf("Teardown error: %v", err)
+               }
+               if done {
+                       break
+               }
+       }
+        return []byte{}, nil
+}
diff --git a/internal/pkg/cniserver/cniserver.go b/internal/pkg/cniserver/cniserver.go
new file mode 100644 (file)
index 0000000..eaa7105
--- /dev/null
@@ -0,0 +1,235 @@
+package cniserver
+
+import (
+       "encoding/json"
+       "fmt"
+       "io/ioutil"
+       "net/http"
+        "strings"
+        "os"
+        "net"
+        "path/filepath"
+        "syscall"
+        "k8s.io/klog"
+
+       "github.com/containernetworking/cni/pkg/types"
+       "github.com/gorilla/mux"
+       "k8s.io/client-go/kubernetes"
+        "ovn4nfv-k8s-plugin/internal/pkg/config"
+        utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+       utilwait "k8s.io/apimachinery/pkg/util/wait"
+)
+
+const CNIServerRunDir string = "/var/run/ovn4nfv-k8s-plugin/cniserver"
+const CNIServerSocketName string = "ovn4nfv-k8s-plugin-cni-server.sock"
+const CNIServerSocketPath string = CNIServerRunDir + "/" + CNIServerSocketName
+
+
+type CNIcommand string
+
+const CNIAdd CNIcommand = "ADD"
+const CNIUpdate CNIcommand = "UPDATE"
+const CNIDel CNIcommand = "DEL"
+
+type CNIServerRequest struct {
+       Command CNIcommand
+       PodNamespace string
+       PodName string
+       SandboxID string
+       Netns string
+       IfName string
+       CNIConf *types.NetConf
+}
+
+type cniServerRequestFunc func(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error)
+
+type CNIEndpointRequest struct {
+       ArgEnv    map[string]string `json:"env,omitempty"`
+       NetConfig []byte            `json:"config,omitempty"`
+}
+type CNIServer struct {
+       http.Server
+       requestFunc  cniServerRequestFunc
+       serverrundir string
+       k8sclient      kubernetes.Interface
+}
+
+func NewCNIServer(serverRunSir string, k8sclient kubernetes.Interface) *CNIServer {
+       klog.Infof("Setting up CNI server in nfn-agent")
+       if len(serverRunSir) == 0 {
+               serverRunSir = CNIServerRunDir
+       }
+
+       router := mux.NewRouter()
+       cs := &CNIServer{
+               Server: http.Server{
+                       Handler: router,
+               },
+               serverrundir: serverRunSir,
+                k8sclient: k8sclient,
+       }
+       router.NotFoundHandler = http.HandlerFunc(http.NotFound)
+       router.HandleFunc("/", cs.handleCNIShimRequest).Methods("POST")
+       return cs
+}
+
+func loadCNIShimArgs(env map[string]string) (map[string]string, error) {
+       cnishimArgs, ok := env["CNI_ARGS"]
+       if !ok {
+               return nil, fmt.Errorf("cnishim req missing CNI_ARGS: '%s'", env)
+       }
+
+       mapArgs := make(map[string]string)
+       for _, arg := range strings.Split(cnishimArgs, ";") {
+               parts := strings.Split(arg, "=")
+               if len(parts) != 2 {
+                       return nil, fmt.Errorf("invalid CNI_ARG from cnishim '%s'", arg)
+               }
+               mapArgs[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+       }
+       return mapArgs, nil
+}
+
+func loadCNIRequestToCNIServer(r *CNIEndpointRequest) (*CNIServerRequest, error) {
+       cmd, ok := r.ArgEnv["CNI_COMMAND"]
+       if !ok {
+               return nil, fmt.Errorf("cnishim req missing CNI_COMMAND")
+       }
+
+       cnishimreq := &CNIServerRequest{
+               Command: CNIcommand(cmd),
+       }
+
+        cnishimreq.SandboxID, ok = r.ArgEnv["CNI_CONTAINERID"]
+        if !ok {
+                return nil, fmt.Errorf("cnishim req missing CNI_CONTAINERID")
+        }
+
+        cnishimreq.Netns, ok = r.ArgEnv["CNI_NETNS"]
+        if !ok {
+                return nil, fmt.Errorf("cnishim req missing CNI_NETNS")
+        }
+
+        cnishimreq.IfName, ok = r.ArgEnv["CNI_IFNAME"]
+        if !ok {
+                return nil, fmt.Errorf("cnishim req missing CNI_IFNAME")
+        }
+
+        cnishimArgs, err := loadCNIShimArgs(r.ArgEnv)
+        if err != nil {
+                return nil, err
+        }
+
+        cnishimreq.PodNamespace, ok = cnishimArgs["K8S_POD_NAMESPACE"]
+        if !ok {
+                return nil, fmt.Errorf("cnishim req missing K8S_POD_NAMESPACE")
+        }
+
+        cnishimreq.PodName, ok = cnishimArgs["K8S_POD_NAME"]
+       if !ok {
+               return nil, fmt.Errorf("cnishim req missing K8S_POD_NAME")
+       }
+
+        netconf, err := config.ConfigureNetConf(r.NetConfig)
+        if err != nil {
+                return nil, fmt.Errorf("cnishim req CNI arg configuration failed:%v",err)
+        }
+
+        cnishimreq.CNIConf = netconf
+       return cnishimreq, nil
+}
+
+func (cs *CNIServer) handleCNIShimRequest(w http.ResponseWriter, r *http.Request) {
+       var cr CNIEndpointRequest
+       b, _ := ioutil.ReadAll(r.Body)
+       if err := json.Unmarshal(b, &cr); err != nil {
+               http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+               return
+       }
+
+       req, err := loadCNIRequestToCNIServer(&cr)
+       if err != nil {
+               http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+               return
+       }
+
+       klog.Infof("Waiting for %s result for CNI server pod %s/%s", req.Command, req.PodNamespace, req.PodName)
+       result, err := cs.requestFunc(req, cs.k8sclient)
+       if err != nil {
+               http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+       } else {
+               w.Header().Set("Content-Type", "application/json")
+               if _, err := w.Write(result); err != nil {
+                        klog.Warningf("Error writing %s HTTP response: %v", req.Command, err)
+               }
+        }
+}
+
+func HandleCNIcommandRequest(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error) {
+        var result []byte
+       var err error
+        klog.Infof("[PodNamespace:%s/PodName:%s] dispatching pod network request %v", request.PodNamespace, request.PodName, request)
+        klog.Infof("k8sclient  %s", fmt.Sprintf("%v",k8sclient))
+       switch request.Command {
+       case CNIAdd:
+               result, err = request.cmdAdd(k8sclient)
+       case CNIDel:
+               result, err = request.cmdDel()
+       default:
+       }
+       klog.Infof("[PodNamespace:%s/PodName:%s] CNI request %v, result %q, err %v", request.PodNamespace, request.PodName, request, string(result), err)
+       if err != nil {
+               return nil, fmt.Errorf("[PodNamespace:%s/PodName:%s] CNI request %v %v", request.PodNamespace, request.PodName, request, err)
+       }
+       return result, nil
+}
+
+func (cs *CNIServer) Start(requestFunc cniServerRequestFunc) error {
+       if requestFunc == nil {
+               return fmt.Errorf("no CNI request handler")
+       }
+       cs.requestFunc = requestFunc
+       socketPath := filepath.Join(cs.serverrundir, CNIServerSocketName)
+       if err := os.RemoveAll(cs.serverrundir); err != nil && !os.IsNotExist(err) {
+               info, err := os.Stat(cs.serverrundir)
+               if err != nil {
+                       return fmt.Errorf("failed to stat old cni server info socket directory %s: %v", cs.serverrundir, err)
+               }
+               tmp := info.Sys()
+               statt, ok := tmp.(*syscall.Stat_t)
+               if !ok {
+                       return fmt.Errorf("failed to read CNI Server info socket directory stat info: %T", tmp)
+               }
+               if statt.Uid != 0 {
+                       return fmt.Errorf("insecure owner of CNI Server info socket directory %s: %v", cs.serverrundir, statt.Uid)
+               }
+
+               if info.Mode()&0777 != 0700 {
+                       return fmt.Errorf("insecure permissions on CNI Server info socket directory %s: %v", cs.serverrundir, info.Mode())
+               }
+
+               if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+                       return fmt.Errorf("failed to remove old CNI Server info socket %s: %v", socketPath, err)
+               }
+       }
+       if err := os.MkdirAll(cs.serverrundir, 0700); err != nil {
+               return fmt.Errorf("failed to create CNI Server info socket directory %s: %v", cs.serverrundir, err)
+       }
+
+       unixListener, err := net.Listen("unix", socketPath)
+       if err != nil {
+               return fmt.Errorf("failed to listen on CNI Server info socket: %v", err)
+       }
+       if err := os.Chmod(socketPath, 0600); err != nil {
+               unixListener.Close()
+               return fmt.Errorf("failed to set CNI Server info socket mode: %v", err)
+       }
+
+       cs.SetKeepAlivesEnabled(false)
+       go utilwait.Forever(func() {
+               if err := cs.Serve(unixListener); err != nil {
+                       utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
+               }
+       }, 0)
+        return nil
+}
diff --git a/internal/pkg/cnishim/cnishim.go b/internal/pkg/cnishim/cnishim.go
new file mode 100644 (file)
index 0000000..ffedbfb
--- /dev/null
@@ -0,0 +1,111 @@
+package cni
+
+import (
+       "bytes"
+       "encoding/json"
+       "fmt"
+       "io/ioutil"
+       "net"
+       "net/http"
+       "os"
+       "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
+       "ovn4nfv-k8s-plugin/internal/pkg/config"
+       "strings"
+
+       "github.com/containernetworking/cni/pkg/skel"
+       "github.com/containernetworking/cni/pkg/types"
+       "github.com/containernetworking/cni/pkg/types/current"
+       "github.com/sirupsen/logrus"
+)
+
+const CNIEndpointURLReq string = "http://dummy/"
+
+type Endpoint struct {
+       cniServerSocketPath string
+}
+
+func CNIEndpoint(cniServerSocketPath string) *Endpoint {
+       if len(cniServerSocketPath) == 0 {
+               cniServerSocketPath = cniserver.CNIServerSocketPath
+       }
+       return &Endpoint{cniServerSocketPath: cniServerSocketPath}
+}
+
+func cniEndpointRequest(args *skel.CmdArgs) *cniserver.CNIEndpointRequest {
+       osEnvMap := make(map[string]string)
+       for _, item := range os.Environ() {
+               idx := strings.Index(item, "=")
+               if idx > 0 {
+                       osEnvMap[strings.TrimSpace(item[:idx])] = item[idx+1:]
+               }
+       }
+
+       return &cniserver.CNIEndpointRequest{
+               ArgEnv:    osEnvMap,
+               NetConfig: args.StdinData,
+       }
+}
+
+func (ep *Endpoint) sendCNIServerReq(req *cniserver.CNIEndpointRequest) ([]byte, error) {
+       cnireqdata, err := json.Marshal(req)
+       if err != nil {
+               return nil, fmt.Errorf("sendCNIServerReq: failed to Marshal CNIShim Req %v:%v", req, err)
+       }
+
+       httpc := http.Client{
+               Transport: &http.Transport{
+                       Dial: func(proto, addr string) (net.Conn, error) {
+                               return net.Dial("unix", ep.cniServerSocketPath)
+                       },
+               },
+       }
+
+       reponse, err := httpc.Post(CNIEndpointURLReq, "application/json", bytes.NewReader(cnireqdata))
+       if err != nil {
+               return nil, fmt.Errorf("Failed to send CNIServer request: %v", err)
+       }
+       defer reponse.Body.Close()
+
+       rbody, err := ioutil.ReadAll(reponse.Body)
+       if err != nil {
+               return nil, fmt.Errorf("Failed to read the CNI Server reponse:%v", err)
+       }
+
+       if reponse.StatusCode != 200 {
+               return nil, fmt.Errorf("CNI Server request is failed with reponse status %v and reponse body %s", reponse.StatusCode, string(rbody))
+       }
+
+       return rbody, nil
+}
+
+func (ep *Endpoint) CmdAdd(args *skel.CmdArgs) error {
+       logrus.Infof("ovn4nfvk8s-cni: cmdAdd ")
+       conf, err := config.ConfigureNetConf(args.StdinData)
+       if err != nil {
+               return fmt.Errorf("invalid stdin args")
+       }
+       logrus.Infof("ovn4nfvk8s-cni: cmdAdd configure net conf details -%+v", conf)
+       req := cniEndpointRequest(args)
+        logrus.Infof("ovn4nfvk8s-cni: cmdAdd CNIEndpoint Request:%+v",req)
+       reponsebody, err := ep.sendCNIServerReq(req)
+       if err != nil {
+               return err
+       }
+       result, err := current.NewResult(reponsebody)
+       if err != nil {
+               return fmt.Errorf("failed to unmarshall CNIServer Result reponse %v - err:%v", string(reponsebody), err)
+       }
+
+       return types.PrintResult(result, conf.CNIVersion)
+}
+
+func (ep *Endpoint) CmdCheck(args *skel.CmdArgs) error {
+       return nil
+}
+
+func (ep *Endpoint) CmdDel(args *skel.CmdArgs) error {
+       logrus.Infof("ovn4nfvk8s-cni: cmdDel ")
+       req := cniEndpointRequest(args)
+       _, err := ep.sendCNIServerReq(req)
+       return err
+}
index 002ad80..e9ad3e1 100644 (file)
@@ -1,6 +1,7 @@
 package config
 
 import (
+        "encoding/json"
        "fmt"
        "os"
        "path/filepath"
@@ -8,6 +9,8 @@ import (
 
        "github.com/sirupsen/logrus"
        "github.com/urfave/cli"
+        "github.com/containernetworking/cni/pkg/types"
+        "github.com/containernetworking/cni/pkg/version"
        gcfg "gopkg.in/gcfg.v1"
 
        "k8s.io/client-go/kubernetes"
@@ -283,3 +286,17 @@ func NewClientset(conf *KubernetesConfig) (*kubernetes.Clientset, error) {
 
        return kubernetes.NewForConfig(kconfig)
 }
+
+func ConfigureNetConf(bytes []byte) (*types.NetConf, error) {
+        conf := &types.NetConf{}
+       if err := json.Unmarshal(bytes, conf); err != nil {
+               return nil, fmt.Errorf("failed to load netconf: %v", err)
+       }
+
+        if conf.RawPrevResult != nil {
+                if err := version.ParsePrevResult(conf); err != nil {
+                        return nil, err
+                }
+        }
+        return conf, nil
+}
index 9b388b7..c2e9142 100644 (file)
@@ -65,7 +65,7 @@ func SetExec(exec kexec.Interface) error {
        if err != nil {
                return err
        }
-       runner.hostIP = os.Getenv("HOST_IP")
+       runner.hostIP = os.Getenv("OVN_NB_TCP_SERVICE_HOST")
        // OVN Host Port
        runner.hostPort = "6641"
        log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort)
index 75ed731..d195782 100644 (file)
@@ -170,7 +170,7 @@ func (r *ReconcilePod) addLogicalPorts(pod *corev1.Pod) error {
                        return err
                }
        }
-       
+
        switch {
        case nfn.Type == "ovn4nfv":
                ovnCtl, err := ovn.GetOvnController()
index 674ee7e..5c91136 100644 (file)
@@ -2,29 +2,12 @@ FROM ubuntu:18.04 as base
 
 USER root
 
-RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat
+RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat jq wget
 
 RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
 RUN bash -xc "\
 pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb; \
+wget -q -nv -O- https://api.github.com/repos/akraino-icn/ovs/releases/tags/v2.12.0 2>/dev/null | jq -r '.assets[] | select(.browser_download_url | contains("\""deb"\"")) | .browser_download_url' | wget -i -; \
 dpkg-scanpackages . | gzip -c9  > Packages.gz; \
 popd; \
 "
diff --git a/utilities/kernel/debian/install_kernel_modules.sh b/utilities/kernel/debian/install_kernel_modules.sh
new file mode 100755 (executable)
index 0000000..20228e8
--- /dev/null
@@ -0,0 +1,32 @@
+!#/bin/bash
+
+apt-get install apt-transport-https dpkg-dev -y
+
+mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
+pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb
+dpkg-scanpackages . | gzip -c9  > Packages.gz
+popd
+
+sudo apt-get install apt-transport-https
+echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null
+cp /etc/apt/sources.list /etc/apt/sources.list~
+sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list
+apt-get update && apt-get build-dep dkms -y && apt-get install openvswitch-datapath-dkms=2.12.0-1 -y