From ce14abc0128d4c55a7805c52ea199e2ce6159428 Mon Sep 17 00:00:00 2001 From: Kuralamudhan Ramakrishnan Date: Mon, 6 Jul 2020 16:10:19 -0700 Subject: [PATCH] Adding node interface, SNAT and OVN Node switch port - Adding nfn proto for node interface MAC and IP address - Adding node switch ports in NFN Operator - Adding grpc client in nfn agent to get mac and IP address for node interface - Adding feature to create Node interface with OVS internal port - Make sure pod controller requeue the pod request for empty node field in pod spec - Unique node interface name using SHA and maintain 15 charactor for OVS switch port - Adding SNAT for default interface in each node - Adding iptables modules for SNAT rules Signed-off-by: Kuralamudhan Ramakrishnan Change-Id: I6cfa36e45007e796eb651345f9f0751329defcf7 --- build/Dockerfile | 2 +- build/bin/entrypoint | 10 +++ cmd/nfn-agent/nfn-agent.go | 54 +++++++++---- cmd/ovn4nfvk8s-cni/app/helper_linux.go | 63 ++++++++++++++- deploy/ovn4nfv-k8s-plugin.yaml | 1 + go.mod | 5 +- go.sum | 4 + internal/pkg/cniserver/cni.go | 76 +++++++++--------- internal/pkg/cniserver/cniserver.go | 101 +++++++++++------------ internal/pkg/config/config.go | 30 ++++--- internal/pkg/network/iptables.go | 124 ++++++++++++++++++++++++++++ internal/pkg/nfnNotify/proto/nfn.pb.go | 75 ++++++++++------- internal/pkg/nfnNotify/proto/nfn.proto | 2 + internal/pkg/nfnNotify/server.go | 13 ++- internal/pkg/node/node.go | 31 +++++++ internal/pkg/ovn/ovn.go | 142 +++++++++++++++++++++++++++++++-- pkg/controller/pod/pod_controller.go | 16 +++- 17 files changed, 592 insertions(+), 157 deletions(-) create mode 100644 internal/pkg/network/iptables.go create mode 100644 internal/pkg/node/node.go diff --git a/build/Dockerfile b/build/Dockerfile index af88d13..eefdeb3 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -7,7 +7,7 @@ ENV http_proxy $HTTP_PROXY ENV https_proxy $HTTPS_PROXY ENV no_proxy $NO_PROXY -RUN apt-get update && apt-get install -y -qq apt-transport-https make curl net-tools iproute2 \ +RUN apt-get update && apt-get install -y -qq apt-transport-https make curl net-tools iproute2 iptables \ wget software-properties-common setpriv dpkg-dev netcat jq RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb diff --git a/build/bin/entrypoint b/build/bin/entrypoint index c9646a0..b8ac7db 100755 --- a/build/bin/entrypoint +++ b/build/bin/entrypoint @@ -66,6 +66,15 @@ install_cni_plugins() { rm -rf cni-plugins-linux-$IMAGE_ARC-$CNI_VERSION.tgz } +set_snat_default_inteface() { + default_interface=$(awk '$2 == 00000000 { print $1 }' /proc/net/route) + + # Checking the SNAT for default interfaces + if ! iptables -t nat -C POSTROUTING -o $default_interface -j MASQUERADE 2>/dev/null ; then + iptables -t nat -A POSTROUTING -o $default_interface -j MASQUERADE + fi +} + cmd=${1:-""} case ${cmd} in @@ -80,6 +89,7 @@ case ${cmd} in cp -f $OVN4NFV_BIN_FILE $CNI_BIN_DIR cp -f $OVN4NFV_CONF_FILE $OVN4NFV_CONF_DIR cp -f $OVN4NFV_NET_CONF_FILE $CNI_CONF_DIR + set_snat_default_inteface create_kubeconfig install_cni_plugins # Sleep forever. diff --git a/cmd/nfn-agent/nfn-agent.go b/cmd/nfn-agent/nfn-agent.go index 8d33778..b2ee630 100644 --- a/cmd/nfn-agent/nfn-agent.go +++ b/cmd/nfn-agent/nfn-agent.go @@ -3,21 +3,26 @@ package main import ( "context" "fmt" - "google.golang.org/grpc" "io" - kexec "k8s.io/utils/exec" "os" "os/signal" + cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver" pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto" - cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver" "ovn4nfv-k8s-plugin/internal/pkg/ovn" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" "strings" "syscall" "time" + + "google.golang.org/grpc" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + kexec "k8s.io/utils/exec" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + //"google.golang.org/grpc/keepalive" + + "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app" + "google.golang.org/grpc/status" "sigs.k8s.io/controller-runtime/pkg/log/zap" ) @@ -201,6 +206,19 @@ DIRECTPRNETWORK: } } +func createNodeOVSInternalPort(payload *pb.Notification_InSync) error { + nodeIntfIPAddr := strings.Trim(strings.TrimSpace(payload.InSync.GetNodeIntfIpAddress()), "\"") + nodeIntfMacAddr := strings.Trim(strings.TrimSpace(payload.InSync.GetNodeIntfMacAddress()), "\"") + nodeName := os.Getenv("NFN_NODE_NAME") + + err := app.CreateNodeOVSInternalPort(nodeIntfIPAddr, nodeIntfMacAddr, nodeName) + if err != nil { + return err + } + + return nil +} + func handleNotif(msg *pb.Notification) { switch msg.GetCniType() { case "ovn4nfv": @@ -239,6 +257,12 @@ func handleNotif(msg *pb.Notification) { } case *pb.Notification_InSync: + if payload.InSync.GetNodeIntfIpAddress() != "" && payload.InSync.GetNodeIntfMacAddress() != "" { + err := createNodeOVSInternalPort(payload) + if err != nil { + return + } + } inSyncVlanProvidernetwork() inSyncDirectProvidernetwork() pnCreateStore = nil @@ -301,10 +325,10 @@ func main() { client := pb.NewNfnNotifyClient(conn) errorChannel = make(chan string) - // creates the in-cluster config + // creates the in-cluster config config, err := rest.InClusterConfig() if err != nil { - log.Error(err, "Unable to create in-cluster config") + log.Error(err, "Unable to create in-cluster config") return } @@ -312,15 +336,15 @@ func main() { clientset, err := kubernetes.NewForConfig(config) if err != nil { log.Error(err, "Unable to create clientset for in-cluster config") - return + return } - cniserver := cs.NewCNIServer("",clientset) - err = cniserver.Start(cs.HandleCNIcommandRequest) - if err != nil { - log.Error(err, "Unable to start cni server") - return - } + cniserver := cs.NewCNIServer("", clientset) + err = cniserver.Start(cs.HandleCNIcommandRequest) + if err != nil { + log.Error(err, "Unable to start cni server") + return + } // Run client in background go subscribeNotif(client) shutdownHandler(errorChannel) diff --git a/cmd/ovn4nfvk8s-cni/app/helper_linux.go b/cmd/ovn4nfvk8s-cni/app/helper_linux.go index 1702597..cfcd4e9 100644 --- a/cmd/ovn4nfvk8s-cni/app/helper_linux.go +++ b/cmd/ovn4nfvk8s-cni/app/helper_linux.go @@ -6,13 +6,15 @@ import ( "fmt" "net" "os/exec" + "ovn4nfv-k8s-plugin/internal/pkg/config" + "ovn4nfv-k8s-plugin/internal/pkg/network" "strconv" "strings" - "github.com/sirupsen/logrus" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/ip" "github.com/containernetworking/plugins/pkg/ns" + "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" ) @@ -35,6 +37,65 @@ func renameLink(curName, newName string) error { return nil } +//Todo Comments +func CreateNodeOVSInternalPort(nodeintfipaddr, nodeintfmacaddr, node string) error { + nodeName := strings.ToLower(node) + nodeOVSInternalIntfName := config.GetNodeIntfName(nodeName) + + hwAddr, err := net.ParseMAC(nodeintfmacaddr) + if err != nil { + logrus.Errorf("Error is converting %q to net hwaddr: %v", nodeOVSInternalIntfName, err) + return fmt.Errorf("Error is converting %q to net hwaddr: %v", nodeOVSInternalIntfName, err) + } + + ovsArgs := []string{ + "add-port", "br-int", nodeOVSInternalIntfName, "--", "set", + "interface", nodeOVSInternalIntfName, "type=internal", + fmt.Sprintf("mac_in_use=%s", strings.ReplaceAll(hwAddr.String(), ":", "\\:")), + fmt.Sprintf("mac=%s", strings.ReplaceAll(hwAddr.String(), ":", "\\:")), + fmt.Sprintf("external_ids:iface-id=%s", nodeOVSInternalIntfName), + } + logrus.Infof("ovs-vsctl args - %v", ovsArgs) + + //var out []byte + out, err := exec.Command("ovs-vsctl", ovsArgs...).CombinedOutput() + if err != nil { + logrus.Errorf("failure in creating Node OVS internal port - %s: %v - %q", nodeOVSInternalIntfName, err, string(out)) + return fmt.Errorf("failure in creating Node OVS internal port - %s: %v - %q", nodeOVSInternalIntfName, err, string(out)) + } + logrus.Infof("ovs-vsctl args - %v output:%v", ovsArgs, string(out)) + + link, err := netlink.LinkByName(nodeOVSInternalIntfName) + if err != nil { + logrus.Errorf("failed to get netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err) + return fmt.Errorf("failed to get netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err) + } + + if err := netlink.LinkSetUp(link); err != nil { + logrus.Errorf("failed to set up netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err) + return fmt.Errorf("failed to set up netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err) + } + + addr, err := netlink.ParseAddr(nodeintfipaddr) + if err != nil { + logrus.Errorf("failed to parse IP addr %s: %v", nodeintfipaddr, err) + return fmt.Errorf("failed to parse IP addr %s: %v", nodeintfipaddr, err) + } + err = netlink.AddrAdd(link, addr) + if err != nil { + logrus.Errorf("failed to parse IP addr %s: %v", nodeintfipaddr, err) + return fmt.Errorf("failed to add IP addr %s to %s: %v", nodeintfipaddr, nodeOVSInternalIntfName, err) + } + + err = network.SetupAndEnsureIPTables(network.MasqRules(nodeOVSInternalIntfName)) + if err != nil { + logrus.Errorf("failed to apply snat rule for %s: %v", nodeOVSInternalIntfName, err) + return fmt.Errorf("failed to apply snat rule for %s: %v", nodeOVSInternalIntfName, err) + } + + return nil +} + func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress, gatewayIP, defaultGateway string, idx, mtu int) (*current.Interface, *current.Interface, error) { hostIface := ¤t.Interface{} contIface := ¤t.Interface{} diff --git a/deploy/ovn4nfv-k8s-plugin.yaml b/deploy/ovn4nfv-k8s-plugin.yaml index 5d5017e..73d3ab3 100644 --- a/deploy/ovn4nfv-k8s-plugin.yaml +++ b/deploy/ovn4nfv-k8s-plugin.yaml @@ -490,6 +490,7 @@ spec: - name: ovn4nfv image: integratedcloudnative/ovn4nfv-k8s-plugin:master command: ["/usr/local/bin/entrypoint", "cni"] + imagePullPolicy: IfNotPresent resources: requests: cpu: "100m" diff --git a/go.mod b/go.mod index 985c693..e0f14a5 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/ajstarks/svgo v0.0.0-20191124160048-bd5c74aaa11c // indirect github.com/containernetworking/cni v0.7.1 github.com/containernetworking/plugins v0.8.1 - github.com/coreos/go-iptables v0.4.2 // indirect + github.com/coreos/go-iptables v0.4.2 github.com/creack/pty v1.1.9 // indirect github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect @@ -17,6 +17,7 @@ require ( github.com/go-openapi/spec v0.19.5 github.com/go-openapi/swag v0.19.6 // indirect github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect github.com/golang/protobuf v1.3.5 github.com/google/btree v1.0.0 // indirect @@ -72,7 +73,7 @@ require ( k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a k8s.io/utils v0.0.0-20190801114015-581e00157fb1 sigs.k8s.io/controller-runtime v0.2.0-beta.4 - sigs.k8s.io/controller-tools v0.1.10 + sigs.k8s.io/controller-tools v0.1.10 // indirect sigs.k8s.io/structured-merge-diff v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4963c29..4fee0cb 100644 --- a/go.sum +++ b/go.sum @@ -382,6 +382,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -435,6 +436,7 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -445,6 +447,7 @@ github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/openshift/origin v0.0.0-20160503220234-8f127d736703/go.mod h1:0Rox5r9C8aQn6j1oAOQ0c1uC86mYbUFObzjBRvUKHII= @@ -814,6 +817,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/internal/pkg/cniserver/cni.go b/internal/pkg/cniserver/cni.go index 2c91f04..95a41d5 100644 --- a/internal/pkg/cniserver/cni.go +++ b/internal/pkg/cniserver/cni.go @@ -1,26 +1,28 @@ package cniserver import ( - "encoding/json" - "k8s.io/apimachinery/pkg/util/wait" - "fmt" - "net" - "strconv" - "net/http" - "time" - "k8s.io/klog" + "encoding/json" + "fmt" + "net" + "net/http" + "strconv" + "time" - "k8s.io/client-go/kubernetes" - "github.com/containernetworking/cni/pkg/types" - "github.com/containernetworking/cni/pkg/types/current" - "ovn4nfv-k8s-plugin/internal/pkg/kube" - "k8s.io/apimachinery/pkg/api/errors" - "ovn4nfv-k8s-plugin/internal/pkg/config" - "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" + + "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app" + "ovn4nfv-k8s-plugin/internal/pkg/config" + "ovn4nfv-k8s-plugin/internal/pkg/kube" + + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" ) const ( - ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces" + ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces" ) func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) { @@ -85,12 +87,12 @@ func prettyPrint(i interface{}) string { } func isNotFoundError(err error) bool { - statusErr, ok := err.(*errors.StatusError) - return ok && statusErr.Status().Code == http.StatusNotFound + statusErr, ok := err.(*errors.StatusError) + return ok && statusErr.Status().Code == http.StatusNotFound } func (cr *CNIServerRequest) addMultipleInterfaces(ovnAnnotation, namespace, podName string) types.Result { - klog.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ") + klog.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ") var ovnAnnotatedMap []map[string]string ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) if err != nil { @@ -224,26 +226,26 @@ func (cr *CNIServerRequest) addRoutes(ovnAnnotation string, dstResult types.Resu } func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) { - klog.Infof("ovn4nfvk8s-cni: cmdAdd") + klog.Infof("ovn4nfvk8s-cni: cmdAdd") namespace := cr.PodNamespace - podname := cr.PodName + podname := cr.PodName if namespace == "" || podname == "" { return nil, fmt.Errorf("required CNI variable missing") } - klog.Infof("ovn4nfvk8s-cni: cmdAdd for pod podname:%s and namespace:%s", podname, namespace) + klog.Infof("ovn4nfvk8s-cni: cmdAdd for pod podname:%s and namespace:%s", podname, namespace) kubecli := &kube.Kube{KClient: kclient} // Get the IP address and MAC address from the API server. var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1} var annotation map[string]string - var err error + var err error if err = wait.ExponentialBackoff(annotationBackoff, func() (bool, error) { annotation, err = kubecli.GetAnnotationsOnPod(namespace, podname) if err != nil { - if isNotFoundError(err) { - return false, fmt.Errorf("Error - pod not found - %v", err) - } - klog.Infof("ovn4nfvk8s-cni: cmdAdd Warning - Error while obtaining pod annotations - %v", err) - return false,nil + if isNotFoundError(err) { + return false, fmt.Errorf("Error - pod not found - %v", err) + } + klog.Infof("ovn4nfvk8s-cni: cmdAdd Warning - Error while obtaining pod annotations - %v", err) + return false, nil } if _, ok := annotation[ovn4nfvAnnotationTag]; ok { return true, nil @@ -258,7 +260,7 @@ func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) if !ok { return nil, fmt.Errorf("Error while obtaining pod annotations") } - result := cr.addMultipleInterfaces(ovnAnnotation, namespace, podname) + result := cr.addMultipleInterfaces(ovnAnnotation, namespace, podname) //Add Routes to the pod if annotation found for routes ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"] if ok { @@ -266,12 +268,12 @@ func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) result = cr.addRoutes(ovnRouteAnnotation, result) } - if result == nil { - klog.Errorf("result struct the ovn4nfv-k8s-plugin cniserver") - return nil, fmt.Errorf("result is nil from cni server response") - } + if result == nil { + klog.Errorf("result struct the ovn4nfv-k8s-plugin cniserver") + return nil, fmt.Errorf("result is nil from cni server response") + } - responseBytes, err := json.Marshal(result) + responseBytes, err := json.Marshal(result) if err != nil { return nil, fmt.Errorf("failed to marshal pod request response: %v", err) } @@ -280,8 +282,8 @@ func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) } func (cr *CNIServerRequest) cmdDel() ([]byte, error) { - klog.Infof("cmdDel ") - for i := 0; i < 10; i++ { + klog.Infof("cmdDel ") + for i := 0; i < 10; i++ { ifaceName := cr.SandboxID[:14] + strconv.Itoa(i) done, err := app.PlatformSpecificCleanup(ifaceName) if err != nil { @@ -291,5 +293,5 @@ func (cr *CNIServerRequest) cmdDel() ([]byte, error) { break } } - return []byte{}, nil + return []byte{}, nil } diff --git a/internal/pkg/cniserver/cniserver.go b/internal/pkg/cniserver/cniserver.go index eaa7105..7e55767 100644 --- a/internal/pkg/cniserver/cniserver.go +++ b/internal/pkg/cniserver/cniserver.go @@ -4,27 +4,28 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "net/http" - "strings" - "os" - "net" - "path/filepath" - "syscall" - "k8s.io/klog" + "os" + "path/filepath" + "strings" + "syscall" + + "k8s.io/klog" + + "ovn4nfv-k8s-plugin/internal/pkg/config" "github.com/containernetworking/cni/pkg/types" "github.com/gorilla/mux" - "k8s.io/client-go/kubernetes" - "ovn4nfv-k8s-plugin/internal/pkg/config" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilwait "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" ) const CNIServerRunDir string = "/var/run/ovn4nfv-k8s-plugin/cniserver" const CNIServerSocketName string = "ovn4nfv-k8s-plugin-cni-server.sock" const CNIServerSocketPath string = CNIServerRunDir + "/" + CNIServerSocketName - type CNIcommand string const CNIAdd CNIcommand = "ADD" @@ -32,13 +33,13 @@ const CNIUpdate CNIcommand = "UPDATE" const CNIDel CNIcommand = "DEL" type CNIServerRequest struct { - Command CNIcommand + Command CNIcommand PodNamespace string - PodName string - SandboxID string - Netns string - IfName string - CNIConf *types.NetConf + PodName string + SandboxID string + Netns string + IfName string + CNIConf *types.NetConf } type cniServerRequestFunc func(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error) @@ -51,7 +52,7 @@ type CNIServer struct { http.Server requestFunc cniServerRequestFunc serverrundir string - k8sclient kubernetes.Interface + k8sclient kubernetes.Interface } func NewCNIServer(serverRunSir string, k8sclient kubernetes.Interface) *CNIServer { @@ -66,7 +67,7 @@ func NewCNIServer(serverRunSir string, k8sclient kubernetes.Interface) *CNIServe Handler: router, }, serverrundir: serverRunSir, - k8sclient: k8sclient, + k8sclient: k8sclient, } router.NotFoundHandler = http.HandlerFunc(http.NotFound) router.HandleFunc("/", cs.handleCNIShimRequest).Methods("POST") @@ -100,42 +101,42 @@ func loadCNIRequestToCNIServer(r *CNIEndpointRequest) (*CNIServerRequest, error) Command: CNIcommand(cmd), } - cnishimreq.SandboxID, ok = r.ArgEnv["CNI_CONTAINERID"] - if !ok { - return nil, fmt.Errorf("cnishim req missing CNI_CONTAINERID") - } + cnishimreq.SandboxID, ok = r.ArgEnv["CNI_CONTAINERID"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_CONTAINERID") + } - cnishimreq.Netns, ok = r.ArgEnv["CNI_NETNS"] - if !ok { - return nil, fmt.Errorf("cnishim req missing CNI_NETNS") - } + cnishimreq.Netns, ok = r.ArgEnv["CNI_NETNS"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_NETNS") + } - cnishimreq.IfName, ok = r.ArgEnv["CNI_IFNAME"] - if !ok { - return nil, fmt.Errorf("cnishim req missing CNI_IFNAME") - } + cnishimreq.IfName, ok = r.ArgEnv["CNI_IFNAME"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_IFNAME") + } - cnishimArgs, err := loadCNIShimArgs(r.ArgEnv) - if err != nil { - return nil, err - } + cnishimArgs, err := loadCNIShimArgs(r.ArgEnv) + if err != nil { + return nil, err + } - cnishimreq.PodNamespace, ok = cnishimArgs["K8S_POD_NAMESPACE"] - if !ok { - return nil, fmt.Errorf("cnishim req missing K8S_POD_NAMESPACE") - } + cnishimreq.PodNamespace, ok = cnishimArgs["K8S_POD_NAMESPACE"] + if !ok { + return nil, fmt.Errorf("cnishim req missing K8S_POD_NAMESPACE") + } - cnishimreq.PodName, ok = cnishimArgs["K8S_POD_NAME"] + cnishimreq.PodName, ok = cnishimArgs["K8S_POD_NAME"] if !ok { return nil, fmt.Errorf("cnishim req missing K8S_POD_NAME") } - netconf, err := config.ConfigureNetConf(r.NetConfig) - if err != nil { - return nil, fmt.Errorf("cnishim req CNI arg configuration failed:%v",err) - } + netconf, err := config.ConfigureNetConf(r.NetConfig) + if err != nil { + return nil, fmt.Errorf("cnishim req CNI arg configuration failed:%v", err) + } - cnishimreq.CNIConf = netconf + cnishimreq.CNIConf = netconf return cnishimreq, nil } @@ -160,16 +161,16 @@ func (cs *CNIServer) handleCNIShimRequest(w http.ResponseWriter, r *http.Request } else { w.Header().Set("Content-Type", "application/json") if _, err := w.Write(result); err != nil { - klog.Warningf("Error writing %s HTTP response: %v", req.Command, err) + klog.Warningf("Error writing %s HTTP response: %v", req.Command, err) } - } + } } func HandleCNIcommandRequest(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error) { - var result []byte + var result []byte var err error - klog.Infof("[PodNamespace:%s/PodName:%s] dispatching pod network request %v", request.PodNamespace, request.PodName, request) - klog.Infof("k8sclient %s", fmt.Sprintf("%v",k8sclient)) + klog.Infof("[PodNamespace:%s/PodName:%s] dispatching pod network request %v", request.PodNamespace, request.PodName, request) + klog.Infof("k8sclient %s", fmt.Sprintf("%v", k8sclient)) switch request.Command { case CNIAdd: result, err = request.cmdAdd(k8sclient) @@ -231,5 +232,5 @@ func (cs *CNIServer) Start(requestFunc cniServerRequestFunc) error { utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err)) } }, 0) - return nil + return nil } diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index e9ad3e1..b8ab825 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -1,16 +1,18 @@ package config import ( - "encoding/json" + "crypto/sha1" + "encoding/hex" + "encoding/json" "fmt" "os" "path/filepath" "reflect" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/version" "github.com/sirupsen/logrus" "github.com/urfave/cli" - "github.com/containernetworking/cni/pkg/types" - "github.com/containernetworking/cni/pkg/version" gcfg "gopkg.in/gcfg.v1" "k8s.io/client-go/kubernetes" @@ -288,15 +290,23 @@ func NewClientset(conf *KubernetesConfig) (*kubernetes.Clientset, error) { } func ConfigureNetConf(bytes []byte) (*types.NetConf, error) { - conf := &types.NetConf{} + conf := &types.NetConf{} if err := json.Unmarshal(bytes, conf); err != nil { return nil, fmt.Errorf("failed to load netconf: %v", err) } - if conf.RawPrevResult != nil { - if err := version.ParsePrevResult(conf); err != nil { - return nil, err - } - } - return conf, nil + if conf.RawPrevResult != nil { + if err := version.ParsePrevResult(conf); err != nil { + return nil, err + } + } + return conf, nil +} + +func GetNodeIntfName(node string) string { + h := sha1.New() + h.Write([]byte(node)) + bs := h.Sum(nil) + encodednodeStr := hex.EncodeToString(bs) + return fmt.Sprintf("ovn4nfv0-%s", encodednodeStr[:6]) } diff --git a/internal/pkg/network/iptables.go b/internal/pkg/network/iptables.go new file mode 100644 index 0000000..6e71b3f --- /dev/null +++ b/internal/pkg/network/iptables.go @@ -0,0 +1,124 @@ +package network + +import ( + "fmt" + "strings" + + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + + "github.com/coreos/go-iptables/iptables" +) + +var log = logf.Log.WithName("iptables") + +type IPTables interface { + AppendUnique(table string, chain string, rulespec ...string) error + Delete(table string, chain string, rulespec ...string) error + Exists(table string, chain string, rulespec ...string) (bool, error) +} + +type IPTablesRule struct { + table string + chain string + rulespec []string +} + +func MasqRules(ifname string) []IPTablesRule { + return []IPTablesRule{ + // This rule makes sure ifname is SNAT + {"nat", "POSTROUTING", []string{"-o", ifname, "-j", "MASQUERADE"}}, + } +} + +func ForwardRules(ovnNetwork string) []IPTablesRule { + return []IPTablesRule{ + // These rules allow traffic to be forwarded if it is to or from the ovn network range. + {"filter", "FORWARD", []string{"-s", ovnNetwork, "-j", "ACCEPT"}}, + {"filter", "FORWARD", []string{"-d", ovnNetwork, "-j", "ACCEPT"}}, + } +} + +func ipTablesRulesExist(ipt IPTables, rules []IPTablesRule) (bool, error) { + for _, rule := range rules { + exists, err := ipt.Exists(rule.table, rule.chain, rule.rulespec...) + if err != nil { + // this shouldn't ever happen + return false, fmt.Errorf("failed to check rule existence: %v", err) + } + if !exists { + return false, nil + } + } + + return true, nil +} + +func SetupAndEnsureIPTables(rules []IPTablesRule) error { + ipt, err := iptables.New() + if err != nil { + // if we can't find iptables, give up and return + log.Error(err, "Failed to setup IPTables. iptables binary was not found") + return err + } + + // Ensure that all the iptables rules exist every 5 seconds + if err := ensureIPTables(ipt, rules); err != nil { + log.Error(err, "Failed to ensure iptables rules") + return err + } + + return nil + +} + +// DeleteIPTables delete specified iptables rules +func DeleteIPTables(rules []IPTablesRule) error { + ipt, err := iptables.New() + if err != nil { + // if we can't find iptables, give up and return + log.Error(err, "Failed to setup IPTables. iptables binary was not found") + return err + } + teardownIPTables(ipt, rules) + return nil +} + +func ensureIPTables(ipt IPTables, rules []IPTablesRule) error { + exists, err := ipTablesRulesExist(ipt, rules) + if err != nil { + return fmt.Errorf("Error checking rule existence: %v", err) + } + if exists { + // if all the rules already exist, no need to do anything + return nil + } + // Otherwise, teardown all the rules and set them up again + // We do this because the order of the rules is important + log.Info("Some iptables rules are missing; deleting and recreating rules") + teardownIPTables(ipt, rules) + if err = setupIPTables(ipt, rules); err != nil { + return fmt.Errorf("Error setting up rules: %v", err) + } + return nil +} + +func setupIPTables(ipt IPTables, rules []IPTablesRule) error { + for _, rule := range rules { + log.Info("Adding iptables rule: ", "rule", strings.Join(rule.rulespec, " ")) + err := ipt.AppendUnique(rule.table, rule.chain, rule.rulespec...) + if err != nil { + return fmt.Errorf("failed to insert IPTables rule: %v", err) + } + } + + return nil +} + +func teardownIPTables(ipt IPTables, rules []IPTablesRule) { + for _, rule := range rules { + log.Info("Deleting iptables rule: ", "rule", strings.Join(rule.rulespec, " ")) + // We ignore errors here because if there's an error it's almost certainly because the rule + // doesn't exist, which is fine (we don't need to delete rules that don't exist) + ipt.Delete(rule.table, rule.chain, rule.rulespec...) + } +} diff --git a/internal/pkg/nfnNotify/proto/nfn.pb.go b/internal/pkg/nfnNotify/proto/nfn.pb.go index d419af8..750d55b 100644 --- a/internal/pkg/nfnNotify/proto/nfn.pb.go +++ b/internal/pkg/nfnNotify/proto/nfn.pb.go @@ -371,6 +371,8 @@ func (m *DirectInfo) GetProviderIntf() string { } type InSync struct { + NodeIntfIpAddress string `protobuf:"bytes,1,opt,name=node_intf_ip_address,json=nodeIntfIpAddress,proto3" json:"node_intf_ip_address,omitempty"` + NodeIntfMacAddress string `protobuf:"bytes,2,opt,name=node_intf_mac_address,json=nodeIntfMacAddress,proto3" json:"node_intf_mac_address,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -401,6 +403,20 @@ func (m *InSync) XXX_DiscardUnknown() { var xxx_messageInfo_InSync proto.InternalMessageInfo +func (m *InSync) GetNodeIntfIpAddress() string { + if m != nil { + return m.NodeIntfIpAddress + } + return "" +} + +func (m *InSync) GetNodeIntfMacAddress() string { + if m != nil { + return m.NodeIntfMacAddress + } + return "" +} + func init() { proto.RegisterType((*SubscribeContext)(nil), "SubscribeContext") proto.RegisterType((*Notification)(nil), "Notification") @@ -416,34 +432,37 @@ func init() { } var fileDescriptor_5b809db4a7814953 = []byte{ - // 431 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6f, 0xd3, 0x30, - 0x14, 0x5d, 0xd8, 0x94, 0x34, 0xb7, 0x1d, 0x74, 0x16, 0x1f, 0x05, 0x84, 0x34, 0xbc, 0x97, 0x8a, - 0x87, 0x6c, 0x8c, 0x57, 0x9e, 0x18, 0x42, 0x8b, 0x84, 0x22, 0x94, 0x21, 0x5e, 0x2d, 0xd7, 0x71, - 0x90, 0xb5, 0xf4, 0x3a, 0xf2, 0x4c, 0x4b, 0x7e, 0x00, 0xbf, 0x83, 0xbf, 0xc7, 0xcf, 0x40, 0xb1, - 0x93, 0x35, 0x5d, 0xf7, 0xb2, 0x37, 0xfb, 0x9e, 0xeb, 0x73, 0xce, 0xbd, 0x3a, 0x86, 0x18, 0x4b, - 0x4c, 0x6a, 0xa3, 0xad, 0xa6, 0xa7, 0x30, 0xbd, 0xfa, 0xb5, 0xb8, 0x11, 0x46, 0x2d, 0xe4, 0x85, - 0x46, 0x2b, 0x7f, 0x5b, 0xf2, 0x1a, 0x62, 0xd4, 0x85, 0x64, 0xc8, 0x97, 0x72, 0x16, 0x1c, 0x07, - 0xf3, 0x38, 0x1f, 0xb5, 0x85, 0x8c, 0x2f, 0x25, 0xfd, 0x17, 0xc0, 0x24, 0xd3, 0x56, 0x95, 0x4a, - 0x70, 0xab, 0x34, 0x92, 0x97, 0x30, 0x12, 0xa8, 0x98, 0x6d, 0xea, 0xbe, 0x39, 0x12, 0xa8, 0xbe, - 0x37, 0xb5, 0x24, 0x14, 0x22, 0x85, 0xec, 0xa6, 0x41, 0x31, 0x7b, 0x74, 0x1c, 0xcc, 0xc7, 0xe7, - 0x51, 0x92, 0xe2, 0x55, 0x83, 0xe2, 0x72, 0x2f, 0x0f, 0x95, 0x3b, 0x91, 0x2f, 0x40, 0x6a, 0xa3, - 0x57, 0xaa, 0x90, 0x86, 0xe1, 0x9a, 0x09, 0x23, 0xb9, 0x95, 0xb3, 0x7d, 0xd7, 0xfe, 0x3c, 0xf9, - 0xd6, 0x41, 0x99, 0xb4, 0x6b, 0x6d, 0xae, 0x2f, 0x1c, 0x7a, 0xb9, 0x97, 0x4f, 0xfb, 0x37, 0xd9, - 0xda, 0xd7, 0xee, 0xf2, 0x18, 0xb9, 0xd4, 0x2b, 0x39, 0x3b, 0xb8, 0x9f, 0x27, 0x77, 0xe8, 0x36, - 0x8f, 0xaf, 0x7d, 0x8a, 0x21, 0xaa, 0x79, 0x53, 0x69, 0x5e, 0xd0, 0x3f, 0x01, 0x3c, 0xbb, 0xd7, - 0x00, 0x99, 0xc3, 0x74, 0x28, 0x36, 0x58, 0xd4, 0xe3, 0x0d, 0x61, 0xbb, 0x2e, 0xf2, 0x06, 0x0e, - 0x56, 0x15, 0xc7, 0x6e, 0xfe, 0x38, 0xf9, 0x51, 0x71, 0x4c, 0xb1, 0xd4, 0xb9, 0x2b, 0x93, 0x13, - 0x08, 0x0b, 0x65, 0xa4, 0xb0, 0xdd, 0xc4, 0xe3, 0xe4, 0xb3, 0xbb, 0xba, 0x96, 0x0e, 0xa2, 0x7f, - 0x77, 0x7d, 0x78, 0xb3, 0x0f, 0xf0, 0xf1, 0x0e, 0x8e, 0x5a, 0x41, 0x56, 0xe9, 0x9f, 0x4a, 0xf0, - 0x8a, 0x29, 0xb4, 0xa5, 0x33, 0x15, 0xe7, 0x4f, 0x5a, 0xe0, 0xab, 0xaf, 0xa7, 0x68, 0x4b, 0x72, - 0x06, 0x4f, 0xbd, 0x32, 0xbb, 0x25, 0x77, 0xed, 0xfb, 0xae, 0x9d, 0x78, 0xac, 0x37, 0xd4, 0xbe, - 0xa0, 0xd7, 0x30, 0xea, 0x07, 0x23, 0x2f, 0x20, 0x72, 0x4a, 0xaa, 0xe8, 0xac, 0x84, 0xed, 0x35, - 0x2d, 0xc8, 0x09, 0x1c, 0x6e, 0xf3, 0x79, 0xf9, 0x49, 0x3d, 0x60, 0x22, 0x6f, 0x61, 0xb2, 0x65, - 0xd1, 0x6b, 0x8e, 0xab, 0x8d, 0x3d, 0xfa, 0x1e, 0x60, 0xb3, 0xa4, 0x5d, 0xd6, 0x60, 0x97, 0x95, - 0x8e, 0x20, 0xf4, 0xc1, 0x3b, 0xff, 0xe8, 0xc2, 0xef, 0x02, 0xdc, 0x90, 0x53, 0x88, 0x6f, 0xc3, - 0x4f, 0x8e, 0x92, 0xbb, 0x1f, 0xe1, 0xd5, 0x61, 0x32, 0x4c, 0xfa, 0x59, 0xb0, 0x08, 0xdd, 0xa7, - 0xf9, 0xf0, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x24, 0xa2, 0x9f, 0x85, 0x41, 0x03, 0x00, 0x00, + // 472 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xdd, 0x6e, 0xd3, 0x30, + 0x14, 0x5e, 0xd8, 0xd4, 0x34, 0xa7, 0x1d, 0xb4, 0x47, 0x1b, 0x14, 0x10, 0xd2, 0xc8, 0x6e, 0x2a, + 0x2e, 0xd2, 0x6d, 0xdc, 0x72, 0x03, 0x43, 0x68, 0x91, 0xa0, 0x42, 0x19, 0xe2, 0xd6, 0x72, 0x1d, + 0x07, 0x59, 0x4b, 0x8f, 0xa3, 0xd4, 0xb4, 0xe4, 0x01, 0x78, 0x0e, 0x5e, 0x8f, 0xc7, 0x40, 0xb1, + 0x93, 0xfe, 0xac, 0xbb, 0xe1, 0x2e, 0x39, 0xdf, 0xf9, 0x7e, 0x6c, 0xf9, 0x83, 0x80, 0x32, 0x8a, + 0x8a, 0x52, 0x1b, 0x1d, 0x4e, 0x60, 0x70, 0xfb, 0x73, 0xb6, 0x10, 0xa5, 0x9a, 0xc9, 0x6b, 0x4d, + 0x46, 0xfe, 0x32, 0xf8, 0x12, 0x02, 0xd2, 0xa9, 0x64, 0xc4, 0xe7, 0x72, 0xe4, 0x9d, 0x79, 0xe3, + 0x20, 0xe9, 0xd6, 0x83, 0x29, 0x9f, 0xcb, 0xf0, 0xaf, 0x07, 0xfd, 0xa9, 0x36, 0x2a, 0x53, 0x82, + 0x1b, 0xa5, 0x09, 0x9f, 0x43, 0x57, 0x90, 0x62, 0xa6, 0x2a, 0xda, 0x65, 0x5f, 0x90, 0xfa, 0x56, + 0x15, 0x12, 0x43, 0xf0, 0x15, 0xb1, 0x45, 0x45, 0x62, 0xf4, 0xe8, 0xcc, 0x1b, 0xf7, 0xae, 0xfc, + 0x28, 0xa6, 0xdb, 0x8a, 0xc4, 0xcd, 0x41, 0xd2, 0x51, 0xf6, 0x0b, 0x3f, 0x01, 0x16, 0xa5, 0x5e, + 0xaa, 0x54, 0x96, 0x8c, 0x56, 0x4c, 0x94, 0x92, 0x1b, 0x39, 0x3a, 0xb4, 0xeb, 0x4f, 0xa3, 0xaf, + 0x0d, 0x34, 0x95, 0x66, 0xa5, 0xcb, 0xbb, 0x6b, 0x8b, 0xde, 0x1c, 0x24, 0x83, 0x96, 0x33, 0x5d, + 0xb9, 0xd9, 0x7d, 0x9d, 0x52, 0xce, 0xf5, 0x52, 0x8e, 0x8e, 0x1e, 0xd6, 0x49, 0x2c, 0xba, 0xab, + 0xe3, 0x66, 0x1f, 0x02, 0xf0, 0x0b, 0x5e, 0xe5, 0x9a, 0xa7, 0xe1, 0x6f, 0x0f, 0x4e, 0x1f, 0x0c, + 0x80, 0x63, 0x18, 0x6c, 0x9b, 0x6d, 0x5d, 0xd4, 0xe3, 0x8d, 0x60, 0x7d, 0x5d, 0xf8, 0x0a, 0x8e, + 0x96, 0x39, 0xa7, 0xe6, 0xfc, 0x41, 0xf4, 0x3d, 0xe7, 0x14, 0x53, 0xa6, 0x13, 0x3b, 0xc6, 0x73, + 0xe8, 0xa4, 0xaa, 0x94, 0xc2, 0x34, 0x27, 0xee, 0x45, 0x1f, 0xed, 0xaf, 0x5d, 0x69, 0xa0, 0xf0, + 0xcf, 0x7e, 0x0e, 0x17, 0xf6, 0x3f, 0x72, 0xbc, 0x81, 0x61, 0x6d, 0xc8, 0x72, 0xfd, 0x43, 0x09, + 0x9e, 0x33, 0x45, 0x26, 0xb3, 0xa1, 0x82, 0xe4, 0x49, 0x0d, 0x7c, 0x76, 0xf3, 0x98, 0x4c, 0x86, + 0x17, 0x70, 0xe2, 0x9c, 0xd9, 0x5a, 0xdc, 0xae, 0x1f, 0xda, 0x75, 0x74, 0x58, 0x1b, 0xa8, 0x66, + 0x84, 0x77, 0xd0, 0x6d, 0x0f, 0x86, 0xcf, 0xc0, 0xb7, 0x4e, 0x2a, 0x6d, 0xa2, 0x74, 0xea, 0xdf, + 0x38, 0xc5, 0x73, 0x38, 0xde, 0xd5, 0x73, 0xf6, 0xfd, 0x62, 0x4b, 0x09, 0x5f, 0x43, 0x7f, 0x27, + 0xa2, 0xf3, 0xec, 0xe5, 0x9b, 0x78, 0xe1, 0x25, 0xc0, 0xe6, 0x92, 0xf6, 0x55, 0xbd, 0x7d, 0xd5, + 0x30, 0x87, 0x8e, 0x7b, 0x78, 0x38, 0x81, 0x13, 0xfb, 0xb6, 0xeb, 0x55, 0xa6, 0x0a, 0xc6, 0xd3, + 0xb4, 0x94, 0x8b, 0x45, 0xc3, 0x1a, 0xd6, 0x58, 0xcd, 0x88, 0x8b, 0xf7, 0x0e, 0xc0, 0x4b, 0x38, + 0xdd, 0x10, 0xe6, 0x5c, 0xac, 0x19, 0x2e, 0x3d, 0xb6, 0x8c, 0x2f, 0x5c, 0x34, 0x94, 0xab, 0x77, + 0xb6, 0x60, 0xb6, 0x24, 0x15, 0x4e, 0x20, 0x58, 0x17, 0x0c, 0x87, 0xd1, 0xfd, 0xb2, 0xbd, 0x38, + 0x8e, 0xb6, 0xdb, 0x74, 0xe1, 0xcd, 0x3a, 0xb6, 0x98, 0x6f, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, + 0xad, 0x54, 0x7e, 0xa2, 0xa5, 0x03, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/pkg/nfnNotify/proto/nfn.proto b/internal/pkg/nfnNotify/proto/nfn.proto index 02855e7..567df29 100644 --- a/internal/pkg/nfnNotify/proto/nfn.proto +++ b/internal/pkg/nfnNotify/proto/nfn.proto @@ -47,4 +47,6 @@ message DirectInfo { } message InSync { + string node_intf_ip_address = 1; + string node_intf_mac_address = 2; } diff --git a/internal/pkg/nfnNotify/server.go b/internal/pkg/nfnNotify/server.go index ac22d68..a201618 100644 --- a/internal/pkg/nfnNotify/server.go +++ b/internal/pkg/nfnNotify/server.go @@ -4,13 +4,14 @@ import ( "fmt" "net" pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto" + "ovn4nfv-k8s-plugin/internal/pkg/node" v1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1" clientset "ovn4nfv-k8s-plugin/pkg/generated/clientset/versioned" "strings" "google.golang.org/grpc" "google.golang.org/grpc/reflection" - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" @@ -45,6 +46,11 @@ func (s *serverDB) Subscribe(sc *pb.SubscribeContext, ss pb.NfnNotify_SubscribeS if nodeName == "" { return fmt.Errorf("Node name can't be empty") } + + nodeIntfIPAddr, nodeIntfMacAddr, err := node.AddNodeLogicalPorts(nodeName) + if err != nil { + return fmt.Errorf("Error in creating node logical port for node- %s: %v", nodeName, err) + } cp := client{ context: sc, stream: ss, @@ -61,7 +67,10 @@ func (s *serverDB) Subscribe(sc *pb.SubscribeContext, ss pb.NfnNotify_SubscribeS inSyncMsg := pb.Notification{ CniType: "ovn4nfv", Payload: &pb.Notification_InSync{ - InSync: &pb.InSync{}, + InSync: &pb.InSync{ + NodeIntfIpAddress: nodeIntfIPAddr, + NodeIntfMacAddress: nodeIntfMacAddr, + }, }, } log.Info("Send Insync") diff --git a/internal/pkg/node/node.go b/internal/pkg/node/node.go new file mode 100644 index 0000000..e989e07 --- /dev/null +++ b/internal/pkg/node/node.go @@ -0,0 +1,31 @@ +package node + +import ( + "ovn4nfv-k8s-plugin/internal/pkg/ovn" + + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" +) + +var log = logf.Log.WithName("node") + +//AddNodeLogicalPorts return nodeIntfMacAddr and nodeIntfIPAddr +func AddNodeLogicalPorts(node string) (nodeIntfMacAddr, nodeIntfIPAddr string, err error) { + ovnCtl, err := ovn.GetOvnController() + if err != nil { + return "", "", err + } + + log.Info("Calling CreateNodeLogicalPorts") + nodeIntfMacAddr, nodeIntfIPAddr, err = ovnCtl.AddNodeLogicalPorts(node) + if err != nil { + return "", "", err + } + return nodeIntfMacAddr, nodeIntfIPAddr, nil +} + +//DeleteNodeLogicalPorts return nil +func DeleteNodeLogicalPorts(name, namesapce string) error { + // Run delete for all controllers; + // Todo + return nil +} diff --git a/internal/pkg/ovn/ovn.go b/internal/pkg/ovn/ovn.go index 6f7951a..97dc99d 100644 --- a/internal/pkg/ovn/ovn.go +++ b/internal/pkg/ovn/ovn.go @@ -2,14 +2,16 @@ package ovn import ( "fmt" - "github.com/mitchellh/mapstructure" - kapi "k8s.io/api/core/v1" - kexec "k8s.io/utils/exec" "math/rand" "os" + "ovn4nfv-k8s-plugin/internal/pkg/config" k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1" "strings" "time" + + "github.com/mitchellh/mapstructure" + kapi "k8s.io/api/core/v1" + kexec "k8s.io/utils/exec" ) type Controller struct { @@ -32,22 +34,23 @@ const ( var ovnConf *OVNNetworkConf +//GetOvnNetConf return error func GetOvnNetConf() error { ovnConf = &OVNNetworkConf{} ovnConf.Subnet = os.Getenv("OVN_SUBNET") if ovnConf.Subnet == "" { - fmt.Errorf("OVN subnet is not set in nfn-operator configmap env") + return fmt.Errorf("OVN subnet is not set in nfn-operator configmap env") } ovnConf.GatewayIP = os.Getenv("OVN_GATEWAYIP") if ovnConf.GatewayIP == "" { - fmt.Errorf("OVN gatewayIP is not set in nfn-operator configmap env") + log.Info("No Gateway IP address provided - 1st IP address of the subnet range will be used as Gateway", "Subnet", ovnConf.Subnet) } ovnConf.ExcludeIPs = os.Getenv("OVN_EXCLUDEIPS") if ovnConf.ExcludeIPs == "" { - fmt.Errorf("OVN excludeIPs is not set in nfn-operator configmap env") + log.Info("No IP addresses are excluded in the subnet range", "Subnet", ovnConf.Subnet) } return nil @@ -98,6 +101,20 @@ func GetOvnController() (*Controller, error) { return nil, fmt.Errorf("OVN Controller not initialized") } +func (oc *Controller) AddNodeLogicalPorts(node string) (ipAddr, macAddr string, err error) { + nodeName := strings.ToLower(node) + portName := config.GetNodeIntfName(nodeName) + + log.V(1).Info("Creating Node logical port", "node", nodeName, "portName", portName) + + ipAddr, macmacAddr, err := oc.addNodeLogicalPortWithSwitch(Ovn4nfvDefaultNw, portName) + if err != nil { + return "", "", err + } + + return ipAddr, macmacAddr, nil +} + // AddLogicalPorts adds ports to the Pod func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]interface{}) (key, value string) { @@ -344,6 +361,110 @@ func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string return gatewayIP, mask, nil } +func (oc *Controller) addNodeLogicalPortWithSwitch(logicalSwitch, portName string) (ipAddr, macAddr string, r error) { + var out, stderr string + var err error + + log.V(1).Info("Creating Node logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch) + + out, stderr, err = RunOVNNbctl("--wait=sb", "--", + "--may-exist", "lsp-add", logicalSwitch, portName, + "--", "lsp-set-addresses", + portName, "dynamic") + if err != nil { + log.Error(err, "Error while creating logical port %s ", "portName", portName, "stdout", out, "stderr", stderr) + return "", "", err + } + + count := 30 + for count > 0 { + out, stderr, err = RunOVNNbctl("get", + "logical_switch_port", portName, "dynamic_addresses") + + if err == nil && out != "[]" { + break + } + if err != nil { + log.Error(err, "Error while obtaining addresses for", "portName", portName) + return "", "", err + } + time.Sleep(time.Second) + count-- + } + if count == 0 { + log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr) + return "", "", err + } + + // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while + // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3". + outStr := strings.TrimLeft(out, `[`) + outStr = strings.TrimRight(outStr, `]`) + outStr = strings.Trim(outStr, `"`) + addresses := strings.Split(outStr, " ") + if len(addresses) != 2 { + log.Info("Error while obtaining addresses for", "portName", portName) + return "", "", err + } + + _, mask, err := oc.getGatewayFromSwitch(logicalSwitch) + if err != nil { + log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch) + return "", "", err + } + + ipAddr = fmt.Sprintf("%s/%s", addresses[1], mask) + macAddr = fmt.Sprintf("%s", addresses[0]) + + return ipAddr, macAddr, nil +} + +func (oc *Controller) getNodeLogicalPortIPAddr(pod *kapi.Pod) (ipAddress string, r error) { + var out, stderr, nodeName, portName string + var err error + + nodeName = strings.ToLower(pod.Spec.NodeName) + portName = config.GetNodeIntfName(nodeName) + + log.V(1).Info("Get Node logical port", "pod", pod.GetName(), "node", nodeName, "portName", portName) + + count := 30 + for count > 0 { + out, stderr, err = RunOVNNbctl("get", + "logical_switch_port", portName, "dynamic_addresses") + + if err == nil && out != "[]" { + break + } + if err != nil { + log.Error(err, "Error while obtaining addresses for", "portName", portName) + return "", err + } + time.Sleep(time.Second) + count-- + } + if count == 0 { + log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr) + return "", err + } + + // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while + // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3". + outStr := strings.TrimLeft(out, `[`) + outStr = strings.TrimRight(outStr, `]`) + outStr = strings.Trim(outStr, `"`) + addresses := strings.Split(outStr, " ") + if len(addresses) != 2 { + log.Info("Error while obtaining addresses for", "portName", portName) + return "", err + } + + ipAddr := fmt.Sprintf("%s", addresses[1]) + log.V(1).Info("Get Node logical port", "pod", pod.GetName(), "node", nodeName, "portName", portName, "Node port IP", ipAddr) + + return ipAddr, nil +} + func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, portName string) (annotation string) { var out, stderr string var err error @@ -425,11 +546,18 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA return } - gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) + _, mask, err := oc.getGatewayFromSwitch(logicalSwitch) if err != nil { log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch) return } + + gatewayIP, err := oc.getNodeLogicalPortIPAddr(pod) + if err != nil { + log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch) + return + } + annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) return annotation diff --git a/pkg/controller/pod/pod_controller.go b/pkg/controller/pod/pod_controller.go index d195782..1f1aa1b 100644 --- a/pkg/controller/pod/pod_controller.go +++ b/pkg/controller/pod/pod_controller.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" + "ovn4nfv-k8s-plugin/internal/pkg/ovn" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "ovn4nfv-k8s-plugin/internal/pkg/ovn" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -32,6 +33,7 @@ type nfnNetwork struct { } var enableOvnDefaultIntf bool = true + // Add creates a new Pod Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager) error { @@ -137,6 +139,12 @@ func (r *ReconcilePod) Reconcile(request reconcile.Request) (reconcile.Result, e return reconcile.Result{}, nil } + if instance.Spec.NodeName == "" { + return reconcile.Result{ + Requeue: true, + }, nil + } + err = r.addLogicalPorts(instance) if err != nil && err.Error() == "Failed to add ports" { // Requeue the object @@ -163,7 +171,7 @@ func (r *ReconcilePod) addLogicalPorts(pod *corev1.Pod) error { nfn, err := r.readPodAnnotation(pod) if err != nil { // No annotation for multiple interfaces - nfn = &nfnNetwork {Interface: nil} + nfn = &nfnNetwork{Interface: nil} if enableOvnDefaultIntf == true { nfn.Type = "ovn4nfv" } else { @@ -177,7 +185,7 @@ func (r *ReconcilePod) addLogicalPorts(pod *corev1.Pod) error { if err != nil { return err } - if _, ok := pod.Annotations[ovn.Ovn4nfvAnnotationTag]; ok { + if _, ok := pod.Annotations[ovn.Ovn4nfvAnnotationTag]; ok { return fmt.Errorf("Pod annotation found") } key, value := ovnCtl.AddLogicalPorts(pod, nfn.Interface) @@ -187,7 +195,7 @@ func (r *ReconcilePod) addLogicalPorts(pod *corev1.Pod) error { return fmt.Errorf("Failed to add ports") default: return fmt.Errorf("Unsupported Networking type %s", nfn.Type) - // Add other types here + // Add other types here } } -- 2.16.6