From a6c37bf8c9c1e9f5072bfc3e43b6ec0061ee2108 Mon Sep 17 00:00:00 2001 From: Ritu Sood Date: Fri, 9 Aug 2019 11:24:26 -0700 Subject: [PATCH] Add CRD Controller for Network Add CRD controller functionality for Network creation and deletion. Related generated code is uploaded in patch: https://gerrit.opnfv.org/gerrit/#/c/ovn4nfv-k8s-plugin/+/68324/ Change-Id: Ibd3e652edc56aa1084f684438597e1b978977bbf Signed-off-by: Ritu Sood --- go.mod | 1 + internal/pkg/ovn/common.go | 17 --- internal/pkg/ovn/ovn.go | 139 ++++++++++++++++++-- internal/pkg/ovn/utils.go | 8 +- pkg/apis/k8s/v1alpha1/network_types.go | 6 +- pkg/controller/add_network.go | 10 ++ pkg/controller/network/network_controller.go | 189 +++++++++++++++++++++++++++ pkg/utils/finalizer_utils.go | 20 +++ 8 files changed, 357 insertions(+), 33 deletions(-) create mode 100644 pkg/controller/add_network.go create mode 100644 pkg/controller/network/network_controller.go create mode 100644 pkg/utils/finalizer_utils.go diff --git a/go.mod b/go.mod index 4d0d716..b838d2b 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/containernetworking/cni v0.7.1 github.com/containernetworking/plugins v0.8.1 github.com/coreos/go-iptables v0.4.2 // indirect + github.com/go-logr/logr v0.1.0 github.com/google/btree v1.0.0 // indirect github.com/gophercloud/gophercloud v0.2.0 // indirect github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect diff --git a/internal/pkg/ovn/common.go b/internal/pkg/ovn/common.go index 60cd202..09d770b 100644 --- a/internal/pkg/ovn/common.go +++ b/internal/pkg/ovn/common.go @@ -7,7 +7,6 @@ import ( "math/rand" "net" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" - "strings" "time" ) @@ -65,22 +64,6 @@ func setupDistributedRouter(name string) error { return nil } -// Find if switch exists -func findLogicalSwitch(name string) bool { - // get logical switch from OVN - output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading", - "--columns=name", "find", "logical_switch", "name="+name) - if err != nil { - log.Error(err, "Error in obtaining list of logical switch", "stderr", stderr) - return false - } - - if strings.Compare(name, output) == 0 { - return true - } - return false -} - // generateMac generates mac address. func generateMac() string { prefix := "00:00:00" diff --git a/internal/pkg/ovn/ovn.go b/internal/pkg/ovn/ovn.go index dad4641..82fe837 100644 --- a/internal/pkg/ovn/ovn.go +++ b/internal/pkg/ovn/ovn.go @@ -5,6 +5,9 @@ import ( "github.com/mitchellh/mapstructure" kapi "k8s.io/api/core/v1" kexec "k8s.io/utils/exec" + "math/rand" + "net" + k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1" "strings" "time" ) @@ -14,7 +17,8 @@ type Controller struct { } const ( - ovn4nfvRouterName = "ovn4nfv-master" + ovn4nfvRouterName = "ovn4nfv-master" + // Ovn4nfvAnnotationTag tag on already processed Pods Ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces" ) @@ -35,12 +39,10 @@ func NewOvnController(exec kexec.Interface) (*Controller, error) { if exec == nil { exec = kexec.New() } - if err := SetExec(exec); err != nil { log.Error(err, "Failed to initialize exec helper") return nil, err } - if err := SetupOvnUtils(); err != nil { log.Error(err, "Failed to initialize OVN State") return nil, err @@ -71,7 +73,7 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int } if _, ok := pod.Annotations[Ovn4nfvAnnotationTag]; ok { - log.Info("AddLogicalPorts : Pod annotation found") + log.V(1).Info("AddLogicalPorts : Pod annotation found") return } @@ -86,7 +88,7 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int return } - if !findLogicalSwitch(ns.Name) { + if !oc.FindLogicalSwitch(ns.Name) { log.Info("Logical Switch not found") return } @@ -111,7 +113,6 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int } } - outStr = oc.addLogicalPortWithSwitch(pod, ns.Name, ns.IPAddress, ns.MacAddress, ns.Interface, ns.NetType) if outStr == "" { return @@ -131,9 +132,9 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int return key, value } +// DeleteLogicalPorts deletes the OVN ports for the pod func (oc *Controller) DeleteLogicalPorts(name, namespace string) { - log.Info("Deleting pod", "name", name) logicalPort := fmt.Sprintf("%s_%s", namespace, name) // get the list of logical ports from OVN @@ -143,12 +144,11 @@ func (oc *Controller) DeleteLogicalPorts(name, namespace string) { log.Error(err, "Error in obtaining list of logical ports ", "stdout", stdout, "stderr", stderr) return } - log.Info("Deleting", "Existing Ports", stdout) existingLogicalPorts := strings.Fields(stdout) for _, existingPort := range existingLogicalPorts { if strings.Contains(existingPort, logicalPort) { // found, delete this logical port - log.Info("Deleting", "existingPort", existingPort) + log.V(1).Info("Deleting", "Port", existingPort) stdout, stderr, err := RunOVNNbctl("--if-exists", "lsp-del", existingPort) if err != nil { @@ -159,11 +159,128 @@ func (oc *Controller) DeleteLogicalPorts(name, namespace string) { return } +// CreateNetwork in OVN controller +func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error { + var stdout, stderr string + + // Currently only these fields are supported + name := cr.Name + subnet := cr.Spec.Ipv4Subnets[0].Subnet + gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway + excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps + + output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading", + "--columns=name", "find", "logical_switch", "name="+name) + if err != nil { + log.Error(err, "Error in reading logical switch", "stderr", stderr) + return nil + } + + if strings.Compare(name, output) == 0 { + log.V(1).Info("Logical Switch already exists, delete first to update/recreate", "name", name) + return nil + } + + _, cidr, err := net.ParseCIDR(subnet) + if err != nil { + log.Error(err, "ovnNetwork '%s' invalid subnet CIDR", "name", name) + return err + + } + firstIP := NextIP(cidr.IP) + n, _ := cidr.Mask.Size() + + var gatewayIPMask string + var gwIP net.IP + if gatewayIP != "" { + gwIP, _, err = net.ParseCIDR(gatewayIP) + if err != nil { + // Check if this is a valid IP address + gwIP = net.ParseIP(gatewayIP) + } + } + // If no valid Gateway use the first IP address for GatewayIP + if gwIP == nil { + gatewayIPMask = fmt.Sprintf("%s/%d", firstIP.String(), n) + } else { + gatewayIPMask = fmt.Sprintf("%s/%d", gwIP.String(), n) + } + + // Create a logical switch and set its subnet. + if excludeIps != "" { + stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "ls-add", name, "--", "set", "logical_switch", name, "other-config:subnet="+subnet, "external-ids:gateway_ip="+gatewayIPMask, "other-config:exclude_ips="+excludeIps) + } else { + stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "ls-add", name, "--", "set", "logical_switch", name, "other-config:subnet="+subnet, "external-ids:gateway_ip="+gatewayIPMask) + } + if err != nil { + log.Error(err, "Failed to create a logical switch", "name", name, "stdout", stdout, "stderr", stderr) + return err + } + + routerMac, stderr, err := RunOVNNbctl("--if-exist", "get", "logical_router_port", "rtos-"+name, "mac") + if err != nil { + log.Error(err, "Failed to get logical router port", "stderr", stderr) + return err + } + if routerMac == "" { + prefix := "00:00:00" + newRand := rand.New(rand.NewSource(time.Now().UnixNano())) + routerMac = fmt.Sprintf("%s:%02x:%02x:%02x", prefix, newRand.Intn(255), newRand.Intn(255), newRand.Intn(255)) + } + + _, stderr, err = RunOVNNbctl("--wait=hv", "--may-exist", "lrp-add", ovn4nfvRouterName, "rtos-"+name, routerMac, gatewayIPMask) + if err != nil { + log.Error(err, "Failed to add logical port to router", "stderr", stderr) + return err + } + + // Connect the switch to the router. + stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "stor-"+name, "--", "set", "logical_switch_port", "stor-"+name, "type=router", "options:router-port=rtos-"+name, "addresses="+"\""+routerMac+"\"") + if err != nil { + log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout) + return err + } + + return nil +} + +// DeleteNetwork in OVN controller +func (oc *Controller) DeleteNetwork(cr *k8sv1alpha1.Network) error { + + name := cr.Name + stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "lrp-del", "rtos-"+name) + if err != nil { + log.Error(err, "Failed to delete router port", "name", name, "stdout", stdout, "stderr", stderr) + return err + } + stdout, stderr, err = RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name) + if err != nil { + log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr) + return err + } + return nil +} + +// FindLogicalSwitch returns true if switch exists +func (oc *Controller) FindLogicalSwitch(name string) bool { + // get logical switch from OVN + output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading", + "--columns=name", "find", "logical_switch", "name="+name) + if err != nil { + log.Error(err, "Error in obtaining list of logical switch", "stderr", stderr) + return false + } + if strings.Compare(name, output) == 0 { + return true + } + return false +} + func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string, error) { var gatewayIPMaskStr, stderr string var ok bool var err error - log.Info("getGatewayFromSwitch", "logicalSwitch", logicalSwitch) + log.V(1).Info("getGatewayFromSwitch", "logicalSwitch", logicalSwitch) if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok { gatewayIPMaskStr, stderr, err = RunOVNNbctl("--if-exists", "get", "logical_switch", logicalSwitch, @@ -203,7 +320,7 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA return } - log.Info("Creating logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch) + log.V(1).Info("Creating logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch) if ipAddress != "" && macAddress != "" { isStaticIP = true diff --git a/internal/pkg/ovn/utils.go b/internal/pkg/ovn/utils.go index 2478ac2..615c2f9 100644 --- a/internal/pkg/ovn/utils.go +++ b/internal/pkg/ovn/utils.go @@ -84,15 +84,17 @@ func run(cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { cmd := runner.exec.Command(cmdPath, args...) cmd.SetStdout(stdout) cmd.SetStderr(stderr) - log.Info("exec:", "cmdPath", cmdPath, "args", strings.Join(args, " ")) + log.V(1).Info("exec:", "cmdPath", cmdPath, "args", strings.Join(args, " ")) err := cmd.Run() if err != nil { - log.Error(err, "exec:", "cmdPath", cmdPath, "args", strings.Join(args, " ")) + log.Error(err, "Error:", "cmdPath", cmdPath, "args", strings.Join(args, " "), "stdout", stdout, "stderr", stderr) + } else { + log.V(1).Info("output:", "stdout", stdout) } return stdout, stderr, err } -// RunOVNSbctlWithTimeout runs command via ovn-nbctl with a specific timeout +// RunOVNNbctlWithTimeout runs command via ovn-nbctl with a specific timeout func RunOVNNbctlWithTimeout(timeout int, args ...string) (string, string, error) { var cmdArgs []string if len(runner.hostIP) > 0 { diff --git a/pkg/apis/k8s/v1alpha1/network_types.go b/pkg/apis/k8s/v1alpha1/network_types.go index a52dd58..a606dbb 100644 --- a/pkg/apis/k8s/v1alpha1/network_types.go +++ b/pkg/apis/k8s/v1alpha1/network_types.go @@ -42,8 +42,10 @@ type DnsSpec struct { const ( //Created indicates the status of success Created = "Created" - //Indicates internal Irrecoverable Error - InternalError = "InternalError" + //CreateInternalError indicates create internal irrecoverable Error + CreateInternalError = "CreateInternalError" + //DeleteInternalError indicates delete internal irrecoverable Error + DeleteInternalError = "DeleteInternalError" ) // NetworkStatus defines the observed state of Network diff --git a/pkg/controller/add_network.go b/pkg/controller/add_network.go new file mode 100644 index 0000000..d35657b --- /dev/null +++ b/pkg/controller/add_network.go @@ -0,0 +1,10 @@ +package controller + +import ( + "ovn4nfv-k8s-plugin/pkg/controller/network" +) + +func init() { + // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. + AddToManagerFuncs = append(AddToManagerFuncs, network.Add) +} diff --git a/pkg/controller/network/network_controller.go b/pkg/controller/network/network_controller.go new file mode 100644 index 0000000..2392e3d --- /dev/null +++ b/pkg/controller/network/network_controller.go @@ -0,0 +1,189 @@ +package network + +import ( + "context" + "fmt" + k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1" + + // corev1 "k8s.io/api/core/v1" + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "ovn4nfv-k8s-plugin/internal/pkg/ovn" + "ovn4nfv-k8s-plugin/pkg/utils" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +var log = logf.Log.WithName("network_controller") + +// Add creates a new Network Controller and adds it to the Manager. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + return add(mgr, newReconciler(mgr)) +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + return &ReconcileNetwork{client: mgr.GetClient(), scheme: mgr.GetScheme()} +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := controller.New("network-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + // Watch for changes to primary resource Network + err = c.Watch(&source.Kind{Type: &k8sv1alpha1.Network{}}, &handler.EnqueueRequestForObject{}) + if err != nil { + return err + } + + return nil +} + +// blank assignment to verify that ReconcileNetwork implements reconcile.Reconciler +var _ reconcile.Reconciler = &ReconcileNetwork{} + +// ReconcileNetwork reconciles a Network object +type ReconcileNetwork struct { + // This client, initialized using mgr.Client() above, is a split client + // that reads objects from the cache and writes to the apiserver + client client.Client + scheme *runtime.Scheme +} +type reconcileFun func(instance *k8sv1alpha1.Network, reqLogger logr.Logger) error + +// Reconcile reads that state of the cluster for a Network object and makes changes based on the state read +// and what is in the Network.Spec +// The Controller will requeue the Request to be processed again if the returned error is non-nil or +// Result.Requeue is true, otherwise upon completion it will remove the work from the queue. +func (r *ReconcileNetwork) Reconcile(request reconcile.Request) (reconcile.Result, error) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.V(1).Info("Reconciling Network") + + // Fetch the Network instance + instance := &k8sv1alpha1.Network{} + err := r.client.Get(context.TODO(), request.NamespacedName, instance) + if err != nil { + if errors.IsNotFound(err) { + // Request object not found, could have been deleted after reconcile request. + // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers. + // Return and don't requeue + reqLogger.V(1).Info("Network Object not found") + return reconcile.Result{}, nil + } + // Error reading the object - requeue the request. + return reconcile.Result{}, err + } + for _, fun := range []reconcileFun{ + r.reconcileFinalizers, + r.createNetwork, + } { + if err = fun(instance, reqLogger); err != nil { + return reconcile.Result{}, err + } + } + return reconcile.Result{}, nil +} + +const ( + nfnNetworkFinalizer = "nfnCleanUpNetwork" +) + +func (r *ReconcileNetwork) createNetwork(cr *k8sv1alpha1.Network, reqLogger logr.Logger) error { + + if !cr.DeletionTimestamp.IsZero() { + // Marked for deletion + return nil + } + switch { + case cr.Spec.CniType == "ovn4nfv": + ovnCtl, err := ovn.GetOvnController() + if err != nil { + return err + } + err = ovnCtl.CreateNetwork(cr) + if err != nil { + // Log the error + reqLogger.Error(err, "Error Creating Network") + cr.Status.State = k8sv1alpha1.CreateInternalError + } else { + cr.Status.State = k8sv1alpha1.Created + } + err = r.client.Status().Update(context.TODO(), cr) + if err != nil { + return err + } + // If OVN internal error don't requeue + return nil + // Add other CNI types here + } + reqLogger.Info("CNI type not supported", "name", cr.Spec.CniType) + return fmt.Errorf("CNI type not supported") + +} + +func (r *ReconcileNetwork) deleteNetwork(cr *k8sv1alpha1.Network, reqLogger logr.Logger) error { + + switch { + case cr.Spec.CniType == "ovn4nfv": + ovnCtl, err := ovn.GetOvnController() + if err != nil { + return err + } + err = ovnCtl.DeleteNetwork(cr) + if err != nil { + // Log the error + reqLogger.Error(err, "Error Delete Network") + cr.Status.State = k8sv1alpha1.DeleteInternalError + err = r.client.Status().Update(context.TODO(), cr) + if err != nil { + return err + } + } + // If OVN internal error don't requeue + return nil + // Add other CNI types here + } + reqLogger.Info("CNI type not supported", "name", cr.Spec.CniType) + return fmt.Errorf("CNI type not supported") +} + +func (r *ReconcileNetwork) reconcileFinalizers(instance *k8sv1alpha1.Network, reqLogger logr.Logger) (err error) { + + if !instance.DeletionTimestamp.IsZero() { + // Instance marked for deletion + if utils.Contains(instance.ObjectMeta.Finalizers, nfnNetworkFinalizer) { + reqLogger.V(1).Info("Finalizer found - delete network") + if err = r.deleteNetwork(instance, reqLogger); err != nil { + reqLogger.Error(err, "Delete network") + } + // Remove the finalizer even if Delete Network fails. Fatal error retry will not resolve + instance.ObjectMeta.Finalizers = utils.Remove(instance.ObjectMeta.Finalizers, nfnNetworkFinalizer) + if err = r.client.Update(context.TODO(), instance); err != nil { + reqLogger.Error(err, "Removing Finalize") + return err + } + } + + } else { + // If finalizer doesn't exist add it + if !utils.Contains(instance.GetFinalizers(), nfnNetworkFinalizer) { + instance.SetFinalizers(append(instance.GetFinalizers(), nfnNetworkFinalizer)) + if err = r.client.Update(context.TODO(), instance); err != nil { + reqLogger.Error(err, "Adding Finalize") + return err + } + reqLogger.V(1).Info("Finalizer added") + } + } + return nil +} diff --git a/pkg/utils/finalizer_utils.go b/pkg/utils/finalizer_utils.go new file mode 100644 index 0000000..5f196c0 --- /dev/null +++ b/pkg/utils/finalizer_utils.go @@ -0,0 +1,20 @@ +package utils + +func Contains(slice []string, str string) bool { + for _, item := range slice { + if item == str { + return true + } + } + return false +} + +func Remove(slice []string, str string) (result []string) { + for _, item := range slice { + if item == str { + continue + } + result = append(result, item) + } + return result +} -- 2.16.6