internal/pkg/ovn/ovn.go

   1 package ovn
   2
   3 import (
   4         "fmt"
   5         "math/rand"
   6         "os"
   7         "ovn4nfv-k8s-plugin/internal/pkg/config"
   8         k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1"
   9         "strings"
  10         "time"
  11
  12         "github.com/mitchellh/mapstructure"
  13         kapi "k8s.io/api/core/v1"
  14         kexec "k8s.io/utils/exec"
  15 )
  16
  17 type Controller struct {
  18         gatewayCache map[string]string
  19 }
  20
  21 type OVNNetworkConf struct {
  22         Subnet     string
  23         GatewayIP  string
  24         ExcludeIPs string
  25 }
  26
  27 const (
  28         ovn4nfvRouterName = "ovn4nfv-master"
  29         // Ovn4nfvAnnotationTag tag on already processed Pods
  30         Ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
  31         // OVN Default Network name
  32         Ovn4nfvDefaultNw = "ovn4nfvk8s-default-nw"
  33 )
  34
  35 var ovnConf *OVNNetworkConf
  36
  37 //GetOvnNetConf return error
  38 func GetOvnNetConf() error {
  39         ovnConf = &OVNNetworkConf{}
  40
  41         ovnConf.Subnet = os.Getenv("OVN_SUBNET")
  42         if ovnConf.Subnet == "" {
  43                 return fmt.Errorf("OVN subnet is not set in nfn-operator configmap env")
  44         }
  45
  46         ovnConf.GatewayIP = os.Getenv("OVN_GATEWAYIP")
  47         if ovnConf.GatewayIP == "" {
  48                 log.Info("No Gateway IP address provided - 1st IP address of the subnet range will be used as Gateway", "Subnet", ovnConf.Subnet)
  49         }
  50
  51         ovnConf.ExcludeIPs = os.Getenv("OVN_EXCLUDEIPS")
  52         if ovnConf.ExcludeIPs == "" {
  53                 log.Info("No IP addresses are excluded in the subnet range", "Subnet", ovnConf.Subnet)
  54         }
  55
  56         return nil
  57 }
  58
  59 type netInterface struct {
  60         Name           string
  61         Interface      string
  62         NetType        string
  63         DefaultGateway string
  64         IPAddress      string
  65         MacAddress     string
  66         GWIPaddress    string
  67 }
  68
  69 var ovnCtl *Controller
  70
  71 // NewOvnController creates a new OVN controller for creating logical networks
  72 func NewOvnController(exec kexec.Interface) (*Controller, error) {
  73
  74         if exec == nil {
  75                 exec = kexec.New()
  76         }
  77         if err := SetExec(exec); err != nil {
  78                 log.Error(err, "Failed to initialize exec helper")
  79                 return nil, err
  80         }
  81
  82         if err := GetOvnNetConf(); err != nil {
  83                 log.Error(err, "nfn-operator OVN Network configmap is not set")
  84                 return nil, err
  85         }
  86         if err := SetupOvnUtils(); err != nil {
  87                 log.Error(err, "Failed to initialize OVN State")
  88                 return nil, err
  89         }
  90
  91         ovnCtl = &Controller{
  92                 gatewayCache: make(map[string]string),
  93         }
  94         return ovnCtl, nil
  95 }
  96
  97 // GetOvnController returns OVN controller for creating logical networks
  98 func GetOvnController() (*Controller, error) {
  99         if ovnCtl != nil {
 100                 return ovnCtl, nil
 101         }
 102         return nil, fmt.Errorf("OVN Controller not initialized")
 103 }
 104
 105 func (oc *Controller) AddNodeLogicalPorts(node string) (ipAddr, macAddr string, err error) {
 106         nodeName := strings.ToLower(node)
 107         portName := config.GetNodeIntfName(nodeName)
 108
 109         log.V(1).Info("Creating Node logical port", "node", nodeName, "portName", portName)
 110
 111         ipAddr, macmacAddr, err := oc.addNodeLogicalPortWithSwitch(Ovn4nfvDefaultNw, portName)
 112         if err != nil {
 113                 return "", "", err
 114         }
 115
 116         return ipAddr, macmacAddr, nil
 117 }
 118
 119 // AddLogicalPorts adds ports to the Pod
 120 func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]interface{}) (key, value string) {
 121
 122         if pod.Spec.HostNetwork {
 123                 return
 124         }
 125
 126         if _, ok := pod.Annotations[Ovn4nfvAnnotationTag]; ok {
 127                 log.V(1).Info("AddLogicalPorts : Pod annotation found")
 128                 return
 129         }
 130
 131         var ovnString, outStr string
 132         var defaultInterface bool
 133
 134         ovnString = "["
 135         var ns netInterface
 136         for _, net := range ovnNetObjs {
 137                 err := mapstructure.Decode(net, &ns)
 138                 if err != nil {
 139                         log.Error(err, "mapstruct error", "network", net)
 140                         return
 141                 }
 142                 if !oc.FindLogicalSwitch(ns.Name) {
 143                         log.Info("Logical Switch not found")
 144                         return
 145                 }
 146                 if ns.Name == Ovn4nfvDefaultNw {
 147                         defaultInterface = true
 148                 }
 149                 if ns.Interface == "" && ns.Name != Ovn4nfvDefaultNw {
 150                         log.Info("Interface name must be provided")
 151                         return
 152                 }
 153                 if ns.DefaultGateway == "" {
 154                         ns.DefaultGateway = "false"
 155                 }
 156                 var portName string
 157                 if ns.Interface != "" {
 158                         portName = fmt.Sprintf("%s_%s_%s", pod.Namespace, pod.Name, ns.Interface)
 159                 } else {
 160                         portName = fmt.Sprintf("%s_%s", pod.Namespace, pod.Name)
 161                         ns.Interface = "*"
 162                 }
 163                 outStr = oc.addLogicalPortWithSwitch(pod, ns.Name, ns.IPAddress, ns.MacAddress, ns.GWIPaddress, portName)
 164                 if outStr == "" {
 165                         return
 166                 }
 167                 last := len(outStr) - 1
 168                 tmpString := outStr[:last]
 169                 tmpString += "," + "\\\"defaultGateway\\\":" + "\\\"" + ns.DefaultGateway + "\\\""
 170                 tmpString += "," + "\\\"interface\\\":" + "\\\"" + ns.Interface + "\\\"}"
 171                 ovnString += tmpString
 172                 ovnString += ","
 173         }
 174         var last int
 175         if defaultInterface == false {
 176                 // Add Default interface
 177                 portName := fmt.Sprintf("%s_%s", pod.Namespace, pod.Name)
 178                 outStr = oc.addLogicalPortWithSwitch(pod, Ovn4nfvDefaultNw, "", "", "", portName)
 179                 if outStr == "" {
 180                         return
 181                 }
 182                 last := len(outStr) - 1
 183                 tmpString := outStr[:last]
 184                 tmpString += "," + "\\\"interface\\\":" + "\\\"" + "*" + "\\\"}"
 185                 ovnString += tmpString
 186                 ovnString += ","
 187         }
 188         last = len(ovnString) - 1
 189         ovnString = ovnString[:last]
 190         ovnString += "]"
 191         key = Ovn4nfvAnnotationTag
 192         value = ovnString
 193         return key, value
 194 }
 195
 196 // DeleteLogicalPorts deletes the OVN ports for the pod
 197 func (oc *Controller) DeleteLogicalPorts(name, namespace string) {
 198
 199         log.Info("DeleteLogicalPorts")
 200         logicalPort := fmt.Sprintf("%s_%s", namespace, name)
 201
 202         // get the list of logical ports from OVN
 203         stdout, stderr, err := RunOVNNbctl("--data=bare", "--no-heading",
 204                 "--columns=name", "find", "logical_switch_port", "external_ids:pod=true")
 205         if err != nil {
 206                 log.Error(err, "Error in obtaining list of logical ports ", "stdout", stdout, "stderr", stderr)
 207                 return
 208         }
 209         existingLogicalPorts := strings.Fields(stdout)
 210         for _, existingPort := range existingLogicalPorts {
 211                 if strings.Contains(existingPort, logicalPort) {
 212                         // found, delete this logical port
 213                         log.Info("Deleting", "Port", existingPort)
 214                         stdout, stderr, err := RunOVNNbctl("--if-exists", "lsp-del",
 215                                 existingPort)
 216                         if err != nil {
 217                                 log.Error(err, "Error in deleting pod's logical port ", "stdout", stdout, "stderr", stderr)
 218                         }
 219                 }
 220         }
 221         return
 222 }
 223
 224 // CreateNetwork in OVN controller
 225 func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error {
 226         var stdout, stderr string
 227
 228         // Currently only these fields are supported
 229         name := cr.Name
 230         subnet := cr.Spec.Ipv4Subnets[0].Subnet
 231         gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
 232         excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
 233
 234         gatewayIPMask, err := createOvnLS(name, subnet, gatewayIP, excludeIps)
 235         if err != nil {
 236                 return err
 237         }
 238
 239         routerMac, stderr, err := RunOVNNbctl("--if-exist", "get", "logical_router_port", "rtos-"+name, "mac")
 240         if err != nil {
 241                 log.Error(err, "Failed to get logical router port", "stderr", stderr)
 242                 return err
 243         }
 244         if routerMac == "" {
 245                 prefix := "00:00:00"
 246                 newRand := rand.New(rand.NewSource(time.Now().UnixNano()))
 247                 routerMac = fmt.Sprintf("%s:%02x:%02x:%02x", prefix, newRand.Intn(255), newRand.Intn(255), newRand.Intn(255))
 248         }
 249
 250         _, stderr, err = RunOVNNbctl("--wait=hv", "--may-exist", "lrp-add", ovn4nfvRouterName, "rtos-"+name, routerMac, gatewayIPMask)
 251         if err != nil {
 252                 log.Error(err, "Failed to add logical port to router", "stderr", stderr)
 253                 return err
 254         }
 255
 256         // Connect the switch to the router.
 257         stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "stor-"+name, "--", "set", "logical_switch_port", "stor-"+name, "type=router", "options:router-port=rtos-"+name, "addresses="+"\""+routerMac+"\"")
 258         if err != nil {
 259                 log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout)
 260                 return err
 261         }
 262
 263         return nil
 264 }
 265
 266 // DeleteNetwork in OVN controller
 267 func (oc *Controller) DeleteNetwork(cr *k8sv1alpha1.Network) error {
 268
 269         name := cr.Name
 270         stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "lrp-del", "rtos-"+name)
 271         if err != nil {
 272                 log.Error(err, "Failed to delete router port", "name", name, "stdout", stdout, "stderr", stderr)
 273                 return err
 274         }
 275         stdout, stderr, err = RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name)
 276         if err != nil {
 277                 log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr)
 278                 return err
 279         }
 280         return nil
 281 }
 282
 283 // CreateProviderNetwork in OVN controller
 284 func (oc *Controller) CreateProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error {
 285         var stdout, stderr string
 286
 287         // Currently only these fields are supported
 288         name := cr.Name
 289         subnet := cr.Spec.Ipv4Subnets[0].Subnet
 290         gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
 291         excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
 292         _, err := createOvnLS(name, subnet, gatewayIP, excludeIps)
 293         if err != nil {
 294                 return err
 295         }
 296
 297         // Add localnet port.
 298         stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "server-localnet_"+name, "--",
 299                 "lsp-set-addresses", "server-localnet_"+name, "unknown", "--",
 300                 "lsp-set-type", "server-localnet_"+name, "localnet", "--",
 301                 "lsp-set-options", "server-localnet_"+name, "network_name=nw_"+name)
 302         if err != nil {
 303                 log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout)
 304                 return err
 305         }
 306
 307         return nil
 308 }
 309
 310 // DeleteProviderNetwork in OVN controller
 311 func (oc *Controller) DeleteProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error {
 312
 313         name := cr.Name
 314         stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name)
 315         if err != nil {
 316                 log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr)
 317                 return err
 318         }
 319         return nil
 320 }
 321
 322 // FindLogicalSwitch returns true if switch exists
 323 func (oc *Controller) FindLogicalSwitch(name string) bool {
 324         // get logical switch from OVN
 325         output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading",
 326                 "--columns=name", "find", "logical_switch", "name="+name)
 327         if err != nil {
 328                 log.Error(err, "Error in obtaining list of logical switch", "stderr", stderr)
 329                 return false
 330         }
 331         if strings.Compare(name, output) == 0 {
 332                 return true
 333         }
 334         return false
 335 }
 336
 337 func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string, error) {
 338         var gatewayIPMaskStr, stderr string
 339         var ok bool
 340         var err error
 341         if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok {
 342                 gatewayIPMaskStr, stderr, err = RunOVNNbctl("--if-exists",
 343                         "get", "logical_switch", logicalSwitch,
 344                         "external_ids:gateway_ip")
 345                 if err != nil {
 346                         log.Error(err, "Failed to get gateway IP", "stderr", stderr, "gatewayIPMaskStr", gatewayIPMaskStr)
 347                         return "", "", err
 348                 }
 349                 if gatewayIPMaskStr == "" {
 350                         return "", "", fmt.Errorf("Empty gateway IP in logical switch %s",
 351                                 logicalSwitch)
 352                 }
 353                 oc.gatewayCache[logicalSwitch] = gatewayIPMaskStr
 354         }
 355         gatewayIPMask := strings.Split(gatewayIPMaskStr, "/")
 356         if len(gatewayIPMask) != 2 {
 357                 return "", "", fmt.Errorf("Failed to get IP and Mask from gateway CIDR:  %s",
 358                         gatewayIPMaskStr)
 359         }
 360         gatewayIP := gatewayIPMask[0]
 361         mask := gatewayIPMask[1]
 362         return gatewayIP, mask, nil
 363 }
 364
 365 func (oc *Controller) addNodeLogicalPortWithSwitch(logicalSwitch, portName string) (ipAddr, macAddr string, r error) {
 366         var out, stderr string
 367         var err error
 368
 369         log.V(1).Info("Creating Node logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch)
 370
 371         out, stderr, err = RunOVNNbctl("--wait=sb", "--",
 372                 "--may-exist", "lsp-add", logicalSwitch, portName,
 373                 "--", "lsp-set-addresses",
 374                 portName, "dynamic")
 375         if err != nil {
 376                 log.Error(err, "Error while creating logical port %s ", "portName", portName, "stdout", out, "stderr", stderr)
 377                 return "", "", err
 378         }
 379
 380         count := 30
 381         for count > 0 {
 382                 out, stderr, err = RunOVNNbctl("get",
 383                         "logical_switch_port", portName, "dynamic_addresses")
 384
 385                 if err == nil && out != "[]" {
 386                         break
 387                 }
 388                 if err != nil {
 389                         log.Error(err, "Error while obtaining addresses for", "portName", portName)
 390                         return "", "", err
 391                 }
 392                 time.Sleep(time.Second)
 393                 count--
 394         }
 395         if count == 0 {
 396                 log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr)
 397                 return "", "", err
 398         }
 399
 400         // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while
 401         // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3".
 402         outStr := strings.TrimLeft(out, `[`)
 403         outStr = strings.TrimRight(outStr, `]`)
 404         outStr = strings.Trim(outStr, `"`)
 405         addresses := strings.Split(outStr, " ")
 406         if len(addresses) != 2 {
 407                 log.Info("Error while obtaining addresses for", "portName", portName)
 408                 return "", "", err
 409         }
 410
 411         _, mask, err := oc.getGatewayFromSwitch(logicalSwitch)
 412         if err != nil {
 413                 log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
 414                 return "", "", err
 415         }
 416
 417         ipAddr = fmt.Sprintf("%s/%s", addresses[1], mask)
 418         macAddr = fmt.Sprintf("%s", addresses[0])
 419
 420         return ipAddr, macAddr, nil
 421 }
 422
 423 func (oc *Controller) getNodeLogicalPortIPAddr(pod *kapi.Pod) (ipAddress string, r error) {
 424         var out, stderr, nodeName, portName string
 425         var err error
 426
 427         nodeName = strings.ToLower(pod.Spec.NodeName)
 428         portName = config.GetNodeIntfName(nodeName)
 429
 430         log.V(1).Info("Get Node logical port", "pod", pod.GetName(), "node", nodeName, "portName", portName)
 431
 432         count := 30
 433         for count > 0 {
 434                 out, stderr, err = RunOVNNbctl("get",
 435                         "logical_switch_port", portName, "dynamic_addresses")
 436
 437                 if err == nil && out != "[]" {
 438                         break
 439                 }
 440                 if err != nil {
 441                         log.Error(err, "Error while obtaining addresses for", "portName", portName)
 442                         return "", err
 443                 }
 444                 time.Sleep(time.Second)
 445                 count--
 446         }
 447         if count == 0 {
 448                 log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr)
 449                 return "", err
 450         }
 451
 452         // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while
 453         // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3".
 454         outStr := strings.TrimLeft(out, `[`)
 455         outStr = strings.TrimRight(outStr, `]`)
 456         outStr = strings.Trim(outStr, `"`)
 457         addresses := strings.Split(outStr, " ")
 458         if len(addresses) != 2 {
 459                 log.Info("Error while obtaining addresses for", "portName", portName)
 460                 return "", err
 461         }
 462
 463         ipAddr := fmt.Sprintf("%s", addresses[1])
 464         log.V(1).Info("Get Node logical port", "pod", pod.GetName(), "node", nodeName, "portName", portName, "Node port IP", ipAddr)
 465
 466         return ipAddr, nil
 467 }
 468
 469 func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, gwipAddress, portName string) (annotation string) {
 470         var out, stderr string
 471         var err error
 472         var isStaticIP bool
 473         if pod.Spec.HostNetwork {
 474                 return
 475         }
 476
 477         log.V(1).Info("Creating logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch)
 478
 479         if ipAddress != "" && macAddress != "" {
 480                 isStaticIP = true
 481         }
 482         if ipAddress != "" && macAddress == "" {
 483                 macAddress = generateMac()
 484                 isStaticIP = true
 485         }
 486
 487         if isStaticIP {
 488                 out, stderr, err = RunOVNNbctl("--may-exist", "lsp-add",
 489                         logicalSwitch, portName, "--", "lsp-set-addresses", portName,
 490                         fmt.Sprintf("%s %s", macAddress, ipAddress), "--", "--if-exists",
 491                         "clear", "logical_switch_port", portName, "dynamic_addresses", "--", "set",
 492                         "logical_switch_port", portName,
 493                         "external-ids:namespace="+pod.Namespace,
 494                         "external-ids:logical_switch="+logicalSwitch,
 495                         "external-ids:pod=true")
 496                 if err != nil {
 497                         log.Error(err, "Failed to add logical port to switch", "out", out, "stderr", stderr)
 498                         return
 499                 }
 500         } else {
 501                 out, stderr, err = RunOVNNbctl("--wait=sb", "--",
 502                         "--may-exist", "lsp-add", logicalSwitch, portName,
 503                         "--", "lsp-set-addresses",
 504                         portName, "dynamic", "--", "set",
 505                         "logical_switch_port", portName,
 506                         "external-ids:namespace="+pod.Namespace,
 507                         "external-ids:logical_switch="+logicalSwitch,
 508                         "external-ids:pod=true")
 509                 if err != nil {
 510                         log.Error(err, "Error while creating logical port %s ", "portName", portName, "stdout", out, "stderr", stderr)
 511                         return
 512                 }
 513         }
 514
 515         count := 30
 516         for count > 0 {
 517                 if isStaticIP {
 518                         out, stderr, err = RunOVNNbctl("get",
 519                                 "logical_switch_port", portName, "addresses")
 520                 } else {
 521                         out, stderr, err = RunOVNNbctl("get",
 522                                 "logical_switch_port", portName, "dynamic_addresses")
 523                 }
 524                 if err == nil && out != "[]" {
 525                         break
 526                 }
 527                 if err != nil {
 528                         log.Error(err, "Error while obtaining addresses for", "portName", portName)
 529                         return
 530                 }
 531                 time.Sleep(time.Second)
 532                 count--
 533         }
 534         if count == 0 {
 535                 log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr)
 536                 return
 537         }
 538
 539         // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while
 540         // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3".
 541         outStr := strings.TrimLeft(out, `[`)
 542         outStr = strings.TrimRight(outStr, `]`)
 543         outStr = strings.Trim(outStr, `"`)
 544         addresses := strings.Split(outStr, " ")
 545         if len(addresses) != 2 {
 546                 log.Info("Error while obtaining addresses for", "portName", portName)
 547                 return
 548         }
 549
 550         _, mask, err := oc.getGatewayFromSwitch(logicalSwitch)
 551         if err != nil {
 552                 log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
 553                 return
 554         }
 555
 556         var gatewayIP string
 557         if gwipAddress != "" {
 558                 gatewayIP = gwipAddress
 559         } else {
 560                 gatewayIP, err = oc.getNodeLogicalPortIPAddr(pod)
 561                 if err != nil {
 562                         log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
 563                         return
 564                 }
 565         }
 566
 567         annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP)
 568
 569         return annotation
 570 }