internal/pkg/ovn/ovn.go

   1 package ovn
   2
   3 import (
   4         "fmt"
   5         "math/rand"
   6         "os"
   7         "ovn4nfv-k8s-plugin/internal/pkg/config"
   8         k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1"
   9         "strings"
  10         "time"
  11
  12         "github.com/mitchellh/mapstructure"
  13         kapi "k8s.io/api/core/v1"
  14         kexec "k8s.io/utils/exec"
  15 )
  16
  17 type Controller struct {
  18         gatewayCache map[string]string
  19 }
  20
  21 type OVNNetworkConf struct {
  22         Subnet     string
  23         GatewayIP  string
  24         ExcludeIPs string
  25 }
  26
  27 const (
  28         ovn4nfvRouterName = "ovn4nfv-master"
  29         // Ovn4nfvAnnotationTag tag on already processed Pods
  30         Ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
  31         // OVN Default Network name
  32         Ovn4nfvDefaultNw = "ovn4nfvk8s-default-nw"
  33 )
  34
  35 var ovnConf *OVNNetworkConf
  36
  37 //GetOvnNetConf return error
  38 func GetOvnNetConf() error {
  39         ovnConf = &OVNNetworkConf{}
  40
  41         ovnConf.Subnet = os.Getenv("OVN_SUBNET")
  42         if ovnConf.Subnet == "" {
  43                 return fmt.Errorf("OVN subnet is not set in nfn-operator configmap env")
  44         }
  45
  46         ovnConf.GatewayIP = os.Getenv("OVN_GATEWAYIP")
  47         if ovnConf.GatewayIP == "" {
  48                 log.Info("No Gateway IP address provided - 1st IP address of the subnet range will be used as Gateway", "Subnet", ovnConf.Subnet)
  49         }
  50
  51         ovnConf.ExcludeIPs = os.Getenv("OVN_EXCLUDEIPS")
  52         if ovnConf.ExcludeIPs == "" {
  53                 log.Info("No IP addresses are excluded in the subnet range", "Subnet", ovnConf.Subnet)
  54         }
  55
  56         return nil
  57 }
  58
  59 type netInterface struct {
  60         Name           string
  61         Interface      string
  62         NetType        string
  63         DefaultGateway string
  64         IPAddress      string
  65         MacAddress     string
  66 }
  67
  68 var ovnCtl *Controller
  69
  70 // NewOvnController creates a new OVN controller for creating logical networks
  71 func NewOvnController(exec kexec.Interface) (*Controller, error) {
  72
  73         if exec == nil {
  74                 exec = kexec.New()
  75         }
  76         if err := SetExec(exec); err != nil {
  77                 log.Error(err, "Failed to initialize exec helper")
  78                 return nil, err
  79         }
  80
  81         if err := GetOvnNetConf(); err != nil {
  82                 log.Error(err, "nfn-operator OVN Network configmap is not set")
  83                 return nil, err
  84         }
  85         if err := SetupOvnUtils(); err != nil {
  86                 log.Error(err, "Failed to initialize OVN State")
  87                 return nil, err
  88         }
  89
  90         ovnCtl = &Controller{
  91                 gatewayCache: make(map[string]string),
  92         }
  93         return ovnCtl, nil
  94 }
  95
  96 // GetOvnController returns OVN controller for creating logical networks
  97 func GetOvnController() (*Controller, error) {
  98         if ovnCtl != nil {
  99                 return ovnCtl, nil
 100         }
 101         return nil, fmt.Errorf("OVN Controller not initialized")
 102 }
 103
 104 func (oc *Controller) AddNodeLogicalPorts(node string) (ipAddr, macAddr string, err error) {
 105         nodeName := strings.ToLower(node)
 106         portName := config.GetNodeIntfName(nodeName)
 107
 108         log.V(1).Info("Creating Node logical port", "node", nodeName, "portName", portName)
 109
 110         ipAddr, macmacAddr, err := oc.addNodeLogicalPortWithSwitch(Ovn4nfvDefaultNw, portName)
 111         if err != nil {
 112                 return "", "", err
 113         }
 114
 115         return ipAddr, macmacAddr, nil
 116 }
 117
 118 // AddLogicalPorts adds ports to the Pod
 119 func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]interface{}) (key, value string) {
 120
 121         if pod.Spec.HostNetwork {
 122                 return
 123         }
 124
 125         if _, ok := pod.Annotations[Ovn4nfvAnnotationTag]; ok {
 126                 log.V(1).Info("AddLogicalPorts : Pod annotation found")
 127                 return
 128         }
 129
 130         var ovnString, outStr string
 131         var defaultInterface bool
 132
 133         ovnString = "["
 134         var ns netInterface
 135         for _, net := range ovnNetObjs {
 136                 err := mapstructure.Decode(net, &ns)
 137                 if err != nil {
 138                         log.Error(err, "mapstruct error", "network", net)
 139                         return
 140                 }
 141                 if !oc.FindLogicalSwitch(ns.Name) {
 142                         log.Info("Logical Switch not found")
 143                         return
 144                 }
 145                 if ns.Name == Ovn4nfvDefaultNw {
 146                         defaultInterface = true
 147                 }
 148                 if ns.Interface == "" && ns.Name != Ovn4nfvDefaultNw {
 149                         log.Info("Interface name must be provided")
 150                         return
 151                 }
 152                 if ns.DefaultGateway == "" {
 153                         ns.DefaultGateway = "false"
 154                 }
 155                 var portName string
 156                 if ns.Interface != "" {
 157                         portName = fmt.Sprintf("%s_%s_%s", pod.Namespace, pod.Name, ns.Interface)
 158                 } else {
 159                         portName = fmt.Sprintf("%s_%s", pod.Namespace, pod.Name)
 160                         ns.Interface = "*"
 161                 }
 162                 outStr = oc.addLogicalPortWithSwitch(pod, ns.Name, ns.IPAddress, ns.MacAddress, portName)
 163                 if outStr == "" {
 164                         return
 165                 }
 166                 last := len(outStr) - 1
 167                 tmpString := outStr[:last]
 168                 tmpString += "," + "\\\"defaultGateway\\\":" + "\\\"" + ns.DefaultGateway + "\\\""
 169                 tmpString += "," + "\\\"interface\\\":" + "\\\"" + ns.Interface + "\\\"}"
 170                 ovnString += tmpString
 171                 ovnString += ","
 172         }
 173         var last int
 174         if defaultInterface == false {
 175                 // Add Default interface
 176                 portName := fmt.Sprintf("%s_%s", pod.Namespace, pod.Name)
 177                 outStr = oc.addLogicalPortWithSwitch(pod, Ovn4nfvDefaultNw, "", "", portName)
 178                 if outStr == "" {
 179                         return
 180                 }
 181                 last := len(outStr) - 1
 182                 tmpString := outStr[:last]
 183                 tmpString += "," + "\\\"interface\\\":" + "\\\"" + "*" + "\\\"}"
 184                 ovnString += tmpString
 185                 ovnString += ","
 186         }
 187         last = len(ovnString) - 1
 188         ovnString = ovnString[:last]
 189         ovnString += "]"
 190         key = Ovn4nfvAnnotationTag
 191         value = ovnString
 192         return key, value
 193 }
 194
 195 // DeleteLogicalPorts deletes the OVN ports for the pod
 196 func (oc *Controller) DeleteLogicalPorts(name, namespace string) {
 197
 198         log.Info("DeleteLogicalPorts")
 199         logicalPort := fmt.Sprintf("%s_%s", namespace, name)
 200
 201         // get the list of logical ports from OVN
 202         stdout, stderr, err := RunOVNNbctl("--data=bare", "--no-heading",
 203                 "--columns=name", "find", "logical_switch_port", "external_ids:pod=true")
 204         if err != nil {
 205                 log.Error(err, "Error in obtaining list of logical ports ", "stdout", stdout, "stderr", stderr)
 206                 return
 207         }
 208         existingLogicalPorts := strings.Fields(stdout)
 209         for _, existingPort := range existingLogicalPorts {
 210                 if strings.Contains(existingPort, logicalPort) {
 211                         // found, delete this logical port
 212                         log.Info("Deleting", "Port", existingPort)
 213                         stdout, stderr, err := RunOVNNbctl("--if-exists", "lsp-del",
 214                                 existingPort)
 215                         if err != nil {
 216                                 log.Error(err, "Error in deleting pod's logical port ", "stdout", stdout, "stderr", stderr)
 217                         }
 218                 }
 219         }
 220         return
 221 }
 222
 223 // CreateNetwork in OVN controller
 224 func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error {
 225         var stdout, stderr string
 226
 227         // Currently only these fields are supported
 228         name := cr.Name
 229         subnet := cr.Spec.Ipv4Subnets[0].Subnet
 230         gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
 231         excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
 232
 233         gatewayIPMask, err := createOvnLS(name, subnet, gatewayIP, excludeIps)
 234         if err != nil {
 235                 return err
 236         }
 237
 238         routerMac, stderr, err := RunOVNNbctl("--if-exist", "get", "logical_router_port", "rtos-"+name, "mac")
 239         if err != nil {
 240                 log.Error(err, "Failed to get logical router port", "stderr", stderr)
 241                 return err
 242         }
 243         if routerMac == "" {
 244                 prefix := "00:00:00"
 245                 newRand := rand.New(rand.NewSource(time.Now().UnixNano()))
 246                 routerMac = fmt.Sprintf("%s:%02x:%02x:%02x", prefix, newRand.Intn(255), newRand.Intn(255), newRand.Intn(255))
 247         }
 248
 249         _, stderr, err = RunOVNNbctl("--wait=hv", "--may-exist", "lrp-add", ovn4nfvRouterName, "rtos-"+name, routerMac, gatewayIPMask)
 250         if err != nil {
 251                 log.Error(err, "Failed to add logical port to router", "stderr", stderr)
 252                 return err
 253         }
 254
 255         // Connect the switch to the router.
 256         stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "stor-"+name, "--", "set", "logical_switch_port", "stor-"+name, "type=router", "options:router-port=rtos-"+name, "addresses="+"\""+routerMac+"\"")
 257         if err != nil {
 258                 log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout)
 259                 return err
 260         }
 261
 262         return nil
 263 }
 264
 265 // DeleteNetwork in OVN controller
 266 func (oc *Controller) DeleteNetwork(cr *k8sv1alpha1.Network) error {
 267
 268         name := cr.Name
 269         stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "lrp-del", "rtos-"+name)
 270         if err != nil {
 271                 log.Error(err, "Failed to delete router port", "name", name, "stdout", stdout, "stderr", stderr)
 272                 return err
 273         }
 274         stdout, stderr, err = RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name)
 275         if err != nil {
 276                 log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr)
 277                 return err
 278         }
 279         return nil
 280 }
 281
 282 // CreateProviderNetwork in OVN controller
 283 func (oc *Controller) CreateProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error {
 284         var stdout, stderr string
 285
 286         // Currently only these fields are supported
 287         name := cr.Name
 288         subnet := cr.Spec.Ipv4Subnets[0].Subnet
 289         gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
 290         excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
 291         _, err := createOvnLS(name, subnet, gatewayIP, excludeIps)
 292         if err != nil {
 293                 return err
 294         }
 295
 296         // Add localnet port.
 297         stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "server-localnet_"+name, "--",
 298                 "lsp-set-addresses", "server-localnet_"+name, "unknown", "--",
 299                 "lsp-set-type", "server-localnet_"+name, "localnet", "--",
 300                 "lsp-set-options", "server-localnet_"+name, "network_name=nw_"+name)
 301         if err != nil {
 302                 log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout)
 303                 return err
 304         }
 305
 306         return nil
 307 }
 308
 309 // DeleteProviderNetwork in OVN controller
 310 func (oc *Controller) DeleteProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error {
 311
 312         name := cr.Name
 313         stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name)
 314         if err != nil {
 315                 log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr)
 316                 return err
 317         }
 318         return nil
 319 }
 320
 321 // FindLogicalSwitch returns true if switch exists
 322 func (oc *Controller) FindLogicalSwitch(name string) bool {
 323         // get logical switch from OVN
 324         output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading",
 325                 "--columns=name", "find", "logical_switch", "name="+name)
 326         if err != nil {
 327                 log.Error(err, "Error in obtaining list of logical switch", "stderr", stderr)
 328                 return false
 329         }
 330         if strings.Compare(name, output) == 0 {
 331                 return true
 332         }
 333         return false
 334 }
 335
 336 func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string, error) {
 337         var gatewayIPMaskStr, stderr string
 338         var ok bool
 339         var err error
 340         if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok {
 341                 gatewayIPMaskStr, stderr, err = RunOVNNbctl("--if-exists",
 342                         "get", "logical_switch", logicalSwitch,
 343                         "external_ids:gateway_ip")
 344                 if err != nil {
 345                         log.Error(err, "Failed to get gateway IP", "stderr", stderr, "gatewayIPMaskStr", gatewayIPMaskStr)
 346                         return "", "", err
 347                 }
 348                 if gatewayIPMaskStr == "" {
 349                         return "", "", fmt.Errorf("Empty gateway IP in logical switch %s",
 350                                 logicalSwitch)
 351                 }
 352                 oc.gatewayCache[logicalSwitch] = gatewayIPMaskStr
 353         }
 354         gatewayIPMask := strings.Split(gatewayIPMaskStr, "/")
 355         if len(gatewayIPMask) != 2 {
 356                 return "", "", fmt.Errorf("Failed to get IP and Mask from gateway CIDR:  %s",
 357                         gatewayIPMaskStr)
 358         }
 359         gatewayIP := gatewayIPMask[0]
 360         mask := gatewayIPMask[1]
 361         return gatewayIP, mask, nil
 362 }
 363
 364 func (oc *Controller) addNodeLogicalPortWithSwitch(logicalSwitch, portName string) (ipAddr, macAddr string, r error) {
 365         var out, stderr string
 366         var err error
 367
 368         log.V(1).Info("Creating Node logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch)
 369
 370         out, stderr, err = RunOVNNbctl("--wait=sb", "--",
 371                 "--may-exist", "lsp-add", logicalSwitch, portName,
 372                 "--", "lsp-set-addresses",
 373                 portName, "dynamic")
 374         if err != nil {
 375                 log.Error(err, "Error while creating logical port %s ", "portName", portName, "stdout", out, "stderr", stderr)
 376                 return "", "", err
 377         }
 378
 379         count := 30
 380         for count > 0 {
 381                 out, stderr, err = RunOVNNbctl("get",
 382                         "logical_switch_port", portName, "dynamic_addresses")
 383
 384                 if err == nil && out != "[]" {
 385                         break
 386                 }
 387                 if err != nil {
 388                         log.Error(err, "Error while obtaining addresses for", "portName", portName)
 389                         return "", "", err
 390                 }
 391                 time.Sleep(time.Second)
 392                 count--
 393         }
 394         if count == 0 {
 395                 log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr)
 396                 return "", "", err
 397         }
 398
 399         // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while
 400         // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3".
 401         outStr := strings.TrimLeft(out, `[`)
 402         outStr = strings.TrimRight(outStr, `]`)
 403         outStr = strings.Trim(outStr, `"`)
 404         addresses := strings.Split(outStr, " ")
 405         if len(addresses) != 2 {
 406                 log.Info("Error while obtaining addresses for", "portName", portName)
 407                 return "", "", err
 408         }
 409
 410         _, mask, err := oc.getGatewayFromSwitch(logicalSwitch)
 411         if err != nil {
 412                 log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
 413                 return "", "", err
 414         }
 415
 416         ipAddr = fmt.Sprintf("%s/%s", addresses[1], mask)
 417         macAddr = fmt.Sprintf("%s", addresses[0])
 418
 419         return ipAddr, macAddr, nil
 420 }
 421
 422 func (oc *Controller) getNodeLogicalPortIPAddr(pod *kapi.Pod) (ipAddress string, r error) {
 423         var out, stderr, nodeName, portName string
 424         var err error
 425
 426         nodeName = strings.ToLower(pod.Spec.NodeName)
 427         portName = config.GetNodeIntfName(nodeName)
 428
 429         log.V(1).Info("Get Node logical port", "pod", pod.GetName(), "node", nodeName, "portName", portName)
 430
 431         count := 30
 432         for count > 0 {
 433                 out, stderr, err = RunOVNNbctl("get",
 434                         "logical_switch_port", portName, "dynamic_addresses")
 435
 436                 if err == nil && out != "[]" {
 437                         break
 438                 }
 439                 if err != nil {
 440                         log.Error(err, "Error while obtaining addresses for", "portName", portName)
 441                         return "", err
 442                 }
 443                 time.Sleep(time.Second)
 444                 count--
 445         }
 446         if count == 0 {
 447                 log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr)
 448                 return "", err
 449         }
 450
 451         // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while
 452         // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3".
 453         outStr := strings.TrimLeft(out, `[`)
 454         outStr = strings.TrimRight(outStr, `]`)
 455         outStr = strings.Trim(outStr, `"`)
 456         addresses := strings.Split(outStr, " ")
 457         if len(addresses) != 2 {
 458                 log.Info("Error while obtaining addresses for", "portName", portName)
 459                 return "", err
 460         }
 461
 462         ipAddr := fmt.Sprintf("%s", addresses[1])
 463         log.V(1).Info("Get Node logical port", "pod", pod.GetName(), "node", nodeName, "portName", portName, "Node port IP", ipAddr)
 464
 465         return ipAddr, nil
 466 }
 467
 468 func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, portName string) (annotation string) {
 469         var out, stderr string
 470         var err error
 471         var isStaticIP bool
 472         if pod.Spec.HostNetwork {
 473                 return
 474         }
 475
 476         log.V(1).Info("Creating logical port for on switch", "portName", portName, "logicalSwitch", logicalSwitch)
 477
 478         if ipAddress != "" && macAddress != "" {
 479                 isStaticIP = true
 480         }
 481         if ipAddress != "" && macAddress == "" {
 482                 macAddress = generateMac()
 483                 isStaticIP = true
 484         }
 485
 486         if isStaticIP {
 487                 out, stderr, err = RunOVNNbctl("--may-exist", "lsp-add",
 488                         logicalSwitch, portName, "--", "lsp-set-addresses", portName,
 489                         fmt.Sprintf("%s %s", macAddress, ipAddress), "--", "--if-exists",
 490                         "clear", "logical_switch_port", portName, "dynamic_addresses", "--", "set",
 491                         "logical_switch_port", portName,
 492                         "external-ids:namespace="+pod.Namespace,
 493                         "external-ids:logical_switch="+logicalSwitch,
 494                         "external-ids:pod=true")
 495                 if err != nil {
 496                         log.Error(err, "Failed to add logical port to switch", "out", out, "stderr", stderr)
 497                         return
 498                 }
 499         } else {
 500                 out, stderr, err = RunOVNNbctl("--wait=sb", "--",
 501                         "--may-exist", "lsp-add", logicalSwitch, portName,
 502                         "--", "lsp-set-addresses",
 503                         portName, "dynamic", "--", "set",
 504                         "logical_switch_port", portName,
 505                         "external-ids:namespace="+pod.Namespace,
 506                         "external-ids:logical_switch="+logicalSwitch,
 507                         "external-ids:pod=true")
 508                 if err != nil {
 509                         log.Error(err, "Error while creating logical port %s ", "portName", portName, "stdout", out, "stderr", stderr)
 510                         return
 511                 }
 512         }
 513
 514         count := 30
 515         for count > 0 {
 516                 if isStaticIP {
 517                         out, stderr, err = RunOVNNbctl("get",
 518                                 "logical_switch_port", portName, "addresses")
 519                 } else {
 520                         out, stderr, err = RunOVNNbctl("get",
 521                                 "logical_switch_port", portName, "dynamic_addresses")
 522                 }
 523                 if err == nil && out != "[]" {
 524                         break
 525                 }
 526                 if err != nil {
 527                         log.Error(err, "Error while obtaining addresses for", "portName", portName)
 528                         return
 529                 }
 530                 time.Sleep(time.Second)
 531                 count--
 532         }
 533         if count == 0 {
 534                 log.Error(err, "Error while obtaining addresses for", "portName", portName, "stdout", out, "stderr", stderr)
 535                 return
 536         }
 537
 538         // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while
 539         // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3".
 540         outStr := strings.TrimLeft(out, `[`)
 541         outStr = strings.TrimRight(outStr, `]`)
 542         outStr = strings.Trim(outStr, `"`)
 543         addresses := strings.Split(outStr, " ")
 544         if len(addresses) != 2 {
 545                 log.Info("Error while obtaining addresses for", "portName", portName)
 546                 return
 547         }
 548
 549         _, mask, err := oc.getGatewayFromSwitch(logicalSwitch)
 550         if err != nil {
 551                 log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
 552                 return
 553         }
 554
 555         gatewayIP, err := oc.getNodeLogicalPortIPAddr(pod)
 556         if err != nil {
 557                 log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
 558                 return
 559         }
 560
 561         annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP)
 562
 563         return annotation
 564 }