Route based chaining 56/70656/4
authorRitu Sood <Ritu.Sood@intel.com>
Wed, 6 May 2020 20:47:11 +0000 (13:47 -0700)
committerKuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Fri, 18 Sep 2020 00:09:13 +0000 (17:09 -0700)
1) Adding controller for route based chaining
2) nfn agent code to insert route in container namespace
3) Calculate routes based on the Chaining Routes

Based on the Ritu(ovn4nfv-k8s-plugin committer) patches

Co-authored-by: Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Signed-off-by: Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Change-Id: I6757f269a5df7d98d650f273468c6ccbcc055d3e

12 files changed:
cmd/nfn-agent/nfn-agent.go
deploy/crds/k8s.plugin.opnfv.org_networkchainings_crd.yaml [new file with mode: 0644]
deploy/crds/k8s_v1alpha1_networkchaining_cr.yaml [new file with mode: 0644]
go.mod
go.sum
internal/pkg/nfnNotify/proto/nfn.pb.go
internal/pkg/nfnNotify/proto/nfn.proto
internal/pkg/nfnNotify/server.go
internal/pkg/ovn/common.go
internal/pkg/utils/chain.go [new file with mode: 0644]
pkg/controller/add_networkchaining.go [new file with mode: 0644]
pkg/controller/networkchaining/networkchaining_controller.go [new file with mode: 0644]

index b2ee630..8e7ad78 100644 (file)
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package main
 
 import (
@@ -8,6 +24,7 @@ import (
        "os/signal"
        cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
        pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
+        chaining "ovn4nfv-k8s-plugin/internal/pkg/utils"
        "ovn4nfv-k8s-plugin/internal/pkg/ovn"
        "strings"
        "syscall"
@@ -256,6 +273,18 @@ func handleNotif(msg *pb.Notification) {
                                deleteDirectProvidernetwork(payload)
                        }
 
+               case *pb.Notification_ContainterRtInsert:
+                       id := payload.ContainterRtInsert.GetContainerId()
+                       pid, err := chaining.GetPidForContainer(id)
+                       if err != nil {
+                               log.Error(err, "Failed to get pid", "containerID", id)
+                               return
+                       }
+                       err = chaining.ContainerAddRoute(pid, payload.ContainterRtInsert.GetRoute())
+                       if err != nil {
+                               return
+                       }
+
                case *pb.Notification_InSync:
                        if payload.InSync.GetNodeIntfIpAddress() != "" && payload.InSync.GetNodeIntfMacAddress() != "" {
                                err := createNodeOVSInternalPort(payload)
diff --git a/deploy/crds/k8s.plugin.opnfv.org_networkchainings_crd.yaml b/deploy/crds/k8s.plugin.opnfv.org_networkchainings_crd.yaml
new file mode 100644 (file)
index 0000000..77257c3
--- /dev/null
@@ -0,0 +1,89 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: networkchainings.k8s.plugin.opnfv.org
+spec:
+  group: k8s.plugin.opnfv.org
+  names:
+    kind: NetworkChaining
+    listKind: NetworkChainingList
+    plural: networkchainings
+    singular: networkchaining
+  scope: Namespaced
+  subresources:
+    status: {}
+  validation:
+    openAPIV3Schema:
+      description: NetworkChaining is the Schema for the networkchainings API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: NetworkChainingSpec defines the desired state of NetworkChaining
+          properties:
+            chainType:
+              type: string
+            routingSpec:
+              properties:
+                leftNetwork:
+                  items:
+                    properties:
+                      gatewayIp:
+                        type: string
+                      networkName:
+                        type: string
+                    required:
+                    - gatewayIp
+                    - networkName
+                    type: object
+                  type: array
+                namespace:
+                  type: string
+                networkChain:
+                  type: string
+                rightNetwork:
+                  items:
+                    properties:
+                      gatewayIp:
+                        type: string
+                      networkName:
+                        type: string
+                    required:
+                    - gatewayIp
+                    - networkName
+                    type: object
+                  type: array
+              required:
+              - leftNetwork
+              - namespace
+              - networkChain
+              - rightNetwork
+              type: object
+          required:
+          - chainType
+          - routingSpec
+          type: object
+        status:
+          description: NetworkChainingStatus defines the observed state of NetworkChaining
+          properties:
+            state:
+              type: string
+          required:
+          - state
+          type: object
+      type: object
+  version: v1alpha1
+  versions:
+  - name: v1alpha1
+    served: true
+    storage: true
diff --git a/deploy/crds/k8s_v1alpha1_networkchaining_cr.yaml b/deploy/crds/k8s_v1alpha1_networkchaining_cr.yaml
new file mode 100644 (file)
index 0000000..330cd20
--- /dev/null
@@ -0,0 +1,19 @@
+apiVersion: k8s.plugin.opnfv.org/v1alpha1
+kind: NetworkChaining
+metadata:
+  name: example-networkchaining
+spec:
+  # Add fields here
+  chainType: "Routing"
+  routingSpec:
+    namespace: "default"
+    networkChain: "app=slb,dync-net1,app=ngfw,dync-net2,app=sdwan"
+    leftNetwork:
+    - networkName: "pnet1"
+      gatewayIp: "172.30.10.2"
+      subnet: "172.30.10.0/24"
+    rightNetwork:
+    - networkName: "pnet2"
+      gatewayIp: "172.30.20.2"
+      subnet: "172.30.20.0/24"
+
diff --git a/go.mod b/go.mod
index e0f14a5..ff0bd8f 100644 (file)
--- a/go.mod
+++ b/go.mod
@@ -8,6 +8,9 @@ require (
        github.com/containernetworking/plugins v0.8.1
        github.com/coreos/go-iptables v0.4.2
        github.com/creack/pty v1.1.9 // indirect
+       github.com/docker/distribution v2.7.1+incompatible // indirect
+       github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0
+       github.com/docker/go-connections v0.4.0 // indirect
        github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
        github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 // indirect
        github.com/elazarl/goproxy/ext v0.0.0-20191011121108-aa519ddbe484 // indirect
@@ -19,9 +22,8 @@ require (
        github.com/gogo/protobuf v1.3.1 // indirect
        github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
        github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 // indirect
-       github.com/golang/protobuf v1.3.5
+       github.com/golang/protobuf v1.4.1
        github.com/google/btree v1.0.0 // indirect
-       github.com/google/go-cmp v0.3.1 // indirect
        github.com/gophercloud/gophercloud v0.2.0 // indirect
        github.com/gorilla/mux v1.7.4
        github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
@@ -34,6 +36,8 @@ require (
        github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
        github.com/onsi/ginkgo v1.11.0
        github.com/onsi/gomega v1.8.1
+       github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+       github.com/opencontainers/image-spec v1.0.1 // indirect
        github.com/phpdave11/gofpdi v1.0.8 // indirect
        github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
        github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31 // indirect
@@ -59,6 +63,7 @@ require (
        gonum.org/v1/plot v0.0.0-20191107103940-ca91d9d40d0a // indirect
        google.golang.org/genproto v0.0.0-20200325114520-5b2d0af7952b // indirect
        google.golang.org/grpc v1.28.0
+       google.golang.org/protobuf v1.22.0
        gopkg.in/gcfg.v1 v1.2.3
        gopkg.in/warnings.v0 v0.1.2 // indirect
        gopkg.in/yaml.v2 v2.2.7 // indirect
diff --git a/go.sum b/go.sum
index 4fee0cb..a49ac00 100644 (file)
--- a/go.sum
+++ b/go.sum
@@ -120,8 +120,14 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v0.0.0-20180612054059-a9fbbdc8dd87/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
 github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
@@ -280,6 +286,13 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
 github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1 h1:ZFgWrT+bLgsYPirOnRfKLYJLvssAegOj/hgyMFdJZe0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
@@ -289,6 +302,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
@@ -449,7 +464,10 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/openshift/origin v0.0.0-20160503220234-8f127d736703/go.mod h1:0Rox5r9C8aQn6j1oAOQ0c1uC86mYbUFObzjBRvUKHII=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
@@ -848,6 +866,13 @@ google.golang.org/grpc v1.27.0 h1:rRYRFMVgRv6E0D70Skyfsr28tDXIuuPZyWGMPdMcnXg=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
 google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
index 750d55b..70628c6 100644 (file)
@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: nfn.proto
+// source: proto/nfn.proto
 
 package nfn
 
@@ -35,7 +35,7 @@ func (m *SubscribeContext) Reset()         { *m = SubscribeContext{} }
 func (m *SubscribeContext) String() string { return proto.CompactTextString(m) }
 func (*SubscribeContext) ProtoMessage()    {}
 func (*SubscribeContext) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{0}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{0}
 }
 
 func (m *SubscribeContext) XXX_Unmarshal(b []byte) error {
@@ -69,6 +69,8 @@ type Notification struct {
        //      *Notification_InSync
        //      *Notification_ProviderNwCreate
        //      *Notification_ProviderNwRemove
+       //      *Notification_ContainterRtInsert
+       //      *Notification_ContainterRtRemove
        Payload              isNotification_Payload `protobuf_oneof:"payload"`
        XXX_NoUnkeyedLiteral struct{}               `json:"-"`
        XXX_unrecognized     []byte                 `json:"-"`
@@ -79,7 +81,7 @@ func (m *Notification) Reset()         { *m = Notification{} }
 func (m *Notification) String() string { return proto.CompactTextString(m) }
 func (*Notification) ProtoMessage()    {}
 func (*Notification) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{1}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{1}
 }
 
 func (m *Notification) XXX_Unmarshal(b []byte) error {
@@ -123,12 +125,24 @@ type Notification_ProviderNwRemove struct {
        ProviderNwRemove *ProviderNetworkRemove `protobuf:"bytes,4,opt,name=provider_nw_remove,json=providerNwRemove,proto3,oneof"`
 }
 
+type Notification_ContainterRtInsert struct {
+       ContainterRtInsert *ContainerRouteInsert `protobuf:"bytes,5,opt,name=containter_rt_insert,json=containterRtInsert,proto3,oneof"`
+}
+
+type Notification_ContainterRtRemove struct {
+       ContainterRtRemove *ContainerRouteRemove `protobuf:"bytes,6,opt,name=containter_rt_remove,json=containterRtRemove,proto3,oneof"`
+}
+
 func (*Notification_InSync) isNotification_Payload() {}
 
 func (*Notification_ProviderNwCreate) isNotification_Payload() {}
 
 func (*Notification_ProviderNwRemove) isNotification_Payload() {}
 
+func (*Notification_ContainterRtInsert) isNotification_Payload() {}
+
+func (*Notification_ContainterRtRemove) isNotification_Payload() {}
+
 func (m *Notification) GetPayload() isNotification_Payload {
        if m != nil {
                return m.Payload
@@ -157,12 +171,28 @@ func (m *Notification) GetProviderNwRemove() *ProviderNetworkRemove {
        return nil
 }
 
+func (m *Notification) GetContainterRtInsert() *ContainerRouteInsert {
+       if x, ok := m.GetPayload().(*Notification_ContainterRtInsert); ok {
+               return x.ContainterRtInsert
+       }
+       return nil
+}
+
+func (m *Notification) GetContainterRtRemove() *ContainerRouteRemove {
+       if x, ok := m.GetPayload().(*Notification_ContainterRtRemove); ok {
+               return x.ContainterRtRemove
+       }
+       return nil
+}
+
 // XXX_OneofWrappers is for the internal use of the proto package.
 func (*Notification) XXX_OneofWrappers() []interface{} {
        return []interface{}{
                (*Notification_InSync)(nil),
                (*Notification_ProviderNwCreate)(nil),
                (*Notification_ProviderNwRemove)(nil),
+               (*Notification_ContainterRtInsert)(nil),
+               (*Notification_ContainterRtRemove)(nil),
        }
 }
 
@@ -179,7 +209,7 @@ func (m *ProviderNetworkCreate) Reset()         { *m = ProviderNetworkCreate{} }
 func (m *ProviderNetworkCreate) String() string { return proto.CompactTextString(m) }
 func (*ProviderNetworkCreate) ProtoMessage()    {}
 func (*ProviderNetworkCreate) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{2}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{2}
 }
 
 func (m *ProviderNetworkCreate) XXX_Unmarshal(b []byte) error {
@@ -234,7 +264,7 @@ func (m *ProviderNetworkRemove) Reset()         { *m = ProviderNetworkRemove{} }
 func (m *ProviderNetworkRemove) String() string { return proto.CompactTextString(m) }
 func (*ProviderNetworkRemove) ProtoMessage()    {}
 func (*ProviderNetworkRemove) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{3}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{3}
 }
 
 func (m *ProviderNetworkRemove) XXX_Unmarshal(b []byte) error {
@@ -289,7 +319,7 @@ func (m *VlanInfo) Reset()         { *m = VlanInfo{} }
 func (m *VlanInfo) String() string { return proto.CompactTextString(m) }
 func (*VlanInfo) ProtoMessage()    {}
 func (*VlanInfo) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{4}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{4}
 }
 
 func (m *VlanInfo) XXX_Unmarshal(b []byte) error {
@@ -342,7 +372,7 @@ func (m *DirectInfo) Reset()         { *m = DirectInfo{} }
 func (m *DirectInfo) String() string { return proto.CompactTextString(m) }
 func (*DirectInfo) ProtoMessage()    {}
 func (*DirectInfo) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{5}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{5}
 }
 
 func (m *DirectInfo) XXX_Unmarshal(b []byte) error {
@@ -370,6 +400,147 @@ func (m *DirectInfo) GetProviderIntf() string {
        return ""
 }
 
+type RouteData struct {
+       Dst                  string   `protobuf:"bytes,2,opt,name=dst,proto3" json:"dst,omitempty"`
+       Gw                   string   `protobuf:"bytes,3,opt,name=gw,proto3" json:"gw,omitempty"`
+       XXX_NoUnkeyedLiteral struct{} `json:"-"`
+       XXX_unrecognized     []byte   `json:"-"`
+       XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *RouteData) Reset()         { *m = RouteData{} }
+func (m *RouteData) String() string { return proto.CompactTextString(m) }
+func (*RouteData) ProtoMessage()    {}
+func (*RouteData) Descriptor() ([]byte, []int) {
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{6}
+}
+
+func (m *RouteData) XXX_Unmarshal(b []byte) error {
+       return xxx_messageInfo_RouteData.Unmarshal(m, b)
+}
+func (m *RouteData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+       return xxx_messageInfo_RouteData.Marshal(b, m, deterministic)
+}
+func (m *RouteData) XXX_Merge(src proto.Message) {
+       xxx_messageInfo_RouteData.Merge(m, src)
+}
+func (m *RouteData) XXX_Size() int {
+       return xxx_messageInfo_RouteData.Size(m)
+}
+func (m *RouteData) XXX_DiscardUnknown() {
+       xxx_messageInfo_RouteData.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RouteData proto.InternalMessageInfo
+
+func (m *RouteData) GetDst() string {
+       if m != nil {
+               return m.Dst
+       }
+       return ""
+}
+
+func (m *RouteData) GetGw() string {
+       if m != nil {
+               return m.Gw
+       }
+       return ""
+}
+
+type ContainerRouteInsert struct {
+       ContainerId          string       `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"`
+       Route                []*RouteData `protobuf:"bytes,2,rep,name=route,proto3" json:"route,omitempty"`
+       XXX_NoUnkeyedLiteral struct{}     `json:"-"`
+       XXX_unrecognized     []byte       `json:"-"`
+       XXX_sizecache        int32        `json:"-"`
+}
+
+func (m *ContainerRouteInsert) Reset()         { *m = ContainerRouteInsert{} }
+func (m *ContainerRouteInsert) String() string { return proto.CompactTextString(m) }
+func (*ContainerRouteInsert) ProtoMessage()    {}
+func (*ContainerRouteInsert) Descriptor() ([]byte, []int) {
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{7}
+}
+
+func (m *ContainerRouteInsert) XXX_Unmarshal(b []byte) error {
+       return xxx_messageInfo_ContainerRouteInsert.Unmarshal(m, b)
+}
+func (m *ContainerRouteInsert) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+       return xxx_messageInfo_ContainerRouteInsert.Marshal(b, m, deterministic)
+}
+func (m *ContainerRouteInsert) XXX_Merge(src proto.Message) {
+       xxx_messageInfo_ContainerRouteInsert.Merge(m, src)
+}
+func (m *ContainerRouteInsert) XXX_Size() int {
+       return xxx_messageInfo_ContainerRouteInsert.Size(m)
+}
+func (m *ContainerRouteInsert) XXX_DiscardUnknown() {
+       xxx_messageInfo_ContainerRouteInsert.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ContainerRouteInsert proto.InternalMessageInfo
+
+func (m *ContainerRouteInsert) GetContainerId() string {
+       if m != nil {
+               return m.ContainerId
+       }
+       return ""
+}
+
+func (m *ContainerRouteInsert) GetRoute() []*RouteData {
+       if m != nil {
+               return m.Route
+       }
+       return nil
+}
+
+type ContainerRouteRemove struct {
+       ContainerId          string       `protobuf:"bytes,1,opt,name=container_id,json=containerId,proto3" json:"container_id,omitempty"`
+       Route                []*RouteData `protobuf:"bytes,2,rep,name=route,proto3" json:"route,omitempty"`
+       XXX_NoUnkeyedLiteral struct{}     `json:"-"`
+       XXX_unrecognized     []byte       `json:"-"`
+       XXX_sizecache        int32        `json:"-"`
+}
+
+func (m *ContainerRouteRemove) Reset()         { *m = ContainerRouteRemove{} }
+func (m *ContainerRouteRemove) String() string { return proto.CompactTextString(m) }
+func (*ContainerRouteRemove) ProtoMessage()    {}
+func (*ContainerRouteRemove) Descriptor() ([]byte, []int) {
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{8}
+}
+
+func (m *ContainerRouteRemove) XXX_Unmarshal(b []byte) error {
+       return xxx_messageInfo_ContainerRouteRemove.Unmarshal(m, b)
+}
+func (m *ContainerRouteRemove) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+       return xxx_messageInfo_ContainerRouteRemove.Marshal(b, m, deterministic)
+}
+func (m *ContainerRouteRemove) XXX_Merge(src proto.Message) {
+       xxx_messageInfo_ContainerRouteRemove.Merge(m, src)
+}
+func (m *ContainerRouteRemove) XXX_Size() int {
+       return xxx_messageInfo_ContainerRouteRemove.Size(m)
+}
+func (m *ContainerRouteRemove) XXX_DiscardUnknown() {
+       xxx_messageInfo_ContainerRouteRemove.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ContainerRouteRemove proto.InternalMessageInfo
+
+func (m *ContainerRouteRemove) GetContainerId() string {
+       if m != nil {
+               return m.ContainerId
+       }
+       return ""
+}
+
+func (m *ContainerRouteRemove) GetRoute() []*RouteData {
+       if m != nil {
+               return m.Route
+       }
+       return nil
+}
+
 type InSync struct {
        NodeIntfIpAddress    string   `protobuf:"bytes,1,opt,name=node_intf_ip_address,json=nodeIntfIpAddress,proto3" json:"node_intf_ip_address,omitempty"`
        NodeIntfMacAddress   string   `protobuf:"bytes,2,opt,name=node_intf_mac_address,json=nodeIntfMacAddress,proto3" json:"node_intf_mac_address,omitempty"`
@@ -382,7 +553,7 @@ func (m *InSync) Reset()         { *m = InSync{} }
 func (m *InSync) String() string { return proto.CompactTextString(m) }
 func (*InSync) ProtoMessage()    {}
 func (*InSync) Descriptor() ([]byte, []int) {
-       return fileDescriptor_5b809db4a7814953, []int{6}
+       return fileDescriptor_5ee04cc9cbb38bc3, []int{9}
 }
 
 func (m *InSync) XXX_Unmarshal(b []byte) error {
@@ -424,45 +595,56 @@ func init() {
        proto.RegisterType((*ProviderNetworkRemove)(nil), "ProviderNetworkRemove")
        proto.RegisterType((*VlanInfo)(nil), "VlanInfo")
        proto.RegisterType((*DirectInfo)(nil), "DirectInfo")
+       proto.RegisterType((*RouteData)(nil), "RouteData")
+       proto.RegisterType((*ContainerRouteInsert)(nil), "ContainerRouteInsert")
+       proto.RegisterType((*ContainerRouteRemove)(nil), "ContainerRouteRemove")
        proto.RegisterType((*InSync)(nil), "InSync")
 }
 
 func init() {
-       proto.RegisterFile("nfn.proto", fileDescriptor_5b809db4a7814953)
-}
-
-var fileDescriptor_5b809db4a7814953 = []byte{
-       // 472 bytes of a gzipped FileDescriptorProto
-       0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xdd, 0x6e, 0xd3, 0x30,
-       0x14, 0x5e, 0xd8, 0xd4, 0x34, 0xa7, 0x1d, 0xb4, 0x47, 0x1b, 0x14, 0x10, 0xd2, 0xc8, 0x6e, 0x2a,
-       0x2e, 0xd2, 0x6d, 0xdc, 0x72, 0x03, 0x43, 0x68, 0x91, 0xa0, 0x42, 0x19, 0xe2, 0xd6, 0x72, 0x1d,
-       0x07, 0x59, 0x4b, 0x8f, 0xa3, 0xd4, 0xb4, 0xe4, 0x01, 0x78, 0x0e, 0x5e, 0x8f, 0xc7, 0x40, 0xb1,
-       0x93, 0xfe, 0xac, 0xbb, 0xe1, 0x2e, 0x39, 0xdf, 0xf9, 0x7e, 0x6c, 0xf9, 0x83, 0x80, 0x32, 0x8a,
-       0x8a, 0x52, 0x1b, 0x1d, 0x4e, 0x60, 0x70, 0xfb, 0x73, 0xb6, 0x10, 0xa5, 0x9a, 0xc9, 0x6b, 0x4d,
-       0x46, 0xfe, 0x32, 0xf8, 0x12, 0x02, 0xd2, 0xa9, 0x64, 0xc4, 0xe7, 0x72, 0xe4, 0x9d, 0x79, 0xe3,
-       0x20, 0xe9, 0xd6, 0x83, 0x29, 0x9f, 0xcb, 0xf0, 0xaf, 0x07, 0xfd, 0xa9, 0x36, 0x2a, 0x53, 0x82,
-       0x1b, 0xa5, 0x09, 0x9f, 0x43, 0x57, 0x90, 0x62, 0xa6, 0x2a, 0xda, 0x65, 0x5f, 0x90, 0xfa, 0x56,
-       0x15, 0x12, 0x43, 0xf0, 0x15, 0xb1, 0x45, 0x45, 0x62, 0xf4, 0xe8, 0xcc, 0x1b, 0xf7, 0xae, 0xfc,
-       0x28, 0xa6, 0xdb, 0x8a, 0xc4, 0xcd, 0x41, 0xd2, 0x51, 0xf6, 0x0b, 0x3f, 0x01, 0x16, 0xa5, 0x5e,
-       0xaa, 0x54, 0x96, 0x8c, 0x56, 0x4c, 0x94, 0x92, 0x1b, 0x39, 0x3a, 0xb4, 0xeb, 0x4f, 0xa3, 0xaf,
-       0x0d, 0x34, 0x95, 0x66, 0xa5, 0xcb, 0xbb, 0x6b, 0x8b, 0xde, 0x1c, 0x24, 0x83, 0x96, 0x33, 0x5d,
-       0xb9, 0xd9, 0x7d, 0x9d, 0x52, 0xce, 0xf5, 0x52, 0x8e, 0x8e, 0x1e, 0xd6, 0x49, 0x2c, 0xba, 0xab,
-       0xe3, 0x66, 0x1f, 0x02, 0xf0, 0x0b, 0x5e, 0xe5, 0x9a, 0xa7, 0xe1, 0x6f, 0x0f, 0x4e, 0x1f, 0x0c,
-       0x80, 0x63, 0x18, 0x6c, 0x9b, 0x6d, 0x5d, 0xd4, 0xe3, 0x8d, 0x60, 0x7d, 0x5d, 0xf8, 0x0a, 0x8e,
-       0x96, 0x39, 0xa7, 0xe6, 0xfc, 0x41, 0xf4, 0x3d, 0xe7, 0x14, 0x53, 0xa6, 0x13, 0x3b, 0xc6, 0x73,
-       0xe8, 0xa4, 0xaa, 0x94, 0xc2, 0x34, 0x27, 0xee, 0x45, 0x1f, 0xed, 0xaf, 0x5d, 0x69, 0xa0, 0xf0,
-       0xcf, 0x7e, 0x0e, 0x17, 0xf6, 0x3f, 0x72, 0xbc, 0x81, 0x61, 0x6d, 0xc8, 0x72, 0xfd, 0x43, 0x09,
-       0x9e, 0x33, 0x45, 0x26, 0xb3, 0xa1, 0x82, 0xe4, 0x49, 0x0d, 0x7c, 0x76, 0xf3, 0x98, 0x4c, 0x86,
-       0x17, 0x70, 0xe2, 0x9c, 0xd9, 0x5a, 0xdc, 0xae, 0x1f, 0xda, 0x75, 0x74, 0x58, 0x1b, 0xa8, 0x66,
-       0x84, 0x77, 0xd0, 0x6d, 0x0f, 0x86, 0xcf, 0xc0, 0xb7, 0x4e, 0x2a, 0x6d, 0xa2, 0x74, 0xea, 0xdf,
-       0x38, 0xc5, 0x73, 0x38, 0xde, 0xd5, 0x73, 0xf6, 0xfd, 0x62, 0x4b, 0x09, 0x5f, 0x43, 0x7f, 0x27,
-       0xa2, 0xf3, 0xec, 0xe5, 0x9b, 0x78, 0xe1, 0x25, 0xc0, 0xe6, 0x92, 0xf6, 0x55, 0xbd, 0x7d, 0xd5,
-       0x30, 0x87, 0x8e, 0x7b, 0x78, 0x38, 0x81, 0x13, 0xfb, 0xb6, 0xeb, 0x55, 0xa6, 0x0a, 0xc6, 0xd3,
-       0xb4, 0x94, 0x8b, 0x45, 0xc3, 0x1a, 0xd6, 0x58, 0xcd, 0x88, 0x8b, 0xf7, 0x0e, 0xc0, 0x4b, 0x38,
-       0xdd, 0x10, 0xe6, 0x5c, 0xac, 0x19, 0x2e, 0x3d, 0xb6, 0x8c, 0x2f, 0x5c, 0x34, 0x94, 0xab, 0x77,
-       0xb6, 0x60, 0xb6, 0x24, 0x15, 0x4e, 0x20, 0x58, 0x17, 0x0c, 0x87, 0xd1, 0xfd, 0xb2, 0xbd, 0x38,
-       0x8e, 0xb6, 0xdb, 0x74, 0xe1, 0xcd, 0x3a, 0xb6, 0x98, 0x6f, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff,
-       0xad, 0x54, 0x7e, 0xa2, 0xa5, 0x03, 0x00, 0x00,
+       proto.RegisterFile("proto/nfn.proto", fileDescriptor_5ee04cc9cbb38bc3)
+}
+
+var fileDescriptor_5ee04cc9cbb38bc3 = []byte{
+       // 601 bytes of a gzipped FileDescriptorProto
+       0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0x4f, 0x6f, 0xd3, 0x4e,
+       0x10, 0x6d, 0x92, 0x36, 0xa9, 0x27, 0xfd, 0x93, 0xae, 0xd2, 0xdf, 0x2f, 0x80, 0x90, 0x82, 0x7b,
+       0x89, 0x90, 0x70, 0xda, 0x72, 0xe5, 0x02, 0xad, 0x50, 0x2d, 0x41, 0x84, 0x5c, 0xc4, 0x85, 0xc3,
+       0x6a, 0xbb, 0xde, 0x54, 0xab, 0x3a, 0xbb, 0xd6, 0x66, 0x9b, 0xe0, 0x0f, 0xc0, 0xe7, 0xe0, 0x9b,
+       0x22, 0xb4, 0x7f, 0x6c, 0x27, 0x4d, 0x0e, 0x1c, 0xb8, 0xed, 0xbe, 0x37, 0xf3, 0xe6, 0xed, 0x8c,
+       0x3d, 0x70, 0x9c, 0x2b, 0xa9, 0xe5, 0x58, 0x4c, 0x45, 0x64, 0x4f, 0xe1, 0x18, 0x7a, 0xb7, 0x8f,
+       0x77, 0x73, 0xaa, 0xf8, 0x1d, 0xbb, 0x92, 0x42, 0xb3, 0x1f, 0x1a, 0xbd, 0x80, 0x40, 0xc8, 0x94,
+       0x61, 0x41, 0x66, 0x6c, 0xd0, 0x18, 0x36, 0x46, 0x41, 0xb2, 0x6f, 0x80, 0x09, 0x99, 0xb1, 0xf0,
+       0x77, 0x13, 0x0e, 0x26, 0x52, 0xf3, 0x29, 0xa7, 0x44, 0x73, 0x29, 0xd0, 0x33, 0xd8, 0xa7, 0x82,
+       0x63, 0x5d, 0xe4, 0x65, 0x70, 0x87, 0x0a, 0xfe, 0xb5, 0xc8, 0x19, 0x0a, 0xa1, 0xc3, 0x05, 0x9e,
+       0x17, 0x82, 0x0e, 0x9a, 0xc3, 0xc6, 0xa8, 0x7b, 0xd9, 0x89, 0x62, 0x71, 0x5b, 0x08, 0x7a, 0xb3,
+       0x93, 0xb4, 0xb9, 0x3d, 0xa1, 0x8f, 0x80, 0x72, 0x25, 0x17, 0x3c, 0x65, 0x0a, 0x8b, 0x25, 0xa6,
+       0x8a, 0x11, 0xcd, 0x06, 0x2d, 0x1b, 0xfe, 0x5f, 0xf4, 0xc5, 0x53, 0x13, 0xa6, 0x97, 0x52, 0x3d,
+       0x5c, 0x59, 0xf6, 0x66, 0x27, 0xe9, 0x95, 0x39, 0x93, 0xa5, 0xc3, 0x9e, 0xea, 0x28, 0x36, 0x93,
+       0x0b, 0x36, 0xd8, 0xdd, 0xae, 0x93, 0x58, 0x76, 0x5d, 0xc7, 0x61, 0x28, 0x86, 0x3e, 0x95, 0x42,
+       0x13, 0x2e, 0x34, 0x53, 0x58, 0x69, 0xcc, 0xc5, 0x9c, 0x29, 0x3d, 0xd8, 0xb3, 0x4a, 0xa7, 0xd1,
+       0x95, 0x23, 0x99, 0x4a, 0xe4, 0xa3, 0x66, 0xb1, 0x25, 0x6f, 0x76, 0x12, 0x54, 0x27, 0x25, 0xda,
+       0xa1, 0x9b, 0x52, 0xde, 0x54, 0x7b, 0xab, 0x54, 0xe5, 0x69, 0x4d, 0xca, 0xa1, 0x1f, 0x02, 0xe8,
+       0xe4, 0xa4, 0xc8, 0x24, 0x49, 0xc3, 0x9f, 0x0d, 0x38, 0xdd, 0xda, 0x16, 0x34, 0x82, 0xde, 0x6a,
+       0x0b, 0x56, 0xc6, 0x77, 0x54, 0x3f, 0xd3, 0x0c, 0x11, 0xbd, 0x84, 0xdd, 0x45, 0x46, 0x84, 0x9f,
+       0x4a, 0x10, 0x7d, 0xcb, 0x88, 0x88, 0xc5, 0x54, 0x26, 0x16, 0x46, 0x67, 0xd0, 0x4e, 0xb9, 0x62,
+       0x54, 0xfb, 0x39, 0x74, 0xa3, 0x6b, 0x7b, 0xb5, 0x21, 0x9e, 0x0a, 0x7f, 0x6d, 0xfa, 0xf0, 0x2d,
+       0xfc, 0x7b, 0x1f, 0xaf, 0xe1, 0xc4, 0x14, 0xc4, 0x99, 0xbc, 0xe7, 0x94, 0x64, 0x98, 0x0b, 0x3d,
+       0xb5, 0xa6, 0x82, 0xe4, 0xd8, 0x10, 0x9f, 0x1c, 0x1e, 0x0b, 0x3d, 0x45, 0xe7, 0xd0, 0x77, 0x95,
+       0x71, 0x25, 0x6e, 0xc3, 0x5b, 0x36, 0x1c, 0x39, 0xae, 0x34, 0x64, 0x32, 0xc2, 0x07, 0xd8, 0x2f,
+       0x1f, 0x86, 0xfe, 0x87, 0x8e, 0xad, 0xc4, 0x53, 0x6f, 0xa5, 0x6d, 0xae, 0x71, 0x8a, 0xce, 0xe0,
+       0x70, 0x5d, 0xcf, 0x95, 0x3f, 0xc8, 0x57, 0x94, 0xd0, 0x2b, 0x38, 0x58, 0xb3, 0xe8, 0x6a, 0x76,
+       0xb3, 0xda, 0x5e, 0x78, 0x01, 0x50, 0x37, 0x69, 0x53, 0xb5, 0xb1, 0xa9, 0x1a, 0xbe, 0x81, 0xc0,
+       0x4e, 0xfe, 0x9a, 0x68, 0x82, 0x7a, 0xd0, 0x4a, 0xe7, 0xda, 0x57, 0x37, 0x47, 0x74, 0x04, 0xcd,
+       0xfb, 0xa5, 0x2f, 0xd5, 0xbc, 0x5f, 0x86, 0xdf, 0xa1, 0xbf, 0xed, 0xe3, 0x33, 0xe6, 0x68, 0x89,
+       0xd7, 0xef, 0xeb, 0x56, 0x58, 0x9c, 0xa2, 0x21, 0xec, 0x29, 0x93, 0x31, 0x68, 0x0e, 0x5b, 0xa3,
+       0xee, 0x25, 0x44, 0x55, 0xdd, 0xc4, 0x11, 0x9b, 0xe2, 0x7e, 0x96, 0xff, 0x44, 0x3c, 0x83, 0xb6,
+       0xfb, 0xef, 0xd1, 0x18, 0xfa, 0x76, 0xb5, 0x98, 0x9e, 0x60, 0x9e, 0x63, 0x92, 0xa6, 0x8a, 0xcd,
+       0xe7, 0x5e, 0xf6, 0xc4, 0x70, 0xa6, 0x35, 0x71, 0xfe, 0xde, 0x11, 0xe8, 0x02, 0x4e, 0xeb, 0x84,
+       0x19, 0xa1, 0x55, 0x86, 0x6b, 0x14, 0x2a, 0x33, 0x3e, 0x13, 0xea, 0x53, 0x2e, 0xdf, 0x41, 0x20,
+       0xa6, 0xc2, 0xee, 0xa8, 0x02, 0x8d, 0x21, 0xa8, 0xf6, 0x1b, 0x3a, 0x89, 0x9e, 0xee, 0xba, 0xe7,
+       0x87, 0xd1, 0xea, 0x32, 0x3b, 0x6f, 0xdc, 0xb5, 0xed, 0x5e, 0x7c, 0xfb, 0x27, 0x00, 0x00, 0xff,
+       0xff, 0x47, 0xfb, 0x68, 0x42, 0x2a, 0x05, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -569,5 +751,5 @@ var _NfnNotify_serviceDesc = grpc.ServiceDesc{
                        ServerStreams: true,
                },
        },
-       Metadata: "nfn.proto",
+       Metadata: "proto/nfn.proto",
 }
index 567df29..6bd6704 100644 (file)
@@ -19,6 +19,9 @@ message Notification {
         InSync in_sync = 2;
         ProviderNetworkCreate provider_nw_create = 3;
         ProviderNetworkRemove provider_nw_remove = 4;
+        ContainerRouteInsert containter_rt_insert = 5;
+        ContainerRouteRemove containter_rt_remove = 6;
+
     }
 }
 
@@ -46,6 +49,21 @@ message DirectInfo {
     string provider_intf = 1;
 }
 
+message RouteData {
+    string dst = 2;
+    string gw = 3;
+}
+
+message ContainerRouteInsert {
+    string container_id = 1;
+    repeated RouteData route = 2;
+}
+
+message ContainerRouteRemove {
+    string container_id = 1;
+    repeated RouteData route = 2;
+}
+
 message InSync {
     string node_intf_ip_address = 1;
     string node_intf_mac_address = 2;
index a201618..4257cf9 100644 (file)
@@ -1,9 +1,26 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package nfn
 
 import (
        "fmt"
        "net"
        pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
+       chaining "ovn4nfv-k8s-plugin/internal/pkg/utils"
        "ovn4nfv-k8s-plugin/internal/pkg/node"
        v1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1"
        clientset "ovn4nfv-k8s-plugin/pkg/generated/clientset/versioned"
@@ -269,6 +286,55 @@ func sendMsg(msg pb.Notification, labels string, option string, nodeReq string)
        return nil
 }
 
+//SendProviderNotif to client
+func SendRouteNotif(chainRoutingInfo []chaining.RoutingInfo, msgType string) error {
+       var msg pb.Notification
+       var err error
+       var ins pb.ContainerRouteInsert
+
+       for _, r := range chainRoutingInfo {
+               ins.ContainerId = r.Id
+               ins.Route = nil
+
+               rt := &pb.RouteData{
+                       Dst: r.LeftNetworkRoute.Dst,
+                       Gw:  r.LeftNetworkRoute.GW,
+               }
+               ins.Route = append(ins.Route, rt)
+
+               rt = &pb.RouteData{
+                       Dst: r.RightNetworkRoute.Dst,
+                       Gw:  r.RightNetworkRoute.GW,
+               }
+               ins.Route = append(ins.Route, rt)
+
+               for _, d := range r.DynamicNetworkRoutes {
+                       rt = &pb.RouteData{
+                               Dst: d.Dst,
+                               Gw:  d.GW,
+                       }
+                       ins.Route = append(ins.Route, rt)
+               }
+               if msgType == "create" {
+                       msg = pb.Notification{
+                               CniType: "ovn4nfv",
+                               Payload: &pb.Notification_ContainterRtInsert{
+                                       ContainterRtInsert: &ins,
+                               },
+                       }
+               }
+               client := notifServer.GetClient(r.Node)
+               if client.stream != nil {
+                       if err := client.stream.Send(&msg); err != nil {
+                               log.Error(err, "Failed to send msg", "Node", r.Node)
+                               return err
+                       }
+               }
+               // TODO: Handle Delete
+       }
+       return err
+}
+
 func nodeListIterator(labels string) <-chan string {
        ch := make(chan string)
 
index e38c440..0053e56 100644 (file)
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package ovn
 
 import (
@@ -323,3 +339,61 @@ func ipToInt(ip net.IP) *big.Int {
 func intToIP(i *big.Int) net.IP {
        return net.IP(i.Bytes())
 }
+
+// Get Subnet for a logical bridge
+func GetNetworkSubnet(nw string) (string, error) {
+        stdout, stderr, err := RunOVNNbctl("--if-exists",
+                "get", "logical_switch", nw,
+                "other_config:subnet")
+        if err != nil {
+                log.Error(err, "Failed to subnet for network", "stderr", stderr, "stdout", stdout)
+                return "", err
+        }
+        return stdout, nil
+}
+
+func GetIPAdressForPod(nw string, name string) (string, error) {
+        _, stderr, err := RunOVNNbctl("--data=bare", "--no-heading",
+                "--columns=name", "find", "logical_switch", "name="+nw)
+        if err != nil {
+                log.Error(err, "Error in obtaining list of logical switch", "stderr", stderr)
+                return "", err
+        }
+        stdout, stderr, err := RunOVNNbctl("lsp-list", nw)
+        if err != nil {
+                log.Error(err, "Failed to list ports", "stderr", stderr, "stdout", stdout)
+                return "", err
+        }
+        // stdout format
+        // <port-uuid> (<port-name>)
+        // <port-uuid> (<port-name>)
+        // ...
+        ll := strings.Split(stdout, "\n")
+        if len(ll) == 0 {
+                return "", fmt.Errorf("IPAdress Not Found")
+        }
+        for _, l := range ll {
+                pn := strings.Fields(l)
+                if len(pn) < 2 {
+                        return "", fmt.Errorf("IPAdress Not Found")
+                }
+                if strings.Contains(pn[1], name) {
+                        // Found Port
+                        s := strings.Replace(pn[1], "(", "", -1)
+                        s = strings.Replace(s, ")", "", -1)
+                        dna, stderr, err := RunOVNNbctl("get", "logical_switch_port", s, "dynamic_addresses")
+                        if err != nil {
+                                log.Error(err, "Failed to get dynamic_addresses", "stderr", stderr, "stdout", dna)
+                                return "", err
+                        }
+                        // format - mac:ip
+                        ipAddr := strings.Fields(dna)
+                        if len(ipAddr) < 2 {
+                                return "", fmt.Errorf("IPAdress Not Found")
+                        }
+                        return ipAddr[1], nil
+                }
+        }
+        return "", fmt.Errorf("IPAdress Not Found %s", name)
+}
+
diff --git a/internal/pkg/utils/chain.go b/internal/pkg/utils/chain.go
new file mode 100644 (file)
index 0000000..aa98aa1
--- /dev/null
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nfn
+
+import (
+       "context"
+       "fmt"
+       "ovn4nfv-k8s-plugin/internal/pkg/ovn"
+       k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1"
+       "strings"
+
+       pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
+
+       "github.com/containernetworking/plugins/pkg/ns"
+       "github.com/docker/docker/client"
+       v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+       "k8s.io/client-go/kubernetes"
+       "sigs.k8s.io/controller-runtime/pkg/client/config"
+       logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+)
+
+var log = logf.Log.WithName("chaining")
+
+type RoutingInfo struct {
+       Name                 string            // Name of the pod
+       Namespace            string            // Namespace of the Pod
+       Id                   string            // Container ID for pod
+       Node                 string            // Hostname where Pod is scheduled
+       LeftNetworkRoute     k8sv1alpha1.Route // TODO: Update to support multiple networks
+       RightNetworkRoute    k8sv1alpha1.Route // TODO: Update to support multiple networks
+       DynamicNetworkRoutes []k8sv1alpha1.Route
+}
+
+var chainRoutingInfo []RoutingInfo
+
+// Calcuate route to get to left and right edge networks and other networks (not adjacent) in the chain
+func calculateDeploymentRoutes(namespace, label string, pos int, num int, ln []k8sv1alpha1.RoutingNetwork, rn []k8sv1alpha1.RoutingNetwork, networkList, deploymentList []string) (r RoutingInfo, err error) {
+
+       var nextLeftIP string
+       var nextRightIP string
+
+       r.Namespace = namespace
+       // Get a config to talk to the apiserver
+       cfg, err := config.GetConfig()
+       if err != nil {
+               return RoutingInfo{}, err
+       }
+       var k *kubernetes.Clientset
+       k, err = kubernetes.NewForConfig(cfg)
+       if err != nil {
+               log.Error(err, "Error building Kuberenetes clientset")
+               return RoutingInfo{}, err
+       }
+       lo := v1.ListOptions{LabelSelector: label}
+       // List the Pods matching the Labels
+       pods, err := k.CoreV1().Pods(namespace).List(lo)
+       if err != nil {
+               log.Error(err, "Deloyment with label not found", "label", label)
+               return RoutingInfo{}, err
+       }
+       // LOADBALANCER NOT YET SUPPORTED - Assuming deployment has only one Pod
+       if len(pods.Items) <= 0 {
+               log.Error(err, "Deloyment with label not found", "label", label)
+               return RoutingInfo{}, fmt.Errorf("Pod not found")
+       }
+       // Get the containerID of the first container
+       r.Id = strings.TrimPrefix(pods.Items[0].Status.ContainerStatuses[0].ContainerID, "docker://")
+       r.Name = pods.Items[0].GetName()
+       r.Node = pods.Items[0].Spec.NodeName
+       // Calcluate IP addresses for next neighbours on both sides
+       if pos == 0 {
+               nextLeftIP = ln[0].GatewayIP
+       } else {
+               name := strings.Split(deploymentList[pos-1], "=")
+               nextLeftIP, err = ovn.GetIPAdressForPod(networkList[pos-1], name[1])
+               if err != nil {
+                       return RoutingInfo{}, err
+               }
+       }
+       if pos == num-1 {
+               nextRightIP = rn[0].GatewayIP
+       } else {
+               name := strings.Split(deploymentList[pos+1], "=")
+               nextRightIP, err = ovn.GetIPAdressForPod(networkList[pos], name[1])
+               if err != nil {
+                       return RoutingInfo{}, err
+               }
+       }
+       // Calcuate left right Route to be inserted in Pod
+       r.LeftNetworkRoute.Dst = ln[0].Subnet
+       r.LeftNetworkRoute.GW = nextLeftIP
+       r.RightNetworkRoute.Dst = rn[0].Subnet
+       r.RightNetworkRoute.GW = nextRightIP
+       // For each network that is not adjacent add route
+       for i := 0; i < len(networkList); i++ {
+               if i == pos || i == pos-1 {
+                       continue
+               } else {
+                       var rt k8sv1alpha1.Route
+                       rt.Dst, err = ovn.GetNetworkSubnet(networkList[i])
+                       if err != nil {
+                               return RoutingInfo{}, err
+                       }
+                       if i > pos {
+                               rt.GW = nextRightIP
+                       } else {
+                               rt.GW = nextLeftIP
+                       }
+                       r.DynamicNetworkRoutes = append(r.DynamicNetworkRoutes, rt)
+               }
+       }
+       return
+}
+
+func CalculateRoutes(cr *k8sv1alpha1.NetworkChaining) ([]RoutingInfo, error) {
+       //
+       var deploymentList []string
+       var networkList []string
+
+       // TODO: Add Validation of Input to this function
+       ln := cr.Spec.RoutingSpec.LeftNetwork
+       rn := cr.Spec.RoutingSpec.RightNetwork
+       chains := strings.Split(cr.Spec.RoutingSpec.NetworkChain, ",")
+       i := 0
+       for _, chain := range chains {
+               if i%2 == 0 {
+                       deploymentList = append(deploymentList, chain)
+               } else {
+                       networkList = append(networkList, chain)
+               }
+               i++
+       }
+       num := len(deploymentList)
+       for i, deployment := range deploymentList {
+               r, err := calculateDeploymentRoutes(cr.Namespace, deployment, i, num, ln, rn, networkList, deploymentList)
+               if err != nil {
+                       return nil, err
+               }
+               chainRoutingInfo = append(chainRoutingInfo, r)
+       }
+       return chainRoutingInfo, nil
+}
+
+func ContainerAddRoute(containerPid int, route []*pb.RouteData) error {
+
+       str := fmt.Sprintf("/host/proc/%d/ns/net", containerPid)
+
+       nms, err := ns.GetNS(str)
+       if err != nil {
+               log.Error(err, "Failed namesapce", "containerID", containerPid)
+               return err
+       }
+       defer nms.Close()
+       err = nms.Do(func(_ ns.NetNS) error {
+               for _, r := range route {
+                       dst := r.GetDst()
+                       gw := r.GetGw()
+                       stdout, stderr, err := ovn.RunIP("route", "add", dst, "via", gw)
+                       if err != nil && !strings.Contains(stderr, "RTNETLINK answers: File exists") {
+                               log.Error(err, "Failed to ip route add", "stdout", stdout, "stderr", stderr)
+                               return err
+                       }
+               }
+               return nil
+       })
+       if err != nil {
+               log.Error(err, "Failed Netns Do", "containerID", containerPid)
+               return err
+       }
+       return nil
+}
+
+func GetPidForContainer(id string) (int, error) {
+       cli, err := client.NewEnvClient()
+       if err != nil {
+               fmt.Println("Unable to create docker client")
+               return 0, err
+       }
+       cli.NegotiateAPIVersion(context.Background())
+       cj, err := cli.ContainerInspect(context.Background(), id)
+       if err != nil {
+               fmt.Println("Unable to Inspect docker container")
+               return 0, err
+       }
+       if cj.State.Pid == 0 {
+               return 0, fmt.Errorf("Container not found %s", id)
+       }
+       return cj.State.Pid, nil
+
+}
diff --git a/pkg/controller/add_networkchaining.go b/pkg/controller/add_networkchaining.go
new file mode 100644 (file)
index 0000000..ef54288
--- /dev/null
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controller
+
+import (
+       "ovn4nfv-k8s-plugin/pkg/controller/networkchaining"
+)
+
+func init() {
+       // AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
+       AddToManagerFuncs = append(AddToManagerFuncs, networkchaining.Add)
+}
diff --git a/pkg/controller/networkchaining/networkchaining_controller.go b/pkg/controller/networkchaining/networkchaining_controller.go
new file mode 100644 (file)
index 0000000..c0195f2
--- /dev/null
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2020 Intel Corporation, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package networkchaining
+
+import (
+       "fmt"
+       "context"
+       notif "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify"
+       chaining "ovn4nfv-k8s-plugin/internal/pkg/utils"
+       k8sv1alpha1 "ovn4nfv-k8s-plugin/pkg/apis/k8s/v1alpha1"
+       "ovn4nfv-k8s-plugin/pkg/utils"
+       "k8s.io/apimachinery/pkg/api/errors"
+       "k8s.io/apimachinery/pkg/runtime"
+       "sigs.k8s.io/controller-runtime/pkg/client"
+       "sigs.k8s.io/controller-runtime/pkg/controller"
+       "sigs.k8s.io/controller-runtime/pkg/handler"
+       "github.com/go-logr/logr"
+       logf "sigs.k8s.io/controller-runtime/pkg/log"
+       "sigs.k8s.io/controller-runtime/pkg/manager"
+       "sigs.k8s.io/controller-runtime/pkg/reconcile"
+       "sigs.k8s.io/controller-runtime/pkg/source"
+)
+
+var log = logf.Log.WithName("controller_networkchaining")
+
+// Add creates a new NetworkChaining Controller and adds it to the Manager. The Manager will set fields on the Controller
+// and Start it when the Manager is Started.
+func Add(mgr manager.Manager) error {
+       return add(mgr, newReconciler(mgr))
+}
+
+// newReconciler returns a new reconcile.Reconciler
+func newReconciler(mgr manager.Manager) reconcile.Reconciler {
+       return &ReconcileNetworkChaining{client: mgr.GetClient(), scheme: mgr.GetScheme()}
+}
+
+// add adds a new Controller to mgr with r as the reconcile.Reconciler
+func add(mgr manager.Manager, r reconcile.Reconciler) error {
+       // Create a new controller
+       c, err := controller.New("networkchaining-controller", mgr, controller.Options{Reconciler: r})
+       if err != nil {
+               return err
+       }
+
+       // Watch for changes to primary resource NetworkChaining
+       err = c.Watch(&source.Kind{Type: &k8sv1alpha1.NetworkChaining{}}, &handler.EnqueueRequestForObject{})
+       if err != nil {
+               return err
+       }
+       return nil
+}
+
+// blank assignment to verify that ReconcileNetworkChaining implements reconcile.Reconciler
+var _ reconcile.Reconciler = &ReconcileNetworkChaining{}
+
+// ReconcileNetworkChaining reconciles a NetworkChaining object
+type ReconcileNetworkChaining struct {
+       // This client, initialized using mgr.Client() above, is a split client
+       // that reads objects from the cache and writes to the apiserver
+       client client.Client
+       scheme *runtime.Scheme
+}
+type reconcileFun func(instance *k8sv1alpha1.NetworkChaining, reqLogger logr.Logger) error
+// Reconcile reads that state of the cluster for a NetworkChaining object and makes changes based on the state read
+// and what is in the NetworkChaining.Spec
+// TODO(user): Modify this Reconcile function to implement your Controller logic.  This example creates
+// a Pod as an example
+// Note:
+// The Controller will requeue the Request to be processed again if the returned error is non-nil or
+// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
+func (r *ReconcileNetworkChaining) Reconcile(request reconcile.Request) (reconcile.Result, error) {
+       reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
+       reqLogger.Info("Reconciling NetworkChaining")
+
+       // Fetch the NetworkChaining instance
+       instance := &k8sv1alpha1.NetworkChaining{}
+       err := r.client.Get(context.TODO(), request.NamespacedName, instance)
+       if err != nil {
+               if errors.IsNotFound(err) {
+                       // Request object not found, could have been deleted after reconcile request.
+                       // Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
+                       // Return and don't requeue
+                       return reconcile.Result{}, nil
+               }
+               // Error reading the object - requeue the request.
+               return reconcile.Result{}, err
+       }
+       for _, fun := range []reconcileFun{
+               r.reconcileFinalizers,
+               r.createChain,
+       } {
+               if err = fun(instance, reqLogger); err != nil {
+                       return reconcile.Result{}, err
+               }
+       }
+
+
+       return reconcile.Result{}, nil
+}
+const (
+       nfnNetworkChainFinalizer = "nfnCleanUpNetworkChain"
+)
+
+func (r *ReconcileNetworkChaining) createChain(cr *k8sv1alpha1.NetworkChaining, reqLogger logr.Logger) error {
+
+       if !cr.DeletionTimestamp.IsZero() {
+               // Marked for deletion
+               return nil
+       }
+       switch {
+       case cr.Spec.ChainType == "Routing":
+               routeList, err := chaining.CalculateRoutes(cr)
+               if err != nil {
+                       return err
+               }
+               err = notif.SendRouteNotif(routeList, "create")
+               if err != nil {
+                       cr.Status.State = k8sv1alpha1.CreateInternalError
+                       reqLogger.Error(err, "Error Sending Message")
+               } else {
+                       cr.Status.State = k8sv1alpha1.Created
+               }
+
+               err = r.client.Status().Update(context.TODO(), cr)
+               if err != nil {
+                       return err
+               }
+               return nil
+       // Add other Chaining types here
+       }
+       reqLogger.Info("Chaining type not supported", "name", cr.Spec.ChainType)
+       return fmt.Errorf("Chaining type not supported")
+}
+
+func (r *ReconcileNetworkChaining) deleteChain(cr *k8sv1alpha1.NetworkChaining, reqLogger logr.Logger) error {
+
+       reqLogger.Info("Delete Chain not implementated")
+       return fmt.Errorf("Not Implemented")
+}
+
+func (r *ReconcileNetworkChaining) reconcileFinalizers(instance *k8sv1alpha1.NetworkChaining, reqLogger logr.Logger) (err error) {
+
+       if !instance.DeletionTimestamp.IsZero() {
+               // Instance marked for deletion
+               if utils.Contains(instance.ObjectMeta.Finalizers, nfnNetworkChainFinalizer) {
+                       reqLogger.V(1).Info("Finalizer found - delete chain")
+                       if err = r.deleteChain(instance, reqLogger); err != nil {
+                               reqLogger.Error(err, "Delete chain")
+                       }
+                       // Remove the finalizer even if Delete Network fails. Fatal error retry will not resolve
+                       instance.ObjectMeta.Finalizers = utils.Remove(instance.ObjectMeta.Finalizers, nfnNetworkChainFinalizer)
+                       if err = r.client.Update(context.TODO(), instance); err != nil {
+                               reqLogger.Error(err, "Removing Finalizer")
+                               return err
+                       }
+               }
+
+       } else {
+               // If finalizer doesn't exist add it
+               if !utils.Contains(instance.GetFinalizers(), nfnNetworkChainFinalizer) {
+                       instance.SetFinalizers(append(instance.GetFinalizers(), nfnNetworkChainFinalizer))
+                       if err = r.client.Update(context.TODO(), instance); err != nil {
+                               reqLogger.Error(err, "Adding Finalizer")
+                               return err
+                       }
+                       reqLogger.V(1).Info("Finalizer added")
+               }
+       }
+       return nil
+}