Add support for native Kubernetes CPU Manager 86/71986/1 0.0.1
authorYury Kylulin <yury.kylulin@intel.com>
Thu, 11 Feb 2021 13:58:39 +0000 (16:58 +0300)
committerYury Kylulin <yury.kylulin@intel.com>
Thu, 11 Feb 2021 14:04:36 +0000 (17:04 +0300)
For Kubernetes environment core ids in the config and test files are relative and
automatically remapped to the allowed cores allocated for container.

There is no change to the baremetal or VM environment.

Signed-off-by: Yury Kylulin <yury.kylulin@intel.com>
Change-Id: I63e499723e8213de1b05d4175eb3eddc4492ccf5

33 files changed:
VNFs/DPPD-PROX/helper-scripts/rapid/configs/gen.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/gen_gw.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/genv6.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/impair.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/irq.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2gen.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2gen_bare.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/l2swap.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/secgw1.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/secgw2.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/swap.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/configs/swapv6.cfg
VNFs/DPPD-PROX/helper-scripts/rapid/pod-rapid.yaml
VNFs/DPPD-PROX/helper-scripts/rapid/rapid.pods
VNFs/DPPD-PROX/helper-scripts/rapid/rapid_generator_machine.py
VNFs/DPPD-PROX/helper-scripts/rapid/rapid_machine.py
VNFs/DPPD-PROX/helper-scripts/rapid/rapid_parser.py
VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput_64B_64F.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009_Throughput_acaeab_16384F.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/TST009ipV6.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/bare.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/basicrapid.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/corestats.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/impair.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/increment_till_fail.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/ipv6.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/irq.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/l2framerate.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/l2zeroloss.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/l3framerate.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/portstats.test
VNFs/DPPD-PROX/helper-scripts/rapid/tests/secgw.test

index 1827395..d0ccc13 100644 (file)
@@ -41,7 +41,7 @@ mempool size=8K
 name=${name}
 heartbeat timeout=${heartbeat}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]
index fc3b6a6..2843d35 100644 (file)
@@ -40,7 +40,7 @@ mempool size=8K
 name=${name}
 heartbeat timeout=${heartbeat}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]
index 650e284..32fadbc 100644 (file)
@@ -38,7 +38,7 @@ mempool size=8K
 name=${name}
 heartbeat timeout=${heartbeat}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]
index 95d6516..3eaf80e 100644 (file)
@@ -31,14 +31,13 @@ vlan=yes
 vdev=impair_tap
 local ipv4=${local_ip1}
 
-
 [defaults]
 mempool size=8K
 
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index 4e9af96..0f26e6e 100644 (file)
@@ -34,7 +34,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index e094221..3af0ac9 100644 (file)
@@ -37,7 +37,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $gencores]
index 1739638..0ce3a1a 100644 (file)
@@ -35,7 +35,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index 1897bbd..d941e5e 100644 (file)
@@ -41,7 +41,7 @@ mempool size=16K
 start time=20
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index 2fe3291..9aedc85 100644 (file)
@@ -41,7 +41,7 @@ mempool size=16K
 start time=20
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index 0cca80c..f66322a 100644 (file)
@@ -35,7 +35,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index e073f8f..61c8a59 100644 (file)
@@ -33,7 +33,7 @@ mempool size=8K
 [global]
 name=${name}
 
-[core 0]
+[core $mcore]
 mode=master
 
 [core $cores]
index fed57db..6d5e820 100644 (file)
@@ -7,7 +7,7 @@ metadata:
 spec:
   containers:
   - name: pod-rapid
-    image: localhost:5000/prox_slim:latest
+    image: opnfv/rapid:latest
     imagePullPolicy: Always
     securityContext:
       capabilities:
@@ -19,9 +19,12 @@ spec:
       requests:
         hugepages-2Mi: 512Mi
         memory: 1Gi
+        cpu: 8
         intel.com/intel_sriov_vfio: '1'
       limits:
         hugepages-2Mi: 512Mi
+        memory: 1Gi
+        cpu: 8
         intel.com/intel_sriov_vfio: '1'
   volumes:
   - name: hugepages
index f690872..f568139 100644 (file)
@@ -15,7 +15,7 @@
 ##
 
 [DEFAULT]
-total_number_of_pods=3
+total_number_of_pods=2
 
 [POD1]
 nodeSelector_hostname=k8s-node1
@@ -26,8 +26,3 @@ dp_subnet=24
 nodeSelector_hostname=k8s-node2
 dp_ip=192.168.30.12
 dp_subnet=24
-
-[POD3]
-nodeSelector_hostname=k8s-node2
-dp_ip=192.168.30.13
-dp_subnet=24
index eb611bf..074d4f7 100644 (file)
@@ -81,6 +81,24 @@ class RapidGeneratorMachine(RapidMachine):
         return (self.machine_params['gencores'] +
                 self.machine_params['latcores'])
 
+    def remap_all_cpus(self):
+        """Convert relative cpu ids for different parameters (gencores, latcores)
+        """
+        super().remap_all_cpus()
+
+        if self.cpu_mapping is None:
+            return
+
+        if 'gencores' in self.machine_params.keys():
+            cpus_remapped = super().remap_cpus(self.machine_params['gencores'])
+            RapidLog.debug('{} ({}): gencores {} remapped to {}'.format(self.name, self.ip, self.machine_params['gencores'], cpus_remapped))
+            self.machine_params['gencores'] = cpus_remapped
+
+        if 'latcores' in self.machine_params.keys():
+            cpus_remapped = super().remap_cpus(self.machine_params['latcores'])
+            RapidLog.debug('{} ({}): latcores {} remapped to {}'.format(self.name, self.ip, self.machine_params['latcores'], cpus_remapped))
+            self.machine_params['latcores'] = cpus_remapped
+
     def generate_lua(self, vim, prox_config_file):
         appendix = 'gencores="%s"\n'% ','.join(map(str,
             self.machine_params['gencores']))
@@ -109,6 +127,8 @@ class RapidGeneratorMachine(RapidMachine):
         # Start the generator with the -e option so that the cores don't
         # start automatically
         super().start_prox('-e')
+        if self.vim in ['kubernetes']:
+            self.remap_all_cpus()
 
     def set_generator_speed(self, speed):
         # The assumption is that we only use task 0 for generating
index d9e18b0..fb96760 100644 (file)
@@ -49,6 +49,7 @@ class RapidMachine(object):
                 break
         self.machine_params = machine_params
         self.vim = vim
+        self.cpu_mapping = None
 
     def __del__(self):
         if ((not self.configonly) and self.machine_params['prox_socket']):
@@ -58,6 +59,62 @@ class RapidMachine(object):
     def get_cores(self):
         return (self.machine_params['cores'])
 
+    def expand_cpuset(self, cpuset):
+        """Expand cpuset provided as comma-separated list of CPU numbers and
+        CPU ranges of numbers. For more information please see
+        https://man7.org/linux/man-pages/man7/cpuset.7.html
+        """
+        cpuset_expanded = []
+        for cpu in cpuset.split(','):
+            if '-' in cpu:
+                cpu_range = cpu.split('-')
+                cpuset_expanded += range(int(cpu_range[0]), int(cpu_range[1]) + 1)
+            else:
+                cpuset_expanded.append(int(cpu))
+        return cpuset_expanded
+
+    def read_cpuset(self):
+        """Read list of cpus on which we allowed to execute
+        """
+        cmd = 'cat /sys/fs/cgroup/cpuset/cpuset.cpus'
+        cpuset_cpus = self._client.run_cmd(cmd).decode().rstrip()
+        RapidLog.debug('{} ({}): Allocated cpuset: {}'.format(self.name, self.ip, cpuset_cpus))
+        self.cpu_mapping = self.expand_cpuset(cpuset_cpus)
+        RapidLog.debug('{} ({}): Expanded cpuset: {}'.format(self.name, self.ip, self.cpu_mapping))
+
+        # Log CPU core mapping for user information
+        cpu_mapping_str = ''
+        for i in range(len(self.cpu_mapping)):
+            cpu_mapping_str = cpu_mapping_str + '[' + str(i) + '->' + str(self.cpu_mapping[i]) + '], '
+        cpu_mapping_str = cpu_mapping_str[:-2]
+        RapidLog.debug('{} ({}): CPU mapping: {}'.format(self.name, self.ip, cpu_mapping_str))
+
+    def remap_cpus(self, cpus):
+        """Convert relative cpu ids provided as function parameter to match
+        cpu ids from allocated list
+        """
+        cpus_remapped = []
+        for cpu in cpus:
+            cpus_remapped.append(self.cpu_mapping[cpu])
+        return cpus_remapped
+
+    def remap_all_cpus(self):
+        """Convert relative cpu ids for different parameters (mcore, cores)
+        """
+        if self.cpu_mapping is None:
+            RapidLog.debug('{} ({}): cpu mapping is not defined! Please check the configuration!'.format(self.name, self.ip))
+            return
+
+        if 'mcore' in self.machine_params.keys():
+            cpus_remapped = self.remap_cpus(self.machine_params['mcore'])
+            RapidLog.debug('{} ({}): mcore {} remapped to {}'.format(self.name, self.ip, self.machine_params['mcore'], cpus_remapped))
+            self.machine_params['mcore'] = cpus_remapped
+
+        if 'cores' in self.machine_params.keys():
+            cpus_remapped = self.remap_cpus(self.machine_params['cores'])
+            RapidLog.debug('{} ({}): cores {} remapped to {}'.format(self.name, self.ip, self.machine_params['cores'], cpus_remapped))
+            self.machine_params['cores'] = cpus_remapped
+
     def devbind(self):
         # Script to bind the right network interface to the poll mode driver
         for index, dp_port in enumerate(self.dp_ports, start = 1):
@@ -86,6 +143,8 @@ class RapidMachine(object):
                 LuaFile.write("eal=\"--socket-mem=512,0 --file-prefix %s --pci-whitelist %s\"\n" % (self.name, self.machine_params['dp_pci_dev']))
             else:
                 LuaFile.write("eal=\"\"\n")
+            if 'mcore' in self.machine_params.keys():
+                LuaFile.write('mcore="%s"\n'% ','.join(map(str, self.machine_params['mcore'])))
             if 'cores' in self.machine_params.keys():
                 LuaFile.write('cores="%s"\n'% ','.join(map(str, self.machine_params['cores'])))
             if 'ports' in self.machine_params.keys():
@@ -105,6 +164,9 @@ class RapidMachine(object):
             self._client.connect()
             if self.vim in ['OpenStack']:
                 self.devbind()
+            if self.vim in ['kubernetes']:
+                self.read_cpuset()
+                self.remap_all_cpus()
             _, prox_config_file_name = os.path.split(self.machine_params['config_file'])
             self.generate_lua(self.vim, self.machine_params['config_file'])
             self._client.scp_put(self.machine_params['config_file'], '{}/{}'.format(self.rundir, prox_config_file_name))
index 136bc27..c1ab36a 100644 (file)
@@ -120,7 +120,7 @@ class RapidConfigParser(object):
             for option in options:
                 if option in ['prox_socket','prox_launch_exit','monitor']:
                     machine[option] = testconfig.getboolean(section, option)
-                elif option in ['cores', 'gencores','latcores']:
+                elif option in ['mcore', 'cores', 'gencores','latcores']:
                     machine[option] = ast.literal_eval(testconfig.get(
                         section, option))
                 elif option in ['bucket_size_exp']:
index 2630e5c..19d62fa 100644 (file)
@@ -24,6 +24,7 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -31,6 +32,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 297eff8..66767a0 100644 (file)
@@ -24,6 +24,7 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -31,6 +32,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 5d6692c..31d19bf 100644 (file)
@@ -24,6 +24,7 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -31,6 +32,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 28c4f92..a19289f 100644 (file)
@@ -24,6 +24,7 @@ lat_percentile = 99
 name = Generator
 config_file = configs/genv6.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -31,6 +32,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swapv6.cfg
+mcore = [0]
 cores = [1]
 #prox_socket = true
 #prox_launch_exit = true
index 9cfc490..9aab3fd 100644 (file)
@@ -23,12 +23,14 @@ total_number_of_test_machines = 2
 name = Generator
 config_file = configs/l2gen_bare.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
 [TestM2]
 name = Swap
 config_file = configs/l2swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 6eb56a4..76c1fa0 100644 (file)
@@ -24,6 +24,7 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -31,6 +32,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 #prox_socket = true
 #prox_launch_exit = true
index 4e2a445..1873c38 100644 (file)
@@ -22,6 +22,7 @@ total_number_of_test_machines = 1
 [TestM1]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 88dc599..865394f 100644 (file)
@@ -24,18 +24,21 @@ name = Generator
 config_file = configs/gen_gw.cfg
 gw_vm = 2
 dest_vm = 3
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
 [TestM2]
 name = ImpairGW
 config_file = configs/impair.cfg
+mcore = [0]
 cores = [1]
 monitor = False
 
 [TestM3]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 31aef91..7cee0a3 100644 (file)
@@ -24,6 +24,7 @@ lat_percentile = 99
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -31,6 +32,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 #prox_socket = true
 #prox_launch_exit = true
index 632aa7b..e973bc9 100644 (file)
@@ -25,6 +25,7 @@ ipv6 = True
 name = Generator
 config_file = configs/genv6.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 #bucket_size_exp = 12
@@ -32,6 +33,7 @@ latcores = [3]
 [TestM2]
 name = Swap
 config_file = configs/swapv6.cfg
+mcore = [0]
 cores = [1]
 #prox_socket = true
 #prox_launch_exit = true
index 7879a09..77c9cbe 100644 (file)
@@ -22,12 +22,14 @@ total_number_of_test_machines = 2
 [TestM1]
 name = InterruptTestMachine1
 config_file = configs/irq.cfg
+mcore = [0]
 cores = [1,2,3]
 monitor = False
 
 [TestM2]
 name = InterruptTestMachine2
 config_file = configs/irq.cfg
+mcore = [0]
 cores = [1,2,3]
 monitor = False
 
index c5d9c25..062772f 100644 (file)
@@ -23,12 +23,14 @@ total_number_of_test_machines = 2
 name = Generator
 config_file = configs/l2gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
 [TestM2]
 name = Swap
 config_file = configs/l2swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 4419430..30b807f 100644 (file)
@@ -23,12 +23,14 @@ total_number_of_test_machines = 2
 name = Generator
 config_file = configs/l2gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
 [TestM2]
 name = Swap
 config_file = configs/l2swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 4edf4f7..04aa473 100644 (file)
@@ -23,12 +23,14 @@ total_number_of_test_machines = 2
 name = Generator
 config_file = configs/gen.cfg
 dest_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
 [TestM2]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 
 [test1]
index 4756722..fd2e178 100644 (file)
@@ -22,6 +22,7 @@ total_number_of_test_machines = 1
 [TestM1]
 name = Swap
 config_file = configs/swap.cfg
+mcore = [0]
 cores = [1]
 ports = [0]
 
index 4cd57aa..509727a 100644 (file)
@@ -24,6 +24,7 @@ name = Generator
 config_file = configs/gen_gw.cfg
 dest_vm = 3
 gw_vm = 2
+mcore = [0]
 gencores = [1]
 latcores = [3]
 
@@ -31,11 +32,13 @@ latcores = [3]
 name = GW1
 config_file = configs/secgw1.cfg
 dest_vm = 3
+mcore = [0]
 cores = [1]
 
 [TestM3]
 name = GW2
 config_file = configs/secgw2.cfg
+mcore = [0]
 cores = [1]
 
 [test1]