Improve SampleVNF hugepages setup

The goal of this function is to:
- Read the default hugepage size.
- Set 16GB of hugepages.
- Check if the status of the last action.

According to [1], the default hugepage size could be read in
"/proc/meminfo", always in kB. Then "/proc/sys/vm/nr_hugepages"
could be used to set the number of default hugepages.

[1] https://www.kernel.org/doc/Documentation/vm/hugetlbpage.txt

JIRA: YARDSTICK-997

Change-Id: I762d1b16294ba1c1c2feee56610819ac358c7410
Signed-off-by: Rodolfo Alonso Hernandez <rodolfo.alonso.hernandez@intel.com>

diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
index af941c0..1799353 100644 (file)
--- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
@@ -15,14 +15,17 @@
 # limitations under the License.
 #
 
+from copy import deepcopy
+
 import unittest
 import mock
-from copy import deepcopy
+import six
 
 from tests.unit.network_services.vnf_generic.vnf.test_base import mock_ssh
 from tests.unit import STL_MOCKS
 from yardstick.benchmark.contexts.base import Context
 from yardstick.common import exceptions as y_exceptions
+from yardstick.common import utils
 from yardstick.network_services.nfvi.resource import ResourceProfile
 from yardstick.network_services.vnf_generic.vnf.base import VnfdHelper
 
@@ -36,6 +39,7 @@ stl_patch = mock.patch.dict("sys.modules", STL_MOCKS)
 stl_patch.start()
 
 if stl_patch:
+    from yardstick.network_services.vnf_generic.vnf import sample_vnf
     from yardstick.network_services.vnf_generic.vnf.sample_vnf import VnfSshHelper
     from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNFDeployHelper
     from yardstick.network_services.vnf_generic.vnf.sample_vnf import ScenarioHelper
@@ -528,41 +532,19 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
         result = DpdkVnfSetupEnvHelper._update_traffic_type(ip_pipeline_cfg, traffic_options)
         self.assertEqual(result, expected)
 
-    def test__setup_hugepages(self):
-        vnfd_helper = VnfdHelper(self.VNFD_0)
-        ssh_helper = mock.Mock()
-        ssh_helper.execute.return_value = 0, '', ''
-        scenario_helper = mock.Mock()
-        dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
-
-        result = dpdk_setup_helper._setup_hugepages()
-        expect_start_list = ['awk', 'awk', 'echo']
-        expect_in_list = ['meminfo', 'nr_hugepages', '16']
-        call_args_iter = (args[0][0] for args in ssh_helper.execute.call_args_list)
-        self.assertIsNone(result)
-        self.assertEqual(ssh_helper.execute.call_count, 3)
-        for expect_start, expect_in, arg0 in zip(expect_start_list, expect_in_list,
-                                                 call_args_iter):
-            self.assertTrue(arg0.startswith(expect_start))
-            self.assertIn(expect_in, arg0)
-
-    def test__setup_hugepages_2_mb(self):
-        vnfd_helper = VnfdHelper(self.VNFD_0)
+    @mock.patch.object(six, 'BytesIO', return_value=six.BytesIO(b'100\n'))
+    @mock.patch.object(utils, 'read_meminfo',
+                       return_value={'Hugepagesize': '2048'})
+    def test__setup_hugepages(self, mock_meminfo, *args):
         ssh_helper = mock.Mock()
-        ssh_helper.execute.return_value = 0, '2048kB  ', ''
-        scenario_helper = mock.Mock()
-        dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
-
-        result = dpdk_setup_helper._setup_hugepages()
-        expect_start_list = ['awk', 'awk', 'echo']
-        expect_in_list = ['meminfo', 'nr_hugepages', '8192']
-        call_args_iter = (args[0][0] for args in ssh_helper.execute.call_args_list)
-        self.assertIsNone(result)
-        self.assertEqual(ssh_helper.execute.call_count, 3)
-        for expect_start, expect_in, arg0 in zip(expect_start_list, expect_in_list,
-                                                 call_args_iter):
-            self.assertTrue(arg0.startswith(expect_start))
-            self.assertIn(expect_in, arg0)
+        dpdk_setup_helper = DpdkVnfSetupEnvHelper(
+            mock.ANY, ssh_helper, mock.ANY)
+        with mock.patch.object(sample_vnf.LOG, 'info') as mock_info:
+            dpdk_setup_helper._setup_hugepages()
+            mock_info.assert_called_once_with(
+                'Hugepages size (kB): %s, number claimed: %s, number set: '
+                '%s', 2048, 8192, 100)
+        mock_meminfo.assert_called_once_with(ssh_helper)
 
     @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.open')
     @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.find_relative_file')
@@ -622,7 +604,10 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
         dpdk_vnf_setup_env_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, mock.Mock())
         dpdk_vnf_setup_env_helper._validate_cpu_cfg = mock.Mock(return_value=[])
 
-        self.assertIsInstance(dpdk_vnf_setup_env_helper.setup_vnf_environment(), ResourceProfile)
+        with mock.patch.object(dpdk_vnf_setup_env_helper, '_setup_dpdk'):
+            self.assertIsInstance(
+                dpdk_vnf_setup_env_helper.setup_vnf_environment(),
+                ResourceProfile)
 
     def test__setup_dpdk_early_success(self):
         vnfd_helper = VnfdHelper(self.VNFD_0)

diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index 8604e90..4952901 100644 (file)
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -22,6 +22,7 @@ import ipaddress
 import logging
 import os
 import random
+import re
 import socket
 import subprocess
 import sys
@@ -395,3 +396,17 @@ class Timer(object):
 
     def __getattr__(self, item):
         return getattr(self.delta, item)
+
+
+def read_meminfo(ssh_client):
+    """Read "/proc/meminfo" file and parse all keys and values"""
+
+    cpuinfo = six.BytesIO()
+    ssh_client.get_file_obj('/proc/meminfo', cpuinfo)
+    lines = cpuinfo.getvalue().decode('utf-8')
+    matches = re.findall(r"([\w\(\)]+):\s+(\d+)( kB)*", lines)
+    output = {}
+    for match in matches:
+        output[match[0]] = match[1]
+
+    return output

diff --git a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
index fbaaa0c..18b4f0b 100644 (file)
--- a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
@@ -19,16 +19,19 @@ from multiprocessing import Queue, Value, Process
 import os
 import posixpath
 import re
-from six.moves import cStringIO
 import subprocess
 import time
 
+import six
+from six.moves import cStringIO
+
 from trex_stl_lib.trex_stl_client import LoggerApi
 from trex_stl_lib.trex_stl_client import STLClient
 from trex_stl_lib.trex_stl_exceptions import STLError
 from yardstick.benchmark.contexts.base import Context
 from yardstick.benchmark.scenarios.networking.vnf_generic import find_relative_file
 from yardstick.common import exceptions as y_exceptions
+from yardstick.common import utils
 from yardstick.common.process import check_if_process_failed
 from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelper
 from yardstick.network_services.helpers.samplevnf_helper import PortPairs
@@ -119,6 +122,8 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
 
     APP_NAME = 'DpdkVnf'
     FIND_NET_CMD = "find /sys/class/net -lname '*{}*' -printf '%f'"
+    NR_HUGEPAGES_PATH = '/proc/sys/vm/nr_hugepages'
+    HUGEPAGES_KB = 1024 * 1024 * 16
 
     @staticmethod
     def _update_packet_type(ip_pipeline_cfg, traffic_options):
@@ -155,19 +160,16 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
         self.dpdk_bind_helper = DpdkBindHelper(ssh_helper)
 
     def _setup_hugepages(self):
-        cmd = "awk '/Hugepagesize/ { print $2$3 }' < /proc/meminfo"
-        hugepages = self.ssh_helper.execute(cmd)[1].rstrip()
-
-        memory_path = \
-            '/sys/kernel/mm/hugepages/hugepages-%s/nr_hugepages' % hugepages
-        self.ssh_helper.execute("awk -F: '{ print $1 }' < %s" % memory_path)
-
-        if hugepages == "2048kB":
-            pages = 8192
-        else:
-            pages = 16
-
-        self.ssh_helper.execute("echo %s | sudo tee %s" % (pages, memory_path))
+        meminfo = utils.read_meminfo(self.ssh_helper)
+        hp_size_kb = int(meminfo['Hugepagesize'])
+        nr_hugepages = int(abs(self.HUGEPAGES_KB / hp_size_kb))
+        self.ssh_helper.execute('echo %s | sudo tee %s' %
+                                (nr_hugepages, self.NR_HUGEPAGES_PATH))
+        hp = six.BytesIO()
+        self.ssh_helper.get_file_obj(self.NR_HUGEPAGES_PATH, hp)
+        nr_hugepages_set = int(hp.getvalue().decode('utf-8').splitlines()[0])
+        LOG.info('Hugepages size (kB): %s, number claimed: %s, number set: %s',
+                 hp_size_kb, nr_hugepages, nr_hugepages_set)
 
     def build_config(self):
         vnf_cfg = self.scenario_helper.vnf_cfg

diff --git a/yardstick/tests/unit/common/test_utils.py b/yardstick/tests/unit/common/test_utils.py
index 033bb02..b4907ad 100644 (file)
--- a/yardstick/tests/unit/common/test_utils.py
+++ b/yardstick/tests/unit/common/test_utils.py
@@ -19,6 +19,7 @@ from six.moves import configparser
 import unittest
 
 import yardstick
+from yardstick import ssh
 from yardstick.common import utils
 from yardstick.common import constants
 
@@ -1075,8 +1076,27 @@ class SafeDecodeUtf8TestCase(unittest.TestCase):
         self.assertEqual('this is a byte array', out)
 
 
-def main():
-    unittest.main()
-
-if __name__ == '__main__':
-    main()
+class ReadMeminfoTestCase(unittest.TestCase):
+
+    MEMINFO = (b'MemTotal:       65860500 kB\n'
+               b'MemFree:        28690900 kB\n'
+               b'MemAvailable:   52873764 kB\n'
+               b'Active(anon):    3015676 kB\n'
+               b'HugePages_Total:       8\n'
+               b'Hugepagesize:    1048576 kB')
+    MEMINFO_DICT = {'MemTotal': '65860500',
+                    'MemFree': '28690900',
+                    'MemAvailable': '52873764',
+                    'Active(anon)': '3015676',
+                    'HugePages_Total': '8',
+                    'Hugepagesize': '1048576'}
+
+    def test_read_meminfo(self):
+        ssh_client = ssh.SSH('user', 'host')
+        with mock.patch.object(ssh_client, 'get_file_obj') as \
+                mock_get_client, \
+                mock.patch.object(six, 'BytesIO',
+                                  return_value=six.BytesIO(self.MEMINFO)):
+            output = utils.read_meminfo(ssh_client)
+            mock_get_client.assert_called_once_with('/proc/meminfo', mock.ANY)
+        self.assertEqual(self.MEMINFO_DICT, output)