xena_deadlock: Fix xena2544.exe deadlock

Fix Xena2544.exe execution with mono where sometimes the
Xena process says complete, but the mono process doesn't exit.

Changes call of Xena2544.exe from communicate thread lock to a
wait try/catch block and checks the log periodically to see
if a deadlock has occured and if so, the mono pid is terminated
to prevent deadlock.

* Fix one small issue with typecasting to int, changed to float

JIRA: VSPERF-319

Change-Id: Ie7a86a2b9ff9cdfacba5a912ad17df6cc0581702
Signed-off-by: Christian Trautman <ctrautma@redhat.com>

diff --git a/tools/pkt_gen/xena/xena.py b/tools/pkt_gen/xena/xena.py
index 7dd4b90..67e9984 100755 (executable)
--- a/tools/pkt_gen/xena/xena.py
+++ b/tools/pkt_gen/xena/xena.py
@@ -25,6 +25,7 @@ Xena Traffic Generator Model
 # python imports
 import binascii
 import logging
+import os
 import subprocess
 import sys
 from time import sleep
@@ -50,6 +51,7 @@ from tools.pkt_gen.xena.XenaDriver import (
     XenaManager,
     )
 
+
 class Xena(ITrafficGenerator):
     """
     Xena Traffic generator wrapper class
@@ -65,6 +67,19 @@ class Xena(ITrafficGenerator):
         self._duration = None
         self.tx_stats = None
         self.rx_stats = None
+        self._log_handle = None
+
+        user_home = os.path.expanduser('~')
+        self._log_path = '{}/Xena/Xena2544-2G/Logs/xena2544.log'.format(
+                user_home)
+
+        # make the folder and log file if they doesn't exist
+        if not os.path.exists(self._log_path):
+            os.makedirs(os.path.dirname(self._log_path))
+
+        # empty the file contents
+        open(self._log_path, 'w').close()
+
 
     @property
     def traffic_defaults(self):
@@ -99,7 +114,7 @@ class Xena(ITrafficGenerator):
                 root[0][1][0][0].get('PortRxBpsL1')) + float(
                     root[0][1][0][1].get('PortRxBpsL1')))/ 1000000
             results[ResultsConstants.THROUGHPUT_RX_PERCENT] = (
-                100 - int(root[0][1][0].get('TotalLossRatioPcnt'))) * float(
+                100 - float(root[0][1][0].get('TotalLossRatioPcnt'))) * float(
                     root[0][1][0].get('TotalTxRatePcnt'))/100
             results[ResultsConstants.TX_RATE_FPS] = root[0][1][0].get(
                 'TotalTxRateFps')
@@ -451,6 +466,44 @@ class Xena(ITrafficGenerator):
             self.rx_stats = self.xmanager.ports[1].get_rx_stats()
         sleep(1)
 
+    def _start_xena_2544(self):
+        """
+        Start the xena2544 exe.
+        :return: None
+        """
+        args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c",
+                "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r",
+                "./tools/pkt_gen/xena", "-u",
+                settings.getValue('TRAFFICGEN_XENA_USER')]
+        # Sometimes Xena2544.exe completes, but mono holds the process without
+        # releasing it, this can cause a deadlock of the main thread. Use the
+        # xena log file as a way to detect this.
+        self._log_handle = open(self._log_path, 'r')
+        # read the contents of the log before we start so the next read in the
+        # wait method are only looking at the text from this test instance
+        self._log_handle.read()
+        self.mono_pipe = subprocess.Popen(args, stdout=sys.stdout)
+
+    def _wait_xena_2544_complete(self):
+        """
+        Wait for Xena2544.exe completion.
+        :return: None
+        """
+        data = ''
+        while True:
+            try:
+                self.mono_pipe.wait(60)
+                self._log_handle.close()
+                break
+            except subprocess.TimeoutExpired:
+                # check the log to see if Xena2544 has completed and mono is
+                # deadlocked.
+                data += self._log_handle.read()
+                if 'TestCompletedSuccessfully' in data:
+                    self._log_handle.close()
+                    self.mono_pipe.terminate()
+                    break
+
     def _stop_api_traffic(self):
         """
         Stop traffic through the socket API
@@ -495,13 +548,11 @@ class Xena(ITrafficGenerator):
         See ITrafficGenerator for description
         """
         self._duration = duration
-
         self._params.clear()
         self._params['traffic'] = self.traffic_defaults.copy()
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._start_traffic_api(numpkts)
         return self._stop_api_traffic()
 
@@ -517,7 +568,6 @@ class Xena(ITrafficGenerator):
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._start_traffic_api(-1)
         return self._stop_api_traffic()
 
@@ -533,7 +583,6 @@ class Xena(ITrafficGenerator):
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._start_traffic_api(-1)
 
     def stop_cont_traffic(self):
@@ -554,15 +603,10 @@ class Xena(ITrafficGenerator):
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._setup_json_config(trials, lossrate, '2544_throughput')
+        self._start_xena_2544()
+        self._wait_xena_2544_complete()
 
-        args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c",
-                "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r",
-                "./tools/pkt_gen/xena", "-u",
-                settings.getValue('TRAFFICGEN_XENA_USER')]
-        self.mono_pipe = subprocess.Popen(args, stdout=sys.stdout)
-        self.mono_pipe.communicate()
         root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
         return Xena._create_throughput_result(root)
 
@@ -578,22 +622,15 @@ class Xena(ITrafficGenerator):
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._setup_json_config(trials, lossrate, '2544_throughput')
-
-        args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c",
-                "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r",
-                "./tools/pkt_gen/xena", "-u",
-                settings.getValue('TRAFFICGEN_XENA_USER')]
-        self.mono_pipe = subprocess.Popen(args, stdout=sys.stdout)
+        self._start_xena_2544()
 
     def wait_rfc2544_throughput(self):
         """Wait for and return results of RFC2544 test.
 
         See ITrafficGenerator for description
         """
-        self.mono_pipe.communicate()
-        sleep(2)
+        self._wait_xena_2544_complete()
         root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
         return Xena._create_throughput_result(root)
 
@@ -610,16 +647,9 @@ class Xena(ITrafficGenerator):
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._setup_json_config(trials, lossrate, '2544_b2b')
-
-        args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c",
-                "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r",
-                "./tools/pkt_gen/xena", "-u",
-                settings.getValue('TRAFFICGEN_XENA_USER')]
-        self.mono_pipe = subprocess.Popen(
-            args, stdout=sys.stdout)
-        self.mono_pipe.communicate()
+        self._start_xena_2544()
+        self._wait_xena_2544_complete()
         root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
         return Xena._create_throughput_result(root)
 
@@ -630,27 +660,18 @@ class Xena(ITrafficGenerator):
         See ITrafficGenerator for description
         """
         self._duration = duration
-
         self._params.clear()
         self._params['traffic'] = self.traffic_defaults.copy()
         if traffic:
             self._params['traffic'] = merge_spec(self._params['traffic'],
                                                  traffic)
-
         self._setup_json_config(trials, lossrate, '2544_b2b')
-
-        args = ["mono", "./tools/pkt_gen/xena/Xena2544.exe", "-c",
-                "./tools/pkt_gen/xena/profiles/2bUsed.x2544", "-e", "-r",
-                "./tools/pkt_gen/xena", "-u",
-                settings.getValue('TRAFFICGEN_XENA_USER')]
-        self.mono_pipe = subprocess.Popen(
-            args, stdout=sys.stdout)
+        self._start_xena_2544()
 
     def wait_rfc2544_back2back(self):
         """Wait and set results of RFC2544 test.
         """
-        self.mono_pipe.communicate()
-        sleep(2)
+        self._wait_xena_2544_complete()
         root = ET.parse(r'./tools/pkt_gen/xena/xena2544-report.xml').getroot()
         return Xena._create_throughput_result(root)