Minor changes to the code, to prep for code freeze.

the result format of iperf has been made consistent

Change-Id: Iede4b381ca2e5d7643bf6420f17684bbf2cc2921
Signed-off-by: Mofassir <Mofassir_arif@Dellteam.com>

diff --git a/benchmarks/playbooks/fio.yaml b/benchmarks/playbooks/fio.yaml
index 40fd805..9f832ce 100644 (file)
--- a/benchmarks/playbooks/fio.yaml
+++ b/benchmarks/playbooks/fio.yaml
@@ -2,9 +2,9 @@
    connection: local
    tasks:
     - name: making fio  directory
-      file: path={{Dest_dir}}/fio state=directory
+      file: path=../../{{Dest_dir}}/fio state=directory
     - name: making temporary fio directory
-      file: path={{Dest_dir}}/fio/fio_temp state=directory
+      file: path=../../{{Dest_dir}}/fio/fio_temp state=directory
 
 
  - hosts: "{{role}}"
@@ -54,29 +54,29 @@
       shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2
       register: files_to_copy
     - name: copy results
-      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/fio/fio_temp
       with_items: files_to_copy.stdout_lines
     - name: registering log files
       shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2
       register: copy_log_results
     - name: copying log results
-      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/fio/fio_temp
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/fio/fio_temp
       with_items: copy_log_results.stdout_lines
 
 
     - name: copy results
-      fetch:  src=/root/results/{{item}}  dest={{Dest_dir}}/fio
+      fetch:  src=/root/results/{{item}}  dest=../../{{Dest_dir}}/fio
       with_items: files_to_copy.stdout_lines
 
  - hosts: 127.0.0.1
    connection: local
    tasks:
     - name: extracting_json
-      shell: ( find  {{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/fio/)
+      shell: ( find  ../../{{Dest_dir}}/fio/fio_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/fio/)
     - name: making_logs_folder
-      shell: mkdir -p {{Dest_dir}}/fio/logs
+      shell: mkdir -p ../../{{Dest_dir}}/fio/logs
     - name: extracting_log
-      shell: ( find {{Dest_dir}}/fio/fio_temp/ -name "*.log"  | xargs cp -t {{Dest_dir}}/fio/logs)
+      shell: ( find ../../{{Dest_dir}}/fio/fio_temp/ -name "*.log"  | xargs cp -t ../../{{Dest_dir}}/fio/logs)
     - name: removing fio_log
-      shell: rm -rf {{Dest_dir}}/fio/fio_temp
+      shell: rm -rf ../../{{Dest_dir}}/fio/fio_temp
 

diff --git a/benchmarks/playbooks/iperf.yaml b/benchmarks/playbooks/iperf.yaml
index dd5c2b0..11d427f 100644 (file)
--- a/benchmarks/playbooks/iperf.yaml
+++ b/benchmarks/playbooks/iperf.yaml
@@ -59,24 +59,40 @@
       poll: 0
       when: rolename == "1-server"
     - name: Running Iperf on Host
-      shell: iperf3  -Z --{{protocol}} --time {{duration}} -b {{bandwidthGbps}}G  -c {{item}} -J >> ./qtip_result/server{{hostID.stdout}}-{{item}}.json
-      ignore_errors: yes
-      with_items:
-        - "{{ip1}}"
-        - "{{ip2}}"
-      when: rolename == "2-host" and "{{ip2}}" != ''
-    - name: Running Iperf on Host
-      shell: iperf3  -Z --{{protocol}} --time {{duration}} -b {{bandwidthGbps}}G -c {{item}} -J >> ./qtip_result/server{{hostID.stdout}}-{{item}}.json
+      shell: iperf3  --time {{duration}} -b {{bandwidthGbps}}G -c {{privateip1}} -J -O10>> ./qtip_result/iperf_raw.json
       ignore_errors: yes
       with_items: 
         - "{{ip1}}"
       when: rolename == "2-host" and "{{ip2}}" == ''
+    - name: Fetching result transformation script
+      copy: src=./result_transform/iperf/iperf_transform.py dest={{home_dir.stdout}}/qtip_result
+    - name: Transforming result
+      shell: cd $HOME/qtip_result  &&  python iperf_transform.py
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: copy report formation script
+      copy: src=./result_transform/final_report.py dest={{home_dir.stdout}}/qtip_result
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: consolidating report
+      shell: cd $HOME/qtip_result &&  python final_report.py IPERF      
+      when: rolename =="2-host" and "{{ip2}}" == ''
     - name: Files to Copy  
       shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2
       register: files_to_copy
+      when: rolename =="2-host" and "{{ip2}}" == ''
     - name: copy results
       fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/iperf/iperf_temp
       with_items: files_to_copy.stdout_lines
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: registering log files
+      shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2
+      register: copy_log_results
+      when: rolename =="2-host" and "{{ip2}}" == ''
+    - name: copying log results
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest=../../{{Dest_dir}}/iperf/iperf_temp
+      with_items: copy_log_results.stdout_lines
+      when: rolename =="2-host" and "{{ip2}}" == ''
+
+
  - hosts: 127.0.0.1
    connection: local
    tasks:
@@ -86,4 +102,14 @@
         when: role is defined
     - name: extracting_json
       shell: ( find  ../../{{Dest_dir}}/iperf/iperf_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/iperf/)
-      when: rolename == "2-host"
\ No newline at end of file
+      when: rolename == "2-host"
+    - name: making_logs_folder
+      shell: mkdir -p ../../{{Dest_dir}}/iperf/logs
+    - name: extracting_log
+      shell: ( find ../../{{Dest_dir}}/iperf/iperf_temp/ -name "*.log"  | xargs cp -t ../../{{Dest_dir}}/iperf/logs)
+      when: rolename == "2-host"
+    - name: removing iperf_raw file
+      shell: rm -rf ../../{{Dest_dir}}/iperf/iperf_raw.json
+      when: rolename == "2-host"
+    - name: removing iperf_temp
+      shell: rm -rf ../../{{Dest_dir}}/iperf/iperf_temp
\ No newline at end of file

diff --git a/benchmarks/playbooks/netperf.yaml b/benchmarks/playbooks/netperf.yaml
index fb6fd18..3292f06 100644 (file)
--- a/benchmarks/playbooks/netperf.yaml
+++ b/benchmarks/playbooks/netperf.yaml
@@ -71,4 +71,6 @@
         when: role is defined
     - name: extracting_json
       shell: ( find  ../../{{Dest_dir}}/netperf/netperf_temp/ -name "*.json" | xargs cp -t ../../{{Dest_dir}}/netperf/)
-      when: rolename == "2-host"
\ No newline at end of file
+      when: rolename == "2-host"
+    - name: removing netperf_temp
+      shell: rm -rf ../../{{Dest_dir}}/netperf/netperf_temp
\ No newline at end of file

diff --git a/benchmarks/playbooks/result_transform/iperf/iperf_transform.py b/benchmarks/playbooks/result_transform/iperf/iperf_transform.py
new file mode 100644 (file)
index 0000000..39c5956
--- /dev/null
+++ b/benchmarks/playbooks/result_transform/iperf/iperf_transform.py
@@ -0,0 +1,30 @@
+import json\r
+import datetime\r
+import pickle\r
+with open('iperf_raw.json','r') as ifile:\r
+    raw_iperf_data=json.loads(ifile.read().rstrip())\r
+    \r
+    \r
+bits_sent= raw_iperf_data['end']['sum_sent']['bits_per_second']\r
+bits_received= raw_iperf_data['end']['sum_received']['bits_per_second']\r
+total_byte_sent=raw_iperf_data['end']['sum_sent']['bytes']\r
+total_byte_received=raw_iperf_data['end']['sum_received']['bytes']\r
+cpu_host_total_percent=raw_iperf_data['end']['cpu_utilization_percent']['host_total']\r
+cpu_remote_total_percent=raw_iperf_data['end']['cpu_utilization_percent']['remote_total']\r
+\r
+result={}\r
+time_stamp = str(datetime.datetime.utcnow().isoformat())\r
+\r
+result['1. Version']=raw_iperf_data['start']['version']\r
+result['2. Bandwidth']={}\r
+result['2. Bandwidth']['1. throughput Sender (b/s)'] = bits_sent\r
+result['2. Bandwidth']['2. throughput Received (b/s)'] = bits_received\r
+result['3. CPU']={}\r
+result['3. CPU']['1. CPU host total (%)']=cpu_host_total_percent\r
+result['3. CPU']['2. CPU remote total (%)']=cpu_remote_total_percent\r
+\r
+with open('iperf_raw-'+time_stamp+'.log','w+') as ofile:\r
+    ofile.write(json.dumps(raw_iperf_data))\r
+    \r
+with open('./result_temp', 'w+') as result_file:\r
+    pickle.dump(result,result_file)
\ No newline at end of file

diff --git a/data/hosts b/data/hosts
index 5d73834..ac441ab 100644 (file)
--- a/data/hosts
+++ b/data/hosts
@@ -1,3 +1,4 @@
-[1Run]
-172.18.0.100
-172.18.0.99
+[2-host]
+172.18.0.112
+[1-server]
+172.18.0.113

diff --git a/data/my_key.pem b/data/my_key.pem
index 50a6b86..701beaf 100644 (file)
--- a/data/my_key.pem
+++ b/data/my_key.pem
@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAv5Y+wZP4KOzZf3hahb3UVUYf235R21M9lo6xnoie1ph71Pa+
-uHeCAnUhQs/RzCoi6y8U2JrPqmF3/HaP0CSPTyAZTdiKqHspQdLDndTdSMcVOMw0
-1C4QdQPTsejwsLW4ktxw/xjroZEhVyij5tJdCGjFLtPXXmYdPy4zSMqAOMuPvelE
-IlkEqcRkqQ0X0UBjQTDJvHKw1wYH9sFQLwMpuUI4JU1uHiC19LjJxV0Ih67yIXyw
-TSzw35rWPAbumgfJ5wG6k4jw1s8zYet10pXYMIxsAevieb8skSz97ytQKIVUjj/j
-6NfyoMxTuEKT8uvxwlx5QHU2N3A6qHVigWjmVQIDAQABAoIBAQC3KW2/5bYHckCR
-hD+Owyg/ijYkwwSBGXi0+qDG/rhUUx14QZmeg+qQbOiqlkJt3Q7bJ3zXhFHrAxoS
-DEt7Y6h6riZi7RXK80xDOLt5pisUS/h2/4u2Gn/zTwBRg3CwkFrwVL3WtD15jmWA
-jUL9BrqYsolpNnAdOKTXZVT0T1z8bb0FpRRCkKrYAUWF6ansvv7gryTLXjEvdBCp
-THz9QCMdGSm6bOM1uhRQciT4v0of+NiSBnbiWjXK8AFVVXYRmm0dk/9oY5VYVkCS
-RdWblKaag4jDg2I0LCInfWHyuBzaFyRSNfBaIEJyrxMZQ6WwfxvoECFXzaFEif4Q
-j8OYe//hAoGBAP3y7VM8ESkTDaAGjsc1n9W3UNcsl8mmTBOo1rTkP9lV+Wx3aO4W
-DQyeog34+/gKF91OT78l1Mn1hdjsGDD5tYSfYT8Ld+hmw2Ydosz56cZmU5nItEy5
-xe4IiGhPVM0jb3YnHhRgkgG8ZxT/6wba9KcOyxSk+j7lF+fO8eg2SaO9AoGBAMEi
-YFfp8zm8krdiLl0AtS8YVi2kOBdeRUDY9kMtY54QM4JLcOus3/7Z8SNIE+UpuUk3
-793o9gAvEIOtvafS5Orpo6QXIAI9X/RoPewheA9K9G5Jfp99A1bQQwpwWxd9w/+h
-vA7a1XV0SyMfbOVaEnHgQ1ntWmGKj7ekQ1+3y6p5AoGBAKfBe29Zpchpcr28t0L3
-tJ98mqWaKqcl+Omfxj327grHVZ6Gf0WQQYyNmmX6jZC8MimamumdtyS20XWL+SsZ
-OgTjOF1qWQqoH/psJ2iaywip/h3/zMNGkW7+KgYmKdHdz7rQenlhlfIZoErqCDse
-FzHoftUjasBznEggmAMbIahNAoGABu2jxSOYXRbdsIRuI1n2JfVaihKdDIGOPHBM
-DUbgjvgH0QHB0EKIWo0Y8GoShKMFiZHfIffMc1882WxvJEkVezPccfN+a3YmLg9b
-QVBGDU5kDFzIlRIX/5JDBWnYarRRRqdXprTE+YytcVocPvIHPVyNLfmsx1kAJNdY
-r2NZhAkCgYBF3Nfyy2OG7Py/+/fyAghr0Lj9I2zBQE7LvSLKBzfSQQIc6QvWiRwn
-W15h+695z389FnAgXaj8EWrX1Qa75VhNJO2/HgJssNFSfVJFRoWAAz3mVp+LG+vm
-gd+4evGpTG1eSedghYAXyYgXZIzMmKFue925eWRBv6DqDoSXtw5MPw==
+MIIEowIBAAKCAQEA1Smz6MCROcDEx4VIw3/6aJkjIdNYzv6u4UFvupvmyXjLO1N3
+ADUHj2hBSdB15Jo3QF4/f/M8D4EOIJz5mDhZCfFc3DXY/rMuopvigE5fMRnxPaB5
+9p4xxLFpYh5XXJ8XseHeGGI+QxucIxpg8vUvD23r52xYVmAdpiokiAltuRVwDI4e
+O4wEpXn4nHwZG0hcmWqBVdL7oX83sNoo2W9KJ/uKXiaEuUSXlwaxkl4Kxn4g/Gk8
+S2Eyr6b+msKW3PX0oD8Q4qsMA3iMRQK56qq/PD+apCgRUebavolcqAtpJ4tJnkhv
+O0+4naCjPjxvnVordwILUL7B9kXJITrEuPeo6QIDAQABAoIBAQDQXl6WM78CZHQ2
+0QwKxwxBuQhFpXIXjY/nc27DTGhyIxkO2gzmb+/++BtyC0ti9njx++84pMzeHvpv
+WOMHMHWhKYXrYNvIReFdZOIpJ/MXbVOLOVZfg84EhqQMHLqrOH0SDkdrMIXKVa/w
+BHYmqROikoHai+diL7xr1KA3NnhCRW8rMDQjdx1cCy5EJW9Kj+8jow/NWmlObS9q
+TvnwKO5Qqbba9UWcxNltA1hqGNRPTAVIGltmh71pBs+cnQN1wojCA5up2XpvKQLs
+Mvyl3B3k11+xG6cGTwjbs2GmB5LHMR6FrRTsOSPS+m2BcpcnpPxjvrtYQCp4977K
+KrUHak2tAoGBAP2zK3Nm6JjNc3tt/rVCHCcISify4ZOsKbewHRA66rIaJfPPK1TX
+G0hoLhvqb5ZYLFjPbWeCzPs9xaO28gDc32klR9dib1tFZ/ZhGHjOh/DHjUrx45OI
+qjb5yPq2rFDTdDXerd2gRSXpL2A5Zm0OJdFZbt1u8rXTa+djbZvKbezHAoGBANcY
+cqZ33mFC+lf9KNj2CAi3feY56BKYUtXUE7nU/B9VoaxTjyavMI+3TPXX8XdZ/4ZL
+y3oNo3AWBZa8vyqy6Xy0ZlQa2jaVifUr8cundbKmwRIWSq0+V2ODS4Mh7kdjGIVX
+aXRovhid1vSsKbJ8OwmMQ/X6JJ8PGQEv0rvOhSzPAoGAPD/stb9excsk/c8lik5k
++TASzGH2CGf+kE0W2d/kFN/422n71G4xYIS9TovibEYMYhsA1HBP1erZoQO1fVb3
+B7V+1jveTkKxaeWtNiN2TUxW1dGURllSczPm8wOm8JsHIjP7/oItac9zHNsE5ClT
++d/PYG8U3PGU0qLeSP1OFckCgYB/gRWNiqb/XtXJzN8m5YlTqXnvPMpOI4+PA72B
+vjiKVpeU8J9+HgGaX902KoEP8ACh2lJPEsCck+FSPqbPp+jyrHmwR46tJq318MvP
+noANm/39O8uesMLa0Zy0qit2k86YVH3FrMVrED25IklBbZUNjuFGcba9txRqeK/h
+9oVJTwKBgEWKRVYpBK7SzR+7RQZYcO681FXHC4l3e4GBZaNXxEnpyy+qwYpHML/3
+5jgTZ8OusMosfdWWP87ce9D97iXADJ5gaMZr3c7w1VbJqxSu7dX96lNmvzjdGW1M
+WFCHXmPXElZtmcPXCMoA9G5rUSBnwEgNe5ILdejz/l9TNp1OD0+F
 -----END RSA PRIVATE KEY-----

diff --git a/test_cases/dell_santa_clara/network/iperf_topology_1.yaml b/test_cases/dell_santa_clara/network/iperf_topology_1.yaml
index 20a433a..18eba8a 100644 (file)
--- a/test_cases/dell_santa_clara/network/iperf_topology_1.yaml
+++ b/test_cases/dell_santa_clara/network/iperf_topology_1.yaml
@@ -4,8 +4,8 @@ Scenario:
   server : virtualmachine_1
   client: virtualmachine_2
   benchmark_details:
-    duration: 10 
-    protocol: udp
+    duration: 20 
+    protocol: tcp
     bandwidthGbps: 10
 
 Context:
@@ -14,16 +14,16 @@ Context:
 
   Virtual_Machines:
     virtualmachine_1:
-      availability_zone: compute2
+      availability_zone: compute1
       OS_image: QTIP_CentOS
-      public_network: 'provider_network'
+      public_network: 'net04_ext'
       role: 1-server
       flavor: m1.large   
 
     virtualmachine_2:
-      availability_zone: compute2
+      availability_zone: compute1
       OS_image: QTIP_CentOS
-      public_network: 'provider_network'
+      public_network: 'net04_ext'
       role: 2-host
       flavor: m1.large
     

diff --git a/test_cases/dell_santa_clara/network/iperf_topology_2.yaml b/test_cases/dell_santa_clara/network/iperf_topology_2.yaml
index 4392f32..301a991 100644 (file)
--- a/test_cases/dell_santa_clara/network/iperf_topology_2.yaml
+++ b/test_cases/dell_santa_clara/network/iperf_topology_2.yaml
@@ -5,8 +5,8 @@ Scenario:
   client: virtualmachine_2
   benchmark_details:
     duration: 20
-    protocol: udp
-    bandwidthGbps: 1
+    protocol: tcp
+    bandwidthGbps: 10
 
 Context:
   Host_Machines:
@@ -16,14 +16,14 @@ Context:
     virtualmachine_1:
       availability_zone: compute1
       OS_image: QTIP_CentOS
-      public_network: 'provider_network'
+      public_network: 'net04_ext'
       role: 1-server
       flavor: m1.large   
 
     virtualmachine_2:
       availability_zone: compute2
       OS_image: QTIP_CentOS
-      public_network: 'provider_network'
+      public_network: 'net04_ext'
       role: 2-host
       flavor: m1.large
     

diff --git a/test_cases/dell_santa_clara/network/netperf.yaml b/test_cases/dell_santa_clara/network/netperf.yaml
index 60ee4a1..267b288 100644 (file)
--- a/test_cases/dell_santa_clara/network/netperf.yaml
+++ b/test_cases/dell_santa_clara/network/netperf.yaml
@@ -15,14 +15,14 @@ Context:
     virtualmachine_1:
       availability_zone: compute2
       OS_image: QTIP_CentOS
-      public_network: 'provider_network'
+      public_network: 'net04_ext'
       role: 1-server
       flavor: m1.large   
 
     virtualmachine_2:
       availability_zone: compute2
       OS_image: QTIP_CentOS
-      public_network: 'provider_network'
+      public_network: 'net04_ext'
       role: 2-host
       flavor: m1.large
     

diff --git a/test_cases/dell_santa_clara/storage/fio_parallel.yaml b/test_cases/dell_santa_clara/storage/fio_parallel.yaml
index e058af2..9f11af0 100644 (file)
--- a/test_cases/dell_santa_clara/storage/fio_parallel.yaml
+++ b/test_cases/dell_santa_clara/storage/fio_parallel.yaml
@@ -6,11 +6,11 @@ Scenario:
 Context:
   Host_Machines:
     machine_1:
-      ip: 10.20.0.4
+      ip: 10.20.0.6
       pw: r00tme
       role: host
     machine_2:
-      ip: 10.20.0.5
+      ip: 10.20.0.7
       pw: r00tme
       role: host
 

diff --git a/test_list/network.txt b/test_list/network.txt
index e69de29..3ed6f82 100644 (file)
--- a/test_list/network.txt
+++ b/test_list/network.txt
@@ -0,0 +1 @@
+iperf_topology_1.yaml

diff --git a/test_list/storage.txt b/test_list/storage.txt
index e69de29..7884a43 100644 (file)
--- a/test_list/storage.txt
+++ b/test_list/storage.txt
@@ -0,0 +1 @@
+fio_parallel.yaml