RAMSpeed benchmark playbooks, result transformation scripts included.
System Information also included for Network Testcases
System Information Playbook checks for the type of testcase, and gathers
benchmark relevant information only (No Network Info for Compute TestCase)
JIRA: QTIP-54
JIRA: QTIP-38
Change-Id: I8612f06fedfaf7314552c06f388ea6c0cd23f1ee
Signed-off-by: Nauman_Ahad <Nauman_Ahad@dell.com>
     - name: make directory
       shell: mkdir $HOME/qtip_result
     - include: ./sys_info_pbook.yaml
+      vars:
+          network: false
     - name: Installing UnixBench dependencies if CentOS
       shell: yum install git gcc  patch perl-Time-HiRes -y
       when: ansible_os_family == "RedHat"
 
     - name: make qtip_result
       shell: mkdir $HOME/qtip_result
     - include: ./sys_info_pbook.yaml
+      vars:
+          network: false
     - name: Installing nDPI dependencies if CentOS
       shell: yum install git gcc patch  perl-Time-HiRes autofconf automake libpcap-devel libtool -y
       when: ansible_os_family == "RedHat"
 
 import time
 import datetime
 import json
+import sys
 
 os.system('inxi -b -c0 -n > $PWD/est_2')
-est_ob = open("est_2", "r+")
-est_ob2 = open("est_1", "w+")
-in_string = est_ob.read().replace('\n', ' ')
-
-est_ob2.write(in_string)
+est_ob=open("est_2","r+")
+est_ob2=open("est_1","w+")
+in_string= est_ob.read().replace('\n',' ')
+cpu_idle=float(os.popen("""top -bn1 | grep "Cpu(s)" | awk '{print $8}'""").read().rstrip())
+cpu_usage= 100-cpu_idle
+est_ob2.write(in_string);
 est_ob.close()
 est_ob2.close()
 
-Info_dict = {}
-inxi_host = os.popen(
-    """cat $PWD/est_1 | grep -o -P '(?<=Host:).*(?=Kernel)' """).read().lstrip().rstrip()
-inxi_mem = os.popen(
-    """inxi -c0 | grep -o -P '(?<=Mem~).*(?=HDD)' """).read().lstrip().rstrip()
-inxi_cpu = os.popen(
-    """cat $PWD/est_1 | grep -o -P '(?<=CPU).*(?=speed)' | cut -f2 -d':'""").read().lstrip().rstrip()
-
-inxi_distro = os.popen(
-    """ cat $PWD/est_1 | grep -o -P '(?<=Distro:).*(?=Machine:)' """).read().rstrip().lstrip()
-inxi_kernel = os.popen(
-    """ cat $PWD/est_1 | grep -o -P '(?<=Kernel:).*(?=Console:)' """).read().rstrip().lstrip()
-inxi_HD = os.popen(
-    """ cat $PWD/est_1 | grep -o -P '(?<=HDD Total Size:).*(?=Info:)' """).read().rstrip().lstrip()
-
-inxi_product = os.popen(
-    """ cat $PWD/est_1 | grep -o -P '(?<=product:).*(?=Mobo:)' """).read().rstrip().lstrip()
+Info_dict={};
+inxi_host=os.popen("""cat $PWD/est_1 | grep -o -P '(?<=Host:).*(?=Kernel)' """).read().lstrip().rstrip()
+inxi_mem=os.popen("""cat $PWD/est_1 | grep -o -P '(?<=Memory:).*(?=Init)' """).read().lstrip().rstrip()
+inxi_cpu=os.popen("""cat $PWD/est_1 | grep -o -P '(?<=CPU).*(?=speed)' | cut -f2 -d':'""").read().lstrip().rstrip()
+inxi_distro=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Distro:).*(?=Machine:)' """).read().rstrip().lstrip()
+inxi_kernel=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Kernel:).*(?=Console:)' """).read().rstrip().lstrip()
+inxi_HD=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=HDD Total Size:).*(?=Info:)' """).read().rstrip().lstrip()
+inxi_product=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=product:).*(?=Mobo:)' """).read().rstrip().lstrip()
 
 
 
-Info_dict['1_Hostname'] = inxi_host
-Info_dict['2_Product'] = inxi_product
-Info_dict['3_OS Distribution'] = inxi_distro
-Info_dict['4_Kernel'] = inxi_kernel
-Info_dict['5_CPU'] = inxi_cpu
-Info_dict['6_Memory Usage'] = inxi_mem
-Info_dict['7_Disk usage'] = inxi_HD
-Info_dict['8_Network_Interfaces'] = {}
+Info_dict['1_Hostname']=inxi_host
+Info_dict['2_Product']=inxi_product
+Info_dict['3_OS Distribution']=inxi_distro
+Info_dict['4_Kernel']=inxi_kernel
+Info_dict['5_CPU']=inxi_cpu
+Info_dict['6_CPU_Usage']=str(round(cpu_usage,3))+'%'
+Info_dict['7_Memory Usage']=inxi_mem
+Info_dict['8_Disk usage']=inxi_HD
+network_flag=str(sys.argv[1]).rstrip()
 
-tem_2 = """ cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
-print os.system(tem_2 + ' > Hello')
-i = int(os.popen(tem_2 + " | grep -o 'Card' | wc -l ").read())
-print i
+if (network_flag == 'n'):
+    
+    Info_dict['9_Network_Interfaces']={};
+    tem_2=""" cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
+    print os.system(tem_2+' > Hello')
+    i=int(os.popen(tem_2+" | grep -o 'Card' | wc -l ").read())
+    print i
 
 
-for x in range(1, i + 1):
-    tem = """ cat $PWD/est_1 | grep -o -P '(?<=Card-""" + str(
-        x) + """:).*(?=Card-""" + str(x + 1) + """)'"""
-    if i == 1:
-        tem = """ cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
-        inxi_card_1 = (
-            (os.popen(
-                tem +
-                " | grep -o -P '(?<=Card:).*(?=Drives:)'|sed 's/ *driver:.*//'").read().rstrip().lstrip()))
-        print inxi_card_1
-        Info_dict['8_Network_Interfaces']['Interface_' + str(x)] = {}
-        Info_dict['8_Network_Interfaces'][
-            'Interface_' + str(x)]['1_Network_Card'] = inxi_card_1
-        inxi_card_2 = (
-            (os.popen(
-                tem +
-                "| grep -o -P '(?<=Card:).*(?=Drives:)'|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
-        Info_dict['8_Network_Interfaces'][
-            'Interface_' + str(x)]['2_Interface_info'] = inxi_card_2
-    elif x < (i):
-        print "two"
-        #inxi_Card_temp=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Card-"""+str(x+1)+""")' """).read().rstrip().lstrip()))
-        inxi_card_1 = (
-            (os.popen(tem + "| sed 's/ *driver:.*//'").read().rstrip().lstrip()))
-        Info_dict['8_Network_Interfaces']['Interface_' + str(x)] = {}
-        Info_dict['8_Network_Interfaces'][
-            'Interface_' + str(x)]['1_Network_Card'] = inxi_card_1
-        inxi_card_2 = (
-            (os.popen(tem + "|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
-        Info_dict['8_Network_Interfaces'][
-            'Interface_' + str(x)]['2_Interface_info'] = inxi_card_2
-    elif x == i:
-        print "Three"
-        Info_dict['8_Network_Interfaces']['Interface_' + str(x)] = {}
-        inxi_card_1 = (
-            (os.popen(
-                """ cat $PWD/est_1 | grep -o -P '(?<=Card-""" +
-                str(x) +
-                """:).*(?=Drives:)'| sed 's/ *driver:.*//' """).read().rstrip().lstrip()))
-        Info_dict['8_Network_Interfaces'][
-            'Interface_' + str(x)]['1_Network_Card'] = inxi_card_1
-        inxi_card_2 = (
-            (os.popen(
-                """ cat $PWD/est_1 | grep -o -P '(?<=Card-""" +
-                str(x) +
-                """:).*(?=Drives:)'| sed -e 's/^.*IF: //' """).read().rstrip().lstrip()))
-        Info_dict['8_Network_Interfaces'][
-            'Interface_' + str(x)]['2_Interface_info'] = inxi_card_2
-    else:
-        print "No network cards"
+    for x in range (1,i+1):
+            tem=""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Card-"""+str(x+1)+""")'"""
+            if i == 1:
+                tem=""" cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
+                inxi_card_1=((os.popen(tem+" | grep -o -P '(?<=Card:).*(?=Drives:)'|sed 's/ *driver:.*//'").read().rstrip().lstrip()))
+                print inxi_card_1
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]={};
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['1_Network_Card']=inxi_card_1
+                inxi_card_2=((os.popen(tem+"| grep -o -P '(?<=Card:).*(?=Drives:)'|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['2_Interface_info']=inxi_card_2
+            elif x < (i):
+                print "two"
+                #inxi_Card_temp=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Card-"""+str(x+1)+""")' """).read().rstrip().lstrip()))
+                inxi_card_1=((os.popen(tem+"| sed 's/ *driver:.*//'").read().rstrip().lstrip()))
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]={};
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['1_Network_Card']=inxi_card_1
+                inxi_card_2=((os.popen(tem+"|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['2_Interface_info']=inxi_card_2
+            elif x == i:
+                print "Three"
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]={};
+                inxi_card_1=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Drives:)'| sed 's/ *driver:.*//' """).read().rstrip().lstrip()))
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['1_Network_Card']=inxi_card_1
+                inxi_card_2=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Drives:)'| sed -e 's/^.*IF: //' """).read().rstrip().lstrip()))
+                Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['2_Interface_info']=inxi_card_2 
+            else:
+                print "No network cards"
+    os.system("bwm-ng -c 1 | grep -v '=' | grep -v 'iface' | grep -v '-'   > bwm_dump")
+    n_interface=int(os.popen(" cat bwm_dump | grep -v 'total' |  wc -l ").read().rstrip())
+    interface={};
+    for x in range (1,n_interface):
+        interface_name=os.popen(" cat bwm_dump | awk 'NR=="+str(x)+"' | awk '{print $1}' ").read().rstrip().replace(':','')
+        interface[str(interface_name)]={};
+        interface[str(interface_name)]['Rx (KB/s)']=os.popen(" cat bwm_dump | awk 'NR=="+str(x)+"' | awk '{print $2}' ").read().rstrip()
+        interface[str(interface_name)]['Tx (KB/s)']=os.popen(" cat bwm_dump | awk 'NR=="+str(x)+"' | awk '{print $4}' ").read().rstrip()
+        interface[str(interface_name)]['Total (KB/s)']=os.popen(" cat bwm_dump | awk 'NR== "+str(x)+"' | awk '{print $6}' ").read().rstrip()
+ 
+    Info_dict['Interface I/O']=interface
 
-home_dir = str(os.popen("echo $HOME").read().rstrip())
-with open('./sys_info_temp', 'w+')as out_info:
-    pickle.dump(Info_dict, out_info)
+print Info_dict
+    
+with open('./sys_info_temp','w+')as out_info:
+    pickle.dump(Info_dict,out_info)
 
+with open('temp','w+') as result_json:
+    json.dump(Info_dict,result_json,indent=4,sort_keys=True)
 
 
--- /dev/null
+ - hosts: 127.0.0.1
+   connection: local
+   tasks:
+    - name: making ramspeed  directory
+      file: path={{Dest_dir}}/ramspeed state=directory
+    - name: making temporary ramspeed directory
+      file: path={{Dest_dir}}/ramspeed/ramspeed_temp state=directory
+ 
+
+ - hosts:  "{{role}}"
+   tasks:
+   
+    - name: checking_home directory
+      shell: echo $HOME
+      register: home_dir
+    - name: cleaning
+      shell: rm -rf $HOME/ramspeed
+    - name: cleaning previous results
+      shell: rm -rf $HOME/qtip_result
+    - name: making ramspeed temporary directory
+      shell: mkdir $HOME/ramspeed  
+    - name: making  results temporary directory
+      shell: mkdir $HOME/qtip_result 
+    - include: ./sys_info_pbook.yaml
+      vars:
+          network: false
+    - name: Installing RAM_Speed dependencies when CentOS
+      shell: yum install wget gcc -y
+      when: ansible_os_family == "RedHat"
+    - name: Installing RAM_Speed dependencies when Ubuntu
+      shell: apt-get  install wget gcc -y
+      when: ansible_os_family == "Debian"
+    - name: make dummy file
+      shell: touch $HOME/ramspeed/ramspeed.tar.gz
+    - name: Fetching RAM_Speed
+      shell: cd $HOME/ramspeed/ && wget -O ramspeed.tar.gz https://docs.google.com/uc?id=0B92Bp5LZTM7gRFctalZLMktTNDQ
+    - name: Untar RAM_SPeed
+      shell: cd $HOME/ramspeed/ && tar -zxvf ramspeed.tar.gz
+    - name: configure
+      shell: cd $HOME/ramspeed/ramsmp-3.5.0 && ./build.sh
+    - name: Benchmarking IntMem Bandwidth
+      shell: cd $HOME/ramspeed/ramsmp-3.5.0  &&  ./ramsmp -b 3 -l 5 -p 1 >> $HOME/qtip_result/Intmem
+    - name: Benchmarking FloatMem Bandwidth
+      shell: cd $HOME/ramspeed/ramsmp-3.5.0  &&  ./ramsmp -b 6 -l 5 -p 1 >> $HOME/qtip_result/Floatmem
+    - name: Fetching result transformation script
+      copy: src=./result_transform/ramspd/ramspd_transform.py dest={{home_dir.stdout}}/qtip_result
+    - name: Transforming result
+      shell: cd $HOME/qtip_result  &&  python ramspd_transform.py
+    - name: copy report formation script
+      copy: src=./result_transform/final_report.py dest={{home_dir.stdout}}/qtip_result
+    - name: consolidating report
+      shell: cd $HOME/qtip_result &&  python final_report.py RamSpeed
+    - name: registering files
+      shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2
+      register: files_to_copy
+    - name: copy results
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ramspeed/ramspeed_temp
+      with_items: files_to_copy.stdout_lines
+    - name: registering log files
+      shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2
+      register: copy_log_results
+    - name: copying log results
+      fetch:  src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ramspeed/ramspeed_temp
+      with_items: copy_log_results.stdout_lines
+
+  
+    - name: copy results
+      fetch:  src=/root/results/{{item}}  dest={{Dest_dir}}/ssl
+      with_items: files_to_copy.stdout_lines
+
+ - hosts: 127.0.0.1
+   connection: local
+   tasks:
+    - name: extracting_json
+      shell: ( find  {{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/ramspeed/)
+    - name: making_logs_folder
+      shell: mkdir -p {{Dest_dir}}/ramspeed/logs
+    - name: extracting_log
+      shell: ( find {{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.log"  | xargs cp -t {{Dest_dir}}/ramspeed/logs)
+    - name: removing ramspeed_log
+      shell: rm -rf {{Dest_dir}}/ramspeed/ramspeed_temp
+
 
+++ /dev/null
- 
- - hosts: "{{ role }}"
-
-   tasks:
-    - name: clean
-      shell: rm -rf /usr/bin/phoronix-test-suite
-      shell: rm -rf  /usr/share/doc/phoronix-test-suite/
-      shell: rm -rf  /usr/share/phoronix-test-suite
-      shell: rm -rf  /root/photmp/
-    - name: clean previous results
-      file: path=/root/results path=absent
-    - name: make direc
-      file: path=/root/photmp/ state=directory
-    - name: Fetch Phornonix 
-      shell: cd /root/photmp/ && wget https://www.dropbox.com/s/5kks513ozxy7vvs/phoronix-suite.tar.gz
-    - name: unarchive Phoronix
-      shell: cd /root/photmp/ && tar -zxvf  phoronix-suite.tar.gz
-    - name: Phornonix Dependencies
-      shell: yum install php php-xml php-client php-process -y
-    - name: Install Phornonix
-      shell: cd /root/photmp/phoronix-test-suite-master/&&./install-sh    
-    - name: batchSetup
-      shell: phoronix-test-suite batch-setup
-    - name: install
-      shell: phoronix-test-suite batch-install ramspeed
-    - name: run
-      shell:  phoronix-test-suite batch-run ramspeed
-    - name: making directory
-      file: path=/root/results state=directory
-    - name: copying result to temp directory
-      shell: cp -r /var/lib/phoronix-test-suite/test-results/* /root/results/
-    - name: registering files
-      shell: (cd /root/results/; find . -maxdepth 1 -type f) | cut -d'/' -f2
-      register: files_to_copy
-    - name: copy results
-      fetch:  src=/root/results/{{item}} dest={{Dest_dir}}/dhrystone
-      with_items: files_to_copy.stdout_lines
-
 
--- /dev/null
+import os
+import json
+import pickle
+import datetime
+
+
+intmem_copy = os.popen("cat Intmem | grep 'BatchRun   Copy' | awk '{print $4}'").read().rstrip()
+intmem_scale = os.popen("cat Intmem | grep 'BatchRun   Scale' | awk '{print $4}'").read().rstrip()
+intmem_add = os.popen("cat Intmem | grep 'BatchRun   Add' | awk '{print $4}'").read().rstrip()
+intmem_triad = os.popen("cat Intmem | grep 'BatchRun   Triad' | awk '{print $4}'").read().rstrip()
+intmem_average = os.popen("cat Intmem | grep 'BatchRun   AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print intmem_copy
+print intmem_average
+
+floatmem_copy = os.popen("cat Floatmem | grep 'BatchRun  Copy' | awk '{print $4}'").read().rstrip()
+floatmem_scale = os.popen("cat Floatmem | grep 'BatchRun  Scale' | awk '{print $4}'").read().rstrip()
+floatmem_add = os.popen("cat Floatmem | grep 'BatchRun  Add' | awk '{print $4}'").read().rstrip()
+floatmem_triad = os.popen("cat Floatmem | grep 'BatchRun  Triad' | awk '{print $4}'").read().rstrip()
+floatmem_average = os.popen("cat Floatmem | grep 'BatchRun  AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print floatmem_copy
+print floatmem_average
+
+
+hostname = os.popen("hostname").read().rstrip()
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+
+os.system("mv Intmem " + hostname + "-" + time_stamp + ".log")
+os.system("cp  Floatmem >> " + hostname + "-" + time_stamp + ".log")
+
+
+result = {};
+
+result['1. INTmem bandwidth'] = {};
+result['1. INTmem bandwidth']['1. Copy (MB/s)'] = [intmem_copy]
+result['1. INTmem bandwidth']['2. Add (MB/s)'] = [intmem_add]
+result['1. INTmem bandwidth']['3. Scale (MB/s)'] = [intmem_scale]
+result['1. INTmem bandwidth']['4.Triad (MB/s)']= [intmem_triad]
+result['1. INTmem bandwidth']['5. Average (MB/s)'] = [intmem_average]
+
+
+result['2. FLOATmem bandwidth'] = {};
+result['2. FLOATmem bandwidth']['1.Copy (MB/s)']=[floatmem_copy]
+result['2. FLOATmem bandwidth']['2.Add (MB/s)']=[floatmem_add]
+result['2. FLOATmem bandwidth']['3.Scale (MB/s)']=[floatmem_scale]
+result['2. FLOATmem bandwidth']['4.Triad (MB/s)']=[floatmem_triad]
+result['2. FLOATmem bandwidth']['4.Copy (MB/s)']=[floatmem_average]
+
+
+
+with open('./result_temp', 'w+') as result_file:
+    pickle.dump(result, result_file)
+
+
 
     - name: making  results temporary directory
       shell: mkdir $HOME/qtip_result 
     - include: ./sys_info_pbook.yaml
+      vars:
+          network: false
     - name: Installing OpenSSL dependencies when CentOS
       shell: yum install git  wget gcc patch  perl-Time-HiRes autofconf automake libpcap-devel libtool -y
       when: ansible_os_family == "RedHat"
 
    - name: system info collection tool install when Ubuntu
      shell: apt-get install inxi -y
      when: ansible_os_family  == "Debian"
+   - name: Install ansible copy dependencies if remote host has selinux enabled
+     shell: yum  install libselinux-python -y
+     when: ansible_os_family == "RedHat"
+   - name: Install ansiblle copy dependencies if remote host has selinux enaled
+     shell: apt-get install libselinux-python -y
+     when: ansible_os_family == "Debian"
    - name: system_info script copy
      copy: src=./info_script/info_collect.py dest={{home_dir.stdout}}/qtip_result/
-   - name: collecting_sys_info
-     shell: cd $HOME/qtip_result &&  python info_collect.py
+   - name: collecting system informaton for non-network test cases
+     shell: cd $HOME/qtip_result &&  python info_collect.py c
+     when: not network
+   - name: collecting system information for network test  cases
+     shell: cd $HOME/qtip_result && python info_collect.py n
+     when: network
    
 
 
     - name: make directory
       shell: mkdir $HOME/qtip_result
     - include: ./sys_info_pbook.yaml
+      vars:
+          network: false
     - name: Installing UnixBench dependencies if CentOS
       shell: yum install git gcc patch perl-Time-HiRes -y
       when: ansible_os_family == "RedHat"
 
     type: string
     label: Publi_N_ID
     description: Public
-    default: 'provider_network'
+    default: 'net04_ext'
 
   private_net_name:
     type: string