RAMSpeed benchmark playbooks, result transformation scripts included.
System Information also included for Network Testcases
System Information Playbook checks for the type of testcase, and gathers
benchmark relevant information only (No Network Info for Compute TestCase)
JIRA: QTIP-54
JIRA: QTIP-38
Change-Id: I8612f06fedfaf7314552c06f388ea6c0cd23f1ee
Signed-off-by: Nauman_Ahad <Nauman_Ahad@dell.com>
- name: make directory
shell: mkdir $HOME/qtip_result
- include: ./sys_info_pbook.yaml
+ vars:
+ network: false
- name: Installing UnixBench dependencies if CentOS
shell: yum install git gcc patch perl-Time-HiRes -y
when: ansible_os_family == "RedHat"
- name: make qtip_result
shell: mkdir $HOME/qtip_result
- include: ./sys_info_pbook.yaml
+ vars:
+ network: false
- name: Installing nDPI dependencies if CentOS
shell: yum install git gcc patch perl-Time-HiRes autofconf automake libpcap-devel libtool -y
when: ansible_os_family == "RedHat"
import time
import datetime
import json
+import sys
os.system('inxi -b -c0 -n > $PWD/est_2')
-est_ob = open("est_2", "r+")
-est_ob2 = open("est_1", "w+")
-in_string = est_ob.read().replace('\n', ' ')
-
-est_ob2.write(in_string)
+est_ob=open("est_2","r+")
+est_ob2=open("est_1","w+")
+in_string= est_ob.read().replace('\n',' ')
+cpu_idle=float(os.popen("""top -bn1 | grep "Cpu(s)" | awk '{print $8}'""").read().rstrip())
+cpu_usage= 100-cpu_idle
+est_ob2.write(in_string);
est_ob.close()
est_ob2.close()
-Info_dict = {}
-inxi_host = os.popen(
- """cat $PWD/est_1 | grep -o -P '(?<=Host:).*(?=Kernel)' """).read().lstrip().rstrip()
-inxi_mem = os.popen(
- """inxi -c0 | grep -o -P '(?<=Mem~).*(?=HDD)' """).read().lstrip().rstrip()
-inxi_cpu = os.popen(
- """cat $PWD/est_1 | grep -o -P '(?<=CPU).*(?=speed)' | cut -f2 -d':'""").read().lstrip().rstrip()
-
-inxi_distro = os.popen(
- """ cat $PWD/est_1 | grep -o -P '(?<=Distro:).*(?=Machine:)' """).read().rstrip().lstrip()
-inxi_kernel = os.popen(
- """ cat $PWD/est_1 | grep -o -P '(?<=Kernel:).*(?=Console:)' """).read().rstrip().lstrip()
-inxi_HD = os.popen(
- """ cat $PWD/est_1 | grep -o -P '(?<=HDD Total Size:).*(?=Info:)' """).read().rstrip().lstrip()
-
-inxi_product = os.popen(
- """ cat $PWD/est_1 | grep -o -P '(?<=product:).*(?=Mobo:)' """).read().rstrip().lstrip()
+Info_dict={};
+inxi_host=os.popen("""cat $PWD/est_1 | grep -o -P '(?<=Host:).*(?=Kernel)' """).read().lstrip().rstrip()
+inxi_mem=os.popen("""cat $PWD/est_1 | grep -o -P '(?<=Memory:).*(?=Init)' """).read().lstrip().rstrip()
+inxi_cpu=os.popen("""cat $PWD/est_1 | grep -o -P '(?<=CPU).*(?=speed)' | cut -f2 -d':'""").read().lstrip().rstrip()
+inxi_distro=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Distro:).*(?=Machine:)' """).read().rstrip().lstrip()
+inxi_kernel=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Kernel:).*(?=Console:)' """).read().rstrip().lstrip()
+inxi_HD=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=HDD Total Size:).*(?=Info:)' """).read().rstrip().lstrip()
+inxi_product=os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=product:).*(?=Mobo:)' """).read().rstrip().lstrip()
-Info_dict['1_Hostname'] = inxi_host
-Info_dict['2_Product'] = inxi_product
-Info_dict['3_OS Distribution'] = inxi_distro
-Info_dict['4_Kernel'] = inxi_kernel
-Info_dict['5_CPU'] = inxi_cpu
-Info_dict['6_Memory Usage'] = inxi_mem
-Info_dict['7_Disk usage'] = inxi_HD
-Info_dict['8_Network_Interfaces'] = {}
+Info_dict['1_Hostname']=inxi_host
+Info_dict['2_Product']=inxi_product
+Info_dict['3_OS Distribution']=inxi_distro
+Info_dict['4_Kernel']=inxi_kernel
+Info_dict['5_CPU']=inxi_cpu
+Info_dict['6_CPU_Usage']=str(round(cpu_usage,3))+'%'
+Info_dict['7_Memory Usage']=inxi_mem
+Info_dict['8_Disk usage']=inxi_HD
+network_flag=str(sys.argv[1]).rstrip()
-tem_2 = """ cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
-print os.system(tem_2 + ' > Hello')
-i = int(os.popen(tem_2 + " | grep -o 'Card' | wc -l ").read())
-print i
+if (network_flag == 'n'):
+
+ Info_dict['9_Network_Interfaces']={};
+ tem_2=""" cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
+ print os.system(tem_2+' > Hello')
+ i=int(os.popen(tem_2+" | grep -o 'Card' | wc -l ").read())
+ print i
-for x in range(1, i + 1):
- tem = """ cat $PWD/est_1 | grep -o -P '(?<=Card-""" + str(
- x) + """:).*(?=Card-""" + str(x + 1) + """)'"""
- if i == 1:
- tem = """ cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
- inxi_card_1 = (
- (os.popen(
- tem +
- " | grep -o -P '(?<=Card:).*(?=Drives:)'|sed 's/ *driver:.*//'").read().rstrip().lstrip()))
- print inxi_card_1
- Info_dict['8_Network_Interfaces']['Interface_' + str(x)] = {}
- Info_dict['8_Network_Interfaces'][
- 'Interface_' + str(x)]['1_Network_Card'] = inxi_card_1
- inxi_card_2 = (
- (os.popen(
- tem +
- "| grep -o -P '(?<=Card:).*(?=Drives:)'|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
- Info_dict['8_Network_Interfaces'][
- 'Interface_' + str(x)]['2_Interface_info'] = inxi_card_2
- elif x < (i):
- print "two"
- #inxi_Card_temp=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Card-"""+str(x+1)+""")' """).read().rstrip().lstrip()))
- inxi_card_1 = (
- (os.popen(tem + "| sed 's/ *driver:.*//'").read().rstrip().lstrip()))
- Info_dict['8_Network_Interfaces']['Interface_' + str(x)] = {}
- Info_dict['8_Network_Interfaces'][
- 'Interface_' + str(x)]['1_Network_Card'] = inxi_card_1
- inxi_card_2 = (
- (os.popen(tem + "|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
- Info_dict['8_Network_Interfaces'][
- 'Interface_' + str(x)]['2_Interface_info'] = inxi_card_2
- elif x == i:
- print "Three"
- Info_dict['8_Network_Interfaces']['Interface_' + str(x)] = {}
- inxi_card_1 = (
- (os.popen(
- """ cat $PWD/est_1 | grep -o -P '(?<=Card-""" +
- str(x) +
- """:).*(?=Drives:)'| sed 's/ *driver:.*//' """).read().rstrip().lstrip()))
- Info_dict['8_Network_Interfaces'][
- 'Interface_' + str(x)]['1_Network_Card'] = inxi_card_1
- inxi_card_2 = (
- (os.popen(
- """ cat $PWD/est_1 | grep -o -P '(?<=Card-""" +
- str(x) +
- """:).*(?=Drives:)'| sed -e 's/^.*IF: //' """).read().rstrip().lstrip()))
- Info_dict['8_Network_Interfaces'][
- 'Interface_' + str(x)]['2_Interface_info'] = inxi_card_2
- else:
- print "No network cards"
+ for x in range (1,i+1):
+ tem=""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Card-"""+str(x+1)+""")'"""
+ if i == 1:
+ tem=""" cat $PWD/est_1 | grep -o -P '(?<=Network:).*(?=Info:)'"""
+ inxi_card_1=((os.popen(tem+" | grep -o -P '(?<=Card:).*(?=Drives:)'|sed 's/ *driver:.*//'").read().rstrip().lstrip()))
+ print inxi_card_1
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]={};
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['1_Network_Card']=inxi_card_1
+ inxi_card_2=((os.popen(tem+"| grep -o -P '(?<=Card:).*(?=Drives:)'|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['2_Interface_info']=inxi_card_2
+ elif x < (i):
+ print "two"
+ #inxi_Card_temp=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Card-"""+str(x+1)+""")' """).read().rstrip().lstrip()))
+ inxi_card_1=((os.popen(tem+"| sed 's/ *driver:.*//'").read().rstrip().lstrip()))
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]={};
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['1_Network_Card']=inxi_card_1
+ inxi_card_2=((os.popen(tem+"|sed -e 's/^.*IF: //'").read())).rstrip().lstrip()
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['2_Interface_info']=inxi_card_2
+ elif x == i:
+ print "Three"
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]={};
+ inxi_card_1=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Drives:)'| sed 's/ *driver:.*//' """).read().rstrip().lstrip()))
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['1_Network_Card']=inxi_card_1
+ inxi_card_2=((os.popen(""" cat $PWD/est_1 | grep -o -P '(?<=Card-"""+str(x)+""":).*(?=Drives:)'| sed -e 's/^.*IF: //' """).read().rstrip().lstrip()))
+ Info_dict['9_Network_Interfaces']['Interface_'+str(x)]['2_Interface_info']=inxi_card_2
+ else:
+ print "No network cards"
+ os.system("bwm-ng -c 1 | grep -v '=' | grep -v 'iface' | grep -v '-' > bwm_dump")
+ n_interface=int(os.popen(" cat bwm_dump | grep -v 'total' | wc -l ").read().rstrip())
+ interface={};
+ for x in range (1,n_interface):
+ interface_name=os.popen(" cat bwm_dump | awk 'NR=="+str(x)+"' | awk '{print $1}' ").read().rstrip().replace(':','')
+ interface[str(interface_name)]={};
+ interface[str(interface_name)]['Rx (KB/s)']=os.popen(" cat bwm_dump | awk 'NR=="+str(x)+"' | awk '{print $2}' ").read().rstrip()
+ interface[str(interface_name)]['Tx (KB/s)']=os.popen(" cat bwm_dump | awk 'NR=="+str(x)+"' | awk '{print $4}' ").read().rstrip()
+ interface[str(interface_name)]['Total (KB/s)']=os.popen(" cat bwm_dump | awk 'NR== "+str(x)+"' | awk '{print $6}' ").read().rstrip()
+
+ Info_dict['Interface I/O']=interface
-home_dir = str(os.popen("echo $HOME").read().rstrip())
-with open('./sys_info_temp', 'w+')as out_info:
- pickle.dump(Info_dict, out_info)
+print Info_dict
+
+with open('./sys_info_temp','w+')as out_info:
+ pickle.dump(Info_dict,out_info)
+with open('temp','w+') as result_json:
+ json.dump(Info_dict,result_json,indent=4,sort_keys=True)
--- /dev/null
+ - hosts: 127.0.0.1
+ connection: local
+ tasks:
+ - name: making ramspeed directory
+ file: path={{Dest_dir}}/ramspeed state=directory
+ - name: making temporary ramspeed directory
+ file: path={{Dest_dir}}/ramspeed/ramspeed_temp state=directory
+
+
+ - hosts: "{{role}}"
+ tasks:
+
+ - name: checking_home directory
+ shell: echo $HOME
+ register: home_dir
+ - name: cleaning
+ shell: rm -rf $HOME/ramspeed
+ - name: cleaning previous results
+ shell: rm -rf $HOME/qtip_result
+ - name: making ramspeed temporary directory
+ shell: mkdir $HOME/ramspeed
+ - name: making results temporary directory
+ shell: mkdir $HOME/qtip_result
+ - include: ./sys_info_pbook.yaml
+ vars:
+ network: false
+ - name: Installing RAM_Speed dependencies when CentOS
+ shell: yum install wget gcc -y
+ when: ansible_os_family == "RedHat"
+ - name: Installing RAM_Speed dependencies when Ubuntu
+ shell: apt-get install wget gcc -y
+ when: ansible_os_family == "Debian"
+ - name: make dummy file
+ shell: touch $HOME/ramspeed/ramspeed.tar.gz
+ - name: Fetching RAM_Speed
+ shell: cd $HOME/ramspeed/ && wget -O ramspeed.tar.gz https://docs.google.com/uc?id=0B92Bp5LZTM7gRFctalZLMktTNDQ
+ - name: Untar RAM_SPeed
+ shell: cd $HOME/ramspeed/ && tar -zxvf ramspeed.tar.gz
+ - name: configure
+ shell: cd $HOME/ramspeed/ramsmp-3.5.0 && ./build.sh
+ - name: Benchmarking IntMem Bandwidth
+ shell: cd $HOME/ramspeed/ramsmp-3.5.0 && ./ramsmp -b 3 -l 5 -p 1 >> $HOME/qtip_result/Intmem
+ - name: Benchmarking FloatMem Bandwidth
+ shell: cd $HOME/ramspeed/ramsmp-3.5.0 && ./ramsmp -b 6 -l 5 -p 1 >> $HOME/qtip_result/Floatmem
+ - name: Fetching result transformation script
+ copy: src=./result_transform/ramspd/ramspd_transform.py dest={{home_dir.stdout}}/qtip_result
+ - name: Transforming result
+ shell: cd $HOME/qtip_result && python ramspd_transform.py
+ - name: copy report formation script
+ copy: src=./result_transform/final_report.py dest={{home_dir.stdout}}/qtip_result
+ - name: consolidating report
+ shell: cd $HOME/qtip_result && python final_report.py RamSpeed
+ - name: registering files
+ shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.json") | cut -d'/' -f2
+ register: files_to_copy
+ - name: copy results
+ fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ramspeed/ramspeed_temp
+ with_items: files_to_copy.stdout_lines
+ - name: registering log files
+ shell: (cd $HOME/qtip_result/; find . -maxdepth 1 -name "*.log") | cut -d'/' -f2
+ register: copy_log_results
+ - name: copying log results
+ fetch: src={{home_dir.stdout}}/qtip_result/{{item}} dest={{Dest_dir}}/ramspeed/ramspeed_temp
+ with_items: copy_log_results.stdout_lines
+
+
+ - name: copy results
+ fetch: src=/root/results/{{item}} dest={{Dest_dir}}/ssl
+ with_items: files_to_copy.stdout_lines
+
+ - hosts: 127.0.0.1
+ connection: local
+ tasks:
+ - name: extracting_json
+ shell: ( find {{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.json" | xargs cp -t {{Dest_dir}}/ramspeed/)
+ - name: making_logs_folder
+ shell: mkdir -p {{Dest_dir}}/ramspeed/logs
+ - name: extracting_log
+ shell: ( find {{Dest_dir}}/ramspeed/ramspeed_temp/ -name "*.log" | xargs cp -t {{Dest_dir}}/ramspeed/logs)
+ - name: removing ramspeed_log
+ shell: rm -rf {{Dest_dir}}/ramspeed/ramspeed_temp
+
+++ /dev/null
-
- - hosts: "{{ role }}"
-
- tasks:
- - name: clean
- shell: rm -rf /usr/bin/phoronix-test-suite
- shell: rm -rf /usr/share/doc/phoronix-test-suite/
- shell: rm -rf /usr/share/phoronix-test-suite
- shell: rm -rf /root/photmp/
- - name: clean previous results
- file: path=/root/results path=absent
- - name: make direc
- file: path=/root/photmp/ state=directory
- - name: Fetch Phornonix
- shell: cd /root/photmp/ && wget https://www.dropbox.com/s/5kks513ozxy7vvs/phoronix-suite.tar.gz
- - name: unarchive Phoronix
- shell: cd /root/photmp/ && tar -zxvf phoronix-suite.tar.gz
- - name: Phornonix Dependencies
- shell: yum install php php-xml php-client php-process -y
- - name: Install Phornonix
- shell: cd /root/photmp/phoronix-test-suite-master/&&./install-sh
- - name: batchSetup
- shell: phoronix-test-suite batch-setup
- - name: install
- shell: phoronix-test-suite batch-install ramspeed
- - name: run
- shell: phoronix-test-suite batch-run ramspeed
- - name: making directory
- file: path=/root/results state=directory
- - name: copying result to temp directory
- shell: cp -r /var/lib/phoronix-test-suite/test-results/* /root/results/
- - name: registering files
- shell: (cd /root/results/; find . -maxdepth 1 -type f) | cut -d'/' -f2
- register: files_to_copy
- - name: copy results
- fetch: src=/root/results/{{item}} dest={{Dest_dir}}/dhrystone
- with_items: files_to_copy.stdout_lines
-
--- /dev/null
+import os
+import json
+import pickle
+import datetime
+
+
+intmem_copy = os.popen("cat Intmem | grep 'BatchRun Copy' | awk '{print $4}'").read().rstrip()
+intmem_scale = os.popen("cat Intmem | grep 'BatchRun Scale' | awk '{print $4}'").read().rstrip()
+intmem_add = os.popen("cat Intmem | grep 'BatchRun Add' | awk '{print $4}'").read().rstrip()
+intmem_triad = os.popen("cat Intmem | grep 'BatchRun Triad' | awk '{print $4}'").read().rstrip()
+intmem_average = os.popen("cat Intmem | grep 'BatchRun AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print intmem_copy
+print intmem_average
+
+floatmem_copy = os.popen("cat Floatmem | grep 'BatchRun Copy' | awk '{print $4}'").read().rstrip()
+floatmem_scale = os.popen("cat Floatmem | grep 'BatchRun Scale' | awk '{print $4}'").read().rstrip()
+floatmem_add = os.popen("cat Floatmem | grep 'BatchRun Add' | awk '{print $4}'").read().rstrip()
+floatmem_triad = os.popen("cat Floatmem | grep 'BatchRun Triad' | awk '{print $4}'").read().rstrip()
+floatmem_average = os.popen("cat Floatmem | grep 'BatchRun AVERAGE' | awk '{print $4}'").read().rstrip()
+
+print floatmem_copy
+print floatmem_average
+
+
+hostname = os.popen("hostname").read().rstrip()
+time_stamp = str(datetime.datetime.utcnow().isoformat())
+
+
+os.system("mv Intmem " + hostname + "-" + time_stamp + ".log")
+os.system("cp Floatmem >> " + hostname + "-" + time_stamp + ".log")
+
+
+result = {};
+
+result['1. INTmem bandwidth'] = {};
+result['1. INTmem bandwidth']['1. Copy (MB/s)'] = [intmem_copy]
+result['1. INTmem bandwidth']['2. Add (MB/s)'] = [intmem_add]
+result['1. INTmem bandwidth']['3. Scale (MB/s)'] = [intmem_scale]
+result['1. INTmem bandwidth']['4.Triad (MB/s)']= [intmem_triad]
+result['1. INTmem bandwidth']['5. Average (MB/s)'] = [intmem_average]
+
+
+result['2. FLOATmem bandwidth'] = {};
+result['2. FLOATmem bandwidth']['1.Copy (MB/s)']=[floatmem_copy]
+result['2. FLOATmem bandwidth']['2.Add (MB/s)']=[floatmem_add]
+result['2. FLOATmem bandwidth']['3.Scale (MB/s)']=[floatmem_scale]
+result['2. FLOATmem bandwidth']['4.Triad (MB/s)']=[floatmem_triad]
+result['2. FLOATmem bandwidth']['4.Copy (MB/s)']=[floatmem_average]
+
+
+
+with open('./result_temp', 'w+') as result_file:
+ pickle.dump(result, result_file)
+
+
- name: making results temporary directory
shell: mkdir $HOME/qtip_result
- include: ./sys_info_pbook.yaml
+ vars:
+ network: false
- name: Installing OpenSSL dependencies when CentOS
shell: yum install git wget gcc patch perl-Time-HiRes autofconf automake libpcap-devel libtool -y
when: ansible_os_family == "RedHat"
- name: system info collection tool install when Ubuntu
shell: apt-get install inxi -y
when: ansible_os_family == "Debian"
+ - name: Install ansible copy dependencies if remote host has selinux enabled
+ shell: yum install libselinux-python -y
+ when: ansible_os_family == "RedHat"
+ - name: Install ansiblle copy dependencies if remote host has selinux enaled
+ shell: apt-get install libselinux-python -y
+ when: ansible_os_family == "Debian"
- name: system_info script copy
copy: src=./info_script/info_collect.py dest={{home_dir.stdout}}/qtip_result/
- - name: collecting_sys_info
- shell: cd $HOME/qtip_result && python info_collect.py
+ - name: collecting system informaton for non-network test cases
+ shell: cd $HOME/qtip_result && python info_collect.py c
+ when: not network
+ - name: collecting system information for network test cases
+ shell: cd $HOME/qtip_result && python info_collect.py n
+ when: network
- name: make directory
shell: mkdir $HOME/qtip_result
- include: ./sys_info_pbook.yaml
+ vars:
+ network: false
- name: Installing UnixBench dependencies if CentOS
shell: yum install git gcc patch perl-Time-HiRes -y
when: ansible_os_family == "RedHat"
type: string
label: Publi_N_ID
description: Public
- default: 'provider_network'
+ default: 'net04_ext'
private_net_name:
type: string