From a73d1e7f10933de0b614148674c2ea2611c973c8 Mon Sep 17 00:00:00 2001 From: Morgan Richomme Date: Wed, 4 May 2016 16:19:09 +0200 Subject: [PATCH] Add global scenario reporting Save date when a scenario is validated Change-Id: I0895ae56812c7059972a60bf017fdb76d40de25c Signed-off-by: Morgan Richomme --- utils/test/reporting/functest/img/icon-nok.png | Bin 0 -> 2317 bytes utils/test/reporting/functest/img/icon-ok.png | Bin 0 -> 4063 bytes utils/test/reporting/functest/reporting-status.py | 64 +++++++++++++++++++-- .../functest/template/index-status-tmpl.html | 13 ++++- 4 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 utils/test/reporting/functest/img/icon-nok.png create mode 100644 utils/test/reporting/functest/img/icon-ok.png diff --git a/utils/test/reporting/functest/img/icon-nok.png b/utils/test/reporting/functest/img/icon-nok.png new file mode 100644 index 0000000000000000000000000000000000000000..526b5294b28342faafaf7eb5e744b4e5d84e346c GIT binary patch literal 2317 zcmV+o3G()dP)Ac)|_QSb%=eFFL zxx*IO51!=azRcX`IsfN>&Uwyro(sQ8digHHZ_U@JPi-t43xOXf1p>eXpu)Tc=mXk; z0rR>y2*cMM$iIPkDFE|-z-TT(ECK2?;P=4f2OfahT2Ve0b5_vlStekH0Ui(R*+a(< zKhSCV5Af$8)MW#sxdd@U15YJTv~wpcU!F1_w2FyP`v!XJN3OCn)0GdBp|=fp@zP9?x;4JK9s zMYXj8A{M*4k#@_Fote%~!iNuA%ya>NFelQSOzWV6M<)YQUVRl-uNJ2qD-&|_o?$?H zI~|RUToh#w1)(|@Mm8$=pct4wcP_mB_DHjFQZO2o2;^4bcwE?3Dm$=Kr)cf&<`+@; zFG1Lo5r}D0MZg&kl>ZUXj~ zEC!1f!KziD>oQ2YcV8LC?Z6BJy`qBRwl;d8T!Y0y_@`ZWkAr1xRlu~;QW@oNSb#(# zBafjWnMY>N77$)96cxdJ_ra7Y;!nfFsUs4RXLIJjLl4O$>hrvIKpB2Hjj#k<;%CzPyxUCHEEL@ z39C{22!+EC36}`LU*?$mIDKD`7-qM zz@5L3sXo zaq2Bw;D#I0&%TieeETi@^i#&1N;i)?^!HyGO_E7Ce;)Sj zO99%p4JJ->?sf8H*s()W`b;1O2H@qFrQIxIt$E@yVV43O{{_+vL?ECbPyjG>s+g6X zjbswO`U<}JMj{!&fdkOnD+;6oQ&yH)vw;EFvco7yx^iFESxn>r0$`k+yugGz9h|*F0lKb>*;tMp zjlzi&;;`xDzyVmlJ~uEH6K}nhGyc3(ieKRrwE_S>k4G4=!rp}oaO_z2*6Zrvg%>31 zWRw1Wc=5%YnQ)lJ15Pu0%4VWb0R!;aXW}Dnq_`OJ^0NOg8ofGG%pi6~`T-KjWNI?1 ztCMRt^7!M~L##aj)YM3-aus45);=d7?HUlb3J8}jL0g-Oa?#!w<5{?X7NA=eEPufLuf2s@bV+jC|nVix*=w3(01o0aP8QyHbNPpR(i%>U`=P-@K@cYT6t1{IQ0(G=REY)!CUH%>DOg?KcMMlmKZl)2fj^V&*IjakG?}Rc6ePR6N@}v15n$LbiFP zv=r{TD{Hql{pzhe%ZZiwjiAoEVZ`fIYTGuYqETlio_p@fch;_TW+oa{s-{NTbn46J zQ) zyz&aPw!+@M?qcZj<+9d)?KR2mM$7c+5|;Dx!RsYnQ$rJ^Ek0x~2VIZ7yF2cHyYCiW z^7CQeK3dz_ta8xCAneV!X17JKJsKSP#agxen)29VaPPeW;Km!_%o#dTAPzZ!a8ojE z)?iYlX>Zq5CQgL4Yg375em=Cc5Ps*K|E*+7YER^(23%7xOP0XGg;FQbb*aQOHxu5z zot9+MO3VKggp*^ptuU4Vii@GTS_*q!udMC;emHiF@O$r#>9&FeW-0J54VZ*ef^!v8 zT`gr(i-mFHWQqO88+4pLJ$4&CS-0NYhUU@ru359Bvgr2#k_`>C_x9R1 zLl2pop}k|a8Jg4fbYUTSX{i+FrcQ;53SmJryRoV>r%w~FuP1Tgg46c&--7VLHQAod z`GWzoWRs`s_zThMAFJX1aU2d;0b3?zb=Zp8Ro__g;68 zP_C7#+^(+M@7>$?{l4e>`~J4`;FooL^k@4x`I$WB7U0Td@BIS~O^OJ46q=AWmL-;` zum?ME?@hq&Ulx$tfnD+6H{uF%lAvCxth`=Ozpg}WRp=2Bg&-iFjys&VDN^1!bjS@kj5K-y5V_VdN>V3_yabvUHc#`W0 zyeo3FZms>C82K<~NvJ-qQD+HSIZBgpvgF-DYjFYwOPvbatuHtB0UhPi>+h`fyK`JfrZZeJDa=f-v;0!;4?oZBXXN) zTeY87q53l=O1sJ<=jO;b89eRdY8Ov?5fJ0j4Fe{YCaAX?sV0<2YN(0ZvC(=EMC@YGy%8v1XS|##zOnO$*3rh_)}H=xi+y4s zkH+WU;w~NPKT@K|IXUuf9^d%*CWB{U5Zc4nK1MlARwk*msy(BmZ{-JGOCJ;cAbR(U zC_Lrqtj@iq`ufUr_uLY@EI`|mfTuh>4Ix82~ zxVXxr)F^Ry;Yd%wA_#*T=z)k|r%dR9|5NldrIn)EU0OO)JNSFGtE%70T{%SUmc;>+ z)d~+Q(vBwJ%;=tGO-J}u{*ii#ZLsTU1n>?nXi{) zWBPo4xzVxzN7rMm#F47fUZfe+(F&T~I@L}MUuQ8&qtO(cp|Q5K*456f-0z=Ju-zxx zCt&2mz+R9*KBUm68POEH0vVHyb&$a~9-fZe%QKNSn%x%DmFZPia}Y$b;?frg1OaC2 z$9U`MZ{(p1zsy&5+{=CEe}T)+x|ID3&k}@PL=;*Ko3mRlEcnA`Hvgppx2?v=b-?w& z2DX~ed`yYX=G_9r{&1Y&e0<~LYCi%I=L@Yg&zGO4+Nz?|^5~PelkQ_e^_UO^)-uyL z#s%B9^U>eFm7-r{C^JOCEAX0iujW_Ryn-(sxR0VYibCON&us>cHA`PBf1ac{*Zbt}(|#V*#af3ZOqz_n47^*AWxKOyd}rpS_D) z-mnKWIF5tXnlKD$cALCz(`y;ejL->NC{bw9tn)YiAvbc)iN$^^z7}<{LdZEeawZ!y z;o)i@*F<{ow2Nn4YV8^?%pWEM;*1bSpMEAJ{!zV1Bmx$u>yuotZ9A8oxf7)oN=4>$ zyIs27E&y4RWkY@g?VuG$GsUnw#-{AnOZjtzTUGeynj)Y?@X<-{%-1(Wre zO~0wP=`~3o6al#l_$(OAb|spulcnJ0@k|tz+yp=zwXl|h^FN}}T0#-|zZ!6z9uvz& zZaFa%)>=B9P6WV4h2m7@7_;>f8B<7k6(DbhHjFxJUk*_X7CqtJz!8dC^edG}U!?K} z$`8+Y2*Pu7`4wetICbJ%aRD)6D=HMql0F%}#p<1iZIjszNCzlF40a5QX*gLn6 zg~nnky5*T#^_A&~Rz!3P;_|b1vGY|sQAVXqB;szjd&#Zf6xe?M4h}6&vZZipUm}%( zA~d73zp^4~39?p+avbH5b+Tz$uEj7@o#m*0`^(R>A-5Tg?gJ6)E&hC8XV0sh+vA804>Dn$iH^W1J`~as8pNfqqIG4WWQttTP$N0qe zKgovNCIpzS_n6RHBT5FMQlgKDiG0Mw!sO~8`YQ!=Ux2|95G(^Ci%OJLXo+6;mlDxK zO7-L1`}<$ul;KlAM?e95$LET(FX!;$5$-?m5F(Zx+s@~bGk1bO5C#ODAY~u{VXY+y z1I~Nkf<7ktlWH0R6J_>V1cbnL+jOZ60I_0Af^ z7Gn&71bqB=?_f)DGZ~ZN(pO(X5C-Wz{l+Ip(Jiw5-t%KkBuw-I#X#T~Xx5@=*bZ9@ z9Aaq@2;mGuL!q3(5E_>m){=Fy0EA)KD>*AB&|Own+w~%RwYeY=2+} z2g--oJitWXK!RSBN{i04vxH_)Kgj3k4gl$dow-oic~Pp^4{!vkX9!diHvU@@s(g^C`8O$U93Ir^6ZRcR`A-Y#$ zpwUXRKD&{>dhp`ZWma45)s6usY{2#}?O=cT5F2t^R)V0fk0-KWI5Wf<<1gcsku5mN zN%Rr6gXUw&>@5QF7r=FN8)5ycR!IG`*Lz0~FSteal@9W*`!Ay47glwd{@>M@xagtx zaA;wIlM63fIkfBV*<*Ja@{4R9K8d`WV=Oz$aApW4ntD(xFSJX~Kr}!cDC0X}>+|iP z*-Cve$*8H&G$Y|-L^V?xoV2Q;2`8PAQf zb$BydhBh(e7cgSb@#ldsUoQi~k=nuMD$gu@ZuM682FSf@4b4vD$!d4;akaeH=nIQU z$#PPD3r|e{H}C!G#SFW{E0`GdNBQgjyktNV>FUQi$T}I;7uIs}$Y$2(*O7Dc7!|pf zR?&8a(V9xDMy=bT6*j`@#>92pP3Nr5Z&`h=)p@!OUngk1uKqt+lYghDeMi!sL-$Jb zKDP2sfxV@@d}H!4&e`}In?Gul zH!un8jWqkW0ffcYJkTZYh~lcC=`6 zrhsn>fI@I6xT6%EQB0DLL{sz&tj&#+ck&o*(p6V#kb&f^lw!78qFi61-mL^rmA-k| z;uZ7voqX*XEL>fBk=ZX$Ijs!WP@ic|yra~b*(a;cr3QO-5y4T0jLwlWc^q-loZuOc zb@?@nXUFighgJ%$Or#m5Bd~GO)v@ad!c65j$15c|VUwrkzx8hui!a;>aCl_@OW){G zAJK$Y2bi19wk`f?501KPwyJ?XeA&*^ifTDxqTuBi$&BFY$c>C>v^Hp>!9(IgM)laR z!0~E{xoVk~ZSv&YH@~p=-ls3*5yBJNd!h^B)d6y|rh8R)-;u__H_x`F_8}}A``EOL+vUa+rElDJbl3COyp-)h{bDzS4}_>a3cK4@7hhk#r`2th9p%0*XYxiZ z-~Wi+tKbzF_KT_E$*P+ej8Q#cp|vbFDoj_7v(&864qH68^vuC$=D+)nV+W7k#rMGM zaR?s{e>xxl_F)Tmz{m^hJM%YA|JUp}Qx96~R^MbdI@)oxn6y$A6pl9JoNN?{5-@Rs z6QUBl*=@5>Uu33QVyRW3*=;b_nw@*1^!S}WJoe<<>mR7@8~=k1)E}+BD2Si#D;ziA z4*O%TD!9Wt$GvqIj=5`nPiN9T!ZRMOa?skOImcSCHYBw12V1BH6{c!OrjEC!K6$)( z?C#2yi-!R89*5xlFa6sOKQWMB<83^{m!lWgbJ9?5^uqO-lioDqtvNlb^P`S(GfIpx zq9qB#Si+F7(+xYdYIkXRwlVR8Bh?>$u`yl0pF3#AI|6XUFZ$a-%fA(L1%@Gt!~WQ~ zud?fmIAcoGkO; # I know it is uggly... -test_match_matrix = {'vPing': 'vping_ssh', +test_match_matrix = {'healthcheck': 'healthcheck', + 'vPing': 'vping_ssh', 'vPing_userdata': 'vping_userdata', 'ODL': 'odl', 'ONOS': 'onos', @@ -102,6 +105,20 @@ class TestCase(object): self.isRunnable = is_runnable +class ScenarioResult(object): + def __init__(self, status, score=0): + self.status = status + self.score = score + + def getStatus(self): + return self.status + + def getScore(self): + return self.score + +# ***************************************************************************** + + def getApiResults(case, installer, scenario, version): case = case.getName() results = json.dumps([]) @@ -245,7 +262,10 @@ functest_yaml_config = yaml.load(response.text) print "****************************************" print "* Generating reporting..... *" +print ("* Data retention = %s days *" % PERIOD) +print "* *" print "****************************************" + # For all the versions for version in versions: # For all the installers @@ -253,11 +273,16 @@ for version in versions: # get scenarios scenario_results = getScenarios(tempest, installer, version) scenario_stats = getScenarioStats(scenario_results) - items = {} + scenario_result_criteria = {} + # For all the scenarios get results for s, s_result in scenario_results.items(): testCases = [] + # Green or Red light for a given scenario + nb_test_runnable_for_this_scenario = 0 + scenario_score = 0 + # For each scenario declare the test cases # Functest cases for test_case in functest_test_list: @@ -276,31 +301,58 @@ for version in versions: test_case.checkRunnable(installer, s, functest_yaml_config) # print "testcase %s is %s" % (test_case.getName(), # test_case.isRunnable) - print "--------------------------" print ("installer %s, version %s, scenario %s:" % (installer, version, s)) for testCase in testCases: time.sleep(1) if testCase.isRunnable: + nb_test_runnable_for_this_scenario += 1 print (" Searching results for case %s " % (testCase.getName())) result = getResult(testCase, installer, s, version) testCase.setCriteria(result) items[s] = testCases - print "--------------------------" + scenario_score = scenario_score + result except: print ("installer %s, version %s, scenario %s" % (installer, version, s)) print "No data available , error %s " % (sys.exc_info()[0]) - print "****************************************" - templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + # the validation criteria = nb runnable tests x 3 + scenario_criteria = nb_test_runnable_for_this_scenario * 3 + # if 0 runnable tests set criteria at a high value + if scenario_criteria < 1: + scenario_criteria = MAX_SCENARIO_CRITERIA + + s_score = str(scenario_score) + "/" + str(scenario_criteria) + s_status = "KO" + if scenario_score < scenario_criteria: + print (">>>> scenario not OK, score = %s/%s" % + (scenario_score, scenario_criteria)) + s_status = "KO" + else: + print ">>>>> scenario OK, save the information" + s_status = "OK" + with open("./release/" + version + + "/validated_scenario_history.txt", "a") as f: + time_format = "%Y-%m-%d %H:%M" + info = (datetime.datetime.now().strftime(time_format) + + ";" + installer + ";" + s + "\n") + f.write(info) + + scenario_result_criteria[s] = ScenarioResult(s_status, s_score) + print "--------------------------" + + templateLoader = jinja2.FileSystemLoader(os.path.dirname + (os.path.abspath + (__file__))) templateEnv = jinja2.Environment(loader=templateLoader) TEMPLATE_FILE = "./template/index-status-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_stats=scenario_stats, + scenario_results=scenario_result_criteria, items=items, installer=installer, period=PERIOD, diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index 604f2c8e4..7a0656b74 100644 --- a/utils/test/reporting/functest/template/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -40,12 +40,20 @@

List of last scenarios ({{version}}) run over the last {{period}} days

- - + + + + {% for scenario,iteration in scenario_stats.iteritems() -%} + + {%- endfor %} @@ -53,7 +61,6 @@ - {% for scenario, iteration in scenario_stats.iteritems() -%}
-- 2.16.6
ScenarioIterationScenarioStatusScoreIteration
{{scenario}}{%if scenario_results[scenario].getStatus() is sameas "OK" -%} + + {%- else -%} + + {%- endif %}{{scenario_results[scenario].getScore()}} {{iteration}}