From ae6e172471fef8bef60d497eedaf199966ffb799 Mon Sep 17 00:00:00 2001 From: fmenguy Date: Tue, 11 Feb 2020 11:07:52 +0100 Subject: [PATCH] NFVBENCH-160 xtesting integration Change-Id: Id600a1277059e84bc2d46fa20294fb22729f9c1f Signed-off-by: fmenguy --- docker/Dockerfile | 14 +- docker/nfvbench-entrypoint.sh | 10 +- .../user/userguide/images/nfvbench-xtesting.png | Bin 0 -> 17342 bytes docs/testing/user/userguide/index.rst | 1 + docs/testing/user/userguide/xtesting.rst | 237 +++++++++++++++++++++ requirements.txt | 1 + xtesting/ansible/host_vars/127.0.0.1 | 6 + xtesting/ansible/site.yml | 20 ++ xtesting/testcases.yaml | 20 ++ 9 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 docs/testing/user/userguide/images/nfvbench-xtesting.png create mode 100644 docs/testing/user/userguide/xtesting.rst create mode 100644 xtesting/ansible/host_vars/127.0.0.1 create mode 100644 xtesting/ansible/site.yml create mode 100644 xtesting/testcases.yaml diff --git a/docker/Dockerfile b/docker/Dockerfile index 1cb7c71..cdd7f92 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,14 +23,16 @@ RUN apt-get update && apt-get install -y \ iproute2 \ libelf1 \ && ln -s /usr/bin/python3.6 /usr/local/bin/python3 \ - && mkdir -p /opt/trex \ + && mkdir -p /opt/trex/$TREX_VER \ && mkdir /var/log/nfvbench \ - && wget --no-cache https://trex-tgn.cisco.com/trex/release/$TREX_VER.tar.gz \ - && tar xzf $TREX_VER.tar.gz -C /opt/trex \ + && mkdir /tmp/trex \ + && wget --no-cache https://github.com/cisco-system-traffic-generator/trex-core/archive/$TREX_VER.tar.gz \ + && tar xzf $TREX_VER.tar.gz --strip-components=1 -C /tmp/trex \ && rm -f /$TREX_VER.tar.gz \ - && rm -f /opt/trex/$TREX_VER/trex_client_$TREX_VER.tar.gz \ - && cp -a /opt/trex/$TREX_VER/automation/trex_control_plane/interactive/trex /usr/local/lib/python3.6/dist-packages/ \ + && cp -a /tmp/trex/scripts/automation/trex_control_plane/interactive/trex /usr/local/lib/python3.6/dist-packages/ \ + && cp -a /tmp/trex/scripts/* /opt/trex/$TREX_VER/ \ && rm -rf /opt/trex/$TREX_VER/automation/trex_control_plane/interactive/trex \ + && rm -rf /tmp/trex \ && wget https://bootstrap.pypa.io/get-pip.py \ && python3 get-pip.py \ && pip3 install -U pbr \ @@ -39,6 +41,8 @@ RUN apt-get update && apt-get install -y \ && git clone https://gerrit.opnfv.org/gerrit/nfvbench \ && cd /nfvbench && pip3 install -e . \ && wget -O nfvbenchvm-$VM_IMAGE_VER.qcow2 http://artifacts.opnfv.org/nfvbench/images/nfvbenchvm_centos-$VM_IMAGE_VER.qcow2 \ + # Override Xtesting testcases.yaml file by NFVbench default one + && cp xtesting/testcases.yaml /usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml \ && python3 ./docker/cleanup_generators.py \ && rm -rf /nfvbench/.git \ && apt-get remove -y wget git \ diff --git a/docker/nfvbench-entrypoint.sh b/docker/nfvbench-entrypoint.sh index a7195a3..812816d 100755 --- a/docker/nfvbench-entrypoint.sh +++ b/docker/nfvbench-entrypoint.sh @@ -13,9 +13,15 @@ # License for the specific language governing permissions and limitations # under the License. # - -if [ -z "$1" ] || [ $1 != 'start_rest_server' ]; then +if [ -z "$1" ] || ([ $1 != 'start_rest_server' ] && [ $1 != 'run_tests' ]); then tail -f /dev/null +elif [ $1 == 'run_tests' ]; then + PARAMS="" + for var in "${@:2}" + do + PARAMS+="$var " + done + eval "run_tests $PARAMS" else PARAMS="--server" if [ -n "$HOST" ]; then diff --git a/docs/testing/user/userguide/images/nfvbench-xtesting.png b/docs/testing/user/userguide/images/nfvbench-xtesting.png new file mode 100644 index 0000000000000000000000000000000000000000..d5dca8dcee74658032164308d077e9a4441d35ab GIT binary patch literal 17342 zcmeHvcTkgCw|7*Epn}RlRHRrqs7NmX2_QA}&`XdG2_+;zfP~OAs9*y-a*&{+U;|Wo zH=-hk5&@+{0w|ye2{m-?4xs0K@67%F{N~P`cjl;1c3XR|@>^@KwdY*4vV`y8-_H+$ zKz5iI8`wY~nB4@!w#BMwkuMO5Fez5$L3AYv!zPYhg-QUj*wJw6Ta4;DjvLC}$CpLx7ieh=q!w zm4c_0y0Z}h?G*{Pj37ddkQjSKgjc9f5YpGi0&T5lfyCRJA)qK_H4M_g&<5qN@2wGp zw6d^MQ1fv%QbXAUCyI;g3eSRq3Vk;*pCrhyTbihfSv zPR=T7zP5gT=1>KwwXKq_sxr|a%r3yt&sG743(*Ku)3>+rgQLUMRd9gIdN>uRae#t_ zeXzHQfxfqj8QKPJ9~o{Ks$gdyj>n-q9q|fA)E&y)fL%>8h!6Ws2f|X6o6+^K^10NG}xUC7^M#at+ ziZik@Rrm7M@U+H;z>!Wt#sP5saBEM0I}><>nin?E-#Ea=Av8qa(n?L=5~1o~8W0$) zuj(CXWufln>4mg4v9q)YLfS=Q;kF9sa7T^+ZIJ3#Ufu+QU{8BED$)R9pomjaG4-;u z3iJuF!>Ty=1tBcFG|ZeyI3-mjtglaq3PQ=q#K(ew*9bu=n_yM#ZJiB0jfr>_bH^Y9 z8w02vJiA=ISW*P#dDX6T&Xs#z#** z)FRLu?hWVYVPJ41!4U6+3Bs!a$>`y{BP78*7|7Lf>yuY$FWDKsKNLERMV z5a{4zVpV%a0>AuM3^YqSo&F$ z;125kKm(D!f#zliFFj8MYXfBmDk#0qjJyyI4hr_BDBA#AC0j2i4uI?V`5SOFz{XRV z16q2v-rz@swLb!GYU8C5?g=&aMMawU7+ZuwgLLusYL=ePq2QQ$2tSM+(cIex4@Gha zM-YP*ywHwnC{-ntfw!8T3Ka0mDAdWA7;dEGpkic(^;9%c3kB}M$Vr{!FO)gR`pZYG zISlZ7!@nvU;y)jH1%XIHObm4Gi1)@S1B1oBr7{=Ici87YFFI@b_8#JNMdrzH%k=b< zPyfEhZ?YFR@XWTYE`@dQo>vmeX71;aKASM=A*}JeFj7*W#k@|PGH`5!X3{LUC zyy`osF!{RRTs$?U@x9vC%VxW!Cl%gp>DZ%opl^zbZGQeGr?8-aRN0#xTm8)a?47cx znddWAG&+uuhrPZ%nC3AWPSB}*5x$(8)IU?MY1yOML0ZVoPSJP=BQiZkzlWE1xw)4= zpUD`HB{`33#pkroi`zo3y-C-MW#!D2cez&)SPAIW?vZ4AHy)EFDkc${5F7PIES)H< zI8}9W;*#cE+Hmj4+%-;bl*ek;R3NVlEvI*;Mr6EaA|5v{?hOg%>+M{6JX4O7St25a ztAA=QoMR81bqXX%#Jmma$RC*QqPDqzZyJ)H7so@U{?a5YWv7kr!=(1i>WFI(m#a%= zjm@USj&qtS@U?8*q}D8CNNSWvbEO{4#Ovn?lH|80&V-K4=oUx3oau4ZOp#@4@oX9r zCXCOmSa3Ep788@g);r^bV^>wx=ZIl+;)at}-!<^w7asw1dzCws>dbkE^TASOo9_++ftNZ;yG>|+ z+?7A{Tqv)koZ>bV%^}q@Gp)qjazOFo4EwNUULba z;@=5Ntoy$>KpDzDK>Tp=;WRmq5E0b{0K=97x!_IUh~)b>Ax(pW)rf6 zmxKcy%+Dh#$-J#M8JmnZk+Vs8&cv%OuTX|RN5PCIm}|yG|X0ci1gx={4pe!W}}1fmw`FRKF^=rftXg$h-H>E$*_FkB_J% zSw`XBbcWN>6{;<0kKveS^W*R_!^lg9{!AwKQEKn84`QCPgI$;YSJK+ZV+% zAwwmL-+gm*qOgMrRlFKrZP6UK<)HhQ^?3kl0JLchuOUide2xUbHBvf<+6@5n0!$9> zo=v(dyoONpFDtK^ulPLf0JvqiaB%Dm+hYUK7)@e2b9Xqse}QGHyH;9gm4ZYAIyUb6 zFt$cn=2BJdku#BD=&6+P&dyLQz+L7RiRKgzdPYSQ-D^*8bf47vbJGy_I{p$0Eyp?d zyZImd)glZhG3j5UPHb+F8Sg5^%y&^8P}=Hw2S_oW0N9-@#l9S@o)eFUOx@5VM6YH= z@w#Xa+!Ji#Ev}x%6KEdwN0h}NG!Ay>&|_RYYbe6mrJwi9mMxA5VO#SY(w(api#893 zL6qM8c4ThA^NpWfPHW=u_G*$~DG&^$9GT5H3l$*Jc?16kXV%@62Uw+)V!p-aZWND8 zUO7hw9Es!sferLg+gb@)l?|Rju1l*+=h-nYM=IlUn0eh_W)~J*#cII77XwLL5CsuC zgVV+{rONoP{JZ8EEpeM5l9^9B_$U%4E8P%?hq$mP6v0|^3Ax?~y~pWTB=a2AA|;<2tnv+R_*J=?ZGuIbuM?t|TKinu!i zf%t5{y~It!Y)rXexr~2`9RHcM+_|{{y|J~UR6aer0uRx>d+!#dOZt(&iSFNTBWMtpaU8-SC~xeq*$2E*;laXZqjEeZ z9;KTlxJ*vz>&MINx!hEGxnEg9L7_0mBXU?+FolZ`U!~RrE?yNiX5SO<~{m`=9exFBc3?J*mNe<>Xi^I?$~v&8+qavacMN_a`~4cSw4F~J(hVS1eAjjDRB`Pn zsqBPao9AT*A5w2l7s<-XwxZx*nw0Y64bVN}D-{l2KO63GK9)CK?kF@0_b zo{E2*c<;vdexcLBtcueWpGg^~F@aeYIX|+@cfy=Mu+f_ERUc?Jk6Q{;XSZi6#gU~6222Q}2Tj2Ge@7 zll@lPRR)p$eUIXo=#jH}M%#Nn@CF87*tSFB7U{oRLq}8?rh`R5DX1 zlARG!)WLf)rPm~3i>b$m0|J*D|0JT2Ri|0EG?_HF#&uJ&G|@QCfIoQJ%7wLR0^MwJe6m?TNcWAs5X^)rRyFYi z-zqqK9_^$o7I3-Ss8Xl?SZj1PuTlbs>kp|ya<9LUrN6&wYreVK|L_3e@TXMr zk?0y4>-{#9dTDM%OOz(txx=V?cNOvA-CjfaY=PNhSkalK1EUSvp62==tRLJ&-Fats zR%386-(r{W2NjGyK5D zU+X?1mUz8V?CfgZMVLv03^7`gSZ8}WBaF13S;5BLbhz#l0`;G zM^|82pZVjo_TKke+sD1M~=TJh2-%Fxy2dIpRRqK7mY0BJl26=ex|uw%rj-KoaLYH2Ys(!Rnaz zl$pdme-x8g^x*O1$38rJR}LHxg0m>PPq@d7tFt#Z?BndW!S32%;-2pw>Gy-V4Yo*j zH{iRoPeLN&c%n%{*W1OmmKK}PlAH$*Jdb}S&+veq#ftQ1 zJeuWR+!w3m=Sb8HsfTw97mu6AJue9|N)hb(W93B+_2%cKk_g^&V-pi0Pz8mZn#x;o<&Ph~znYxvs4~d6 zDpz)TdYHBHXQ6yWg#kb5E^rv{zSSJbNt-SPA@?aiw8NQcI-L%iA_TRDR)iR)7~Veb2UKB_ZYNN}x7rQfH}8 zxln!_w-@U;UE$O-={eA5vEb5J(8yZozr2HdS@myjgiSf0I0#7qr;T4UsCwn52=XSJ zlkA=wuZ0NT=NzVBn@u5A=tYW3_gPGTxx^N`GyK#o|3P!%lmB0QnBr#>O|>zXo;Fv% z@t?9-=P&Lja(~KK^>#u-e-!og3u=;PV?bwc;+^Gy!JPhypH*GN=j<5M)+9=d1iejj zssB>db%Nup2;D7abz~~2cd@LyY+IvP+N|vC=^K)pZXe*Ky`G`Db+&2s6jTttrifM6 z+)R%-EWy?anRbtT#U8`c4xIcpTRmH)NgTKm^*!B$u*4SW>3Aodm=)w64mV*HpLFWpqKoL9-DXoUT(g(P+dMgAD!tzR^f3!4@|$T7v{eO=hWtsJ7? zOMR$PcAYd1`trdK)EFJaY72 zROmDTyL>o-&KXzstXjRR?CGbjvXGIJw8t2?%o{fu-+vy94T)ra)CV-1`q=f3eWKBx zZ^bn0PVgw6px*rX3ha}yO$a-66HX({?%-@G@=1R7oFVOTklVu>6Ab5~l};H=s0cF< zkXU#_5xRM92S{)ue8n{*san&|L)zPH=L&*a`{V>_zKGo3&P)4yrCIYNDS_5kJ=VdS zK!=T1<{v`2lF;9p z!O6boX|6%>qoAqKubbbrr$SHFKIa^yBFddMF-9}XwTro{2qkUIL z#6%^cz85eB;{BY-*|D>*mC5t7<1x=$pQQr&`m7T!t3PiS_d=tSL}L$S$xv-|*b;H{ z@M(dJZ`b+r;g1EVt)-WCs&w`LY*GK@(E(Vx)_eJ) znLo6@MS@+Lu|Bug;3(liTIUgwH@^S0GSML3VJE?+ftE&F|M5y-H*AJ|R!jO%N9?|s zg0{j|rQ^qV5ThR+jNV~iz4tBTOXAFR#?$IrOSjP*f?18e}J2POEKMyTSFHnMT z^wf(qu4DpJd*#HaSRim=U?m0mDim$tEbDc(pcufUgHz zyzPTDod`33g=3osr21Arw1E}PShqNL)N{|5s+OC$xe{T+(PL_4&9=DKz;IDZV@xt6~?7?Rd5 zhqwco^U!q{6L=*yUn#o=p!=c~J6->kuH z8@%p1Khpm)l`K8C>XH74EZ%Dr=1%?uvKi;}X>D**x8y4KIV+czm2<_r+(7!}`oGOx z?Ml92TgtdtC0^W^^|;Z}z!ZgSW+hIG6q6dup86h9QNH8?JpgN7_D6QXf_MyB4Q}Mx zYA!@12*u5h2~&ighO^G0mHWp&oQIb#r`#Np}*=LG57>7 z)}4r#ULwuCbexMJ6=~Z{QbLb;kcw=2?(c^{L>D-30koB5RZya^0|C#k{=+~8^#)}p22up>5gT=vRQk=t&u4D>i~7T%((ngA~P#4cs0o5(++{^p90kraORk~{_o@c zrTNmk!*%tlXz#Fg+?N7pb0_*@s#JUp@oaym8`2HmE7D8vEvysMiR_D73S%Xgx2EH7F%SKG@QU3tETlJ#-8W?rVf92b{e7c(;o4NXGd zX&XO4+s}@JF2jrcLNXUKR@>v4drrq1{v7_&HSbVZXO+PMEH95PFoe?+fO_(WM#de|F*~!TydUe2Wy-&SK{YLe~ zmCfe`u4wnXP!@AvUFK##mR~ij8|M6Tp1u6c;WrO8BA21qdupvFgE0Z&`As7cHQjux z+6$ErQ;T%Em+Su0ydY4jGk;SjwTo6SSKhFwZ7=hSkFI{_@}opF;zBXl`owbSd;iBCF8*-Jx~kiaU0C5YM=F}(# zr4^YvcE#e8biU+2dnjsgMo_dJ7anb3seQd9t@w5WX;P(oU)p^HO856cr*Ngth(2xG ze;Zn@+(?>K`|m@8?y8K|4Gx?9>uJ@lq|JN{DN)`CjX8{u}N8?<^PB z2yTgU3*f^mS@rMlc zrYs+YxUnM;P&tyNU3SAJHq`PCh{3oJMgL1TGAaghjidMP-{a;wiL811y!Q>Ft!$tu z}gJ7M^n|8RIlanOHhsNJ=lPB7=TuFruYwp~SXA;hKr zjo~p|2;!fI9$ee1J;l0mv`6GaTxHCUncQmT(RmIf%Wk8&81gEvbY|$OFPXKC*MUWv zUfwER{V%+$YMS7Bua!H;s{=hzLHfboqT{SNwiBuH@nN?yKV@W={D%Ln%vNc#wC2A~ zB1<1#>YtMD|CFfdv*+G6e!U}ddqHyIrRCi4NK|7%47oP>9|=CsV_Y}OPiA=*f`=>n z8_qne7DUN>;i7SNSkBM|KH6kMKY_|DY^B_B!9`r9FO}h|<+bkZ*+KE!dn8pQ%JYLa z{2w(b?4t_(F@OIFcqXG+E0kXMNAYr4*LGjEVd4O}cA0{QIsc2+<)B$y@3o_K&n_;% zuYVpd$*0X`5c;cja_Ord*jzJwiQJA7g7xbYxDc~P)d#-DY?B6@%m=z(F-nW=~G%}xXAS#6wcDVbfWB6z%9E}41Tot!J;hUSEdqYVYI zJ)^oiX$3k|8kL@l9z878y%lw%e;*CgEwbcFrul&ICv|Ii0Eno3KRvCUc`Q#<11k1x zL~zL`s4hjTCRt`{p6IfT{~S&P^Smc>#{fCICv(?Bf{`E4{Orw2CD;8urb~6HuS3S9 zh+8QneEBUNT5{m4W>Yd69NwjXSZBBCo&^iU#}SwJ$RF$M`*ilz(-|_|5Do8t!_Oa4 z@ChC#@I2uN!EBT8ygKV994_%6w=SN%&&7}(U;ANvn z(}4Zl1b~lU*QR#oX zh4H<5y6nlCYZCw0;I39V^}CMowTng|6n^+B28oN{X>D=o7gvaq;7~{9eDuRAsf0 z5T2KQ6E~b2iErPB0MXfk45^F!ewei*5cTNW18lc^ki~3|k4bM_IVU*7?H~jTi4zF~ zRjE(?a3pW_6Lj`m-vC-)HJRj^^{;hczX20Wz_U(qaFh>Sg@@hdL9n!`h7fSp z)D^ilvNY+f*3l2m(pMbz!*usHge9SY{EKOvlH)!pS<&_lrrx}Dp+40tua+C}ev?W! zV_({wPn$9YB;uJ{-n&)`$7BS%kiA@T72q@t`_~*58=RFU*>{`k?!`!NO}nqv>})Fq zD$|5ldtthwHHL(x_HOTtAIA;}a~lJ-uY8uX2iyiCPT%G7JbcKO@FFQqOsX)2agg|< z@!Tz*`s$R__-qU$WO%cA4U#g*r`cYAX$qCoWybNcb{}+1{O>Oz4Ycn}Qg7}(#U+^K zxt}Ve4I9$j`9t_*VSxOar`-oN`F7CO1mdd8^4K4mSG4Y;?V6@+(pVvfCaI31xbOt} zTqd}S0x!A&SQ@Jdi7+{Q#>5kaLF%nv&B{?#$3<%5@vyrZztGDLF_atdyxlc*ul*ppN?nr!KWWy2rs-CqeWl)<<(~%k%q~yE_UirBjTm39+Wo~?A%J7%F|8nz3W!JMvm~Kyw*W2X%TZ`HY3QE7GE#^lW z=(Xgajz_{aWy$}-*dHciHB2X-W5pWkUu4u#egzgVYdPZ30+C$805{B^07Vf|YKWf0yYEhX zwf(g{%9F1LXc!3usNRsL~P8MwFQ2E*^pe0JR!EJO8P z3p^*dsPU1ldQ|PufdZG{d10wBr(lhI3dxK`Br~WL{s)NO-4K(t(Gt5*=7qe3F~wA!t&F{zCI25O$SHC&t26i z>%UHy^9M#|>>nZU7p+~G)({%YD~-tP>ELmOt_JX#&%ZPLHewSzug zg~*=>ghuE4tmdtLxC+Kw`e%<1IloH z6JMCIc|pict6-t9@lWFseIhG1^?kkn|0#8WR&ZBGb2}Bnm~Gr zWB+LAw9bfj#!hW#_BO|54SMgy1|W@da#n=u_8(fts6p`WkU%BXycXnx0R$EdfPkC; zvP0&u7vH)$4CDN0=zUDTvj#l(4c*ma|6UGdX%s(L)s-J6Z01lAmxyi>Q~!{npMR7( zc#%`slhBEzM)969c#&O;a0?y-#enE`Mr2)3WsOADbkifY<(Wh$ggXh{BFhxh9xUgf znVnoTvJ_iSJ%d@iheP** zalNTK*-5dCav{k>ZLNzLCnKtI%0K;F+XttZ*$JI|#k4%*lw)C*mRm=6!wCG=XGY6W}b7O=48DzJZ`vBS+eQ9U=?)(sHLUH59~%!CSf$vNs>8=w)4zo{QNjPJP`xLk`}5 zQbA2Pn7a4^WZf6(xT;Q0g|iDR_cdpP{BgYkYdbf+x44OVPsaDH^nL}v8&{BFztL&Q z_@O$Q#a-WF;Ln=I#Q1d04zz8P7jcUNOyEMKYeh_wIfcqH%yPaX=|s8Yp_b-+X6Ht+ zrqn6-8to5Q$Z$}=J{a{Rs7mm<-wiINyTcgeL*&E)?D2wGt%)BARhrxbnC&9d_d)%# zd0@Lhp`qZ_^MdEkyUUBfz!`VJI`N!(X)zd2k||>C<5X-%e~MDR28#|gvc7%pKP0Cp zvY(48bnmO698V%vnsoh?4Y&k0M2Bnw!A5d=1>@x61!)(kfa~zn&w97MRUQ`Plv0`L zTNGtS!|PO0t-Q1M?k>?>GNS$_k#(~PJ&#-yNt?ygO2BQ3<_(RWLuk3tCbWc1#h}BiNMwtjxLo59cTJ!A3!CiMBL`m|cZu%>04U zG5$Y?%cL zcDsA^bMsU*TDtZHD927*e^6XOIC4+o!sSqqV1R~sym+_|F-K_r_E0CKT*EIcvh)u~ z2s>;W`3QD?By@MFe)x1uyzddE@`FUwa1AX9(HFal&-+|#qh~SSpQY7)f>TpY=Urh` z>OY97p$avSKoJ|%r%dIJY^BAb_$ehdj+~p<9YJ}a9hA_7Mg#4rXUD0wrJ90E)SM33 zWbJI#+b>lCe|}EnRHloRO zCSImLfW# zdzE9WwcZ80LUPw0LA)G(=i`FtZ*6LPzep_D^sc4Tc4=U;?|K5q$T`L1CGf)H+|EM| z|NXT^TtR64NG43D}x{uG^`15ay` z*A~o)D!EwWe~q8>Uz^K79KRL$wRKjimlMqPI4`t|I_{_;v(j60b`_ and the common test case execution proposed by `Xtesting `_. +Thanks to a simple test case list, this tool deploys anywhere plug-and-play `CI/CD toolchains in a few commands `_. +In addition, it supports multiple components such as Jenkins and Gitlab CI (test schedulers) and multiple deployment models such as all-in-one or centralized services. + +NFVbench using Xtesting and Xtesting CI will permit: + +- smoothly assemble multiple heterogeneous test cases +- generate the Jenkins jobs +- deploy local CI/CD toolchains everywhere +- dump all test case results and logs for third-party conformance review + +Xtesting CI only requires GNU/Linux as Operating System and asks for a few dependencies as described in Deploy your own Xtesting CI/CD toolchains: + +- python-virtualenv +- docker.io +- git + +Please note the next two points depending on the GNU/Linux distributions and the network settings: + +- SELinux: you may have to add --system-site-packages when creating the virtualenv ("Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!") +- Proxy: you may set your proxy in env for Ansible and in systemd for Docker https://docs.docker.com/config/daemon/systemd/#httphttps-proxy + +Here is the default NFVbench tree as proposed in ``xtesting/ansible/host_vars/127.0.0.1`` file: + +- ``/home/opnfv/nfvbench`` + +File content: + +.. code-block:: bash + + docker_args: + env: {} + volumes: + - /lib/modules/$(uname -r):/lib/modules/$(uname -r) + - /usr/src/kernels:/usr/src/kernels -v /dev:/dev + - /home/opnfv/nfvbench:/tmp/nfvbench + +Please note: replace ``/home/opnfv/nfvbench`` with appropriate path to permit NFVbench container to access config file + +To deploy your own CI toolchain running NFVbench: + +.. code-block:: bash + + virtualenv nfvbench + . nfvbench/bin/activate + pip install ansible + ansible-galaxy install collivier.xtesting + git clone https://gerrit.opnfv.org/gerrit/nfvbench nfvbench + ansible-playbook nfvbench/xtesting/ansible/site.yml + + +NFVbench Xtesting test cases and existing CI chain +-------------------------------------------------- + +For test automation purpose, Xtesting framework can be used as an executor of NFVbench test cases and called by a CI chain (Jenkins, Gitlab CI ...). +Xtesting use a testcases.yaml file to list and run test case. One basic testcases.yaml is provided by NFVbench natively but can be override. + +Example of CI scenario: + +.. image:: images/nfvbench-xtesting.png + +1. Run NFVbench container using Xtesting python library + +The NFVbench container can be started using docker run command. + +To run NFVbench using docker run: + +.. code-block:: bash + + docker run --rm \ + -e S3_ENDPOINT_URL=http://127.0.0.1:9000 \ + -e S3_DST_URL=s3://xtesting/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ + -e HTTP_DST_URL=http://127.0.0.1:8181/$BUILD_TAG/$JOB_NAME-$BUILD_ID \ + -e AWS_ACCESS_KEY_ID=xtesting \ + -e AWS_SECRET_ACCESS_KEY=xtesting \ + -e TEST_DB_URL=http://127.0.0.1:8000/api/v1/results \ + -e NODE_NAME=nfvbench \ + -e BUILD_TAG=$BUILD_TAG \ + --privileged \ + -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) \ + -v /usr/src/kernels:/usr/src/kernels -v /dev:/dev \ + -v $HOME/nfvbench:/tmp/nfvbench \ + -v $HOME/workspace/$JOB_NAME/results:/var/lib/xtesting/results \ + opnfv/nfvbench run_tests -t 10kpps-pvp-run -r -p + ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| Docker options | Description | ++===============================================================+============================================================================+ +| --rm | clean up container after execution | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e S3_ENDPOINT_URL | (Xtesting) Environnement variable used to store NFVbench artifacts to Minio| ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e S3_DST_URL | (Xtesting) Environnement variable used for S3 storage destination | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e HTTP_DST_URL | (Xtesting) Environnement variable used for S3www service | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e AWS_ACCESS_KEY_ID | (Xtesting) Environnement variable used for S3 access key | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e AWS_SECRET_ACCESS_KEY | (Xtesting) Environnement variable used for S3 access secret | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e TEST_DB_URL | (Xtesting) Environnement variable used to export NFVbench results in DB | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e NODE_NAME | (Xtesting) Environnement variable used as result key identifier in DB | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -e BUILD_TAG | (Xtesting) Environnement variable used as result key identifier in DB | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| --privileged | (optional) required if SELinux is enabled on the host | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v /lib/modules:/lib/modules | needed by kernel modules in the container | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v /usr/src/kernels:/usr/src/kernels | needed by TRex to build kernel modules when needed | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v /dev:/dev | needed by kernel modules in the container | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v $HOME/nfvbench:/tmp/nfvbench | folder mapping to pass files between the | +| | host and the docker space (see examples below) | +| | Here we map the $HOME/nfvbench directory on the host | +| | to the /tmp/nfvbench director in the container. | +| | Any other mapping can work as well | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -v $HOME/workspace/$JOB_NAME/results:/var/lib/xtesting/results| (Xtesting) folder mapping to pass files between the | +| | CI chain workspace and the docker space to store Xtesting result files | +| | in orchestrator (Jenkins, Gitlab ...) | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| opnfv/nfvbench | container image name | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| run_tests | (Xtesting) Xtesting command to run test cases | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -t 10kpps-pvp-run | (Xtesting) Xtesting parameter: Test case or tier (group of tests) | +| | to be executed. It will run all the test if not specified. | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -r | (Xtesting) Xtesting parameter: publish result to database | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ +| -p | (Xtesting) Xtesting parameter: publish artifacts to a S3 service | ++---------------------------------------------------------------+----------------------------------------------------------------------------+ + +2. Run Xtesting test cases + +Executed directly by NFVbench docker entrypoint after docker start. + +3. Perform NFVbench test + +Xtesting call NFVbench python script to execute test case scenario and wait for run to be terminated. + +4. Export NFVbench result + +If ``-r`` option is used, results are pushed to a DB through Xtesting. +If ``-p`` option is used, results are pushed to a S3 service through Xtesting. + + +Override testcases.yaml file +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To replace existing testcases.yaml file, using Xtesting CI add the volume mapping in ``xtesting/ansible/host_vars/127.0.0.1`` file: + +.. code-block:: bash + + docker_args: + env: {} + volumes: + - /lib/modules/$(uname -r):/lib/modules/$(uname -r) + - /usr/src/kernels:/usr/src/kernels -v /dev:/dev + - /home/opnfv/nfvbench:/tmp/nfvbench + - /home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml + +* ``/home/opnfv/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside. + + +To replace existing testcases.yaml file, using NFVbench container: + +.. code-block:: bash + + docker run --name nfvbench --detach --privileged -v /lib/modules/$(uname -r):/lib/modules/$(uname -r) -v /usr/src/kernels:/usr/src/kernels -v /dev:/dev -v $HOME/nfvbench:/tmp/nfvbench \ + -v $HOME/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml \ + opnfv/nfvbench + + +* ``$HOME/nfvbench/xtesting/testcases.yaml:/usr/local/lib/python3.6/dist-packages/xtesting/ci/testcases.yaml`` : volume mapping to pass testcases.yaml file between the host and the docker space. Host path required testcases.yaml file inside. + + +Example of Xtesting test case +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + --- + tiers: + - + name: nfvbench + order: 1 + description: 'Data Plane Performance Testing' + testcases: + - + case_name: 10kpps-pvp-run + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'bashfeature' + args: + cmd: + - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps + + +Examples of manual run +~~~~~~~~~~~~~~~~~~~~~~ + +If NFVbench container is already started in CLI mode (see Starting NFVbench in CLI mode dedicated chapter). +To do a single run at 10,000pps bi-directional (or 5kpps in each direction) using the PVP packet path: + +.. code-block:: bash + + docker exec -it nfvbench run_tests -t 10kpps-pvp-run + +Xtesting option used: + +* ``-t 10kpps-pvp-run`` : specify the test case to run + +To pass all test cases: + +.. code-block:: bash + + docker exec -it nfvbench run_tests -t all + +Xtesting option used: + +* ``-t all`` : select all test cases existing in testcases.yaml file + diff --git a/requirements.txt b/requirements.txt index 9eb76c4..430d70c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ tabulate>=0.7.5 flask>=0.12 fluent-logger>=0.5.3 netaddr>=0.7.19 +xtesting>=0.87.0 diff --git a/xtesting/ansible/host_vars/127.0.0.1 b/xtesting/ansible/host_vars/127.0.0.1 new file mode 100644 index 0000000..9e3f1b9 --- /dev/null +++ b/xtesting/ansible/host_vars/127.0.0.1 @@ -0,0 +1,6 @@ +docker_args: + env: {} + volumes: + - /lib/modules/$(uname -r):/lib/modules/$(uname -r) + - /usr/src/kernels:/usr/src/kernels -v /dev:/dev + - /home/opnfv/nfvbench:/tmp/nfvbench \ No newline at end of file diff --git a/xtesting/ansible/site.yml b/xtesting/ansible/site.yml new file mode 100644 index 0000000..4643a32 --- /dev/null +++ b/xtesting/ansible/site.yml @@ -0,0 +1,20 @@ +--- +- hosts: + - 127.0.0.1 + roles: + - role: collivier.xtesting + project: nfvbench + gerrit_project: nfvbench + builds: + steps: + - name: build opnfv/nfvbench + containers: + - name: nfvbench + ref_arg: BRANCH + path: docker + suites: + - container: nfvbench + tests: + - 10kpps-pvp-run + properties: + execution-type: SEQUENTIALLY diff --git a/xtesting/testcases.yaml b/xtesting/testcases.yaml new file mode 100644 index 0000000..cbb5c45 --- /dev/null +++ b/xtesting/testcases.yaml @@ -0,0 +1,20 @@ +--- +tiers: + - + name: nfvbench + order: 1 + description: 'Data Plane Performance Testing' + testcases: + - + case_name: 10kpps-pvp-run + project_name: nfvbench + criteria: 100 + blocking: true + clean_flag: false + description: '' + run: + name: 'bashfeature' + args: + cmd: + - nfvbench -c /tmp/nfvbench/nfvbench.cfg --rate 10kpps + -- 2.16.6