From b2c252cd8913ef15a00d63a391da1c8a8a17d739 Mon Sep 17 00:00:00 2001 From: Bryan Sullivan Date: Tue, 17 Oct 2017 09:55:26 -0700 Subject: [PATCH] Merge AT&T WIP on modeled cloud-native stacks into Models Change-Id: I646825bf7d1a9c1be9c00475028084f920c9d399 Signed-off-by: Bryan Sullivan --- docs/images/models-k8s.png | Bin 0 -> 53813 bytes tools/README.md | 16 + tools/docker/demo_deploy.sh | 60 + tools/docker/docker-cluster.sh | 221 +++ tools/docker/nginx.json | 67 + tools/kubernetes/README.md | 17 + tools/kubernetes/demo_deploy.sh | 74 + tools/kubernetes/k8s-cluster.sh | 438 ++++++ tools/maas/deploy.sh | 75 + tools/prometheus/README.md | 10 + .../dashboards/Docker_Dashboard-1503539375161.json | 712 +++++++++ ..._Host_and_Container_Overview-1503539411705.json | 1618 +++++++++++++++++++ ...Node_Exporter_Server_Metrics-1503539692670.json | 1632 ++++++++++++++++++++ .../Node_exporter_single_server-1503539807236.json | 792 ++++++++++ tools/prometheus/prometheus-tools.sh | 228 +++ tools/rancher/demo_deploy.sh | 65 + tools/rancher/rancher-cluster.sh | 529 +++++++ tools/traffic.sh | 31 + 18 files changed, 6585 insertions(+) create mode 100644 docs/images/models-k8s.png create mode 100644 tools/README.md create mode 100644 tools/docker/demo_deploy.sh create mode 100644 tools/docker/docker-cluster.sh create mode 100644 tools/docker/nginx.json create mode 100644 tools/kubernetes/README.md create mode 100644 tools/kubernetes/demo_deploy.sh create mode 100644 tools/kubernetes/k8s-cluster.sh create mode 100644 tools/maas/deploy.sh create mode 100644 tools/prometheus/README.md create mode 100644 tools/prometheus/dashboards/Docker_Dashboard-1503539375161.json create mode 100644 tools/prometheus/dashboards/Docker_Host_and_Container_Overview-1503539411705.json create mode 100644 tools/prometheus/dashboards/Node_Exporter_Server_Metrics-1503539692670.json create mode 100644 tools/prometheus/dashboards/Node_exporter_single_server-1503539807236.json create mode 100644 tools/prometheus/prometheus-tools.sh create mode 100644 tools/rancher/demo_deploy.sh create mode 100644 tools/rancher/rancher-cluster.sh create mode 100644 tools/traffic.sh diff --git a/docs/images/models-k8s.png b/docs/images/models-k8s.png new file mode 100644 index 0000000000000000000000000000000000000000..c54bcdb444aa865ec2705a3bdbe5e167fa3fa452 GIT binary patch literal 53813 zcmagFbyyrh(>RDN5NtO%1b26L3!1>P)8$>=)zuRLR+K?UAw+?JfkBs(l~RF$LC}DKfs;mtgMopK$S*>KHo!Wo z$Vk9cjuGuc{~%b3zZZvrsfk5>G6q2ZzxpVv?F<8h(f#KK+wTA|gMpbGl9Liw`($w3 zj=D^&?!!WK;)H|UHvQwU{ovfwc18zaqnoVdRo=cA|G8MW_<uGP>9|v(Z{tx1@li z1Tn%t+BDqN_A90 zZXNapEFFHD#~qK0G%1NUcoE94t>wOwl@(-hU)~_NTrn@HZS0Rv7=Dyj{+)zDFu{|~ zt+IP$r?7m1#M!BfWC(@c)2cgDKNNd`M2FYw<8D!f^~_CBR?v$af)^d*#}r1v(EYrz zu9xfx;UpZ-8&FFgXX_hO@?Q&Vv@1dB?{|conyy?dMQX!pf5g>g9&`rD=Xl0LUxi4T z%Pf*iiRW8Xf1W+zv#AgW`AJ%Hw2KXWQ=*jQ$6!59+B#vZ$oOm-)e7Z*NFna!p>|$9 zfQNA0E46AUk~ZR?n4U=B7PIgh1(5u<8>+M+O?L8iEBrh2n+Fv!?;Rtw;|CDfg{M(@ zZSPXFQ9eEpSf)`^@K%QnVcdWh*gb^nt+z|xT>bdKsQJsG*pv!LYN=sBIjsI;>w2P9 zvvRR%zQ-f-B%#b^^ky!_5z+Ik{}SC>YQr0yw9L7P1KM57Ve|CJ1WP!;*)#Lq7&G@I zEo#H;YRJyAQ(2x8D$V{(Nvy+rS!r9=qBMO{k#+9hqxFxugVTk+%K zrEoTZr`f|_cp7Oh+f7DaTpt% z@hw6akrPCF2rtaaBDOnLppVHsS`uUM+mC({ugv`j7+;|AESm>$b(p6yns9KgetCls z3)3Tg`!~Om{lWs;n}oLfpB5HvNlD9sYiBAi1^pYj3?M;n6DI8ry3MzT@ z)s5@4_A1Q@?N9H1qG#Dn!BjBt+SS$Y>sN402j49_sS4z>s$7F3R&hbk>ET3oecFYRyZs)a{c$e!=IulFkYznKG1Fc)g{J z`X%oaG9Xh>MuH-53T!4S33H6lW|t<3Buk1Q zT%&%i$5zb0e{KeB-eO8chdRN>MK5Ia`Xt`r1^K1>(V_vTBbR&-xCMfNFRAgQQ)RxV*e9+-iml9FDPU3YPcY8zNp-X(T*Ro=E zbC5Twa55vApPK0dbZsn~OS2A8=FBJw%f6bf6@BbwBM)^UUol+`kF9HUtdGe)K3mN{ z;2;l`X5#+?Jbqa4FQK%5p-dkX-eN}U*&?~XMf^G^sq4n#ul1clt|k|Q+HqqXduxQ5 z+71YCK%B}jY+53ST4_R}w~klUbU4T*cFAdVnDl&ct z1RxoqA{;9Z(zlLWcRAgT7>LMwz5|-8IsE|_!TS^8|Fpy6yicw^hwkje2z0K=wpQmG zX5dXRO)m;-(>Lm3qUCrGMAvlsLPA)%wQM3=w*2wk8_A$k3Oh7(1E)oN!Zx!pr-kb2 zy39-n#5v8#KetlilKqoZ&z2gH8-i_1v2aupGjT{xkEA;&(KP?@v&+xxll3KKCd?IE z1K^+&4Yijr)o^-}kTtO6P@%5XRd+luOv=H9G}xe&xj1_Ys}%7;j?7FeU$kg1uVOy_ zeT_^EptYhP*p2W=Q6$h&IO+Z1Uy!+7Yg}U4>@T2L2iXP*2jjnYHsF#17BYE+lgMj3 z-3U`*5#uh`S?l9G9_ih7^=sW6lM5;s`Bo3-uC^{zt)vcj2y?j%`gmFGCGZcMcSodc zb5ME>Y#Rv&%O4q9@4P2szS*X>khuR6lbZJGkog)w&dQE8JCiF94O~*`zi#93q9lJ} z%S1&Lkko3Thvw49;1l?G%IaS?E9%$BVP_f)6HqC#Py>2`2?zEyyo!gR@1l)4nIm~Y zj*NiG#B#y~l^>Dl@%B-dzUUr5BQq*k^J{^=Y=p;m;WG%Dcnfl^Sht@n zrbCg%i-bl!M}pWx17#&=C!)#8P{sSCC*8Xx>+_)rWB8<-R`Z8z6EmY zWa;F$L;M!Ln^hn{=fr-LdX92Be(5ijd$8BIpB&(8jsDzn4||BN6H*t~zb+Y7VCPoy#64ALjX7`?gWQ<$kejM5Smzo$++lvr2mi{U|T~ zJIN5tQQiof6mal5gxB}i>wA3E8?Xg_eOR^g)x=0|QXrVnnl+c z-reN zvhgf`%X`Fa&?Um~up(T6+95piU1{`NHff?s=$&T7m(Y?>TuBJES^H?+=X^;Voa_R7c=o`qi|| z%|z}WjwnZ+ZZax943IdLEQ%0AB;}+AsZYp-gFe)-(|!oY-*wTW9?D>&0-*jN3sSm? zQ<%BgpP&uRls@&v(`%2E`aPv}c7U18GK6gwL;jD`b*5CQBq3PgBoHnhYH4@>A3?IUlpL1jG-~X2q2OK)Ryp z%@}NSbiN7tDx9P~I7u>8x|KvX$~0FMgcU9+8<{sLulNTv#>Ji(;!WpKecgH`sJb_@ zfvxzP1~WAFvb*NP-qi$#Ez8m(r&yl8#_suptJ{u4^^m=l(?J7YaBf&FEB{{J5tJKa zPg!Ol$+`jBV6wx=iqm(SD^`QS{nm$9Ip`~la%opJL$bN6Rl-JtL6RR$>6Q{!@P`_= zlK#nQT(?X|R2cS^7gK)&6nc`^K@D7y^noH(6D}ApZnd1BZ_@8$aBf_Q9Tc8+#?`lQfD%UVs(%&Q@`ic7# z6{k3kQ5FlcV0p?m*7})?D+Y)ypkl6gbTgBNO_YU6vo>o2qYa$osjlJ{!&0rN`+e*Y z%C@4gcH?bahQ|2JFFcWV%O4g8G44I|>GK4W1s4LbNA60sSmDDZKYIO!BsL6)}Oyu z`Ngq=U?m1PE3;9xHuWeuM3|8{g2QW#*FhW_{K(`!1e1F56R|F3VzuP;ilU-Ll^2Lb zi_t7T(aYXkmQh{#^pl4h5}_bvo-?T!GU*?2*i2t!}0E5(pFcb2N7}_Qi@szY#b%v}XUIE5zwlURFk<{Ghd6 z6DT==Tq+4&D4sGrKF$$YyE(jQ9`KkXdCOvFR`wU2WD0q3BJV52rv_!mcoFvQ{uAw- z2O881h{Ec{$8K#KcPhOBw?Poa5MuVg>mJzw>R=4Pjqs-Om&VjR$KFVsSDk8;pZ_qv zM=H^3VRL)>emFGX0)+qCa03IP`NbWofoQM0s+~I5fPLN{{zJhJ|KV8f!U^g3nQ$wa*dOf%vHS z5(w;={{hXL!i<{7hSsB*?$3V`t<6$h0WBDZJqQ5*hfWTvW;xdF^133A?2>KA9^=HwjV z>NeB}OM}+3Sv6&|NJ@FcQ1D4LQbhNRT37|{6@z=1Sz_lyolcqPQiIfjk;I-hD?hiS z$nm|B@9gJisaWVm&gU}Dj6I1CAA=#7t$T13oRF!=qtm_iH3@VeRNv{lmc6a~>GXt} zl{&riWoZ|!MK1nnkGAqmWoU$>$qY{U_;lH`DOF2|bY9oyhMS~4J{>PGsf!>3NDsDP z4ffRUxG8;AOB%%EE9VdU@pjg0G%NxjNr#A*XYw6{5aEs^m}Aw1^QYen_zY?;?{XE&jlx0Vwgtp@Y%UpCCDw2y)a@$e3$P4*ZNR|HY<|=Paz})YjsdF-zGG;q~q781t zm!03=jBs8&_xC-9u(J0Gw?^{bkm^Pk$A^$UUd$R6hKTgux@0I$%=Nnz$es4NleARL9v7a&o+A<^_X7S`#?HLZfyMWzv zV9+|N$XSov%V94!=~O@SW<$08 z%Ge`S50n3CpkGSk`j_DEeT@BNg}If33VFFl?u0tZvMI1N#;5&P^(tq-4(%^B3iGCK zdUCv;v~8L`m0^Cm_-ROW?Y*!-dIoKI4um0G)=iGlEm(ATxlEPTpmy9w>R_4?xvK7N zSNBN|+xh|u)FTD+0;c>Gk_qis~w%J)Mgh)6zayL!F zii#r`L5>{z@=>%a*2_!u5PirY=V-ZJ-FM>6(a4ELr~g-9l-?(BL6UR)Wl)@K{5E&j zNj2`9A)QnP;XC1E*SV5!2QQcge6*4fXxK&|$Qp1>zp+$jrbfiXMl;?o{1#Pdj*KH2 z8b@{sA`~T%=#3+%8lVYoiIE+<;u(o()Qb4N@a3tr=4peNu53$&lA6;>3GO&n4nsT? z%x;GKibIwB5;blIsihaLWFAx&bjf9tLS$nGH~utn+ZBXH9ty79iqK==2#zF|lx4F; z1{7P9#M9B*zN#lsd6BC5I$NJ)_Lsg<8mLw?@im$_8GTqj*0z*lgY8EZOAdwa;Wi1j zvac>E$z}o#H0~Xxi&6Ixt*?f3S^tAG6svhX?BbH*pMX~qg=EmX6yD%1k3epxQO`$= z!=$T{FD-PmLqX=xaRJuuTU`rZ-uD}x_9zvi^}O@2>AGp4^S z#N$2sQRbsGpi?{UiN@?aVJ}@#sUn`NY)Y>*p(h?He#f>%%7I=s8ljL)GvP%>qQ?B54 zgveEW6pVIC2m%JS1a--!7`a-4z@-!m4w71)N*t{dbh9A1dX|ZQSnF=&eXChB-UWJ| zk$|#*DP1h<)}R$~UdCIJpQ=UQf8e9%UCi+lN^DnMZk5ELlkZho8_L>&B))c1EH7~I z1ZR4Ir}~SE?YM2xx7AczG}?&5f4<@cph{>nqK+9lN3RMSXrnHE-Kt{a36?k5od+>^ zWO-1eMoxklqU;^ed)TuF(Dir!Ld#GvLkxRxC`~(U0XFYg=g@qf>6r{~#oZQgnsBsXJ>4N1~8GL6M^CV!FEd^`Nk*=3Mi zZA^eR49tJ17YX_ixNFvi#kpHzcRgp&ija)Z-(b;1C)3qneHw?}?yzr;EwMvSrTfLZ zC!I=hXeRJ76pLxI%K730!v}>!+^;~eB99rf#$c^ty$qho0f<34OEnA}aiCX%)jTPz z#7Yx3v6m!AuDjzykt&mQDH1=NKXxTilID)QE$*3TqV)sBVQF5OJ!BXjm0^-vaVKUS zWQjMz{XUbl6!4yUZ_Io&0O~)3&A}eHZmj|i(EU)|xO{M#+ViMnT*_f3ClF82!{x)y z$ShH+{CO#5 zF13M$s+Tev024kc9v|-|yR0VHo7tYWU*8KsuKw1^B#g3?3lC0gI1DmbNUC{G)9JKM zC45@Q$w5N#jG_0T6Ss^AugEWrG1=Y($K4CPO4H>aHxp5Ub3!8zBnyR?3?~bf$+=tj zRwA|g?YH+PjVf~>9nwS3IMAM1`AN^KyNG85T|5cVXVbEdH@sh#ShUClK^YP4jc7#b z>M?qb?cg2xP-y)**0Lf7eoPMc5^&KHR=f|$PK{6F%N9!QyLArNQ2n87nL(q>Uk^7GAThaP`)Ol>VJZ>6r!r0)@8 zBS}*)ko7?NzV}h2Uv1O;50Q8x@kON;*kI31|1f~q2NXw(CEx6CUra$d=Fd>*11j^N zVTsPJ-J0LNA&$_)yI+xoGW8l?#CJ>nOUn-K%POS(E)?)MLwC*ljU?KonGR_vFox8b zd1aPx2+|KiH=F#^2gQAYn42+Z-AAC{!LTT;D3~Hjx<2 zPKgJTd3S4Uk}Ef492oI#+dgmpMIq*Q6+UO2AI2B{izFh)9??sGsc*b*-SR~|f^>8} z{Y3T&a+M2;JC{sC4>1s(u>KhxCjQqJ5}TG%9KPYT0q2sSZpyX0ec!{!mKCR0y51Oj zgtQrH@BZk<3`^AHHYh`Aho3jOay6m6ikc|%(jL~e@ieaDs4%d@>K!vH7 zB*=KDF9SL^t*RsUHCr%9Xsja5djVlZmXEE0IB68we%Or++-C$MZOGJHt(4yrzw^Qx zT*a2B`6KwJXMeI#q|Gp>m))FSmYp^~eM?lj!4%GRIdIOt2>jXJ^N-fexifSd94Kr>0fgRmJJ826f^e8GSZ(zdKz9_%uA;7Htd1GRZv&J-;npZL(a@W%ORC z%49jO+o?0W*EX|*BmGi#d8-hvlBe|KB0&7^eqY4qeYDiI2DJg-FRPzL_Z>U(s!fMmJ)Y%nWpi$neK=rbT5GXO`XcI}iW_gcT zm=HNj`g51$J0W~g1orPS}%;NU&gTcx+o?}_k5@8M)nw!0)+e=c-RXX@Khq};G@Rr_m9eVq<$9FQ^7EiaLk zJTrpz12XYCc7(+pE*11>eZLzdzTp!@Immw~Q=M@-Q?6a;l#tS|jwmM|0{k5}IHLFi zh>F2%0BA2}fRXHcM4u}a%k*#jW2}byE489*HC2kEzt-R%srO_$QgE!D|w(bs4Apz?tZ?&~#p-VJ^EO z-dls94pmDW6w$+j_^wg@ObPE&{@G6((Yoa`EHB5B%9DPth;4Ej>9_`e*$veuZfEw6np`m$Z zczjWIqbF?mJ|(=U9QMF-P8K#_lka!*h*3E#_@br=o0fV;Pd)zUs%ZG4`fR>^$bIAA zzTZWNlb<{He;CUEo;;ci@e(aN=D#St=s?>6eM)F3;W?f*m&+=L(3I2Fz8NuK`fio` zkX89ty*yHKFz(AxhNNx5Oh%I_R{bjP6|YoWjLya>e;I-3ChR6HBA{nnWz#L{5n2-K z&8QS7V_tQ+4t&|!A6Z+`a42Q9?c6Q8(G;ykiO@p)T~X4RrPJv%?!EaVW6=<`W99kU z)5c<(I;UlWp>G^&*9dXg^B`Gpm#WRE-|oS-wAU{|P1e~NFNvKLU7ImYR?$y$eq3|< zZ74Hhab(IJ2r-b1ss@neq01AQ{%Y>W-sI_=bN_R!An2M*^tRBAyb8B#O{cEiJ+Efk zic;d{Q1kO%6D7p`itOzHrEUpcEK|vVxIm+X#lc@K!5*ZzUr+0zUooOjOPt}2es!!~ z&=A!P1!R2{*{E7UiMyqc6}Zq-d?vtxnt_hYw-!K{bIyr-6_Hcrz0FAaF6{Tk&f3d& zBvEL&t=rIhJ;?6`woXsNdtK5G@CBIY4{dJpK9ltOAo31FjWV@$kp|5(U+Jms>@91; z+Qac@Xco5tP31UAX^6r|e5tHN-ob&F3sj#&Yv|t<`5j^)AK&)CcvuxdRrG7W$|ox|0g3%WCORjyg?rMVykw4@ z)_JtJN~pTP9@(N-@MM*z$`D>(=$4g7oRwWz`MBA{=4DXHkpG?TjBb_ZHNgSP&$Mu9 z68n+1=S*k3DfHoZEeGC5{m^49(S7!b>qD+@mP z>?h{!3%HxDTWvAjW$SFe{MIjQ|4y=7hUeV*M6^xbz&vLroVUqOvz*Q#F#F6oHr13s zF>qGTEq(gwsBWBHj(;<@(lc7^(?nycC!A?~E?&nDfy%LXe^kj;B+fCWRa`{QBk8>3 z?4n@UHttf0rT(u9!i1s_jqz6*?!2Th=Zn`<;+GIiUk;b>tu5)c^Ay?=kE#L2S>eMTO@~o0FM&yLKtdm&oblm6j zdm`nEg_|kW%Su||T>$HSmiybOpBE9GXSInrFBGio0pU3*z9a5?0;Fy|uN_*CX6q_t zaVjy?k&Wqcj*D+;H)LE)o7r6i*E}{7{Z6j&o-h5KSz?qmmy}MGdX%J#W%SeTn%y7a zSI7EB7*A1BkchF|PxQg(s-z;mMXy6L6XxNJaW29sSLhX zT13>op+dUKx^@KKwP~ey1Ob%&N|PgWz9{BA=c-J@_@dBaEI@692Dfjl>q{yehc61* zrc4~R=waoh>eu@7I+{%1*+pVZWjwo5XdyR56a72)k~+B{uU+rj*bVU;APLtn60lU* zph>s3+3X2htS^>~+*d+#iwa+$uGJjYc$3+Pi&D)D*6(`9Vxsljw5gVUpwk{Vf}u`l z9)!X(*s1y(@LE)$ZUKxd^dI6Her806Mqt#M^Po;VSD8+f&av``@VgrFEn9OH9(z%U zd%=bbJBx2MZhR2nQZo#+X&Ghqgv`<53$l?-@Q8m{x;#m=(oqTxdOvVS?;k}f-gmBw z_z(H_I|5Mx(cbFm^PtFIReD69sW}D)@o}FRz2?$~F-<=7Y@xmHQ-Xz}tBuGRb2BwM z>NUSs&cM>{Gi(4A45;s0WwQCUsTWd1mGu5eOIZBLg*6E1)>F zi>{VVC7e?Z$of%P@IdZrXg@ePW1+5K)A}wmy1J#GZ&olc6Nm=Z=!;f2NpaF{7Q=A( zBA9;8dHTmN?55=tRGHnFy};7fj}X|YcI;TEZS*Lg>fE`Vn+F5Jf5Nc6dvdYJD$Wl2 zY6p1L^Ia;3o*8Jx=CBfO8-jBwGK)4wWd`6A-e;jpzAM#}y^Iu7e9R*INkolH_ z7J9fF;|@nc_zI5HVm~Zpyp?xjI@UJ)4-N3ub2*L_~64(FvY_6SR>hu1R*~A^aWEN6%4##wJ8`K{zzC(zj z_zR$wbi3~)dHQ86>PUuwrqt5+s0U~d3MG8kVfZmacPQmWfJ|5ZcGwZk8BLxXsZl5GRgcJ7NFa{dp zPMkLacuLbLtTI=L=zO86XTU}=mFv!RkQ&kXL%{J^Qql=gNWP&cUB}oz_zG$$lT`CO zJbcm3-Mw>FA(_~mjdYOct!PUWT5S}-yMDNK0yQe{xkjFV&C7WhHVj=1|As*ueY&4cZIC)K3pT9cuOXws~_HrJScm+bXB@rG@-X~%;Q zZgEcYF@p!Jr(_=Wl_-ENx_1(`3(QNPfWB))0TTTIXe&Rd9Ci7?t)hqtKee27AuJNk zj{g<31G9b>))noYZi|4D%M)Z4!I90$_HAD*(}3^xY2W%z*3YUH!lA$d${;;;Adm8M zuv7-@y?dzwq~&aVg9$OLncV6@oq6diSL^oG6e*Y|sk5-une`dgT=HJGR6t4RDJr7P z7YyU^EcTzE5OGqxo^Farrmq)_$qAFUd{Q5MNz(PVHuFPT2=Va7fUbmj-k|y z^~)0Kp180h*eP0_INaH8MepNWy?R1O^w_U5v@Qm!jVd`b&i7CAb_y!HT+o?=SuU%e zK(tW;gzg&L(jsx{`K81sPP|FsOt!o(xuud~*h=b{*xWgS^jVpk1#*Kgqjwfx^@?Dr=Y&uaZb6E zQaWZjOWG5$M}M&1QA187YSCp8M=#Ydc>{iODZx}HX^w>M54;(Iv%2>~hgPDLHr5&@U0*Zf=1!Zrn z9>cyzd=2k5SCiK(^7>NUFRhu`@V!9A{BzBj;+%>Lx)jYyLJzg#O~>JOKC7psys@A@ zC4vLJi#vAKK$zZ{u46fr1Ezk5DOZd0z*=0ttYQ0?HF18Ir|;{fu`NL}mDi80@qyOE zGKrPE;uh@RtSg8HFE7MI+SX_ zc?nE2J=#>_S&^R3C3@>Op=;my1(5#x$8)KtBB_Xq?N_4Q69wme`uCkN5K+^T%kT%z z6oFJ$UzeHDwe@6-lW)L{ex+0Zmj-Wd1nH{*lMNZS0H3_2iT;Cp=lE$0@QUeAbIF;K z(lP6I2!#v2Z+VJiM;8r@`&Oe&;(gV|Iv)?-#VGBnzt>Tuo;f9r)HmmOQ z`zyK_Y>&Y?3=tohA;)`DlqOtdt~NW5b&{Kwk> zTSV{>d%f2j!ydN1HKw3e_7Lo1Y{DodPB%#PlQ(RAuplV-`g3CA_zfvAVy zv@-n@eLWFb`(t;e%}($?3Bdsu6#tl0%WabdP#_1&N;{&MIqX}?Chf2I5PYHk>~#7} zm53q*LWqwF#74dpEHZifjO?$C7$j5gK32M>=0qwj_H~p7sB0>&F?b2Mdt(L$j=8%3 z+G?sFC!61GJZ32P^p;JS9BOq=!`N@S)q1+?p54 zn8>dk{M3_EdFw7@VP@jXX}y9ouyPjz=*`z3H{$RG0}_TR0my-53G9df6m68(*?a&e zZ#hk>ElRY+0tAgdP8MmZBMG&DuOYO!I8#*rhuoQ5@P7nE#GzyThe8}M^!Xn%X60i0 z4vAUHs z`!ws_o!e{AhJ1?!LVg6fG^lz!e7E?Rx}Dwv?hTw9eOvd|#zgL)ZVgE2(zZ=W5_ps( zkoJ{TzGGs6e!3-U(gmWOA_2?w?w5P_gOm`JI}${=-Ha4&DBs z#b61I+gYonpS2c+E;hd1@`tc}cz2_5-F}Zwm=Tmt7T>0VEFUsfHXS zBHvIWuY=qjrD^|JJ6S!q=ATi~7{CSvaL{o@Wm=14wb>Q)#!?c7#X%fdtM-Q=>8R@Y;rD|KtE4=M=lm~5I3mX@ z$JND=fBpL)C|8MgqOJEb8J)NJOPA~JysIyMoi`2|zQGQjzZG6K!>p52`2z;Lr%g#r zduYFNWQ%4r zJuY)PuZosVmYO?xTIa7gXhG?unH8k&Pp_oCHOfCd4Jv(e7@u0b-_fITrje>aZP2J& z$%BDf>FS3piW&@C80*YZ>={+2RpH8g4Mt|+>Rwl`r!5=n*}H~{;{^vEVcxqn0iZNmQXr9gZj*+Xi1=^?Jxe047@UEd z@wovPmy97Uxx!HEdG*xoV3bGYhM5P?x}x{dK8^o3Viat_2s7`YXWr~l`R~2bTi>R3 zH3#D~ZZL#|Oyv%^d*Jo*T*v9>?p1MrHZoL(cX+I%Uv2GE7M-bZ9R?=60kCIidknr5 zT12n8(1cGzv3hhta1!R*|yT6d{r`anQZ+hxE(JZ0-An`|VlL%+7<& zTZl09!vkfeAv}Nz>rpPuYboo0B4e9ywn9&7E*;0T5QbuhBd8dK<`O;P5`vK-Au73F zJPk=T$N?L$UH*?BLyfZ3N97gXCLs??X8J2@m??P{J6qH}<6yZt#51)vFGnTM#6u8k`mrbA= z-2bK2v?7>dY1M)P{oKOKl|n9osyLu&wYh=7Tenh{4@J$!<3A6iL(ynjdV|?oz^T=G=F| zpBf@?z04|K&WC3=San%}CR_O)GXptqIy}^QiNTvhQUR+vQ<%{~G!I=nUK4t&r5`B2 zY)szwNmOueI2&|#(ZvBqCO$2kxXc>d6L=hremuaEzfm^a$b3AhxGlXkeCMMZ{kj7#n%D~CO&ATm z;Ms?xyZ}VWR_I|Oh80hm_yLgHR_vzl*)(hrKBYDzD8nlgs>l*Cz|b4DTncdj!z+lG z6azGjA>u$}!2$Ac44`-v=u{1D!Bopl@JEp~^bnQp&T1o!cmAk_o9oKhpTGaJn#YV&TCC~HHjrz^O zuqSc3O0CN!_u(}#&%o`aRB2@mT8=nmF>u|VEqW3nzTm29Eb%>Cl1Fyt ztlVmv!GE{v(_?L+^{gKd-Mg_Fhr4AKrnQ&m8OxW;N@&?=v8}+j#cTAwZ0zYt5_&YM zXk7oI(xJ@o@#UbmOr)bB;=OfT+2!o;srT@y-|$3lM+%`e4yGlZySQzx*kOt;MF5W| zt{B}!Kp`#;gzbNfVpVNT7YBh0MnVcap9rK`aT$f4$L`8sr%cHc!}kRx%vI@KKiIgD zXxW?KNFigwojk^ttzDCj7J8yS1jJU$541Vd(A%)Km7rTv^3cuCT@S$Md$@>iYc~~e zIh;y#@AT{J5fIIc4BfVCcE(1kn!x^71M1-1Af$|Aj4QGB6-83OOOOQ_z!aX|jer36 zka-V;TaCCs#5|*Ohk7qg9!JeY2~4rk(sdYTEsJ$?alNIcZW5T+1KJ)HnqKCc*}{c% z_bYY5&y`#8>1$yAN9_OlB6P-+m}}qbWi^PW$Kc+A`z1r-)}ws)**sC&3`zJRc58qY5&B^gG9stW40r)#mq-E%_t@2iVx5P|$ZGCoMZ12`>eXeP%zMW2PBi+ojshj_@Jx*Ej zZ80@G7N7puSe`ucWiR#dc2?veV)z_ycy)1^zf9+GS1CihPP1WjvNUxszB4~SyH#C22eDQJ+Efz@Zv-P><>}6@Y(Ei59^8#9p6-i zzp7VFPOZ5ym2v2t z*W)g{2nDfnLvL+cm1e37a7F-p2j?Qu(X*j&Gi8rV--z!+(7l7_r+xWh!}QL6&d&p} zk&D4U=jP>d;n2~h<51LuAY$o3C{h4IL5LVUJU2AH02#nLxp2w>eHaWA-^I{N#Cn5J zlK3HFYPmH*C|qn{Ne1wFE}S_A0|>|J=>Lsos8Fo)I%TbG{j$^XPu;uYe%!FdeXU$u zneb_J?Wy8kAf+!w@bpXQd3}AG_fD%J(=%MhLqn&^mUw;$>F>M#3|zySjbJQAh(SiD z{dU#D&RbPh7Up!Edl>X^3oi5)lV1AbOb&Glm&1yt>_=c#!mRJErQh+XhGFJym+1YV z5|3TGNv|o%W!eYpBat&4tJ4mjHBuz|=Iuw)$Gd(z5Q^br|9I=ZMR`lF86j`SO_d>U z&_LIF-ru?Ygr1bM1-OA34)-s^s~2chHZNt_A_x=zLYLa(EJc#ARaxxB88{&@VfVAK zZOb-iRo=&M_yO$l&sCOLb~^mu+51Tz^+#-J6b#ak(^2U`p+9^?z1GVc++%`t&u)_} zg_dF>M4zVj9XjqFK4zb*%$$8-{r#RZoXhvw=QsXXBI??`lKTiV_whE|-QAxfhTDU& zmWCd?r-lzfbsC*hZljw@f}_KUys<(%pMGvIY7qHOPdf1Km&(*`xOcFw$-A}rd_HX%I;ux8ill< zu{bO;RLb7=rT#*(6!$?1w&5*aj?kTyx~$j}Cq*gm6MZWa2`N%z*_n)E3LLsLq-!+S zLYJ^B<(Ny;)S}x_$UrPO7h3SOAQQiL{3ySO!FK+4#6i?9cXvyKI;{RQ;43Wi_XO&@O-ZNkLY(rBfGn5hJy zkl{e^RRSW7dvTEaXBgjFXyuKxxh2nJxYn%*Rl>P8tyG%zy<2{>V-^Yg27j@MlDz6x zMi;k0gho}GfvJ|CVRB(5$5~F^TavBOfbJ}4X>(u-Aq*7^GI1rI-XRN?VPHj_pWPBy z&S?DV%_!uSi15E?HsC-6XLL4XuLBdp$ZE<^5-{mN7T%khUr7eivqlY$O8qVH(w5^@ zUlx7DfpLa(_s4-Nsi{c&##oyu2`tO0-$C*HEctZEQ3Fv02r5)zwirU?VylzA&~O3+ z4HwW08skYo+Xq7W3p|(S=mITEcv^Boe-CYF1uEyTLd^4^Nf9z!IZ%RmwNtsMy(m<> zUaC(QyQ*+&dIiG7p^FofWOmT^z`&M0%vANQjwo^CfUc4LpoLeK2RzNOL51w*1bo2W5H_*oag+soVfZgkR66fe1uBbi>3<^vhqEi`*nIWLOl^x;okz-V zMcJyFa&S@|2SCE=-bV`2te*~e6lN?);Y|6EH<_&x2xSX%ScZRAv~TCk_}v>{Al8*r*N!lA2WC}+Gw{q_3QITs9Rerz|uV_62% ztT{slg3{j+_{HJq7OknSmFfRx8GJXx1mbr;wHj@b<&7&|%*nDar=liG!{G)4bykfG z8twMIXVmD^8Ga5@>NaP{M$*PpT}8uzY}*;qNYwwEEBWq2x7}X~UxVL;vcOo# z{BM=Tr$bla|Kr&uf8s#}+_uY%BNI~{oO(&taJiVewI#zuZhfb>^(nmh;K>TvF3D5@ znKHBXIbg{1!n&4`(LOd8jOWLvk!lVE=euvJ! zK#!Yu%!kIWwHbl*kUZ0z5jW3iZDi@>RdDp1T{-(nD)ppr)nyx{(YjZg;HoEVy70mi zNz(Z+7X*!T={>h#&m_rO*GGk>j8TI6jB@U-Bk6JmQ4SeM(wSmB1-dPsVSxBFJ!4_OvD%=PDrusZgGqJY)(C9;C+6;;f6uWu5Eqgo&nLFj|P5$tJVvxSO zYKt+j#cRA&)k%%4v96m4C;a#*4uy*@5gQr1>gRDO6@X(n*Jng0`>TdNi85L1tZzL% zA2oYK*LovCq?sbh7O^786=}3grfG{ls%D%ka*QGmKo1&ZYJyC-1Su#vHGxwFnR( z62S&LG7R6#P6P9!7|w&O4Du!fAR58eaGaTs6uCLwc=-{5dusEQDr{ta-sCN;l@mjspV!8EH|J+S{c6m zc;i#4vppF}eP08u1c}r>y{i)AqdWa%t{ubGqQU&B;`IkfR+H^!V1^-2tiIT*hmJ#y zePZtkJxlre%mLNi>Z}cEh4e?Zm0N5;n5m>DYIoF@bsW1ey#8lbtge!ftc;C z5qXiuAk+tj(j7?#FqT<}0SVZT%cue)SDyRn1_4aR=ZnRnE8D@d;kY_}& zF)uaPT4qQ<2?C&^-lm0K--UBL%<=g)~ z4h-NRIfG;o7;?@sNX|LujAQ{pK)@l3l9S||bIu@0PJ-kNO3spz;J?vxJm>t@efNLg zTlcNiYoKRMS9MqIs@mQA^W7aoOl*k#JR~T_%BD%+DV+Q4k@yW>9!W^wVQU8#1Gpc{ zov(rs)yURMLWANM>)n3Lo-JP`C?VX)CC}DU_HGuuf-2E$b!**Ao~6Zt*aOY0CXGKo ztVY4kp^yi{8!S*y_vNdBsIlG+>3_|8qhhL_^sy_Q^h%GtRtyqZUPtG{qNj&{eX6BL z2D?UYs5g@YS{REC&e>H^B)H8YvoZNyZ=a>l52@B64t+$GioBjy+Fz%^<3bP9gxg-3 zO%XJ%o$6~d7NfRd6TbOASp>B!#6|H;4s<1J0vU&ojR4)Q9K5J3wLT`3vcY6g?soTr=_~k&buD+9-8<^*|9?wc=)_oqr$#ge_W3Bi{ zQlIxqjsNDio;Pu>(K_OSj|xM>ovK{2oR@4|a&42X%;&?| z6;l9DW-gxAaRV)y@vw@8WeT3tn7!Qxi7<^#Lx_xmpVF}wnf=5 zY3dPI*)%8FW$YbwQOJj10Tdo0Qu9$@*nd9;tqPc;abbeF(t1D&Ah zR$=4SP;`#o^!(#|(ak5`g~X~G%KBNds`SuskKLZ!O_PPV>UZ2tw&u0q6b&a=N?oue z>!?1W?u=7%z=#M92M)Nh$wJ`VhJ8|l*Nxnx1bLwxKKW% zFGPMhN3@a!hUZqhjOj~@_HJv$5;P)vbJR~p!gWK3VxC*sOy;?x2_YbTjIIy;; zvLXNzOWv}YCxv#zz8H=v5hq&7Cth3bkM^1`f?|gJhBcCGF@}k$Xg0$aN?=50_CCli zLB*oF_iVADg;!7YY9wcBR4uIB6heYc&L?Vro(oXch)`ytG8^I2(Q<*NP*ZHEmUf)o z8g>3Kf=dOrGWyI=7SYl!cM`5Ic;;b=rN&nGIa%nb8&;U}h(`U~E&clm^?IXl+l`kN zU*Kl&z2pyoX|Q92Wx~v4G`6Mvv7N=hRH{+`>9zdF66{6PIAP&{klAJX1#ke9>@6LG zB6I=dtQ=nmtlw%wOQny^nj7OE3#~2klf`8%7&#{t0gc1ttPU(l zfCs?h2{gGWtF$GB_|bGzFf@GMwup{@lun7WO}1giN_f`1Dm_(fglh}Ecpq9oDHD;> zxVGBQMi(KiS?5mE)goYh{1R;&z|A(BYRHVZWbImEhU=sPOFo$Kc?-1FqI2;@9_c|C%8}vW37+b z>;3p>!y;*hgnuz^Yd|ikT4ZKAyBiy|(W+Smk zgAInQxQ;BmA=z{#d-c+Mur1;w`9uPDITJn6zDoa=w52|yc*^b1QBgCcm8g-y>h#b& zFHYg6;Gvr*fcwIs305%^uu*~c4!$n~MJB$&&LCVbg=IhfZ~btEe9((^n-cx18sxLd z{iPB+fWVMV2*6LDA`o%?b7X-!o?pj`j@ALJ`7lq zK*~+24;0H5 zb{FnZ2Ve+a_lngU0kPue#xKDgyR)`#`LLD?AQrw?g#ZLa#BizUB0ThBA>+~KdoT0R z-{hhKA=bM>tQtmhL8o78kzFTro6#jP5l3-;M_~UBP%2l0j2T#HXo40U^?n8EsGY$O z@8~S#=m4`U=1E!HT;+65cgUXs<~0=x+a~6V3bpVYm3YoS53mH!rh8v$(6y0R3Hzpg zexK&hXkT|zKzB*XCmUO7Z2#3&b$wWVJuiRvAX(y=S3Kdt)0+z2`hvc|hLoxm@^_!# zsGwxC^_F3b^w~NE{u##i!~Gds%%a2qvrF7_=5gur zu7;S=y9N56VJBnSoIdS!&yR`3&DTx54aOdnRJG7j%RZ`&QesRp%<*T>S}lK~d>4BD zPU#Fzln8vF2m2M;%!d(;v>aOf8GZaOqRW4WA%FKV5M_Ua!87OZUV+Lq&3#AwS4knN z6stfMDa#XGUQrHt<)4w(2g>h3FWO*n zMy%-{qF|Y-Ah)|+t+Q&PKg(h+yedU(u-I%xga3?T<8?2?%QK|sjSSF&aL>_%$mDN> zL0e!`z2-Wrpe6khgKksJK40@{x=ne7TyBJ2$ZNzL%)CfPM!1q{f zZ~(vc0&IV@zwOV{dl?Y)wzB9X5R%Sd$lA%1;6X?WlPzeYH(MbH`?32ZL2iy&}|Xv z`^)&(A>(-WuR~ia`Hv>Kvad8grVM4#)T6rEO(B3~aOQC!27Neruwelev<03v@v@E= z4EVMrC2wecosPSHKy)AfdhA5IXMvrE7`(9z4QwhK(ILV>_#A>8m!JiPSbI*sXvaX| zYx@INIX%11L())3<`;<23PCh>eC)!ZZDWE|ckCI39KzJje4;^-gV{UDB&fo}1Lmhq zqV=FS?N*Vln3MPoFtkD*Rn=w~Otc7nT?HPxI7i?_OI|)`L!1%Aj}nR)sd*9?HHIJT zvzgUf!Ki*J9* z>*QQzVh*#BsP^Enl_BK%VrO^2eM}_x_d#Ygoy}`w(2+ns>7Wn(OV)*k<@e2(6j%JoG(>=~;@BWf_Or-|w<$pr;1BpF94>X#x2&v5`v`6mwM)pX$JXgRhGN{2hxf;<(X9 zNhV^-Nq75NUqWzF>d$l2gJgGh_rK19Q3+0)7K5HmR2(pcHVo(Dh!Si*@|zuf(S(qf z1F5i){F4d+)(~n6ok83<8$2%qzUv`QB6I^b+D63{T^lL7pM&jZGMpOr>QtGjNrLF z*{{yixx~6XBEKF2bn+K`SEF=_uRo#-e;$b7`^Av}E4cmTxAlV?lx86e?{mpI;r%z^ zr?{KuWw&bw*CU@2J{;ILdo8rtvG#vmUh+xTa8IcEx{iGQ@Ml2#5fIfXokP0rk~H+c%q3kp5+WgY81gNtM*Pf!MGonKpYR&T>5tnde4n*VP%LXF}DNFt3^x z)KG~cJL0#1?+|SC5)9JW8&=8kGHQs$u_Q>s7@jn6>>ew*9u&MPh{$O<4sG~Zm=H2C z`Sn~I&zG#{UopHZZG1@AfZ|MXnEcMzn8Yy=MZ{R6Kri@1-di}ERC!@mUHVm>3SaHW zDtL6fiNEsNh@EJ@_LX+ zg*fQWuL+36rE@R>In?qy%gSHG8hTG`VHC>h4gs2{kpxl_qaw)-62t4+Q;gjbAwkWc z@(eTR9rQt{Yz+p*pwtKeGQA=LFC??9MO%zwVPaMqASs1moBP++uJue;VKs8VMf#Sx z3qgcIiA#n7-~{1_Bg6)dqVMyy?qrY1fSeitNrWH6;IB2BoPi9Fupo&B$3S_9>BACp zI?OfT6f+V3gE07;IsumCZiKr96p@Wg>284z{ssibfo6J#{Qm^3bmA_@*;r^S8OMR^ z^p3k(o|wel_a(&1hjnIu4I9%a#U3th?#>Pvg@g1fXGjI4+`kf#N%XfE>XqQjIpG#M znnhrwwHi&JRpPdX7HR6Yeq_W!jKsbJUPi4QQu%%a=17nTH94&no<@=*b>;y%xdvaI zxN>Y0(1?q-_wWTOvmJvmyB|2|xwaRcZckyERxUdB6)FmUNV}#8ZU=~( z$Ap`G9?r*u?mK|JT*W6kXf=tQ*)ZLtJh?b(Uqy9{>`SykkSy6ESI-&~DsHOnA#P#8ijB(@(oFW-S_>4s%pc zN{>zRK@mmeW8;ocQEl_hmq3LW+RulvlPB2ZJ0+(0@cK+N=2)~`vic*>#BJMIeV)`n zPAY49Inm=i3nW*&*il7{Bm>&aHv>;9!%a2wfD^w128qD8XP$AjgYd0AQvGGRX zy=7&$MBwiilaKLD4zRHhu~6@tvm8JGIPs4Ywf2w0te0Ctb_fC<}< z4x*Mp-<^}CVx&#+;O%MLOKQdgT_w%wIAXhCG?fTLlE)3MQ;_+0>}>rA;B$C3omq51 z(8e}!)0D2OCE?9TA*^9Mr6mD*Qp?wfLdA903J5~l4zA;CHGvJ^1KJ2_GwCRBrLs{H zGbW_`CFzlqLnp)0@yq!bagPR>YRX4GyhbGmLo|!SjtB;2SZW1yOayW za{BC(&k=_tqyHo6N*%$YeCqBOI`+6Iw|_wHa+E4YJ(5nW<9H9UV3>gSFCDdtiw&t(A}<1Q>8Mcvq|>uUxK^!`nV`r=(oy2TQU#O~ z7G$oRu6cdSaYsTGd|~S$6|w+Z*9u(WCjqnRLY<}i;C#zTlY>By;-K&OzQOG^kA34+ zMweaFjk=lPBFPCV{=5r2hXwo5ivub0ioB?L&#eu`ly;Erv#ZoK;BL9=yB1!4j`wnQ zjax>;70mS0^>qCe>?RT+w667L9xwr|v&k^2hqIbfK3DBCvmZ@`y6@BIVB8#yc%Qs& z3wfAAxJP(wwIaDbURlRYQ$gvm5!t{GaTjB*&WcS@;c)?(8@1Z zvOd;`BvQIbMwGY}aHI-0_p$S+!L`)qRH#5?5DlMN?}<*VDBG{7M_(FR4MW=geM=?01B1)7t^Jgt_%=3 z0P}+V|I$je2^qWHQ$R+?jOh9V zy=K3arL_BK5X`fkj8s`R7tbwHq~w%!${ON3XylwkPG0LJ!jx8fzxLE+UvlgXXE{WR1B(khx;=A^gC}^o7qI#e6b!@r|3;~ zg$fo8?c39{!G768f{@RxYs4?tc4eUziJ|(+uLxv)2H2f)z+8OHnNev^#dr-!gpl=1xdtMVjb&YF4?Ey9 z@NN(3cv>(o`nM^?h%=FB87ytXDq0KUMbvFaV3lP|HTc{?)3(juqY`4%x@xstkNxOG zgx?(ULH}_n8L$XstJj-oPrfKcu-6rrc2RaQPQ{S>LU*5}?++O@Oe#$+0SRFp6-KhV z8l&jUP2yr3dnfo-<6UB?nas}AZH(`20e1+n)o({rNu>2fUZ_I|HIp+NN^l71Z+q-0 z@1Xo}KJE8`SCu{8lB%h=l52J(JO?RE(+)htTR!iv22)NaqWgos@?AxUtyaMG{$}MB zL`G?jisRAHPhZhCx|*?U1q& z$zx57ONJ?E=r=zPww_eZiqXFREa#nqxZ_tVjyi^bhw!JF-cvOPoeC9u__HYyhl|wBD`8t)Rc?4|Cj1b>t!+1#zBH7US zPRgm#g?0~tTmw#BltJ$~GZA_3phBVDav*~_G3w(tz1TqyJ?hCWgPd0h;G*aP*Ab%V ze7n6p08!X;98tf`kw&y2f?fujevu^(PQ+J&I{^@_k`i2^$2{H;e4o{Wl~F<^eNAF+ zoQ4CiHwaW|I30usfsSlm1Syt~-bfJtn!Eq{w*I8DW85OBzRm5jy?0i{vG#1a zb@o#lca^Cj+Q9AXwo{KS+D)a`0Xu%Py0~L)AX3{}ZCd6xhM8T$ON?Drrt1%jIwI6Q zwP#&r`r#TS*3plT;WPZ17nZ!&+CJ*coy}~zKc;D?o&@KXirJXXrs>;q_grvtkL<(z z$89zUP90oBhXd9o;W?9YpZMK=n3z&l*5Vcy#c!u^b8ys+Gd@E*Ub%KU-m^?=a>#&# z9n|lMsb;l1wAqjl3xE8kL1wc0^$)hA4SZB$&FO(Q?_&u$s^jnRAGZ)uw0NFU0+hn{6`V9r(dY{`&*|)jsvG7nn+$skZPur8*){c9wESFL^EY3z@ zR+cT&O>nk}bJKLTXBx(leC4y=#YMY~26*kMT=7Yb*Myg$an>smILdovLw;7DcO0%K zRrG)4+nCv{&Da*5cT9}T+WDxHiIIFYyE&@aW=Jkkq?Psa#Xu)N#JBi zwnRnQ4I$Q+IE)DhtXwPzz-*o5yw^LJQ2lKx?yvv$M`{Np?v~_*f z$rTqGKuqSo8kwf$Jfjwx?eOaB(g=&sQR{iAI4CYIZs|y%?dnLg+2qrrSFEk8z|oJj zoh7>U3)el5&4y2&iws+S2|c)3@(Z{p0wEN2ef;1+^2m{j-prbB}P>tA?i56@OmwN+S`4o z;S6j9vJ}@=ZG2likdH#JXPf^wMTtpie#c#Rb6Ymf@>PmN13QKZ>Qx1{XJ(-;e_Re8_ zAX;htUao8ZzmOSTXTD5#XlByFHHE> zf2#riyuqgIROhO0@hJB?5C`%Epr++Sk(H(nhrT}xDuXSNm#rHC*Tgfl*DH3DYZwo|R;8WGmDMQnZt;peSCAwZcrAPwPd@Bv zG1uf)_H4rPW37JL;{y! z;ChDKZ0mj)M}9kGgzasT;&ps|+e zOHX&FwSnR9ucv136l6=K8g}2DEIO*D7Tr5Xdy)dtUn}pK4}suq-2HNnIa(x2wo~G=U-&xs+4Ini)_n{Yk8Y+PH{=eb8Id&DCo|J~ZVO-6`rcT5 zI_&icQTVhqFMB#WW^kL7%fF$y1*a$K0lQnI^7RwF6D!Y~g@4=1SSlM@_AY-NR#Jz7&%E z?e0_3c~6kJUiG6YXp)zyZScA`RUh0eOTK;KyaD#Jo;$m0c>q4Vv`l@QC2_Fn4BATc z#ogSqsiO!CfAa0VyCR>ivi!SQv^4L=j6naQRu+utiG5Hz{o z)#QCD6rUv&E^tY2HP<`)(0wN^%_Q|c{Rl>g2F1ITvD;&Y7q~w%i3a(0?kTV<0O`{J zr{FL??J9n;F!|+qi?HgRo(CF)Udd}0lKnECqemXmIW0IAG%@H-ZF0W2C+?a4n)kD> zS`TG03`-yY#6+I}K=5SwgkYT}aYN8&mF$i7DpcZYT|G*cWic6E3os;XwAvjjX4Er& zwr2C;?fSC@e>aCKqIG8g@q6Hr1EkqX&L^{?4fZv2x`voFilVnww{T$hYs&Dkk*HMhWB3_Yg-qG=ungXj9LeS=N>M_8TL98zs; zzmaR)DhNOOroud{Ci*4O(WJnAVSgozRI+iOUzjcKZyHwXIasGk-^*OF{`tpzh=0Sh zJv06N`FuBz$Y`oqqIj!}C)UOv^(vrKT52wa+xAmrH#u1D_XbDN&}d#fveL`yAL1I_KB9_uZU3 zQqZK(|9hoYokyniS=Q`4b*EN{z9;QArk=WLhbh7Q`sV-_Jd~o(dcj%0iP(=&2FzUB zB}ah79jdhs38zyslW;wDv9zp`(eH&cP$^RgEW8ap6pW}P_GqY!nh_Maw3t#*)-~64 zrg-jBMh@lrd`YU0MgO#KmX@eto6OYec=2ys3@AOqxsseU0y7d>np4$X46b)CIz|f| z)l%<7^T3s|-|VDsS5}{l(>tdK2A5>J3Rz{_f@@^FN)mza5PjK{4qkORBH?%x^zb1s zUj93v;5&S{_{59i1tr7n6U>+okidjsYGTYJ+Cmt+SJDG?1hC4S{ex;HT)OvPnrPvtv$njNXW?GUKNV30`$iHf1IxjLquxNbE~IZL4l7R zX|2iK#PN#@Q?BQBGDIwv3nY=~fICj?xG`|0RQy|T=@NUq-;w!d8;Rugu)m5_B4L#L z&}i*{0<&%Gp__DI=>`9%e?RN(I*GM8@>F?%bTlU-i6Go1?r>ZcL#b(lr zWY)hiV`4=n7o05#JTm!>TLW<3{~YA*9sK8^=^2Et^wswFOew3mN%!Jy8uA z!{*5IRFI}a^{BD~rz0LpHCeBB*M#SS-n^z^0c=eCFJyZpT~3L@?Tlxj9a;oi&|G*) zhO%oA&@|lKI$MgZ0+;&zyALO7?b3Y;aJkH@BPDSyDo-yK3OuVupc#IV?!@oE2ZmON zEJmYfm20ku>FU&&!gknt+S|W75A%WA*nEc?+jP(kd@{q@Ud-U64{^ z%q*ogo8>|!Rgu&L$*#%PEtYL*U{>`~oAI9|Q-(to{yJDmn`j=ANb%3k%FlkdPfOS> z``@zDe_P`Jja2W#ETMeoO-dWMWni@s-(pxZ5>B#CWvD8j`6`GWck?H4Tr5lP2)8ml-N_C=kF-OEc7$@(C;7n;ek(|1qUX zft}tI@C~0>-7BwZ;3d0?8Yx8%Dell$AzvOIzy5K3~ZCWF0uGo@g=j$h3!6@CMp(Qh=swTL-S}nH!X7w|<{2$W@ z`X#BiszG2x9oL&rPcQPnGpTeZ!3 zpR&wJ9#)wzLj0X(hK+zfQ8UB9C}qtIUzP%<(OjzGbFfo$CoEK&P5e!ZcJ|fLr*;P%hT4zk0>#jRPJgaq{UqvwBU}>1dECdFO=A22vXvx4O$(woY=EW zR2RO5DMhJHB&!y5RRc z8P^%E8cN~fycIK#8@%_>#Qx5F0I34u!V~=2Qs8wKrI3siy=meeq{2Fsw>Wr^;n65Gt#r!X)({&@(P_#t4W4wr}zHw?$ZorF%~H`)P=?I zl_?ezYJmye0F_4?>h~{QWE6m!&fZ4@hAM=f2P>Ld3b$)(y;KGy7n4^Q6j()rmZ1$f zxx2lu$nedTDcIo+LT@lL(8*ZX9&$%2k*7QtU`lm=@&KvVIj1$@Zi0@6R9vbkt4zXMf5+gK#Rbj50zN;PP%A5o;|KwE7apxy@(U0I(DS;DB z@C#w8gDa&SeYJ-37r~ds@bD=6CpPmxHk~-1qbR{21MbFO>abo~tbe-^|1nw68ScBJ zhUkT7zKBD`XS*72ifdE(4HHJb3>Uuoa>RNDNB>41LdrRBba* z)BM9es%$@L#r=;IXsST}yRW(`vx@0B?jErXD zB`m6`ayD@jQtw7liJD^7_Ko&f*0w|~#2&M6Yr+x}FlIl|+G7f!i;#kBEul)SG50L# zA^u?P9TurPB+>#T|L-a;j!-p^Ty=^Yd50RzhYsb%;hdG+wOn$kBTq+^JVfz?xrJ3+ z9^_~G&a}VPI`+nW74zU8wLr&xmi|t|8=>QU%6utQl@8?0x*Fp*O7Wy!lTf-6OD`lP zirQn8J;B$2VMtotRqYQ-z?#{Ey0~gauZkg{LKWQzGp>f-at; z6>yo%jl7u#lt(NZF8AcK-LZ;lL!NEFC#8%lpHH#s2*PxxSW&3PBGE0ff7Lx@nSTYl zlbXy7vjOQ}pD1NVWti7G%X#I)_&5yUzGKL;V?5HPE}-|f&}3t?`k=UIYVKULXsXFG zUt>-aE_Kd8nXs8ivBZKsD=d-{bsiHg>Qka!Xfov-pA@Iu=rDqQ0Ed|X@|!^V*}KmB ziYs-R=JH6zEOu)&n~ne8Lvi>?&V5p+VfDBElOLYGBJqt3o(M?7mP=?iC~si2<-8Xj zEf%f1d4NJvm<8ibA-`hRFCJ!&#CE2zUvS}533{&+@I|=w2{jbJshP5^Hn4!nwisUp zFl#!)ZYX9j67odN@^Y5WdlN9|6ova6wN13HDEl0D%(Ut#-W#o-x~Wa9=zN61AtU;& zO6sI`#MH(3`MqeD5#b2OnzgKlk%kO)A|B(GVqnnB@UvO>KmR5GqPX zTCvNlauJ>-6~;l^ZWrTTIm_E^dat3)ZKLh=gtiyCW{HOed{s}ky#kFV$Q#nhTp@$M z8~(up%(*m9eD{`LREhNL7-%NMetutJ{_MC2 zEQC(MN{C}!vJfAaPmyD(Y`VgWM;Fu@5hqEod96!>0Z}6wuZ12>c$S8QPO||LuK3*k^v_7bCQ=j(ksL%J5>fgLk0s*lymi^TgC4Np+Mk=pQHzJBn5v~0 zUs3gbU6pLSe|or|9QAhjC7zYXta~$rQ|M03lRB#WCBZO{#WM^$p1Uhf4lL6L9PrqL z>-l=V43LZ{P99!3I}A*Og|)y{y@X0KdbuyypBSD~4?n54O;_)tH~k}FEYuiJilFqS z(i1NXg3B7-TCT}-BJj|ySJHMrE<6=l|E;G8Ha0d4G~3oQK*lKbU=eIn+UQXhsE(?` zy4y>r12@ygGq1}81ti!RzOHO&eo7c)LtHEHNMlsKVs5>s0?^FS>X~Ur(qZpLTwVc3(fX4UV zcJm){RyFJWvO+j@Sm7WpilniLMNhE0Qv$omjV+&K z@!ajR_c_2U=gA=p%(v(3Y7CgfviP<(uAZ`J#K$G3va#|$sDt)v7Oy=5RFbjBeREIH z&c3X7R+`0y$K9v5XeW~6216ckRg`@@be)pt(?Z1`Xa8Jbbo`)w6tDM0>ZJYzMCidI z9bV@rtDyTdA`^WwFmv_Wq6?Ln2OUk9)hNBS({mDSFBjJpdvj-xy|lmtI?TBS77k_L z$_y3CKn2~~v*3mKbZNMQI#aNq2$FcL2v4SN(u5a+~Ytx|C zPP4j+;h5(gX|nNG)xAAkByxU6|=dmCb|!nG(a7si6hK$TZ$ANKKO?eA7lu zE}G14)8>eU()U4(ymKQiNuYPn&gvWcd`T*zuy;8v2%>0bN&O^tgCW^x1Fj0~fTmzC zelOW~YzOYx3(wkhwXPpn5DdAXLsbSp4Pl%%7~hRshDPWI$!a9xaQB>TOsC_tlVItJ7E*NIv0e}E%R476Szt$O) zcljh@8z}{wn4z&mlqHNymygsm2=}`jL5_;;Mql}%O!=G%jzFN9Zd0LbJ=?~kE)^#e z!^({J+>Gf{@|1*~>sqWXYBDZv3%ED4`nEPQZ%g_p9s*@Q5#Lp8P)X_N|sSQVs#9 zdU=6~U+J5HEW)K-wk&b(vKi2~(eaV{FC7S=;ReC3LAO>i`vY>RV&y#@fWq$gKLu*( zpU3uw?|(UjsG%!}RfT86)|SPpkAd`Qz^;HQAWh=kf^cFLzXr)8GNi+BME)-_`Tu?? zBe$={P=eZXDH8F2lD8O6X^)KS3o9S z6CS-^9$s>8UIsPSnh*rm_j&VFz$4p_f{X{vKt~ce4GA?q((I$!e2j$P8YiN3y1zcb zMS3knNtj3^v5QDQB35_&^OFNF9|H#$rD(+7qJ(2KX@Au4Ac`eOhV16X`0_07`%!$_ z5~WPHV|%UZl*OFUa-sYxgQ+W>8!G2gX{ZLHLzpQ1q`h${lVC*e)@>(Q@M(3q4$1I= zQX14m*K(>d!%nNrhsKYwcREAC|**6 zj=D$!+Pd9di$C@%wS#vIn`EqmvCZ=A`KX;3A)Enwg81OX4nSXeYODx`tllG1JMV-l zMfA98S%V=d5MzSvtc=3f3)uqWjbQ83&kyo8hQH|th&p%i$a0F)=7wlJJ4-)*5snx` z7JgQyA7O>uV*PHo!XO|i)0J#Ty`CPB>MM~G!`V2gKkhL_Q>ubH?xWvH>4Nonq4`!k zTj*S>=}~D)sX&4!&ZgWwth@!MV67LA>4c{#o6v`dLt3cuSDUuSUGn?S=PO_P+V4+p}Ae zwP@;R7Q^ke!f>@aQhN%AJ+t#{L@q>#DZM|d4dmlp5P2M~;6|-MJeK;h@e(bi_1Koi zilkYyv)@-IOwH6~y%0t*r~SSN1iz~cxX)^UV$u)l!ljHCA!0xR$NzS{{-3*uAK$1^ zynAhw^(KfvA@@%3uH#fKvLN{T=z7}w z7hk;vnu*vy?wJta&^(r`bq-9{wcm2x zAAW1)_|CHWcsDaG;bCH9ithVQ1K|;%jE?u4tb(j!y+@crU&3hQ!VHj{Ts%S_l&2gU z=?OS@^u4z-a~{e*0d$N2=PrVa^pkhL`xjHy|G7@O!p5?luD8?IbMI>ho&}jGdiOCV z;rS-FB{97(j{zhe*kpn$w9~`9SM9&%3f!K|4lTO5OS8j)_h99wU!Y1&n9I6<)m{z(S#%|`F_UP`wKm?#nG7j)Y zPBc6%Uat$UgAXU%PzAdz@f^Blx4vgmsCW7bV0-5a)uPzBw>myA&t^Gibe^pTIkA0X z-dyoqBY>Oe4V|N<EK?*zlZMhKp$g0ZzVsw!+74%4TI4wP&BWin#-mFZDe zZyx4lUI}nyG>Jwoos8S4XAZ@kkZ`S#+n#z4jp467YV?N1eSFXqhO4eF&L`|`ee6O) zdCMc=Ioa$`@|5Tc#BJ8l?std8V2(W}=;eiM_&;IA(04P|muz-+%y(*B8z4*Rc%ghB zUy9JD%{;%balLc5!xoRZ&0#3oX6@zED#;|1LNnC(D7@cxp<+zT7%_j2l9XxXvC1eM z3B1;OV85Z2*_sZO6Q=%gO)`9TceF?%Es=Ht8dur#zQu1;blCr|YwM&-2SYxO=}?$b z>e;NM?Zl6Ra`N4umPFZy{q7eWzk-*T3d3EZmS^o_=BWWT2ahLHhR~0_jj(Hjr%ON%i!l3R|l|!|J-M z{vMrJ_*OjDU~qPqC_kF)11%6Y3E@0Z2Q3f9i*7j0E@6WSl9GU3;$itD!KriqlvTSaUA zqa{x8rP%5jOcY{B4~&m^>D&8U9BY~EGm=D7I)e@va~kuo79wRv@pcT6lMHDdlPc`$ zQ?z6-^}omTL{TaB6kECO&_d1LyhMxL-(PSLGM0NQ?q=;8In7v0EN%BC`tMPJb4H5nGLDa8BfOb>1jL5(?lrPIj&`UV4e*8%Qgx9}iRzT-0B8$O+0!fO?yUU8=b0^=X-HIrMpD<^HJxTSs{-|Ol=)mPM)a+7=>ixr5DNYspm@SO?2G}}DQ;!Tl6 zw*0Gq^hGCr@H_U&M#AuTv&BNOzbhL7$v+(WYK@-Pb&c}BW2G=u&IfB8&H1znkx2EE=#Xuhd}@%!oCn@KUuiJ9@Q z9DAKpBl^$TXS*xn+o8iaaMZN0R&EunU9zePSRN~2QP{HQ?zJ&9CWP|@kgHOsh`)`p z8rDKz`uJ>z-#f%o2N$^kHM?M7cW~HFt(9imtTEu6aYjSSM=qfR*)%Nw(Z4Pb+b@X{ z>mw$c_(HNVx)D72X@_nAQv~7r`m&qXD!zqU{k9i$e5P6wKM?-z-~Yu0=ruka!tG+t zlFyyOcJ{5}%|hGvoV&Ml5qNK%`dk!@eR_3(s(9T%+q@ml^5hp#$I};n_XaIh!=oKZ zGUw|M)-e9FPR-kuC49XE!QU&&JCIJg^5uZfq2?#AAr2!?llv?#0Ii0oo5{X$AY|ke zCb*7h>hb09{a5mH&-mL9bJmrAAM5)~H(j>1@;�SB7a#)Ia)HyI6i)h5t6W3^of?&boOIAgOCl z>+G4(!1vT#BCNB=p%!)>4O5~Nw?cKHv{P>oY_VC9vPUzH|k1T=*0lKct zA1(#U9r*J4S0&#?2pxw4%Hf8!b(O3I+tDAtd@W1_Q$c)`4-E{%hPo+!q4>>8dFb48 zIGs9Gp;dW@4Ti7AI1Js|kA2R|ytg!mz0aHtFRSBIfcGaJ z)H__I<^);T0GY&(2LO$4(d6M<;*>)6)5&Wmn@_JuyVrkQbu{(}-L6uH3-({6;xXKT zryJLK1f@UrAFEkt*0JviLZiu_zbB+6WEyhK>mvEf(N-c%0z1J~XQa)V7M|7e%buRi ztJnZ7>UXs*>{;!)r#abn9hxeLZkZ>ngkb#rGqJ2Aw_rZw^1i5QOsP z`45{TzY}nR=~J@Wc(G7Wt2-dBPy^ zq<>i+^qk8qlpf5#;OVxq-w={1>R&Jyz#wb^=LAmCUn5h1k>P)#5@ipLc2*x?B)_M& zj0UC#oUT88sgV7b#0*?Rf00`8GqA}2!efJ59=y{ilKz^B8dP-F*JEw>n2|W@xu1v)-!Iz#*91dE;8|NP+^tf5&mCIF z-O#e9Z02`Ygz>B$yUhY^=RS`Gk=cZ!oUd)a*0?{<7D{ZF0g{HL_N!uYR_|obGO)P7 zRSw4NM_Jdr5{!-|Y45`dOZU1OPNbo<_-wFnjK42gc)_*OMw@c0bXqwop&W7GizzZZ=@X#ZTQH z*q3LDa>54xN5z`u$<)!f*`6r4sWtf)JB}1NAtdRen*Fy_U5Hw%9=r1Q_d+e+J{x`s z`ZKZTvrnA3B#$4lKe3-JO&j#w^yhu%Z(=vna2RauH!fjn)&EK;n|WwVU&oGR{D_c- zR^JKD>JhE|)u-Z{KH5}e!Z1lbn~85X*r15q`uK_u)~xh0;WG(@@JR!}vf_JOS-oavHtZe*oL>j?vd^AiFxUnSFWygbrKRJP`!ux$@Fm(u24(OgboknNge4}0Gduj zEnd-?Q>$jyYqWb&WENW~b7y-8YRjNDdZXIh5zZT&57g)U+<#XR|IUm<_t!arwKf&`J%0f7gtNOJ58i(W<_18&EU35Zr$Z?(Sv#;8f*Nk#6=?lKf;9hhCD`R z8^4yCDf`BtD&#hL%wd^tc+;^9@a0Tz3R~^ueA=Au5!LiYZn6BXWj%KI(t({^3ciY0 z;P_*j3KE#J4mVoLoBBlku18SDz@3gU&zyEbOtUc{u~WaVjrjkCgHaMZb7* zK?A@j1Yp#k&_jZ5L{ks?cgV&R45;-L+pnyl&HfMriRvDxB*CAx-(N2s9|s%@?4IM8 zro2aK>6|RtY-cM-bBk zh~4>5YFwnpG^8{Kha<=?5ZgXOVBX8n~)=&Dj9i zcnC1gG+3oBQbj$OGDMT|;q=DXaJshY%qn~Yz3CkPj=E3aT;Zm`hPCK|ZQ|>0WiyH9 zjX&A~_SwHMbqC8=2`lrr=sQx6d%UmT{3e~@8OGlLHowBQw7nmS8Ru?WXc_Q(t!03haP&Px6 zX(1Tk=Q$n-J81#^-=>l5Url3_2c*h!U(ZdwpsBRN6gOp6PVn60aW%D53#}IdIr@mdq`2nS9msN6K9Eu6GQREin&k^1 zO(^5{Z@aAy;^~F$rW;R+c$P`Tlzao3aj_fn=X7Ksi0ppE-;vL8nLfkm!ZRAyRff0U zE_o9ef}z-`bgil^;UMcDag)$O>e#qp7Yr`&|jlXOxZx%lVLY?XBt-TNhwZeqh!Gk zWx*a=SKW`;vCI45k_yBeb~i%F2c7&cXaf$nN=TU5v%08?B){&1+Wo!!LwB~!|R zI8XzfcY6+$3@Hvrd2IV@qQ6X7Q|Sqvt-m{FEo3BO zB=cSpXXjis{^K2nKeMwC6LmxIS|3%Ui@Z`}xmxi1KTMhXS-BNcQ|h)$Z8eADDl=cF zaraDKtK{n1zu_@b?3B;?fP&xkCHz4EQJPr`RsblyzyCeQw|;7S zlgDTC@Y^S|^4a1i(^#&|wruyM&15lByxuv33JJA7+v>8QZx8L)Ug=ocJkrno$3&c~ z-=>oV92MB#-g+M>QEb8(8k0sdnPj!}!v|H&%WS2{ea3m$Dfjh3pn*~ywA(UIXuC6$ zQ8WVOA#)pA^%UASy(7}J;mNa{OEK|h4bI5SL!{d}nbP zgAfowjDujR0mr?N22ku46c#0Xacn@n9@4V6S%7X)D5y-}A!_)5aL)q*gYobcxQf9i zF{*;KEg>Njfrl_*0Q&6!*z)ZdF>ob?fytpwY2*Zl-vJLvxD61eHVZ{2po-Af9CAJy_JtMbKE+SqGaytRdQL?Y#y)K$8y~EowWDz|Xcc%A9}3DC!$v zv->vY8)f>zY(){m4>1OE{Q!iB^cM9Wf-0ne#pN!iZujp7&4WTo{F~eD+SYjfnXYQ@ z?Iydfn!?d{Yl8Ag%azvZl=u3LTh`!k11(@e`hxJ9h!JYF{adZXe1?z`Zdo{|HL>y7 z`SXyqcy98raG$U-4j373p7Su=I9%!Sw%fZ;)jZGL$$y^pDbQghmOuXHEJH^4-~i_f z`V)JNUQqv{Iz9A;&r8fiBlN}fn&V2q{oS!#qjyaOgxa+3x=Qg)_5@nq{a6gDyS+ht zrEVv+egbn7F8#DjH z2rC#3LhTWn9p5nm)ZWSE9k^eF`fGhQPouHjrjdgdv5+CGc=7`ZPH^ZVlx)8 zPCKJD7PA++u8%O*(GpYjuh6;cmTta}r0G~aMBe9_#_^XCt~s{v=XT56Q7vjv$MMDI zUb(zqUF;haj3ZKxm^74q^f3023@^~$J#0k^}bpg*yW zkx=J)hHvk))8@Oo@4|ey3i_|Y`|Fm+3rcG@ww&kS>R|tvzK}wk6C$>|fz==X*wQjc zRdpX>>;g4xPwpP7IzT=WZ*?|3b>&CTxSeP)&p##XV?uy`!bbJ=^tQBr>&J%ZfHbs3 z_t1~05@IvDXEjkwkB%~St5%KbEM`?+S4!~2fpY<~| zA;_+eQ?*Xvhpv3+b7UE}S0!HibX3Fo_?UIA74NRg^^s11m!EAN3PeqonZnb-r7*&S zOOx_3Xe!y4THy1oTpqSP zEwsC3=)ughRQF^#JpKwclSowlUj7Z!r}eZ#!&^GgafQ_+^UI}M%?G7^N!75;!`E-y z>v?mM_g!BWE+XX{QOw;5p9pqrSxbOs0zAdUeF~t`5P^$wSO9(h#6BhI7u+YzxC0l4 zq7Hs#55o6}D9^jTqttthh&uG5jGzz`AH8(=0Q)ICamNiwW1f54O3m+z;ekqgiY1dU zwPMfHv(g)j91~s9%@@@=qDSZO3q!clR_Tqi9#8Xf%FQQ;P7xE7?sn(0c2<8qQrBDg zXU@BVoBCxnW5iMq$`p3BnvSJxYrjBMIsER16f*yX`Rq?#`~%l%Z%sYdL0Q(RuTc$% zBX>a$iSu59bHVuo{MWRzsyD=W9_VU4)33Ay<%9+K<|cfz%FUbLMuy~8Ucp6A_`8Dp zoUEVP^%{8Yii*j6p0zcNzBO>0kr$&?-&`z#z+2=Y!(yizFNx?%t$}u;yq~&)syec% zk$Y6%7{$0gZbkl=mLnz#qR-U~@=V+fg*)ghqdG}t>VOP@^KUg&bei8#7_i3rTc!)h zi?4|-wv-C^x3MGO0`TBU&nGy61#CV;_RmynMB3uku2s@Wq@TIJu7V7^8Hrp_*uHMl zFcjMowF%94<3E;$nsOTmFHt={qS5=I_w(-hPjac-}mJ1G`)NV7m0bxG;17SejfA2PgS+4>`2d198 zzv6^CFs|51)YAXF)%dbbl+!?DAY@@A|FJ2_=%VuntO}Vjj=N(`b}#Vt6(7w zcbiY!AIZE1HA)a=O!yauTBCRmKeXZqZzO{nC;QQ2IARrAin!wFOh(v(mtEkNEs!(L zcUhPVsuks>9^F>9T7yy$G~H|U8hD1B>@1{=V&FjGG%=r?ve$fXT2cu0B^mPUd%6%& zhou-vzZI@b#@76=dzzGW!{b)&dWR2zBn$C@RSZQ1P^ATIi|Oa`&Ij&iQ|~t9uTZdz zAssI{w6b3I-26Dz|Jiakg^rOCh{1>w+)021)=~i|TF(pPD8U_q%qFX*@p~7}`(S0g zuepIjt3d#<5`PKs1MJNZ&OeCPh81-^4YaN*s;}SFm!q}=_nQ>FRSw*LnZb(+SW-z4 z%AMI!)Au5KfGL2viD2FaJS+#$C^cZY#Yuq)2Syx0$T+@&DKJ66TrjgxIEmM2Z_e^9 z!a>PRBQ4KiV@03up;7!Ot^ykzooOyTaf*Cr@tqlmrvwTDjInWBt&5Suw^lp!9%?}| z4kzNs-4_-!9hQtEg@_%Udg49+8B028q>yZo`~*%7#;Ei4q9XM8wDS#AAM1xq|5+!w z`*%Fao}f^oR}b7KKpBO;J$ixhUNMc>D8aqFPNAKpI0e5JaE9Rq#`PHPRutHCq&NQT zJ;?-%_?a6e)FDeFE7}L$*Ue(AhJh?;v-|QN%$xQ=MHb)%+55YknbR&F3x=I+>s>d< zvibMw3|_CdhJ#X_vkl%wf$A(=`J~Oxn~HwdqbV03$nrcUyY48p_h#hVHYCmNS62$} z?1b=n%sI=oAF7XA`S(aUM=38hbRiumStlv;M0 zFyd%ImOs+E?VY$YQyBq0V=NfByy$4CHnLgtd8@5khI)|O+k8>&4BFy7Qd@GeNeW$i z1B$b|4U9Xi2-`Cc5jNo|OgFu4!mCoU-qyF;*0r$UmTi|@C32^*(DRy{LKL9Wz1RdG z34S1OuUne#=hqRTv237y-`+PU@f?N0oT4BrErfkVe4_Bs^&=3nMJ<= z85LCS$h%@!k~-BpLQ2m%^_o&%Z#_*p;E?zI#J`#=#u0i3R>MiU3=x__T|9s3+1}e&Yihe zcI#desACG@^T2&I7kt;s)!@~=lHu}MwdZy&LZIPf<$z}=e0kic5oKBmWzR`1d}L_K0Z&uW20~88yE4sM~TnTJUG4Hudr%lMKTX;yM`R_pg_a zd=QF{VsaL`6GvXy;dreM0_!%yXZ5NzLge#pg3rAP!0CY&l?dSYmQTHdi;^K}(c z&7SQ{p%EoQa|MO;VCbm?)n=(&4xS-GRv*XQ4cky6?iyFZ2yWHs1kGxDkC_iZPacHi zOUO=ZRA#KQn6Bb+}t$;U_If${uv44#v z1$j&aD|;w3)LsRvg2kb`5dU)W5;C(K7ywjwAzZSDuZS(g*SPb5%a3^0nh^+|WJ%xV zgCJbK&>lr4UP5ikhk^(PEk+FT_@T@FVI+B!p?4uTE*vblEJHL%=_m*XP#iIjJ1n>{ zUrrw5DpG+J=Y=I9SKti~KIr{{J*g?c0QsA71MrvsY={5RH za1~7T)*;gZvM#XykjJ4gb2!2|R6fJF^9ndedEg8OpT@}oE)i%arkz#vhZoTvob@eA z4$2$xtV$TdbQE6R4id{oc{mK969XSw1)~D0Ne=Q@ z3y6sdGFti;rT$vN7!EcB&Oc3fBQ!sMx%vF3{(!*6sve@lA~@ux{fVd+eYa_aw4nJ| zo!?!s?S%|-AAgd*-`OJQ9!Z29DYTNjDzFOG+4&T1tI4sIn(n)))qIkc)StG8HW(9q zmQa~_RdsV14|%2=Vo~hl4hrM26r{UNDN8;uMLiEAdPU)P3OY>gAK72t7f-(T+|MAs z>BRT0QQH)hf|_*kti=ANvGL+qJqsC1rO3rDB6#6Bc$z=wwX(_TA=Gi4I(rkFbTh!^ ze{HF#=RG!MC8yLt?zKi#=FyHppJpIY-C;~|F>%0F3&X(e9oI{H*IHLTs<R+x(Ofi_TvPyO`iB!#?Bp}PSG1E()nJr*aIDf~7L=>?ZV z&&vlfHoY)9tcBdS_c~wMU5h(zId)zxWq7BSU_kVe#8;#$U=}~o+_q|6W=cTQOXo0; zWnKI0uA|OOryL$h51(#(p~ZZA-y5xG-#x2qpp2mQP;2Cx5Z)-Hpz+Z43!TnT@=QPd z>)p!&e!H{zsO7qiH97TlrKy1Cg3J>f#hZK~;x)!W@1C;eU5YWf;@mL}0!2Y14rGBn zC>s}YJx?UxPF^VQhH#_XL5IA1ats-LJzrzx@=ggw(+;=&boX)(j?dXrfzX%x<>Z+c zZwhWeVf5EE^KA|vFi4SIE9gyRBZgUoCyNy~$D$)eJGjj3`{rk>$0+Fv%`baW${Mm( zS-gIxXI^~-Rf?vj9YhOJH+l!IKhH}Fbzm-#^De9{;;q=%=(x%z(=0GAz|q_a_iQ@Y zIPkr+Ddp~z9Dmt95oR|f?z-I}>$e)9f4QCUOn&w}CMk12!d!L$^yifw3$`Fm+a(9& z$?QC|)mVY|%r;5g#6?8N{`yPQ-AxX*zCs^{bl?2*i>jpNmA;c}b2b%b|K<0qf;WBc zygKs5NbURX8vgfRPGWBIlDU2lHTU}=%XC1*`?I0P=kxVRqnj(UU6$PJ?a%pV@Yt~+(B|c3{RLeI73O0q`#bjhduH0or{gXMG0`&1f zd~>{$>n-?laJ~Bd-StorTzmN8Cx;KSTUjLqLr8`_9v4Jb;~qT_zrDIL<+*TQ(4ZD; z&L@_rXu3~rZIC7xoOHT$=9 zQ!vX3amd)u?2mwA-d8U&{Rcv#k~6LQkNbrSb5K+cRNUV;fqK+Jd#f2g5ue9=_xB*` z;3oT;dUKht=rbOd;lEP1wpTfppJLD&nwtT4G;-To%flL)xo>FC{h_uOQq{0+?;6Dv6wcMh+~3YBIanhokODjBnI9lT$_81Eun99O`yHeT~ip&8=eAD1J= zv#K69^A*qQRjxS(S&`7CUHwUUTOdmPa%bwvm%eXfiVJO;l)gQh4yK*J0X}*&CBt zNk{0Qf~BwO`K9P0ID_RqQ0HxMlvu297T?OzI1)RaG@gX#7ogmGPzCE&Q+-)! zGeT5i73=st$6W7SuKRk!poq7^nx(isDsdsSofv(zAgU$0t^SKjjOsh|nl3FThZmfq zM-NbM2b_DaPG*NN0HTispWzZFEgyD2&h5l_@Ok73mCi5{#x~~r^goF6Hq+5G`$&=} z=1cB+VMNFAepOJ^$+8vBMd`Uy1rsa`5JyXhBt8nFpwFROP&2!%vMtn>8~h-3YfR9+ zTQwc9uRE3Z`A5WEN54#HR7>{XXHs6lTD8e-OOt?%n!0~Sb#9U#b#C%W(=2ZNipGmV zP(jNQEhZS-zn`a|v|cFbP_fVW7Ka?29lWd)eddwg)g6a(@Lhmc}SPyzXS2hJHPWDpnP zU}O$#h|A1Qy4MDVrjEPr2dh>x6*s%7ss1#2(89NMhuaPvH2x@kM>tfp@yGXa4u&r~ zkAi2+Qgjweiuu0T>{gfP?Fgdg@mKE-QAk>7N{`e zUmqrO zyVZ{JK#RNnC|BweZRJkd)Z1!76Y6h{1!fy-i2KKD$SSR)sPqjVyw7_YSvGm{`?-h-#IwlQ z!M${6a0)up$h^&K)(2h~_k>Ol@lWqK;_nVYRio0rpS&x`3nbw)jhgoUp5m(lonjJq zw^GgfXiU1aK8L<{sNI@9`UN{#={_o1z9EtKuCKu`Qp^}2&|0+MLj56&QZ>Vl2ydPG z@T$uQ50&A&5mR2lj(SW7)bAhfa4a7Rx<(2pMA1_vbLj;D0Z~>2(!a8s@7PM zRdR|;+~0PeqeGnS)Im6UFM=wZSXdy5nU9oeVaM$tR@A&K5>cduaJ&G{w&_2_rwv`+Tw{zsE@ zx@(^u6(|H5(qk>)^^CzWPHAr;T+hwfbmpw`Omc8A>X~P=vF1L6h3;4@{Y}CoFHwro zmv8f3kNV`0Z0aq%7FXEvzt4nYwV5z^>u+~wq1=mff)tXxQu<@&!nnlSmfaKXZyi2E8k6eHfMh&(N)}Pzv*ZwgD z2R6J`FTZwTp3;e5inSR?DIb5~+RZham~BhqdXIvHY)@P%9jGGcch|z^&+$d8t6dR)VY9fq}^njiJ*=9ots!Vp^2eHb(X;PbYUjdz+v$i6KIQ3he)kkKnaE zR{E|yNB_NR{%^n)KwNk(cap*5MDrh>{IDDIKS&{jh6{lV|F%TR*gkOJ@NdWjKH}g6 zA9ep5lmOL@-4_#lv*>s0(6zz+_~Hx2jd_rS`5S%Gdx`bc1jpBWe|u-({A&u|z@UvI zm0**z(OCWlTk*8uRZbc+9lVv6es(%#jO1Nq<3y$h^w9~5Vh zAT+cm9}M*f*2SlhuBL>K*JQwlFaM_1>J_k+YO%6PH(v#ca@cj9_h1=5w7nyJ?qr6^ z&b_I1y?GONQBE`h=XNXqu&2?dSKiO}mR&5FhwRc>yvjNRVn{l(`t-Ibz_zCer~hZ@ zPtMsz$_I&B1=mf=q2(~f>J5AN@D^WRGUuYNL2Vkj9>;U-o(7m^Vw&Talurap(^bQk z^A_#j_Ca`<7Uff@>&};Jcet1BPFlDmi&9w0g4=5Pa>2yMPLgjXz6OQ2uO8%eQHhfn z8w;RbI59`()cG%aWt{4|@T?odjqcU;{S6V6MIevW3BNsNR1C+04H_k6{O#3(AWz0M zvLBJbW$KQM{NE(^{{k6trOp1%a;jkJ=9%t_ag!k8MCn!bvP^(}&BGENOxfvdDGhmi z8=4{eCzV^`SdnynO#cJTYC#`M5~7AqKaS*z=HrO;qiIjEV-=(~eaF1j+B`U2Vzp$u z{#8F_1Fy^Y&TZ|ioUXm+w=Lme4?CUlKwM9I2+4CaW*z0z+00-?1BmW%IEk`ElXHDw%#oJ0CkI%C1zg|_ zdt^C`TUqjYKuPKaX7NT}sQ(vLc&Qdz5dv?w2jrnb&{Ht)PS(ycrEk9@q%F112V8BC#eB z6Znea8Qo>QGR+}pr;YXTnjOiT3&LD3spvIAA|B87Z`{wv)PiK8p~O->p=t(vyq?5T z*2UA=!>QSj8TE9!0$;kB6S<8 zJPyu92a@P}@O6?(V8-R(u#TDIJ&KIFN0^hq+eg(*04ql;7X^qS9&E&}bx0N4a4ZVQ z@f&!x8$7Qs`uSa=`KL$GX7}|g%CXJ6zn7i^mVP{WgZtK0K=Gfh|9Wc6w4u#orZ;Q* z>no$v*mm&Gr!8Whc}Bq^Z5?CW_&*u%;OtPo;#^}BM_KO5BQ}$Jq#hS%0vAv8a*?;# z3X+HyV}q%f$c`qzP~m9TY+T1nohgw>oL~@Zkj48E#sQ~PELRlIlgj{;h>Tr?{#eqY zj9g`aW*`&W!-+Rb1P`zdvMHho&iY^ZN$K18ffKN9OkOQ__^?V~!S2=^k4{cuQUGI= z%N^djGO;zq$kp{lnpqSXpit|^;Ojup-+uqt?I(qk-|o4TB=7P%J4(+un0Pi5&%=g$8M5ff?9;`|*uO*h=G7x#?;^62X>h zTxT!pF;n#)zkI9rZ&4l=M5$N9mJZ8%KuWC;u^Ifu*yirZ@$9BsKg=DT!;|zQtFON0 zKM*dEp&bAo;9yr|DRhFmFJ*QB#aj>WC;B^Dm3u(cZ2z+_2cWTgbPn%OPgg!Ns0t=@ zhBDqw!=fYr&7hB~g6%DNWPpQS1VJ%kw9kz2zL2@ZNu`S;E__Tj3Hr+t!;Lod6-X#X zZs^!jVq5!ANja#21mxVH9eZgp;I+^-kS%(={pDY_#}XIj^mQP4B8y4LuB#{Fr$Rd! z%C)p9Bh6KuBNm~ui2rC6>G-G0W*I?WpoCo&E&jCG)S|pn_FJ7;9*}RidZg({L@8+) zN!&#z<33oNxyOxpTMj1!eP%eA;=c5vlAo$Tp8;lg+raptRzPd?8n#f^ldCz-B;Pu@ z*kd97gpR|OVjY91#KYp5W=R;d%Hh?q-SDU_(F3ZNRf;5QV0SdsT^sGaF%;1i6|V!sa|G|g+E#s@tP$&nZ2!8+#o+@ri^;WyYw> z8E;X`^TBkU-XSaWb@IbyZJ&1*z!}yra^veq$cNB0taa6cJg-@0+vQCs7x1~VP?!>D zU8!HqXKirIGw~U&^b5^OwkLN*q&(@0N$J(4jN*i!e|zO+OR{rL)hSWzm`qbXKNXzs zh?oeg>-VEl(%-70=r0_5oNaZ<0I2f z0tGXS3`dzwdufYcL(ApZByV*BAU`*q0`xo?yR9n7D=NX#^#Y3zz?>s?)?AcuYPGiM6O#bc+#9 zH4Yz+c)b48mdG$Or^6OL{4wLYYcI{IU_>CJ^I!k@`G1d-$)%Y(yYxoElfXKI*hAN=?kF&&Bl~=r9f= z5s5^u=14)*0MiIZIPpNCNG;)XVC$d7;_DCsgAb^Pavlm(0NLQvKO;g35#abxT06@} zI+P`>3idR5v5w&%10jqrs=?v2=yVGE$SYfe)q?aRvU10U3`VbIj0#sXYko0L6**SH zp1C^9nw#m?n9SNp{!)i(y@UdF2yZ`e;izMbm&dCKmg>&kkY~?~EP+`EF(&})zvEH) z`M0g2YGZjZ74O7aP;0eJq~euMm%NRzsA^#Jb$h^W*80H6?>koRU8x1giOl_a4b|47 zk1*P(aKcs{CM^)I;*B!fTKNP~GejUYhe90ysZx-wj!<2hu=JmF5LsjH1ri&4{si&W ztXiAw=)}p+Z0T?JcfMDR*%9%7kycjV);;*ymrpudZC~;>W5fzBLwnBgrv>|H8wQItk7FmPnN)zZ8-|_7iTsMF@B`m}PXj={Y z^L-Mv*Fpo{V-`#xy_V>B+THbyDndMRHP&&1WyoU5gB7R)LN>}vYATk)NLbR~7I2h| z516Thc2*2vg`B_a!+*=W`>(r=;rKd~SAw5@Zl-p}Osvu-q&z+)X%>Wz-{K;NsAZFM zUq0BBMoF|POnOhyiO?1AxJ6Ii$Pr+FPv|-i91xLB0$aG(CuN~aSzg+L>*b2JIlA7k z{fiz9F43Zubny!XbCpvOp$wH47Nh&3N*P`O+lLs1I$I-#8Ac+dBi}bck$Q_iZU1|W^(r81vO%#ARluyI zEPA2?L4^bN$V<%=dFqngfrmNwrJl<3<2xAXs(#OYnKx55-HvP43P~QtD88;V{I^)C z6PscK@!7rKp_}p1BpXFWDm`)*KOIoBg}FMagLC-jj`;cHRpAa9t{AV0pEb5|^Cj7vefEQS4;Ce(2&tF%gHzQs zupUry%n;5w&f6Lvv^|y26u}>S+?r+I&`uOPqJ*>iXpC-eQJ6aM3z(2Q={<76iJF;( zJ1Mby&r!i!_-<<9acD5-7XHXYrUPOB7@ARHQ`tEdr6oSyMa6H-c4T|0myDaqAwtCU z-Pmm-ahLt^%fnK!=T#(|M4efjwoyuumCB-uis3Z==`#_{q=nTW8;9H*Tk(|RYa8yU7r?*%fNJK z##KouW|ce0L=}%Mf1akledA8^Rn7;N)y~WYad&HKp*a!(+mP%U2r}Xuji^pBB z9je~*YelpG?i?Nnvn=BQ0M z47ffu*yJ&ZWP_xCf|z>3xv0Wg09p#QCA>o>^0*J!G$bbhbAVC!(Th;7|4EDjC;aZ38-RCeJN>ovDW>ZSJAYfYtUgnbxsu}`KIWS=S#o3ZYL7@&Oe9y&(12I_8 zAhXt0VdAGJbK!cP9R4e>sN1pGRpXt^NMi^r;3Vjnfg}p=!7wWxk@IZ`svNmW^5(m5Ss7Ben?0oPw&zXr?36m7P zI$8?Lw18Ze0$tSv4nhGokkC;jw~g|{Mb9vB12n^yD&}5~f?(oro=lk}XhzHyz2qKg zi{a>q&tK`mpYZO1URCGw=hCWvNT%Ne3y4+?Y>iHb&Y$s@2D2LLOzv7u%02=iYDTuYoYeG8`H;c=%aw9gf` zwxYND9!`EJkwcpob>VO0a_q0Hj6GRR2k7_Sp~xkWI}Br6h`mdd;Volpb-&0 z2wO}+Y}#y;SWhZD6Isf-IECjHXOj#2CPI~YljFI4blg!k3-Q$3xU)%}@eSjX0mtze zy(#J5RT7hCiFi~~qRj?B5dtN8-^OqM^(RF^)7FucDpq$X? z&o!UL7-TFtT;6VK>EY#8=hu=e_q{2byHLdA~JHH5j+{7zF&gPY2YVe)~Me4D2%34 zAwo{BI@yUs$&qhf$VL?NH4P+hOd-8d1Y#}knjc8htvUY2HZW9UGWzBgiOkaN@^Phw zDXT~C2;O~=YP(aLDv1klOkR zt7bmKrOqipsfZ4JNz;AZ);w##5JMn#x%o=TFP52O:9090 +#. Grafana dashboard: http://:3000 +#. +#. Prerequisites: +#. - Ubuntu server for cluster nodes (admin/master and worker nodes) +#. - MAAS server as cluster admin for Rancher master/worker nodes +#. - Password-less ssh key provided for node setup +#. Usage: on the MAAS server +#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. $ bash ~/models/tools/docker/demo_deploy.sh "" +#. "" [] +#. : name of private key for cluster node ssh (in current folder) +#. : space separated list of hostnames managed by MAAS +#. : IP of master node +#. : space separated list of worker node IPs +#. : optional name of script for extra setup functions as needed + +key=$1 +nodes="$2" +master=$3 +workers="$4" +extras=$5 + +source ~/models/tools/maas/deploy.sh $1 "$2" $5 +eval `ssh-agent` +ssh-add $key +echo "Setting up Docker..." +bash ~/models/tools/docker/docker-cluster.sh all $master "$workers" +# TODO: Figure this out... Have to break the setup into two steps as something +# causes the ssh session to end before the prometheus setup, if both scripts +# (k8s-cluster and prometheus-tools) are in the same ssh session +echo "Setting up Prometheus..." +scp -o StrictHostKeyChecking=no $key ubuntu@$master:/home/ubuntu/$key +ssh -x -o StrictHostKeyChecking=no ubuntu@$master < "" +#. Automate setup and start demo services. +#. : master node IPs +#. : space-separated list of worker node IPs +#. $ bash docker_cluster.sh setup "" +#. Installs and starts master and worker nodes. +#. $ bash docker_cluster.sh create +#. : Demo service name to start. +#. Currently supported: nginx +#. $ bash docker_cluster.sh delete +#. : Service name to delete. +#. $ bash docker_cluster.sh clean [] +#. : optional IP address of node to clean. +#. By default, cleans the entire cluster. +#. + +# Setup master and worker hosts +function setup() { + # Per https://docs.docker.com/engine/swarm/swarm-tutorial/ + cat >/tmp/env.sh </tmp/prereqs.sh <<'EOF' +#!/bin/bash +# Per https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ +sudo apt-get remove -y docker docker-engine docker.io docker-ce +sudo apt-get update +sudo apt-get install -y \ + linux-image-extra-$(uname -r) \ + linux-image-extra-virtual +sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" +sudo apt-get update +sudo apt-get install -y docker-ce +EOF + + # jq is used for parsing API reponses + sudo apt-get install -y jq + scp -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$master:/home/ubuntu/prereqs.sh + ssh -x -o StrictHostKeyChecking=no ubuntu@$master bash /home/ubuntu/prereqs.sh + # activate docker API + # Per https://www.ivankrizsan.se/2016/05/18/enabling-docker-remote-api-on-ubuntu-16-04/ + ssh -x -o StrictHostKeyChecking=no ubuntu@$master <:9090 +#. Grafana dashboard: http://:3000 +#. +#. Prerequisites: +#. - Ubuntu server for kubernetes cluster nodes (admin/master and agent nodes) +#. - MAAS server as cluster admin for kubernetes master/agent nodes +#. - Password-less ssh key provided for node setup +#. Usage: on the MAAS server +#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. $ bash ~/models/tools/kubernetes/demo_deploy.sh "" +#. "" [] +#. : name of private key for cluster node ssh (in current folder) +#. : space separated list of hostnames managed by MAAS +#. : IP of cluster admin node +#. : space separated list of agent node IPs +#. : CID formatted public network +#. : CIDR formatted private network (may be same as pub-net) +#. : optional name of script for extra setup functions as needed + +key=$1 +nodes="$2" +admin_ip=$3 +agent_ips="$4" +extras=$5 + +source ~/models/tools/maas/deploy.sh $1 "$2" $5 +eval `ssh-agent` +ssh-add $key +if [[ "x$extras" != "x" ]]; then source $extras; fi +scp -o StrictHostKeyChecking=no $key ubuntu@$admin_ip:/home/ubuntu/$key +echo "Setting up kubernetes..." +ssh -x ubuntu@$admin_ip <" +#. nodes: space-separated list of ceph node IPs +#. $ bash k8s-cluster.sh ceph "" [ceph_dev] +#. nodes: space-separated list of ceph node IPs +#. cluster-net: CIDR of ceph cluster network e.g. 10.0.0.1/24 +#. public-net: CIDR of public network +#. ceph_dev: disk to use for ceph. ***MUST NOT BE USED FOR ANY OTHER PURPOSE*** +#. if not provided, ceph data will be stored on osd nodes in /ceph +#. $ bash k8s-cluster.sh helm +#. Setup helm as app kubernetes orchestration tool +#. $ bash k8s-cluster.sh demo +#. Install helm charts for mediawiki and dokuwiki +#. $ bash k8s-cluster.sh all "" [ceph_dev] +#. Runs all the steps above +#. +#. Status: work in progress, incomplete +# + +function setup_prereqs() { + echo "${FUNCNAME[0]}: Create prerequisite setup script" + cat <<'EOG' >/tmp/prereqs.sh +#!/bin/bash +# Basic server pre-reqs +sudo apt-get -y remove kubectl kubelet kubeadm +sudo apt-get update +sudo apt-get upgrade -y +# Set hostname on agent nodes +if [[ "$1" == "agent" ]]; then + echo $(ip route get 8.8.8.8 | awk '{print $NF; exit}') $HOSTNAME | sudo tee -a /etc/hosts +fi +# Install docker 1.12 (default for xenial is 1.12.6) +sudo apt-get install -y docker.io +sudo service docker start +export KUBE_VERSION=1.7.5 +# per https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/ +# Install kubelet, kubeadm, kubectl per https://kubernetes.io/docs/setup/independent/install-kubeadm/ +sudo apt-get update && sudo apt-get install -y apt-transport-https +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +cat <>/tmp/kubeadm.out + cat /tmp/kubeadm.out + export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out) + echo "${FUNCNAME[0]}: Cluster join command for manual use if needed: $k8s_joincmd" + + # Start cluster + echo "${FUNCNAME[0]}: Start the cluster" + mkdir -p $HOME/.kube + sudo cp -f /etc/kubernetes/admin.conf $HOME/.kube/config + sudo chown $(id -u):$(id -g) $HOME/.kube/config + # Deploy pod network + echo "${FUNCNAME[0]}: Deploy calico as CNI" + sudo kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml +} + +function setup_k8s_agents() { + agents="$1" + export k8s_joincmd=$(grep "kubeadm join" /tmp/kubeadm.out) + echo "${FUNCNAME[0]}: Installing agents at $1 with joincmd: $k8s_joincmd" + + setup_prereqs + + kubedns=$(kubectl get pods --all-namespaces | grep kube-dns | awk '{print $4}') + while [[ "$kubedns" != "Running" ]]; do + echo "${FUNCNAME[0]}: kube-dns status is $kubedns. Waiting 60 seconds for it to be 'Running'" + sleep 60 + kubedns=$(kubectl get pods --all-namespaces | grep kube-dns | awk '{print $4}') + done + echo "${FUNCNAME[0]}: kube-dns status is $kubedns" + + for agent in $agents; do + echo "${FUNCNAME[0]}: Install agent at $agent" + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no /tmp/prereqs.sh ubuntu@$agent:/tmp/prereqs.sh + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent bash /tmp/prereqs.sh agent + # Workaround for "[preflight] Some fatal errors occurred: /var/lib/kubelet is not empty" per https://github.com/kubernetes/kubeadm/issues/1 + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent sudo kubeadm reset + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ubuntu@$agent sudo $k8s_joincmd + done + + echo "${FUNCNAME[0]}: Cluster is ready when all nodes in the output of 'kubectl get nodes' show as 'Ready'." +} + +function setup_ceph() { + node_ips=$1 + cluster_net=$2 + public_net=$3 + ceph_dev=$4 + echo "${FUNCNAME[0]}: Deploying ceph-mon on localhost $HOSTNAME" + echo "${FUNCNAME[0]}: Deploying ceph-osd on nodes $node_ips" + echo "${FUNCNAME[0]}: Setting cluster-network=$cluster_net and public-network=$public_net" + mon_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + all_nodes="$mon_ip $node_ips" + # Also caches the server fingerprints so ceph-deploy does not prompt the user + # Note this loop may be partially redundant with the ceph-deploy steps below + for node_ip in $all_nodes; do + echo "${FUNCNAME[0]}: Install ntp and ceph on $node_ip" + ssh -x -o StrictHostKeyChecking=no ubuntu@$node_ip <>ceph.conf +osd max object name len = 256 +osd max object namespace len = 64 +EOF + cat ceph.conf + + echo "${FUNCNAME[0]}: Deploy ceph packages on other nodes" + ceph-deploy install $mon_ip $node_ips + + echo "${FUNCNAME[0]}: Deploy the initial monitor and gather the keys" + ceph-deploy mon create-initial + + if [[ "x$ceph_dev" == "x" ]]; then + n=1 + for node_ip in $node_ips; do + echo "${FUNCNAME[0]}: Prepare ceph OSD on node $node_ip" + echo "$node_ip ceph-osd$n" | sudo tee -a /etc/hosts + # Using ceph-osd$n here avoids need for manual acceptance of the new server hash + ssh -x -o StrictHostKeyChecking=no ubuntu@ceph-osd$n </tmp/ceph-sc.yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: slow +provisioner: kubernetes.io/rbd +parameters: + monitors: $mon_ip:6789 + adminId: admin + adminSecretName: ceph-secret-admin + adminSecretNamespace: "kube-system" + pool: kube + userId: kube + userSecretName: ceph-secret-user +EOF + # TODO: find out where in the above ~/.kube folders became owned by root + sudo chown -R ubuntu:ubuntu ~/.kube/* + kubectl create -f /tmp/ceph-sc.yaml + + echo "${FUNCNAME[0]}: Create storage pool 'kube'" + # https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md method + sudo ceph osd pool create kube 32 32 + + echo "${FUNCNAME[0]}: Authorize client 'kube' access to pool 'kube'" + sudo ceph auth get-or-create client.kube mon 'allow r' osd 'allow rwx pool=kube' + + echo "${FUNCNAME[0]}: Create ceph-secret-user secret in namespace 'default'" + kube_key=$(sudo ceph auth get-key client.kube) + kubectl create secret generic ceph-secret-user --from-literal=key="$kube_key" --namespace=default --type=kubernetes.io/rbd + # A similar secret must be created in other namespaces that intend to access the ceph pool + + # Per https://github.com/kubernetes/examples/blob/master/staging/persistent-volume-provisioning/README.md + + echo "${FUNCNAME[0]}: Create andtest a persistentVolumeClaim" + cat </tmp/ceph-pvc.yaml +{ + "kind": "PersistentVolumeClaim", + "apiVersion": "v1", + "metadata": { + "name": "claim1", + "annotations": { + "volume.beta.kubernetes.io/storage-class": "slow" + } + }, + "spec": { + "accessModes": [ + "ReadWriteOnce" + ], + "resources": { + "requests": { + "storage": "3Gi" + } + } + } +} +EOF + kubectl create -f /tmp/ceph-pvc.yaml + while [[ "x$(kubectl get pvc -o jsonpath='{.status.phase}' claim1)" != "xBound" ]]; do + echo "${FUNCNAME[0]}: Waiting for pvc claim1 to be 'Bound'" + kubectl describe pvc + sleep 10 + done + echo "${FUNCNAME[0]}: pvc claim1 successfully bound to $(kubectl get pvc -o jsonpath='{.spec.volumeName}' claim1)" + kubectl get pvc + kubectl delete pvc claim1 + kubectl describe pods +} + +function wait_for_service() { + echo "${FUNCNAME[0]}: Waiting for service $1 to be available" + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + echo "${FUNCNAME[0]}: Service $1 is at pod $pod" + ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) + while [[ "$ready" != "true" ]]; do + echo "${FUNCNAME[0]}: $1 container is not yet ready... waiting 10 seconds" + sleep 10 + # TODO: figure out why transient pods sometimes mess up this logic, thus need to re-get the pods + pod=$(kubectl get pods --namespace default | awk "/$1/ { print \$1 }") + ready=$(kubectl get pods --namespace default -o jsonpath='{.status.containerStatuses[0].ready}' $pod) + done + echo "${FUNCNAME[0]}: pod $pod container status is $ready" + host_ip=$(kubectl get pods --namespace default -o jsonpath='{.status.hostIP}' $pod) + port=$(kubectl get --namespace default -o jsonpath="{.spec.ports[0].nodePort}" services $1) + echo "${FUNCNAME[0]}: pod $pod container is at host $host_ip and port $port" + while ! curl http://$host_ip:$port ; do + echo "${FUNCNAME[0]}: $1 service is not yet responding... waiting 10 seconds" + sleep 10 + done + echo "${FUNCNAME[0]}: $1 is available at http://$host_ip:$port" +} + +function demo_chart() { + cd ~ + rm -rf charts + git clone https://github.com/kubernetes/charts.git + cd charts/stable + case "$1" in + mediawiki) + # NOT YET WORKING + # mariadb: Readiness probe failed: mysqladmin: connect to server at 'localhost' failed + mkdir ./mediawiki/charts + cp -r ./mariadb ./mediawiki/charts + # LoadBalancer is N/A for baremetal (public cloud only) - use NodePort + sed -i -- 's/LoadBalancer/NodePort/g' ./mediawiki/values.yaml + # Select the storageClass created in the ceph setup step + sed -i -- 's/# storageClass:/storageClass: "slow"/g' ./mediawiki/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./mediawiki/charts/mariadb/values.yaml + helm install --name mw -f ./mediawiki/values.yaml ./mediawiki + wait_for_service mw-mediawiki + ;; + dokuwiki) + sed -i -- 's/# storageClass:/storageClass: "slow"/g' ./dokuwiki/values.yaml + sed -i -- 's/LoadBalancer/NodePort/g' ./dokuwiki/values.yaml + helm install --name dw -f ./dokuwiki/values.yaml ./dokuwiki + wait_for_service dw-dokuwiki + ;; + wordpress) + # NOT YET WORKING + # mariadb: Readiness probe failed: mysqladmin: connect to server at 'localhost' failed + mkdir ./wordpress/charts + cp -r ./mariadb ./wordpress/charts + sed -i -- 's/LoadBalancer/NodePort/g' ./wordpress/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./wordpress/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./wordpress/charts/mariadb/values.yaml + helm install --name wp -f ./wordpress/values.yaml ./wordpress + wait_for_service wp-wordpress + ;; + redmine) + # NOT YET WORKING + # mariadb: Readiness probe failed: mysqladmin: connect to server at 'localhost' failed + mkdir ./redmine/charts + cp -r ./mariadb ./redmine/charts + cp -r ./postgresql ./redmine/charts + sed -i -- 's/LoadBalancer/NodePort/g' ./redmine/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./redmine/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./redmine/charts/mariadb/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./redmine/charts/postgresql/values.yaml + helm install --name rdm -f ./redmine/values.yaml ./redmine + wait_for_service rdm-redmine + ;; + owncloud) + # NOT YET WORKING: needs resolvable hostname for service + mkdir ./owncloud/charts + cp -r ./mariadb ./owncloud/charts + sed -i -- 's/LoadBalancer/NodePort/g' ./owncloud/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./owncloud/values.yaml + sed -i -- 's/# storageClass: "-"/storageClass: "slow"/g' ./owncloud/charts/mariadb/values.yaml + helm install --name oc -f ./owncloud/values.yaml ./owncloud + wait_for_service oc-owncloud + ;; + *) + echo "${FUNCNAME[0]}: demo not implemented for $1" + esac +# extra useful commands +# kubectl describe pvc +# kubectl get pvc +# kubectl describe pods +# kubectl get pods --namespace default +# kubectl get pods --all-namespaces +# kubectl get svc --namespace default dw-dokuwiki +# kubectl describe svc --namespace default dw-dokuwiki +# kubectl describe pods --namespace default dw-dokuwiki +} + +function setup_helm() { + echo "${FUNCNAME[0]}: Setup helm" + # Install Helm + cd ~ + curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh + chmod 700 get_helm.sh + ./get_helm.sh + helm init + helm repo update + # TODO: Workaround for bug https://github.com/kubernetes/helm/issues/2224 + # For testing use only! + kubectl create clusterrolebinding permissive-binding --clusterrole=cluster-admin --user=admin --user=kubelet --group=system:serviceaccounts; + # TODO: workaround for tiller FailedScheduling (No nodes are available that match all of the following predicates:: PodToleratesNodeTaints (1).) + # kubectl taint nodes $HOSTNAME node-role.kubernetes.io/master:NoSchedule- + # Wait till tiller is running + tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}') + while [[ "$tiller_deploy" != "Running" ]]; do + echo "${FUNCNAME[0]}: tiller-deploy status is $tiller_deploy. Waiting 60 seconds for it to be 'Running'" + sleep 60 + tiller_deploy=$(kubectl get pods --all-namespaces | grep tiller-deploy | awk '{print $4}') + done + echo "${FUNCNAME[0]}: tiller-deploy status is $tiller_deploy" + + # Install services via helm charts from https://kubeapps.com/charts + # e.g. helm install stable/dokuwiki +} + +export WORK_DIR=$(pwd) +case "$1" in + master) + setup_k8s_master + ;; + agents) + setup_k8s_agents "$2" + ;; + ceph) + setup_ceph "$2" $3 $4 $5 + ;; + helm) + setup_helm + ;; + demo) + demo_chart $2 + ;; + all) + setup_k8s_master + setup_k8s_agents "$2" + setup_ceph "$2" $3 $4 $5 + setup_helm + demo_chart dokuwiki + ;; + clean) + # TODO + ;; + *) + if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then grep '#. ' $0; fi +esac diff --git a/tools/maas/deploy.sh b/tools/maas/deploy.sh new file mode 100644 index 0000000..ae89893 --- /dev/null +++ b/tools/maas/deploy.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright 2017 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#. What this is: Scripted deployment of servers using MAAS. Currently it deploys +#. the default host OS as configured in MAAS. +#. +#. Prerequisites: +#. - MAAS server configured to admin a set of servers +#. - Password-less ssh key provided for node setup +#. Usage: on the MAAS server +#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. $ source ~/models/tools/maas/demo_deploy.sh "" [] +#. : name of private key for cluster node ssh (in current folder) +#. : space separated list of hostnames managed by MAAS +#. : optional name of script for extra setup functions as needed + +function wait_node_status() { + status=$(maas opnfv machines read hostname=$1 | jq -r ".[0].status_name") + while [[ "x$status" != "x$2" ]]; do + echo "$1 status is $status ... waiting for it to be $2" + sleep 30 + status=$(maas opnfv machines read hostname=$1 | jq -r ".[0].status_name") + done + echo "$1 status is $status" +} + +function release_nodes() { + nodes=$1 + for node in $nodes; do + echo "Releasing node $node" + id=$(maas opnfv machines read hostname=$node | jq -r '.[0].system_id') + maas opnfv machines release machines=$id + done +} + +function deploy_nodes() { + nodes=$1 + for node in $nodes; do + echo "Deploying node $node" + id=$(maas opnfv machines read hostname=$node | jq -r '.[0].system_id') + maas opnfv machines allocate system_id=$id + maas opnfv machine deploy $id + done +} + +function wait_nodes_status() { + nodes=$1 + for node in $nodes; do + wait_node_status $node $2 + done +} + +key=$1 +nodes="$2" +extras=$3 + +release_nodes "$nodes" +wait_nodes_status "$nodes" Ready +deploy_nodes "$nodes" +wait_nodes_status "$nodes" Deployed +eval `ssh-agent` +ssh-add $key +if [[ "x$extras" != "x" ]]; then source $extras; fi diff --git a/tools/prometheus/README.md b/tools/prometheus/README.md new file mode 100644 index 0000000..a3dfcc5 --- /dev/null +++ b/tools/prometheus/README.md @@ -0,0 +1,10 @@ +This folder contains scripts etc to setup [prometheus](https://github.com/prometheus/prometheus) on a server cluster. It installs: +* a prometheus server (on the host OS) and [grafana](https://grafana.com/) (in docker) +* prometheus exporters on a set of other nodes, to be monitored + * [node exporter](https://github.com/prometheus/node_exporter) for node basic analytics + * [haproxy exporter](https://github.com/prometheus/haproxy_exporter) for load-balancer stats from haproxy e.g. as use by Rancher +* several sample grafana dashboards... for more see [grafana dashboards for prometheus](https://grafana.com/dashboards?dataSource=prometheus) + +See comments in [prometheus-tools.sh](prometheus-tools.sh) for more info. + +This is a work in progress! diff --git a/tools/prometheus/dashboards/Docker_Dashboard-1503539375161.json b/tools/prometheus/dashboards/Docker_Dashboard-1503539375161.json new file mode 100644 index 0000000..afc69a2 --- /dev/null +++ b/tools/prometheus/dashboards/Docker_Dashboard-1503539375161.json @@ -0,0 +1,712 @@ +{ +"dashboard": { + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "Docker Monitoring Template", + "editable": true, + "gnetId": 179, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_memory_MemTotal) - sum(node_memory_MemFree+node_memory_Buffers+node_memory_Cached) ) / sum(node_memory_MemTotal) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 30 + } + ], + "thresholds": "65, 90", + "title": "Memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(sum by (container_name)( rate(container_cpu_usage_seconds_total{image!=\"\"}[1m] ) )) / count(node_cpu{mode=\"system\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 30 + } + ], + "thresholds": "65, 90", + "title": "CPU usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_fs_limit_bytes - container_fs_usage_bytes) / sum(container_fs_limit_bytes)", + "interval": "10s", + "intervalFactor": 1, + "metric": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": "65, 90", + "title": "Filesystem usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(rate(container_cpu_user_seconds_total{image!=\"\"}[1m])) by (name))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "metric": "container_cpu_user_seconds_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Container CPU usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(container_memory_usage_bytes{image!=\"\"}) by (name))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Container Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum by (name) (rate(container_network_receive_bytes_total{image!=\"\"}[1m] ) ))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ name }}", + "metric": "container_network_receive_bytes_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Container Network Input", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\"}[1m] ) ))", + "intervalFactor": 2, + "legendFormat": "{{ name }}", + "metric": "container_network_transmit_bytes_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Container Network Output", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Docker Dashboard", + "version": 1 +} +} diff --git a/tools/prometheus/dashboards/Docker_Host_and_Container_Overview-1503539411705.json b/tools/prometheus/dashboards/Docker_Host_and_Container_Overview-1503539411705.json new file mode 100644 index 0000000..6db3532 --- /dev/null +++ b/tools/prometheus/dashboards/Docker_Host_and_Container_Overview-1503539411705.json @@ -0,0 +1,1618 @@ +{ +"dashboard": { + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "A simple overview of the most important Docker host and container metrics. (cAdvisor/Prometheus)", + "editable": true, + "gnetId": 395, + "graphTooltip": 1, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": 143.625, + "panels": [ + { + "aliasColors": { + "SENT": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 5, + "grid": {}, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\"}[$interval])) by (id)", + "intervalFactor": 2, + "legendFormat": "RECEIVED", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum(rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])) by (id)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "SENT", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic on Node", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Ops-Infrastructure": "#447EBC", + "{}": "#DEDAF7" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "editable": true, + "error": false, + "fill": 3, + "grid": {}, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 3, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 10, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 1.9899973849372385, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "count(rate(container_last_seen{name=~\".+\",container_group=\"monitoring\"}[$interval]))", + "intervalFactor": 2, + "legendFormat": "Monitoring", + "metric": "container_last_seen", + "refId": "A", + "step": 10 + }, + { + "expr": "count(rate(container_last_seen{name=~\".+\",container_group=\"ops-infrastructure\"}[$interval]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Backend-Infrastructure", + "refId": "B", + "step": 10 + }, + { + "expr": "count(rate(container_last_seen{name=~\".+\",container_group=\"backend-infrastructure\"}[$interval]))", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Backend-Workers", + "refId": "C", + "step": 10 + }, + { + "expr": "count(rate(container_last_seen{name=~\".+\",container_group=\"backend-workers\"}[$interval]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Ops-Infrastructure", + "refId": "D", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Running Containers (by Container Group)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "{id=\"/\",instance=\"cadvisor:8080\",job=\"prometheus\"}": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 3, + "grid": {}, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2.0707047594142263, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_system_seconds_total[1m]))", + "hide": true, + "intervalFactor": 2, + "legendFormat": "a", + "refId": "B", + "step": 120 + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m]))", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "nur container", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m]))", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "nur docker host", + "metric": "", + "refId": "A", + "step": 20 + }, + { + "expr": "sum(rate(process_cpu_seconds_total[$interval])) * 100", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "host", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(rate(container_cpu_system_seconds_total{name=~\".+\"}[1m])) + sum(rate(container_cpu_system_seconds_total{id=\"/\"}[1m])) + sum(rate(process_cpu_seconds_total[1m]))", + "hide": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage on Node", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": 120, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Belegete Festplatte": "#BF1B00", + "Free Disk Space": "#7EB26D", + "Used Disk Space": "#BF1B00", + "{}": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 4, + "grid": {}, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_free{fstype=\"aufs\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Free Disk Space", + "refId": "A", + "step": 10 + }, + { + "expr": "node_filesystem_size{fstype=\"aufs\"} - node_filesystem_free{fstype=\"aufs\"}", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Used Disk Space", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Free and Used Disk Space on Node", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Available Memory": "#7EB26D", + "Unavailable Memory": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 4, + "grid": {}, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_rss{name=~\".+\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "D", + "step": 30 + }, + { + "expr": "sum(container_memory_rss{name=~\".+\"})", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "A", + "step": 20 + }, + { + "expr": "container_memory_usage_bytes{name=~\".+\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "B", + "step": 20 + }, + { + "expr": "container_memory_rss{id=\"/\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "C", + "step": 30 + }, + { + "expr": "sum(container_memory_rss)", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "E", + "step": 30 + }, + { + "expr": "node_memory_Buffers", + "hide": true, + "intervalFactor": 2, + "legendFormat": "node_memory_Dirty", + "refId": "N", + "step": 30 + }, + { + "expr": "node_memory_MemFree", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "F", + "step": 30 + }, + { + "expr": "node_memory_MemAvailable", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Available Memory", + "refId": "H", + "step": 10 + }, + { + "expr": "node_memory_MemTotal - node_memory_MemAvailable", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Unavailable Memory", + "refId": "G", + "step": 10 + }, + { + "expr": "node_memory_Inactive", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "I", + "step": 30 + }, + { + "expr": "node_memory_KernelStack", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "J", + "step": 30 + }, + { + "expr": "node_memory_Active", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "K", + "step": 30 + }, + { + "expr": "node_memory_MemTotal - (node_memory_Active + node_memory_MemFree + node_memory_Inactive)", + "hide": true, + "intervalFactor": 2, + "legendFormat": "Unknown", + "refId": "L", + "step": 40 + }, + { + "expr": "node_memory_MemFree + node_memory_Inactive ", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "M", + "step": 30 + }, + { + "expr": "container_memory_rss{name=~\".+\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "O", + "step": 30 + }, + { + "expr": "node_memory_Inactive + node_memory_MemFree + node_memory_MemAvailable", + "hide": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "P", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Available Memory on Node", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": 4200000000, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 1.939297855648535, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_bytes_read[$interval])) by (device)", + "intervalFactor": 2, + "legendFormat": "OUT on /{{device}}", + "metric": "node_disk_bytes_read", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(rate(node_disk_bytes_written[$interval])) by (device)", + "intervalFactor": 2, + "legendFormat": "IN on /{{device}}", + "metric": "", + "refId": "B", + "step": 10 + }, + { + "expr": "", + "intervalFactor": 2, + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O on Node", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 284.609375, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 5, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6.0790694124949285, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\".+\"}[$interval])) by (name) * 100", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_cp", + "refId": "F", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage per Container (Stacked)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "", + "logBase": 1, + "max": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "node_load15": "#CCA300" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 4, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 5.920930587505071, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "{__name__=~\"^node_load.*\"}", + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "metric": "node", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "System Load on Node", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 203.515625, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 9, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])) by (name)", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 2 + }, + { + "expr": "rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])", + "hide": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sent Network Traffic per Container", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 10, + "max": 8, + "min": 0, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 3, + "grid": {}, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{name=~\".+\"}) by (name)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 2 + }, + { + "expr": "container_memory_usage_bytes{name=~\".+\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage per Container (Stacked)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 222.703125, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_network_receive_bytes_total{name=~\".+\"}[$interval])) by (name)", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 2 + }, + { + "expr": "- rate(container_network_transmit_bytes_total{name=~\".+\"}[$interval])", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Received Network Traffic per Container", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 3, + "grid": {}, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_rss{name=~\".+\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 20 + }, + { + "expr": "container_memory_usage_bytes{name=~\".+\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "B", + "step": 20 + }, + { + "expr": "sum(container_memory_cache{name=~\".+\"}) by (name)", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cached Memory per Container (Stacked)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "fontSize": "100%", + "hideTimeOverride": false, + "id": 18, + "links": [], + "pageSize": 100, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 6, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "cadvisor_version_info", + "intervalFactor": 2, + "legendFormat": "cAdvisor Version: {{cadvisorVersion}}", + "refId": "A", + "step": 2 + }, + { + "expr": "prometheus_build_info", + "intervalFactor": 2, + "legendFormat": "Prometheus Version: {{version}}", + "refId": "B", + "step": 2 + }, + { + "expr": "node_exporter_build_info", + "intervalFactor": 2, + "legendFormat": "Node-Exporter Version: {{version}}", + "refId": "C", + "step": 2 + }, + { + "expr": "cadvisor_version_info", + "intervalFactor": 2, + "legendFormat": "Docker Version: {{dockerVersion}}", + "refId": "D", + "step": 2 + }, + { + "expr": "cadvisor_version_info", + "intervalFactor": 2, + "legendFormat": "Host OS Version: {{osVersion}}", + "refId": "E", + "step": 2 + }, + { + "expr": "cadvisor_version_info", + "intervalFactor": 2, + "legendFormat": "Host Kernel Version: {{kernelVersion}}", + "refId": "F", + "step": 2 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Check this out", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 290.98582985381427, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 127, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".+", + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Container Group", + "multi": true, + "name": "containergroup", + "options": [], + "query": "label_values(container_group)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 50, + "auto_min": "50s", + "current": { + "text": "auto", + "value": "$__auto_interval" + }, + "datasource": null, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "7m", + "value": "7m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "30s,1m,2m,3m,5m,7m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "type": "interval" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Docker Host & Container Overview", + "version": 1 +} +} diff --git a/tools/prometheus/dashboards/Node_Exporter_Server_Metrics-1503539692670.json b/tools/prometheus/dashboards/Node_Exporter_Server_Metrics-1503539692670.json new file mode 100644 index 0000000..da65d4a --- /dev/null +++ b/tools/prometheus/dashboards/Node_Exporter_Server_Metrics-1503539692670.json @@ -0,0 +1,1632 @@ +{ +"dashboard": { + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "Dashboard to view multiple servers", + "editable": true, + "gnetId": 405, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "rows": [ + { + "collapse": false, + "height": "25px", + "panels": [ + { + "content": "", + "editable": true, + "error": false, + "id": 11, + "minSpan": 2, + "mode": "html", + "repeat": "node", + "span": 12, + "style": {}, + "title": "$node", + "type": "text" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Title", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "25px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": "node", + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(node_cpu{instance=~\"$node\", mode=\"system\"})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 14400, + "target": "" + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 10, + "grid": {}, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (mode)(irate(node_cpu{mode=\"system\",instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mode}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='user',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "user", + "refId": "B", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='nice',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "nice", + "refId": "C", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='iowait',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "iowait", + "refId": "E", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='steal',instance=~'$node'}[5m]))", + "intervalFactor": 2, + "legendFormat": "steal", + "refId": "H", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='idle',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "idle", + "refId": "D", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='irq',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "irq", + "refId": "F", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='softirq',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "softirq", + "refId": "G", + "step": 1200 + }, + { + "expr": "sum by (mode)(irate(node_cpu{mode='guest',instance=~'$node'}[5m]))", + "interval": "", + "intervalFactor": 2, + "legendFormat": "guest", + "refId": "I", + "step": 1200 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 0 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "%", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "Slab": "#E5A8E2", + "Swap": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/Apps|Buffers|Cached|Free|Slab|SwapCached|PageTables|VmallocUsed/", + "fill": 5, + "stack": true + }, + { + "alias": "Swap", + "fill": 5, + "stack": true + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "( node_memory_MemTotal{instance=~'$node'} - node_memory_MemFree{instance=~'$node'} - node_memory_Buffers{instance=~'$node'} - node_memory_Cached{instance=~'$node'} - node_memory_SwapCached{instance=~'$node'} - node_memory_Slab{instance=~'$node'} - node_memory_PageTables{instance=~'$node'} - node_memory_VmallocUsed{instance=~'$node'} )", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Apps", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "node_memory_Buffers{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "B", + "step": 1200 + }, + { + "expr": "node_memory_Cached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 1200 + }, + { + "expr": "node_memory_MemFree{instance=~'$node'}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "E", + "step": 1200 + }, + { + "expr": "node_memory_Slab{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Slab", + "refId": "F", + "step": 1200 + }, + { + "expr": "node_memory_SwapCached{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "SwapCached", + "refId": "G", + "step": 1200 + }, + { + "expr": "node_memory_PageTables{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "PageTables", + "refId": "H", + "step": 1200 + }, + { + "expr": "node_memory_VmallocUsed{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "VmallocUsed", + "metric": "", + "refId": "I", + "step": 1200 + }, + { + "expr": "(node_memory_SwapTotal{instance=~'$node'} - node_memory_SwapFree{instance=~'$node'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Swap", + "metric": "", + "refId": "C", + "step": 1200 + }, + { + "expr": "node_memory_Committed_AS{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Committed", + "metric": "", + "refId": "J", + "step": 1200 + }, + { + "expr": "node_memory_Mapped{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Mapped", + "refId": "K", + "step": 1200 + }, + { + "expr": "node_memory_Active{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Active", + "metric": "", + "refId": "L", + "step": 1200 + }, + { + "expr": "node_memory_Inactive{instance=~'$node'}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Inactive", + "metric": "", + "refId": "M", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "GB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$node\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "load", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Load", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 3, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100.0 - 100 * (node_filesystem_avail{instance=~'$node',device !~'tmpfs',device!~'by-uuid'} / node_filesystem_size{instance=~'$node',device !~'tmpfs',device!~'by-uuid'})", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Disk Used", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_ms{instance=~\"$node\"}[5m])/10", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Utilization per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "logBase": 1, + "max": 100, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Disk Utilization", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_read$/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed{instance=~'$node'}[5m])", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + }, + { + "expr": "irate(node_disk_writes_completed{instance=~'$node'}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOs per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "IO/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Disk IOs per device", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_read/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_sectors_read{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_read", + "refId": "B", + "step": 2400 + }, + { + "expr": "irate(node_disk_sectors_written{instance=~'$node'}[5m]) * 512", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{device}}_write", + "metric": "", + "refId": "A", + "step": 2400, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Throughput per Device", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes/second read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Disk Throughput per device", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "context switches", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Context Switches", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Traffic", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*_in/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_in", + "metric": "", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes{instance=~'$node'}[5m])*8", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{device}}_out", + "refId": "B", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bits", + "label": "bits in (-) / bits out (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~'$node'}", + "intervalFactor": 2, + "legendFormat": "established", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "Udp_NoPorts", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InDatagrams", + "refId": "A", + "step": 1200, + "target": "" + }, + { + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_InErrors", + "refId": "B", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Udp_OutDatagrams", + "refId": "C", + "step": 1200 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Udp_NoPorts", + "refId": "D", + "step": 1200 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "UDP Stats", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 2, + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": "node", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries_limit{instance=~\"$node\"} - node_nf_conntrack_entries{instance=~\"$node\"}", + "intervalFactor": 2, + "legendFormat": "free", + "refId": "A", + "step": 1200, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Conntrack", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus" + ], + "templating": { + "list": [ + { + "allFormat": "glob", + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "multiFormat": "regex values", + "name": "node", + "options": [], + "query": "label_values(node_boot_time, instance)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Server Metrics", + "version": 1 +} +} diff --git a/tools/prometheus/dashboards/Node_exporter_single_server-1503539807236.json b/tools/prometheus/dashboards/Node_exporter_single_server-1503539807236.json new file mode 100644 index 0000000..5dee4b9 --- /dev/null +++ b/tools/prometheus/dashboards/Node_exporter_single_server-1503539807236.json @@ -0,0 +1,792 @@ +{ +"dashboard": { + "__inputs": [ + { + "name": "Prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "description": "Dashboard to get an overview of one server", + "editable": true, + "gnetId": 22, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=~\"$server\"}[5m])) * 100)", + "hide": false, + "intervalFactor": 10, + "legendFormat": "{{cpu}}", + "refId": "A", + "step": 50 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Idle cpu", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "cpu usage", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$server\"}", + "intervalFactor": 4, + "legendFormat": "load 1m", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "node_load5{instance=~\"$server\"}", + "intervalFactor": 4, + "legendFormat": "load 5m", + "refId": "B", + "step": 20, + "target": "" + }, + { + "expr": "node_load15{instance=~\"$server\"}", + "intervalFactor": 4, + "legendFormat": "load 15m", + "refId": "C", + "step": 20, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "System load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "node_memory_SwapFree{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal{instance=~\"$server\"} - node_memory_MemFree{instance=~\"$server\"}", + "intervalFactor": 2, + "legendFormat": "free memory", + "metric": "memo", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Free memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(node_memory_MemFree{instance=~\"$server\"} / node_memory_MemTotal{instance=~\"$server\"}) * 100", + "intervalFactor": 2, + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "10, 20", + "title": "Free memory", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "{instance=\"172.17.0.1:9100\"}", + "yaxis": 2 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (instance) (irate(node_disk_bytes_read{instance=~\"$server\"}[5m]))", + "hide": false, + "intervalFactor": 4, + "legendFormat": "read", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "sum by (instance) (irate(node_disk_bytes_written{instance=~\"$server\"}[5m]))", + "intervalFactor": 4, + "legendFormat": "written", + "refId": "B", + "step": 20 + }, + { + "expr": "sum by (instance) (irate(node_disk_io_time_ms{instance=~\"$server\"}[5m]))", + "intervalFactor": 4, + "legendFormat": "io time", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(node_filesystem_free{device!=\"rootfs\",instance=~\"$server\"} / node_filesystem_size{device!=\"rootfs\",instance=~\"$server\"})", + "intervalFactor": 2, + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "0.10, 0.25", + "title": "Free disk space (lowest mountpoint)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "transmitted ", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "received", + "refId": "A", + "step": 4, + "target": "" + }, + { + "expr": "irate(node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "transmitted ", + "refId": "B", + "step": 4, + "target": "" + }, + { + "expr": "node_network_transmit_bytes{instance=~\"$server\",device!~\"lo\"}", + "hide": true, + "intervalFactor": 2, + "legendFormat": "transmitted ", + "refId": "C", + "step": 2, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Data transfer", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "server", + "options": [], + "query": "label_values(node_boot_time, instance)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node exporter single server", + "version": 1 +} +} diff --git a/tools/prometheus/prometheus-tools.sh b/tools/prometheus/prometheus-tools.sh new file mode 100644 index 0000000..ed6eb22 --- /dev/null +++ b/tools/prometheus/prometheus-tools.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# Copyright 2017 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#. What this is: Functions for testing with Prometheus and Grafana. Sets up +#. Prometheus and Grafana on a master node (e.g. for kubernetes, docker, +#. rancher, openstack) and agent nodes (where applications run). +#. Prerequisites: +#. - Ubuntu server for master and agent nodes +#. - Docker installed +#. Usage: +#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. $ cd ~/models/tools/prometheus +#. $ bash prometheus-tools.sh setup "" +#. : space separated IP of agent nodes +#. $ bash prometheus-tools.sh grafana +#. Runs grafana in a docker container and connects to prometheus as datasource +#. $ bash prometheus-tools.sh all "" +#. Does all of the above +#. $ bash prometheus-tools.sh clean "" +# + +# Prometheus links +# https://prometheus.io/download/ +# https://prometheus.io/docs/introduction/getting_started/ +# https://github.com/prometheus/prometheus +# https://prometheus.io/docs/instrumenting/exporters/ +# https://github.com/prometheus/node_exporter +# https://github.com/prometheus/haproxy_exporter +# https://github.com/prometheus/collectd_exporter + +# Use this to trigger fail() at the right places +# if [ "$RESULT" == "Test Failed!" ]; then fail "message"; fi +function fail() { + echo "$1" + exit 1 +} + +function setup_prometheus() { + # Prerequisites + echo "${FUNCNAME[0]}: Setting up prometheus master and agents" + sudo apt install -y golang-go jq + + # Install Prometheus server + echo "${FUNCNAME[0]}: Setting up prometheus master" + if [[ -d ~/prometheus ]]; then rm -rf ~/prometheus; fi + mkdir ~/prometheus + mkdir ~/prometheus/dashboards + cp -r dashboards/* ~/prometheus/dashboards + cd ~/prometheus + wget https://github.com/prometheus/prometheus/releases/download/v2.0.0-beta.2/prometheus-2.0.0-beta.2.linux-amd64.tar.gz + tar xvfz prometheus-*.tar.gz + cd prometheus-* + # Customize prometheus.yml below for your server IPs + # This example assumes the node_exporter and haproxy_exporter will be installed on each node + cat <<'EOF' >prometheus.yml +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'codelab-monitor' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: 'prometheus' + + # Override the global default and scrape targets from this job every 5 seconds. + scrape_interval: 5s + + static_configs: +EOF + + for node in $nodes; do + echo " - targets: ['${node}:9100']" >>prometheus.yml + echo " - targets: ['${node}:9101']" >>prometheus.yml + done + + # Start Prometheus + nohup ./prometheus --config.file=prometheus.yml > /dev/null 2>&1 & + # Browse to http://host_ip:9090 + + echo "${FUNCNAME[0]}: Installing exporters" + # Install exporters + # https://github.com/prometheus/node_exporter + cd ~/prometheus + wget https://github.com/prometheus/node_exporter/releases/download/v0.14.0/node_exporter-0.14.0.linux-amd64.tar.gz + tar xvfz node*.tar.gz + # https://github.com/prometheus/haproxy_exporter + wget https://github.com/prometheus/haproxy_exporter/releases/download/v0.7.1/haproxy_exporter-0.7.1.linux-amd64.tar.gz + tar xvfz haproxy*.tar.gz + + # The scp and ssh actions below assume you have key-based access enabled to the nodes + for node in $nodes; do + echo "${FUNCNAME[0]}: Setup agent at $node" + scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + node_exporter-0.14.0.linux-amd64/node_exporter ubuntu@$node:/home/ubuntu/node_exporter + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$node "nohup ./node_exporter > /dev/null 2>&1 &" + scp -r -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + haproxy_exporter-0.7.1.linux-amd64/haproxy_exporter ubuntu@$node:/home/ubuntu/haproxy_exporter + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$node "nohup ./haproxy_exporter > /dev/null 2>&1 &" + done + + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + while ! curl -o /tmp/up http://$host_ip:9090/api/v1/query?query=up ; do + echo "${FUNCNAME[0]}: Prometheus API is not yet responding... waiting 10 seconds" + sleep 10 + done + + exp=$(jq '.data.result|length' /tmp/up) + echo "${FUNCNAME[0]}: $exp exporters are up" + while [[ $exp > 0 ]]; do + ((exp--)) + eip=$(jq -r ".data.result[$exp].metric.instance" /tmp/up) + job=$(jq -r ".data.result[$exp].metric.job" /tmp/up) + echo "${FUNCNAME[0]}: $job at $eip" + done + echo "${FUNCNAME[0]}: Prometheus dashboard is available at http://$host_ip:9090" + echo "Prometheus dashboard is available at http://$host_ip:9090" auto>/tmp/summary +} + +function connect_grafana() { + echo "${FUNCNAME[0]}: Setup Grafana datasources and dashboards" + prometheus_ip=$1 + grafana_ip=$2 + + while ! curl -X POST http://admin:admin@$grafana_ip:3000/api/login/ping ; do + echo "${FUNCNAME[0]}: Grafana API is not yet responding... waiting 10 seconds" + sleep 10 + done + + echo "${FUNCNAME[0]}: Setup Prometheus datasource for Grafana" + cd ~/prometheus/ + cat >datasources.json <>/tmp/summary + echo "${FUNCNAME[0]}: Grafana API is available at http://admin:admin@$host_ip:3000/api/v1/query?query=" + echo "Grafana API is available at http://admin:admin@$host_ip:3000/api/v1/query?query=" >>/tmp/summary +} + +function run_and_connect_grafana() { + # Per http://docs.grafana.org/installation/docker/ + host_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + sudo docker run -d -p 3000:3000 --name grafana grafana/grafana + status=$(sudo docker inspect grafana | jq -r '.[0].State.Status') + while [[ "x$status" != "xrunning" ]]; do + echo "${FUNCNAME[0]}: Grafana container state is ($status)" + sleep 10 + status=$(sudo docker inspect grafana | jq -r '.[0].State.Status') + done + echo "${FUNCNAME[0]}: Grafana container state is $status" + + connect_grafana $host_ip $host_ip + echo "${FUNCNAME[0]}: connect_grafana complete" +} + +nodes=$2 +case "$1" in + setup) + setup_prometheus "$2" + ;; + grafana) + run_and_connect_grafana + ;; + all) + setup_prometheus "$2" + run_and_connect_grafana + ;; + clean) + sudo kill $(ps -ef | grep "\./prometheus" | grep prometheus.yml | awk '{print $2}') + rm -rf ~/prometheus + sudo docker stop grafana + sudo docker rm grafana + for node in $nodes; do + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$node "sudo kill $(ps -ef | grep ./node_exporter | awk '{print $2}')" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$node "rm -rf /home/ubuntu/node_exporter" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$node "sudo kill $(ps -ef | grep ./haproxy_exporter | awk '{print $2}')" + ssh -x -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + ubuntu@$node "rm -rf /home/ubuntu/haproxy_exporter" + done + ;; + *) + grep '#. ' $0 +esac +cat /tmp/summary diff --git a/tools/rancher/demo_deploy.sh b/tools/rancher/demo_deploy.sh new file mode 100644 index 0000000..981b421 --- /dev/null +++ b/tools/rancher/demo_deploy.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Copyright 2017 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +#. What this is: Complete scripted deployment of an experimental Rancher-based +#. cloud-native application platform. When complete, Rancher and the following +#. will be installed: +#. - nginx and dokuwiki as demo applications +#. - prometheus + grafana for cluster monitoring/stats +#. Prometheus dashboard: http://:9090 +#. Grafana dashboard: http://:3000 +#. +#. Prerequisites: +#. - Ubuntu server for Rancher cluster nodes (admin/master and agent nodes) +#. - MAAS server as cluster admin for Rancher master/agent nodes +#. - Password-less ssh key provided for node setup +#. Usage: on the MAAS server +#. $ git clone https://gerrit.opnfv.org/gerrit/models ~/models +#. $ bash ~/models/tools/rancher/demo_deploy.sh "" +#. "" [] +#. : name of private key for cluster node ssh (in current folder) +#. : space separated list of hostnames managed by MAAS +#. : IP of cluster admin node +#. : space separated list of agent node IPs +#. : optional name of script for extra setup functions as needed + +key=$1 +nodes="$2" +admin_ip=$3 +agent_ips="$4" +extras=$5 + +source ~/models/tools/maas/deploy.sh $1 "$2" $5 +eval `ssh-agent` +ssh-add $key +if [[ "x$extras" != "x" ]]; then source $extras; fi +scp -o StrictHostKeyChecking=no $key ubuntu@$admin_ip:/home/ubuntu/$key +echo "Setting up Rancher..." +ssh -x ubuntu@$admin_ip <" +#. Automate setup and start demo blueprints. +#. : space-separated list of agent node IPs +#. $ bash rancher_cluster.sh setup "" +#. Installs and starts master and agent nodes. +#. $ bash rancher_cluster.sh master +#. Setup the Rancher master node. +#. $ bash rancher_cluster.sh agents "" +#. Installs and starts agent nodes. +#. $ bash rancher_cluster.sh demo +#. Start demo blueprints. +#. $ bash rancher_cluster.sh clean "" +#. Removes Rancher and installed blueprints from the master and agent nodes. +#. +#. To call the procedures, directly, e.g. public_endpoint nginx/lb +#. $ source rancher-cluster.sh +#. See below for function-specific usage +#. + +# Install master +function setup_master() { + docker_installed=$(dpkg-query -W --showformat='${Status}\n' docker-ce | grep -c "install ok") + if [[ $docker_installed == 0 ]]; then + echo "${FUNCNAME[0]}: installing and starting docker" + # Per https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/ + sudo apt-get remove -y docker docker-engine docker.io + sudo apt-get update + sudo apt-get install -y \ + linux-image-extra-$(uname -r) \ + linux-image-extra-virtual + sudo apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + software-properties-common + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + sudo apt-get update + sudo apt-get install -y docker-ce + + echo "${FUNCNAME[0]}: installing jq" + sudo apt-get install -y jq + fi + + echo "${FUNCNAME[0]}: installing rancher server (master)" + sudo docker run -d --restart=unless-stopped -p 8080:8080 --name rancher rancher/server + + echo "${FUNCNAME[0]}: wait until server is up at http://$1:8080" + delay=0 + id=$(wget -qO- http://$1:8080/v2-beta/projects/ | jq -r '.data[0].id') + while [[ "$id" == "" ]]; do + echo "${FUNCNAME[0]}: rancher server is not yet up, checking again in 10 seconds" + sleep 10 + let delay=$delay+10 + id=$(wget -qO- http://$1:8080/v2-beta/projects/ | jq -r '.data[0].id') + done + echo "${FUNCNAME[0]}: rancher server is up after $delay seconds" + + rm -rf ~/rancher + mkdir ~/rancher +} + +# Install rancher CLI tools +# Usage example: install_cli_tools 172.16.0.2 +function install_cli_tools() { + echo "${FUNCNAME[0]}: installing rancher CLI tools for master $1" + cd ~ + echo "${FUNCNAME[0]}: install Rancher CLI" + rm -rf rancher-v0.6.3 + wget -q https://releases.rancher.com/cli/v0.6.3/rancher-linux-amd64-v0.6.3.tar.gz + gzip -d -f rancher-linux-amd64-v0.6.3.tar.gz + tar -xvf rancher-linux-amd64-v0.6.3.tar + sudo mv rancher-v0.6.3/rancher /usr/bin/rancher + echo "${FUNCNAME[0]}: install Rancher Compose" + rm -rf rancher-compose-v0.12.5 + wget -q https://releases.rancher.com/compose/v0.12.5/rancher-compose-linux-amd64-v0.12.5.tar.gz + gzip -d -f rancher-compose-linux-amd64-v0.12.5.tar.gz + tar -xvf rancher-compose-linux-amd64-v0.12.5.tar + sudo mv rancher-compose-v0.12.5/rancher-compose /usr/bin/rancher-compose + echo "${FUNCNAME[0]}: setup Rancher CLI environment" + # CLI setup http://rancher.com/docs/rancher/v1.6/en/cli/ + # Under the UI "API" select "Add account API key" and name it. Export the keys: + # The following scripted approach assumes you have 1 project/environment (Default) + # Set the url that Rancher is on + export RANCHER_URL=http://$1:8080/v1 + id=$(wget -qO- http://$1:8080/v2-beta/projects/ | jq -r '.data[0].id') + export RANCHER_ENVIRONMENT=$id + curl -s -o /tmp/keys -X POST -H 'Accept: application/json' -H 'Content-Type: application/json' -d '{"accountId":"reference[account]", "description":"string", "name":"string", "publicValue":"string", "secretValue":"password"}' http://$1:8080/v2-beta/projects/$id/apikeys +# curl -s -o /tmp/keys -X POST -H 'Accept: application/json' -H 'Content-Type: application/json' -d {"type":"apikey","accountId":"1a1","name":"admin","description":null,"created":null,"kind":null,"removed":null,"uuid":null} http://$1:8080/v2-beta/projects/$id/apikey + export RANCHER_ACCESS_KEY=$(jq -r '.publicValue' /tmp/keys) + export RANCHER_SECRET_KEY=$(jq -r '.secretValue' /tmp/keys) + # create the env file ~/.rancher/cli.json + rancher config < 0 && "$health" != "healthy" ]]; do + health=$(rancher inspect $id | jq -r ".healthState") + echo $service is $health + sleep 10 + done + echo $service state is $(rancher inspect $id | jq -r ".state") +} + +# Start service based upon docker image and simple templates +# Usage example: start_simple_service nginx nginx:latest 8081:80 3 +# Usage example: start_simple_service dokuwiki ununseptium/dokuwiki-docker 8082:80 2 +function start_simple_service() { + echo "${FUNCNAME[0]}: starting service $1 with image $2, ports $3, and scale $4" + service=$1 + image=$2 + # port is either a single (unexposed) port, or an source:target pair (source + # is the external port) + ports=$3 + scale=$4 + + echo "${FUNCNAME[0]}: creating service folder ~/rancher/$service" + mkdir ~/rancher/$service + cd ~/rancher/$service + echo "${FUNCNAME[0]}: creating docker-compose.yml" + # Define service via docker-compose.yml + cat <docker-compose.yml +version: '2' +services: + $service: + image: $image + ports: + - "$ports" +EOF + + echo "${FUNCNAME[0]}: syntax checking docker-compose.yml" + docker-compose -f docker-compose.yml config + + echo "${FUNCNAME[0]}: creating rancher-compose.yml" + cat <rancher-compose.yml +version: '2' +services: + # Reference the service that you want to extend + $service: + scale: $scale +EOF + + echo "${FUNCNAME[0]}: starting service $service" + rancher up -s $service -d + + wait_till_healthy "$service/$service" 6 + cd ~/rancher +} + +# Add load balancer to a service +# Usage example: lb_service nginx 8000 8081 +# Usage example: lb_service dokuwiki 8001 8082 +function lb_service() { + echo "${FUNCNAME[0]}: adding load balancer port $2 to service $1, port $3" + service=$1 + lbport=$2 + port=$3 + + cd ~/rancher/$service + echo "${FUNCNAME[0]}: creating docker-compose-lb.yml" + # Define lb service via docker-compose.yml + cat <docker-compose-lb.yml +version: '2' +services: + lb: + ports: + - $lbport + image: rancher/lb-service-haproxy:latest +EOF + + echo "${FUNCNAME[0]}: syntax checking docker-compose-lb.yml" + docker-compose -f docker-compose-lb.yml config + + echo "${FUNCNAME[0]}: creating rancher-compose-lb.yml" + cat <rancher-compose-lb.yml +version: '2' +services: + lb: + scale: 1 + lb_config: + port_rules: + - source_port: $lbport + target_port: $port + service: $service/$service + health_check: + port: 42 + interval: 2000 + unhealthy_threshold: 3 + healthy_threshold: 2 + response_timeout: 2000 +EOF + + echo "${FUNCNAME[0]}: starting service lb" + rancher up -s $service -d --file docker-compose-lb.yml --rancher-file rancher-compose-lb.yml + + wait_till_healthy "$service/lb" 6 + cd ~/rancher +} + +# Change scale of a service +# Usage example: scale_service nginx 1 +function scale_service() { + echo "${FUNCNAME[0]}: scaling service $1 to $2 instances" + id=$(rancher ps | grep " $1 " | awk '{print $1}') + rancher scale $id=$2 + + scale=$(rancher inspect $id | jq -r '.currentScale') + health=$(rancher inspect $id | jq -r '.healthState') + while [[ $scale != $2 || "$health" != "healthy" ]]; do + echo $service is scaled at $scale and is $health + scale=$(rancher inspect $id | jq -r '.currentScale') + health=$(rancher inspect $id | jq -r '.healthState') + sleep 10 + done + echo $service is scaled at $scale and is $health +} + +# Get public endpoint for a service +# Usage example public_endpoint nginx/lb +function public_endpoint() { + id=$(rancher ps | grep " $1 " | awk "{print \$1}") + ip=$(rancher inspect $id | jq -r ".publicEndpoints[0].ipAddress") + port=$(rancher inspect $id | jq -r ".publicEndpoints[0].port") + echo "${FUNCNAME[0]}: $1 is accessible at http://$ip:$port" +} + +# Stop a stack +# Usage example: stop_stack nginx +function stop_stack() { + echo "${FUNCNAME[0]}: stopping stack $1" + rancher stop $(rancher stacks | awk "/$1/{print \$1}") +} + +# Start a stopped stack +# Usage example: start_stack nginx +function start_stack() { + echo "${FUNCNAME[0]}: starting stack $1" + rancher start $(rancher stacks | awk "/$1/{print \$1}") + wait_till_healthy $1 6 +} + +# Delete a stack +# Usage example: delete_stack dokuwiki +function delete_stack() { + id=$(rancher stacks | grep "$1" | awk "{print \$1}") + echo "${FUNCNAME[0]}: deleting stack $1 with id $id" + rancher rm --stop $id +} + +# Delete a service +# Usage example: delete_service nginx/lb +function delete_service() { + id=$(rancher ps | grep "$1" | awk "{print \$1}") + echo "${FUNCNAME[0]}: deleting service $1 with id $id" + rancher rm --stop $id +} + +# Start a complex service, i.e. with yaml file customizations +# Usage example: start_complex_service grafana 3000:3000 1 +function start_complex_service() { + echo "${FUNCNAME[0]}: starting service $1 at ports $2, and scale $3" + service=$1 + # port is either a single (unexposed) port, or an source:target pair (source + # is the external port) + ports=$2 + scale=$3 + + echo "${FUNCNAME[0]}: creating service folder ~/rancher/$service" + mkdir ~/rancher/$service + cd ~/rancher/$service + echo "${FUNCNAME[0]}: creating docker-compose.yml" + # Define service via docker-compose.yml + case "$service" in + grafana) + cat <docker-compose.yml +grafana: + image: grafana/grafana:latest + ports: + - $ports + environment: + GF_SECURITY_ADMIN_USER: "admin" + GF_SECURITY_ADMIN_PASSWORD: "password" + GF_SECURITY_SECRET_KEY: $(uuidgen) +EOF + ;; + + *) + esac + + echo "${FUNCNAME[0]}: starting service $service" + rancher up -s $service -d + + wait_till_healthy "$service/$service" 6 + cd ~/rancher +} + +# Automated demo +# Usage example: rancher_demo start "172.16.0.7 172.16.0.8 172.16.0.9" +# Usage example: rancher_demo clean "172.16.0.7 172.16.0.8 172.16.0.9" +function demo() { + # Deploy apps + # Nginx web server, accessible on each machine port 8081, and via load + # balancer port 8001 + start=`date +%s` + setup "$1" + start_simple_service nginx nginx:latest 8081:80 3 + check_service nginx/nginx http "Welcome to nginx!" + lb_service nginx 8001 80 + check_service nginx/lb http "Welcome to nginx!" + # Dokuwiki server, accessible on each machine port 8082, and via load + # balancer port 8002 + start_simple_service dokuwiki ununseptium/dokuwiki-docker 8082:80 2 + check_service dokuwiki/dokuwiki http "This topic does not exist yet" + lb_service dokuwiki 8002 80 + check_service dokuwiki/lb http "This topic does not exist yet" + # Grafana server, accessible on one machine at port 3000 + start_complex_service grafana 3000:3000 1 + id=$(rancher ps | grep " grafana/grafana " | awk "{print \$1}") + source ~/models/tools/prometheus/prometheus-tools.sh setup "$agents" + grafana_ip=$(rancher inspect $id | jq -r ".publicEndpoints[0].ipAddress") + prometheus_ip=$(ip route get 8.8.8.8 | awk '{print $NF; exit}') + connect_grafana $prometheus_ip $grafana_ip + public_endpoint nginx/lb + public_endpoint dokuwiki/lb + public_endpoint grafana/grafana + + end=`date +%s` + runtime=$((end-start)) + runtime=$((runtime/60)) + echo "${FUNCNAME[0]}: Demo duration = $runtime minutes" +} + +# Automate the installation +function setup() { + # Installation: http://rancher.com/docs/rancher/v1.6/en/ + # Install rancher server (master) at primary interface of host + # Account control is disabled (open access to API), and Default env created + ip=$(ip route get 1 | awk '{print $NF;exit}') + setup_master $ip + # Install rancher CLI tools (rancher, rancher-compose), register with master + # and setup CLI environment (e.g. API access/secret keys) + install_cli_tools $ip + + # Add agent hosts per http://rancher.com/docs/rancher/v1.6/en/hosts/custom/ + agents="$1" + for agent in $agents; do + setup_agent Default $agent + done +} + +# Clean the installation +function clean() { + delete_service nginx/lb + delete_stack nginx + delete_service dokuwiki/lb + delete_stack dokuwiki + agents="$1" + for agent in $agents; do + stop_agent $agent + done + sudo docker stop rancher + sudo docker rm -v rancher + sudo apt-get remove -y docker-ce +} + +export WORK_DIR=$(pwd) +case "$1" in + master) + ip=$(ip route get 1 | awk '{print $NF;exit}') + setup_master $ip + ;; + agents) + agents="$2" + for agent in $agents; do + setup_agent Default $agent + done + ;; + ceph) + # TODO Ceph support for rancher, e.g. re + # http://rancher.com/docs/rancher/latest/en/rancher-services/storage-service/ + # https://github.com/rancher/rancher/issues/8722 + # setup_ceph "$2" $3 $4 $5 + ;; + demo) + demo "$2" + ;; + setup) + setup "$2" + ;; + all) + setup "$2" + demo "$2" + check_service nginx/lb + check_service dokuwiki/lb + check_service grafana/grafana + ;; + clean) + clean "$2" + ;; + *) + if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then grep '#. ' $0; fi +esac diff --git a/tools/traffic.sh b/tools/traffic.sh new file mode 100644 index 0000000..c020b6c --- /dev/null +++ b/tools/traffic.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright 2017 AT&T Intellectual Property, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# What this is: semi-random request generator for a web service +#. +#. How to use: +#. $ git clone https://gerrit.opnfv.org/gerrit/models +# $ bash models/tools/traffic +# : address of the web service + +echo "$0: $(date) Generate some traffic, somewhat randomly" +ns="0 00 000" +while true +do + for n in $ns; do + sleep .$n$[ ( $RANDOM % 10 ) + 1 ]s + curl -s $1 > /dev/null + done +done -- 2.16.6