From 564b01be0a386668911c41907cb6bcf869c7321c Mon Sep 17 00:00:00 2001 From: jagadeeshi2i <46392704+jagadeeshi2i@users.noreply.github.com> Date: Tue, 27 Oct 2020 00:19:05 +0530 Subject: [PATCH] Feature: Add prometheus and grafana for kubernetes setup (#747) - Update image in values.yaml Co-authored-by: Aaqib --- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 2 +- docker/config.properties | 1 + kubernetes/README.md | 58 +++++++++++++++++++++-- kubernetes/config.properties | 3 ++ kubernetes/images/grafana_datasource.png | Bin 0 -> 17577 bytes kubernetes/templates/torchserve.yaml | 8 ++++ kubernetes/values.yaml | 1 + 8 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 kubernetes/images/grafana_datasource.png diff --git a/docker/Dockerfile b/docker/Dockerfile index 8df79c420e..7b89cb6244 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -78,7 +78,7 @@ RUN chmod +x /usr/local/bin/dockerd-entrypoint.sh \ COPY config.properties /home/model-server/config.properties RUN mkdir /home/model-server/model-store && chown -R model-server /home/model-server/model-store -EXPOSE 8080 8081 +EXPOSE 8080 8081 8082 USER model-server WORKDIR /home/model-server diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 1bb04f2c1c..e009861ede 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -58,7 +58,7 @@ RUN if [ "$MACHINE_TYPE" = "gpu" ]; then export USE_CUDA=1; fi \ && cp docker/config.properties /home/model-server/config.properties \ && mkdir /home/model-server/model-store && chown -R model-server /home/model-server/model-store -EXPOSE 8080 8081 +EXPOSE 8080 8081 8082 USER model-server WORKDIR /home/model-server ENV TEMP=/home/model-server/tmp diff --git a/docker/config.properties b/docker/config.properties index 669292b829..2fb279fedb 100644 --- a/docker/config.properties +++ b/docker/config.properties @@ -1,5 +1,6 @@ inference_address=http://0.0.0.0:8080 management_address=http://0.0.0.0:8081 +metrics_address=http://0.0.0.0:8082 number_of_netty_threads=32 job_queue_size=1000 model_store=/home/model-server/model-store \ No newline at end of file diff --git a/kubernetes/README.md b/kubernetes/README.md index fd3a0bc54f..967e63c92a 100644 --- a/kubernetes/README.md +++ b/kubernetes/README.md @@ -482,6 +482,7 @@ ```yaml inference_address=http://0.0.0.0:8080 management_address=http://0.0.0.0:8081 + metrics_address=http://0.0.0.0:8082 NUM_WORKERS=1 number_of_gpu=1 number_of_netty_threads=32 @@ -539,8 +540,9 @@ | Parameter | Description | Default | | ------------------ | ------------------------ | ------------------------------- | | `image` | Torchserve Serving image | `pytorch/torchserve:latest-gpu` | - | `management-port` | TS Inference port | `8080` | - | `inference-port` | TS Management port | `8081` | + | `inference_port` | TS Inference port | `8080` | + | `management_port` | TS Management port | `8081` | + | `metrics_port` | TS Mertics port | `8082` | | `replicas` | K8S deployment replicas | `1` | | `model-store` | EFS mountpath | `/home/model-server/shared/` | | `persistence.size` | Storage size to request | `1Gi` | @@ -568,6 +570,7 @@ torchserve: management_port: 8081 inference_port: 8080 + metrics_port: 8082 pvd_mount: /home/model-server/shared/ n_gpu: 1 n_cpu: 1 @@ -647,7 +650,7 @@ } - curl http://your_elb.us-west-2.elb.amazonaws.com.us-west-2.elb.amazonaws.com:8081/models/squeezenet1_1 + curl http://your_elb.us-west-2.elb.amazonaws.com:8081/models/squeezenet1_1 # You should see something similar to the following [ @@ -710,7 +713,56 @@ } ] ``` + ## Metrics + + ## Install prometheus + ``` + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm install prometheus prometheus prometheus-community/prometheus + ``` + + ## Install grafana + ``` + helm repo add grafana https://grafana.github.io/helm-charts + helm install grafana grafana/grafana + ``` + Get admin user password by running: + ``` + kubectl get secret --namespace default grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo + ``` + + ## Add prometheus as data source in grafana + ``` + kubectl get pods + + NAME READY STATUS RESTARTS AGE + efs-provisioner-1603257008-b6b54d986-gng9g 1/1 Running 0 5h15m + grafana-cbd8775fd-6f8l5 1/1 Running 0 4h12m + model-store-pod 1/1 Running 0 4h35m + prometheus-alertmanager-776df7bfb5-hpsp4 2/2 Running 0 4h42m + prometheus-kube-state-metrics-6df5d44568-zkcm2 1/1 Running 0 4h42m + prometheus-node-exporter-fvsd6 1/1 Running 0 4h42m + prometheus-node-exporter-tmfh8 1/1 Running 0 4h42m + prometheus-pushgateway-85948997f7-4s4bj 1/1 Running 0 4h42m + prometheus-server-f8677599b-xmjbt 2/2 Running 0 4h42m + torchserve-7d468f9894-fvmpj 1/1 Running 0 4h33m + + kubectl get pod prometheus-server-f8677599b-xmjbt -o jsonpath='{.status.podIPs[0].ip}' + 192.168.52.141 + ``` + ![Add data source](images/grafana_datasource.png) + + + ## Expose grafana with loadbalancer + ``` + kubectl patch service grafana -p '{"spec": {"type": "LoadBalancer"}}' + + kubectl get svc grafana -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' + ``` + + ## Login to grafana + http://your.grafana.elb.us-west-2.elb.amazonaws.com:3000 ## Troubleshooting diff --git a/kubernetes/config.properties b/kubernetes/config.properties index 7dbc411a58..9f7ad861a7 100644 --- a/kubernetes/config.properties +++ b/kubernetes/config.properties @@ -1,5 +1,8 @@ inference_address=http://0.0.0.0:8080 management_address=http://0.0.0.0:8081 +metrics_address=http://0.0.0.0:8082 +enable_metrics_api=true +metrics_format=prometheus NUM_WORKERS=1 number_of_gpu=1 number_of_netty_threads=32 diff --git a/kubernetes/images/grafana_datasource.png b/kubernetes/images/grafana_datasource.png new file mode 100644 index 0000000000000000000000000000000000000000..271f29a00d1567777a299db51537b8bbebf5bec2 GIT binary patch literal 17577 zcmeHvRdn54x1EeJJ9c7bj+vR6nVFelW{N2>Gjq($j+vR6?Koy;rk;HNy|-}Nj`pc9 z)!5@mN8`ZO($?0RbIqwA3UcCzaM*AF03b?Ah$sO7xFKl!6b2mh>h8&!1X@8j2}!EJ zz`(3-$^Qnu#c~$aa8|ZAb9OUyGzHA<>}^eHos1n#P3@d4?48dcI{5&A5ReoRRB_KZ zUH8!VwD`tLobA(&i@;u}|}zZ-mT%&0^`AOD14{Q$jn=#3D8Aqx6sC`KIgQD}{U1Vspj z$Sx%owC^K3>qH5uB&rppJk+O~@#$g(2Aa`Yne;=gZ-Us^5GwP(?jeH|^rsZ`-|GuQ ze6R9vMO<85SXCQ3TyyMHzAQa?hLl({XI6Qk_o1gapWXe+JP+{g&RKww~HA7DOTZn?V#A5mkYVcVV!hMx0GV&@b zqTzJ+n(f3<>6)T?Bft9FC zE;uzJGa~)B5#&inAqtvGFR7`i33T&lOTJe({u^k|S?7$Xiv2?lO2)qdzr{%tBALCU z7TMc|K18;+&l&(NVxm9u1QqocyB1ihSrKh?jUnKHtE;s^QA=cdalQ&FtJLPel)k@b zZ;l_qY1!)RSM9b(8hCAhtwDBe1B?80=Pd_E)fe(8T(rE!-oA#__G$e zB|SMK$unb($mCehx4e45zp#wJ#5{Q@%&YWkhq8uq*Y~7knaheg!MEJSF#YF5V)APe z`Id~V)&q@=V?Aae`xe3Ts=K?dY-Dye&--@~6EC)(*zozGH2Qye1@n8qVnlLQaBd2& zUZ#SMB;*PDj#61!Ra^J2zO?vk+Q*|ivS@c1U=T3rX2%{qK)|x$RWz{hx<06lf|4-xVaOJix_q zci?$;_H1-kJGs|F>$2(+`SOKuY&Pm>`f8QP0a5z9uIVzyIng(T%|af9?8xt2I4Ec+ z7ge0+a9k;FL*U5fvoi+}+N?X(XZuYRqH#F) zd~zEPbYz_W+y-Twl!IE|G3>rkUhqmDe&n!KDoYh1KfSi+v*T2V&%Lu54MD%(qY_N6 z9$BZ0^L~>c(PeZ+@)Kkw-ilrn3b+s7P8Pq%?L=Usn}HXnkl(<}GF=rV32U&JSrj!F zr_;~8+<*#GtI{k!I^PWR<+!K{UfM~}*5i#sjr&6c82Bra)bZ18tT3Pc;nZF_Y%aft zaF@I}N!|oLjK*Un061p`PKZI;X<#^05g(ldjMr^swz2&#szXskSw&y|(p0i{LAF5z zIyn;|Y_#9Fq-my^nV6a#IrZdb0RsyY8w(Qzz%i}Ek#l2cNQ7qE3#zPQqT?PWs8EO@ zl_1sIX!0)SVCR?dbw41W*=mExpBaOj{FTw9tSq>;gz|-ulxSgbo%;(HCmqpaU$@Io zGPX)?;z=^n!G7@%V|8{NxlpXmTW8e|JHLX0E`HP71`}psV7biD zkXP_x;v~o5V&K*070^mImb0^2yoxXa{=;EIJJlYt1liTAK6FGTEU-Wq0o&%ghLDbs zqmE7fxy>=34+eeE8+joEFB#>f(7^;PKi^zdqk9+~;NSI@4JHnXlNjy_qcsP?_t^Aa zc1^I9miNyn235ljnCvNNj(0<5IN-qd*bPPV`(FoaRR;b(0twNG{#*Oq{1pw}w|;VF z+xkq&z)zbZ$PzKAJor@pyk568Ki4m%gvX z_rm#gANjDQu6=otBV!E$^pY=EZRmhE>PziFmwLn}ipytyBG3UaiZ52MFM7K1an{Qh+}Evwl;v z6R(h?EqjnVQg4*b4Z6Ggwi#tghkYR|bX}uuT8S6C^M2*`P+@fUDttJ%z=c@j zy>0sNp`f{DPco>{srvZdzl|PxEK@4x)WfQ4OH5wp`{7TW;fUu7pb}GIGjU z$=dcqq(lMeMu2ETt8cQ>&7;d-)j;*8@KD~hn^h;3j?Bzj-nBK#x+;dCR=5|qzpuW* z{M=lGjtUN83!Ekzny0T$a)R>-s3sqO1G?6F;DEEj!nSXpdJy&)Z!VttGSyxSd6G5>flE!UYAT75F~itG&=4V9yZMc^}b*uJ{8 zesBMXlX%%f_0jVaN}!}aRf}Q{H;h5`{pI&Pc1T;M>h0bK?Q-r#-_MzwZWCk{d;DNI zkN`BNyQfdSm)$kLU&~+=635x?l^l)Qn*x%8$DEhC)hr3^t|?|rCVBF#(GLy^?(l2f z8WEc0dll-#vyOtd5(Ge4T_s_3brfwcn4N1mRuUX`KG{1d*sLL;Yl;bwEJ_M)(QaLP zQO@>RID!*n$?~$}a+z#frV!)ekj;J>K7%^5d`@Z}9O6=!40@sutoiA(WjGdCbcMp& zWdd2npH}jsZM<7RQ-9RH%@tAC1uQS~@2MY4Dn1Fk8fK;?4nkzX$tNn1v9O5jHSBWY zjg=8Gsgtqn?BeOFFI0G?)gjA44f4XX#vu#9A%sL^b(ln90bgCx`8qQR3Pj8U9y8O_H}1a%_!VGa`DS5M%3wzW+THleAf5vyASjzN9Pi~hOT$p zg;h5jjdM9nX9xXCcZM&CN;7w}&5h7Tc-H&<`s23TZT$|I$ry=X2-(Lk9j`&6(l+`U zu+-)PZaXRWV|u;QtvnPp@lIF%pq>;;6_;6BT;7Vi@i1>%WTb^G#OKu`?H%qp@mrk* z<4aj_S@Fp&xv@YEwAx>Nou=~~^6yf(nCb4X-2`vSOu+VTGz;%+CEK&>lfxzfW zzXKNl;$2y4ha7FER!Z2!P>-kKg`@D^X1+n@lvz$;S()3;+3izS&wSs$ciw}1XkJSF z9x2{(_MNBEffQ_aS#v&4K1l}K%*b&6sM$Bp(Ibgykml-IpLtp+&)j6+h`PLNs0U2) zX75k5)=9f%&wiUqoaXNC-T|jyCpwwWui?X@p-vX|-MPx+ zcc_i@y|l!=M%@qT_trLa5n3l(sat=}B^vBdD_Mag` zI>P2KC6r&LzD4g2} zW!0Nr=oneNV?45**#0KF0uA>G4eYCpg!;cO5ly#SrtL2wmElSY9G*K@KP3u6!0^Lx zZpXj&hqlgQ0W(9qSJldWJ;8(0fIsdbfmh7R!SBRA;Ja9>{bdHbULKP3Lh#fn7^4!l zKX&`u#YD6$wH&?nuOd&mvU!x=Mv=Ec=grVj`q_#_?PR~(&~sN@$pWyTPm9n`J|+cF z^Y`@qvq%KJNt9>oSW}JfSI4*T0|zC-x{6je8}@;fDy~kwBy-kd<)K4x@QI!GuMjR2 z$gG(xYyF+f$r5a!g$f&csYph4ced2%@n^s9ZOMMl2_7uK@JDCa%l z4a?;=T84jNfVCbW5A-q}xqas|J!jt5ZQ>xk^d?RcFoT{n_)M%s8H4=Xwd^<0-8&b2%L|gxh=MlU)dLINN?C`u zFIDDM41p3@9yC@~BVZT<>Emks+N_SOu(YnSw(Gg~LjU{yOrp_WV01)q*8jl!z&mI2 zA{G1i>ux212On7gN(ELmE6mX*ZVGqqXj#J}5jhUdKHI{B4c++xMsiF7gj`?6WJSB% zF*}nd7E;Y=sJEDtK)9-1H(_|uLCG~EO6ng(yCIV)*&XF_{q*-62a%!wLcrTY62U^3 zPsagz(QNO{aH0_Mq76+{fgg`}z7ZZBA8YHevE3a)urB5n%Cis8!LaT^aI7fI?=-c4 zO#Hy3%8Fa$&0K>pn`2?&0XkG@OY6mjx!)KhK~GHINTH7p85S$)CE>0AFDtv)l=~*fAxl#BjR*Z;#JkM8A;zvZzc~%DjKnlh|P7fLrxJdpK>aHFO z*H0)8(85mhW_?i$QXo*%*VhNZ=xmjyEQz@MH0<1&L8Ljj;Jl73Q$2UwDWIc}yg>A$ z&!uKJ&FI3^y!wJ z#>3vuMNjs&^cJH*xjyj?#Ia0S_a?^D4-OgK`%x4m(BEWqJoL%&9^Z63J<80sK3*k{ zcd=k3y3y^(phEzplnG0m<8SWUM9+rS$xs{JDSyF#YfEyhg2R5(5Vve6PP`L;&CgmR zd9gy~$-S6%8|VTqOPihHguUTGr0figtJ#(4#qlT|Z<@(c(ejyAc>hIvQ8GKP}ZbaqK zmNp*TsK4D9U73`i&d&bXTzLQ0T`@uAOFkGNA!W2%@4QVx{3Qy(&tc~;?c~|MGC9;H z(HX-%Ns+h{UxDw++v@A`>~GxJn5E!DzhZvuODP5#Yq=W>zG5?Oai}YL1eWr`$mXPL z+|{@HFI<2DW*Xf@@}9Z@&)@mtQ;Wx`cyvj8fxfgvx5nR&+}|``Yk@BQTt4vsZnSP6 z9vt1e;{7xHVOIY|aQ(j+-09f+DsiN+tlx^h!9LwtseHjqGSw^`cc$$oM2rM0wOaE+ z5-P5;EUxm?QWoa-m{~$#$J5qM1qB=S{;sFX(WvpN!tx8M#gCA*n z?~K+z3T2A_SGH_@)#cESLo3V*^6%XCRLI8N>kpP@T5E(L5%h$TWH@Tl55e--t{dz7 zgM>u2IvkDFx{kTHxU`xEnpEf&VYHTmZ-y5$^dBPd|0GTwqJ60CHAqpjW*qYATRaQh ziZs*5&9%e1$MgeuuG44DsM@nxc&=OzfrX>35uFI(8+Ouc73wzoEw1t=Q$q0zi8U$h zyrgV@vA@Vg%XClD@B$r9uRJN~AQ@>k#{sXTkYi=IJTuky(<@_(X5VYeIyE)$UlOv3zVo{7@kgWcN^kFp?5Rl;0@M5_83-){ZV=Cyx z)?Bt;oX3L*Ap0l=@iHDxO4*fI<~%arw=NobIl`^Y>+0_*CE6vAd*%LiK?gzWJ>Wqz zHgDQmNUo1Pky#H0roEdhJGNy)(3}K3gC(&FrVcH10{<}?Q81-~!( zA7H_C67`*n!R4Z?g!PCxj&zWVLoS6S_(Kdy;+{kSk3VdDVT8+9@A!}MADk`8|LzP8GfEaPy%-JkJcemaHD}cz=GEDSNCGdk zw$`!a{zI#p`D+Iz;E&zO+BZu_KWBVaTX0Mt>%<1-Yd2@cJ2#~IfnMyr&3tzZWx)1I z?oZ40QdabweW)kR4a~4ji^$x=HQWf$0u7TGRyDKDD>6AW{h#J# zfew*BeiX<}u1;|IXj7TKh|(Od|GRF@c4nikgSo}26F9P2nT17C z?&>slbP)o=`PhVtNP4| ztI_v;BAW@h+R5*mOIR(TY*JR9fwLeCp--V{X|*4`oO@|I$tQX4aSXnjja;P}c|A<~ z*xPYcPv4StYh#BRX8eLak-nQf(T=jZOaq89>qcJRorksQ#$-QwDRhyU&EayxN#LlP zSIqHtC`zU-ONw`_J4he2nP9>K#G&$HDt&g`X$hLdvOF!^rAEJw;nPua*~KSKNR?gR z_Xs4}tFk#iJ7P#&=eD@BxAK8Vy#F=Bmp{m*D;bXkx=1PhjiMv>$M$33mM|5b`mB{WIGv)dP_QD*-}k73EFGEU{Y(q z2Y&Bv;9H>3;-lf>L z4qmp_bU_#Fho>baKq1qC?#RZwQb&{A@{}o2hA(yi9Ej=V!xA6N9@&G7GINGPO!!f} zYmE&dphU{h&t64_&SDEObG40JTwXjEb_eHyWXLli)Si*r%*_Lul==R$sHW#e!wo6k zyvhB_w=g+0dp>+2&E7}&>#mwo*-GrX-Va89r&s4CDhd=pp#OBUs$e+1X=Nu`*7TAQ zDzg+G#1P)yahmj<9jwW{q&EB$*DP2Z+~~TMi=Q`6{|MKJ7&=pP3PH;so5R*HJQg^YysFT9^V)ygwBRWmhJdy$R}i0+~Y^}7@0*&q9?q|_1@0rh%M(wIKe*(}bR z0L0wGOgJonP?FYPyS433Pi}bZ`(`U|+t7nC1_@xaqz*PJy4b0`-RQjBMB#|nXqWxg zGlWac`=`$4G+?PO;-;^-0%t-U*}=hcp|0X|2c!BsPb=f^z3{D(NbFAEOwo~vb7d~= zNAy!$^pLHWsyx5VEiZH(h)L4HSM(hP-!kw6)XFG*)T0(1w4kTxiA~t7k~oM97gjJU zz<&yV>_cO-d}7JS@UP?~6!haZw>$7b5^cJKjKtp4dSFxZm%HU z20b37c(%&B8#3fwkNmk5d9j`Bjk~J>0%dQ*;}dBwkcd-M>q zBZ*SeS!>15Q)u%vfQrteUGDb15Q=1S;QE=w^VkU-P+Vy+7b#KOj8ZCE%8d4)Aat{L zCge=~9&NA9Nn^Z2-mN5JDwQDbn$Ah4pFVl2}<)FK%7VGiMi)7Qs zey6o}{6c4OBQ8GE*1}}W9zT$VHa0PdrJ?`fv#Es%C}8W>iPl&ndF7#DM{rl8+MUBq z+6*C4KNMXN;3=B5_6BsQLnjOmucWXeGCDEE^qX^RLcdweMCC-Myy|g-NQ{Cg8waC~ zmhp$ZPx8S@#5v(qm^t-xScL^>{Zk0_a5!W$uEx#}1AE6@60yM_5852jfntZOZoQr* zR_(m-Rs}T8h{ZKc_;g?NGbykkxUbp=UhT6)&pl>_Qg#XPY9wjcUHMrR)s;INb&+Fg z38PrCy?G8YQV|K3VIDu)ORhel#3_m?BvADxk$M`tznnVSXvqS@j6Wyvh(c{el0+r~Ts7`0(sq*M~aka!N*4`v=6hfzr1x#OkSB~ znLn1?^N$BydQ0g=){OxPl6`u0jzcho#%r-*Iei5-+=k3O9BGnDw=)_XLQ%0K5HAS} zr$^ROFV5*KuJtR_rgrqj*GmS9Yz957KA4yf79h-e+J!TZk8+h?4*sb=zYpmyPhe(b z(z$cITliRH4Cg<=;gg)!Xjc&5pkZOez3Nq#Jh!+oMv+2752Z96BxmLABwg?r zc9}Fio^AgmMJ{ic1$?tseDuurn!vFnt-zbi`-bo%Y0dlBgF|n{P(03^G*Pb>5Qum1m{nk>lAKmG} zHaXDOJid`;xy551|5XbH%>H?(~^zzy-1G=jfrMWfbji$bavKxT43lRxblnF!BY}* zbTQ)_MH8YIschJC?M3KjdT0Q(u}@Qq&Hb8#V-%N4zKHvCzjKLVl)`(@?6j?yEpBj; zV`pw3Dz6hf{H|uXBWEUP5=ccSmF35=i^Ig`8*1AWo@Y!Y@R-@XSrZ*x;Ve*Qnx5}? z@pewb1tj7RufOVeAw6SktFt-o-`yXCfj z5rxTBhd#HA`bVDPN_s@=F}NSexv`ok0F#Sk!2Q2guYklFxe9G>o;e*Q;}bawEEg&& zUS&)$xo1Q$c^w@cP1e)&4kjkN;qrbWvMVPDtSlAg76KpL^#y~T5dR_T<`f<;A;!p4!#>-(uHWcMg5DypmRgE3SXIRc_3 z@j)jbBo6eh{qx^>C0AN!N~|;yK%N zmZj%F-8t^VNh;lgnzxVRR!?fk$xp~cM7L$+@3R;eS*_znOtdEY@S;-4!Fmp{38*_H zj|denezXdo2w(6kS#kWpHu_Gnf}NW?o9=R?t2-_cP%#PvVBST?2I`5rDDaCTXk+01 z7Sg-Ef3T@seIkK!_3iGi;wGvuda76mU&c>bxVxO9FJd3u9Busb;hcW$zF=xym*29L zmFVoZ2uSP|k9X*$_b|UWk$099^6^Mk*E_Avp#1uydy0~U0sE-h*HbGr0I)!ExLab_ z)RSQ%8{S$CG{&#)B4`s-UCnB*44r%=J?PGh~>5 z)u$fBQBrtRU+Z2Eefwgos$%LnR@rq?pQe71I$g$m5`Wm!_KHuWE(H+s+h<1~rVu3m z!W73F@v~KBaByiFM!u#!7#PQsK4@E;C`o6Owq=p**EcRgRTN6pRZ zW`50ha;KvtL<^=k+wXoK3HJL-$ZxoZESmM zU3|T{*x;H!@jOxZ^?pvCqV6}|{=Ewr;Lktjb8OT)veacCIHA5mcdu_Fh2VH`>&iO% zY^iQ=f!H(cw2q+rQRa4qXYp!lGJxApVlnfyVZ!nw40-uAF7jo-2H%G>EV=7D+#=Yg zN*wf6)vT39Uv2l?g0l0lt=Ge-Z=kaX?aze9c3)LHL&B&3FfarFFvB0Lkp~9;N=ZWO zzp@blfXtpadG=BrFGy{2wiJi#N}2(mwKVrbKI$e?7A8Xql#)G8HYi8xk{|C5kS4OX zlgbv_NAmm?mJl3v4((~aQ(B(Zvc5Qj0FRK()YA_mAm&$yE)Rz$d)I z6rkJXf@jsw36k+)X=u9T(b*|a(S~zP-4Tg(M!LPP{A|4UYHJV-E?8v{Kv3%Svx??e z$M#}8D)Kp>^PLz9CFUN}H}9`IT~&*AdQW^Dq-=>q&KudR5o14K_*|}|nZ^BviEIng zzSwih$3ZHmV786)8ta$Vj!VR)3G8RyYj4-QS(q()jVe^dg%HGsp+gdE@cETSvcUqo z%ZHX^2mtHMb2i^!XicFJ9#_q#mJHfIaHo2{DmOL|66?Q|K4cRcGsH>YT3Q8`)8BpQ{Zxy1_rc z*7CA|723jb-i#{JV>p46)OH$PN4tABUaHiLOTG8TVi*T5uxHl&Jgv85`c%Vv(7F8R zfi(@bxjF0ju%xmqO&ET|?@UXpEmn6=tdy$A9HI>5UyeDLJeACa$#{-#re6@UMd`3De4@4lr@V}5}}L91p#1i zp@9Ho#HOxH@xA<#(bf0#Ieod3>#>*jB5>_w8XPapqpu9Zm`n(hZP%K)33M*>-$zo3t5+n#@F z1WTwg4XXUB&4F9Ma1RZI7+Ffo+KazLFM_^du-HGv>>@yfpA0fqHsYn^oS;rbHB;T|c7qV6f>1bh&ZWEk0 z7f@m+MKLlbQN~45*1VaRn>hRExlPK@cWqL;U&kaHM4G^(j;gFFD62<+f50MA?44%S z%(XmzrW9i4pI=>t_NvRGd%C-Yj_Lgs6ht85^Jp8-nng>|hr1)w+h8oj5irq6H~p7-nJR z5ip5bW%x#_Z(#llmMS0UvnYkZ=hjzt_p+bLVYPI-c{*_qWRVfHblNw~H?AV!#CW)` zmaZo3f-n}v!N00NJ1a zco&vH+%g{GndFyd?RNEhp+!q)z5F^uTWtB@%tivsynne8gYSB^*IpQ+yG-w9?kSsM z=DMRWv;gxK!m0O0s{G*feREnTX?z{c;OVw>H+=xe^m167xGv~=L;oe)Hu74>sx47? zV`nq8K4>$aJN~|))6{2vt`vjuhF^PZm70Ng7k+W2^CJ5Z*7@vF_k&V3%JSqdNZc&k z&aZ|Fvl2|&vC_THaV94BVcouvk_7aM1E`XyI`U}d=b2I=ywq#9K{Kx)3*r7*4a8zfB+vN`CdTubD=FNz z%GF6~>5bO@{+1G}nm$EAbqEDX{wEkkjjn7Y%VX#2&AywSBiqL~7&_{2p6_Gq)RR2y zUh~FESG-V_zwjeL~+C#_WaZ5@LStJ^NB3Xzn;Nq{}9raF{9FDv8 zx;Og<@3!Vk*PB9YS@V>!n;l)%(WB8^saJr!OA1O0lNBpjjhjbKLZ~N4s_#}MWL6DWs?d-@2 z_+tXUr-v!{-^!Se-92|L)-!#bPQ@{*qD6{@$)?ZjZq&DJxL~I>avqu1x z7q{^Aq@r_mZoe0}qcEkPZoXa>xWD_omKhGGf84ONrzD9So_v#CDo;VU$TE>Bl)?hY zxlP9Dwz8`HIz$%G3I;Zl0uTY!58kxstEeX7zk@Cz~UVYu-yK_Zr@h|$hdMBTe_tgFP>!87s_czf$(w$BW>RH%?(*@>^ zYTvB|4C5;G81&)m@PL&XO!Jqi--Nahokp!!V<^JD4zTp8YyTy-+>p zzlV_7d#0y^UHM3Id>Td=)B-g~GL2JwwZhg!(;g+~?;{&F`Al1$uq6SiprV6u%Ss)-_-0+C&YHXzu_Dr8%&Auq>o$s2pBKKS_b z1VV0u-BE_AVn7^Wb^isM_1^-s<#niv_Vs6MK_dg1n1>vSe@C1Rtn%9ODtxW7vXKgh ztAifFLW=s6Ozm!QT!TKYAY?}b={|u3TFv4A2aNkG1BrZw6#ZMm<}u%|SooJM#5Q!K z0Wy$<1-iI&tEA#UcHuud#x^=hR+XhShq z2ytn}$gbHZ#rXfG&V4iRA3l!_5Qe3CB_jkAn4ZeX3w!;x-d!{Q^svix;9~Vp+54tS zIvVAOEy>hD25bu-WSLp--oB0Qmbh^o?)nPP1Q;!W_svZ6UW z%vgJfzX3YJmlx|{_k6PMusQ{~(PE!tc0QwRIHVfnnjtGNQ!!c(=h?9_euKceIdRXTjuql+lzP6 zP(MR4XXAomZLZANK=acdT&<5jAgpS*eL1^4RTc6P0KUNnfa=*rNIO~fy7Ii3%ge~3uaht3=Y zHMN$VQFGyg+b0~9LylP(0Kq)U7U!F-P(}PFMq>R;IdhYFpAy`gQv_IG3eU9vLr3Je z0YKQ-*rJe_be&+GoAyDyO0P|^58NU;?eu9Gay@w$Ron4e$=k>bZ9hG2q0X$;cWFFg z4)?I*`q;7sq)k1CgKHJ!@NKrueBI=ZV2*zK-kBDoeBA`+AuAT;p5W%6BA2eT7&iC8ns{z6_L`2YdYVl2xdk9HZFa*^7WAcPJ=#wW9Q6Hz3P zGIDEjox*OV(3+E3cZ$2G*1{djyEFjii>!*k1-}nn4LMyFe)s@(jPqIlIRvtv*IX@n z&)1CbIy8~Njd)~Fzgui%b85aJ0v+BKH~kh}g$qYv-~tXg@SB0N%5`eo zPW5Rw27a}`f&BZ*s?Ui*Ap@&2ps>{P#1$nya1Bq=)|Bl511w%al7Q>(uUL_j)tV{u z@?dbJUFS0KTInl1oDKp4UuA7dbC=6A1>qqb!0>Z-ZkA_uc3P>o#) z(Yh_b3cdk;G)H5FSrPz=7aa5fLK+n+THv-cfdf#QRv%Jd0xmUaqSPvm3$y*Rjyn z^ea2x8qa-iO^6}|0aekb7P&IEsw^WpMyRH^9TVw#Sh30*O?I#EYzc234Kc`Fu72~y zqg?#>l#{PNKO%@7e)9e#-dh@!9lsm$a~wvCV zTafB;##Y|uUN#(%unfW*+d-RmMC-Y?GB^U?wJSkc>Xos=*K(a? z&khcw>P(`ccPV;XPu-5?_R5k~T8VY7H5rX;WeF?K#;d4x* zqZtB~9}iMlW4Sybj;T$qTEc|V5+qpC@9C8M4qDBQR1e+k$v2WqVA9^5RDy+VX&SQk z)>@pd?|MDAa_%~+=lMJdD->p%rWd_^-BhO=F)c}oo8R{%N1M)f0{ZMMX1^A~DLg@x z&b+^n9CjdxENpMosZdMGw>fcF#Iax-(JDa!thV8?y_8RHnf$0qPCF~-5sXC_zSzaq zEz4mnDNM=;WDb%|+m(s_XJeMoBZv9jFSh#fjG@mC6q6hK(cYmpTEP#(+4g40Tl+fU zJRa$IjB!V2rHc2?TfZ(iuIV`~%}fnA20mZW>9s!j*Vi)7HL;OUpG&8ok?T8>pQ!vH z=**P*0v529V_rk(r$TNHmQp%1NaL4s4C#BA8gpUIP9Kaf3o$!PvVu25BKf7=!Fe67 zD)TbhT55C$0l5BEUmC)Yy;n>t|8PGoE%{@M|Ah`Cx@LC128SE-jq9^F)#X#_Peo2! zDdGoGhN}fPY*S6Z=zHBy*ZD^uae6Ycx0drMpXxM5rBEkg06@svI^LBaJ--R}St=#1 z906hgQb{4zy~)jA&Gv5%)fo1UBm2r^wQ)(wP*9JcMoTYkLgvla-=LFRa*U}gQ&KJl zHb#O~L1%Mjr@F|9pZ{gzsq6w)_ci|uNvjZN>k6SUZ(?cvbMUU?(n-j#iqYk}9pA zl8VBEO7ILZ7)&NnAlhT9Qfgaa2;GSLg@|nN>o&3}tA$(QdS8Eo&aGSVv=m-C@wb9v;f3fn~-1KMM$ z2gifKwmvhSoPS@&&QeWiD+h~&X4i=PUQ9C`^56iB=0y-3SZ@wcbV?^rfO}Awaji_t zhv5FXNOS)}J{C{e?%;jiu~e&ejtX+?UEo1Pu^|@>BSHr5c*ne}E=-haazc=H+8Rgw zMkddts*8gHVv-Z58P;N|G=~{uiy?uIR7`(MuR9oV(=QLV&t2IM)aX!<^r5Bo@gMH+ zJs^F8n}gASHCfri!*8sRPY|i%SgdfM{4y*{BMs-OAlfn*j`1rnx5sCD@}Of{Q(6<3?zkV1{}`ra#cfjt&gA(?;sf-o#eycG(5L;w+P*x={}yo$|)oEM)Q#N z?X3Wo?D(}^5Wj08m`Eho-JHST-XF@K{XfuX_X9DQ9#x9!azVwEGPJ=^bQ21c5wbeb?{&M$ZBLwPTF(4|ckF6hCSryXEc7KLtDp?=?3Z?>OiK7Z$Nlmluq;Ha@V5il--)EJR*Q|?Rmr8x1jKL zPIa~8M$6ZFccXs8N@`@cUW0gIc;b(@It|L`t}}AEn6P>_cw8;P=FBh>1gT1h%43Wc z)K~Hxp=m{i#svv;^WhtD4P=j(VosQOXzg}(k@ka$OzvG5IM%HpN?|TtO*93tr>eR7 z8S{5aIOstVMGP0t5}01fPkq4e`a^%^YnC?_^SW$d=>RYw$xv%zBtY$y|L)PnP@;oR z7dwzrb~0ZuJ)tN+5lrB)f<|rCzkF<{-1{)@ue~LWD+-S*MKZd%|A1>U^DSz^KzLQ2 z7h}~)*SVP2wV&Oq4P!a|0?c1=0H4|%=-bA%z1i~B5hB~Xth4MzbU_f~X!YTg1)`Py z;P({wXRoBEL^)>}ZWOkO`@N)$VbU$NbuV}OW{j@5R2q@*nmqk{c7Uz_O)}XZe>(Wu z>(u{oD6IMQ`)(Qx0GLv$8wYmEQw+4UMti%;(Tpl$$sQ+ayPj!m8PE~>8#XIFvyKc; zu(2@hu}2FgB@IbQn~ZEe{+rO?^?ot0(l}pJKf{4}>yBU~dq*ejBNo!1fV{I|<*BI! zi;cy!chgd992Us&_MbUdwV5?S9B&C! zq#)I8gG<9d4WXAIQ`R~tp=D!T5>wV?4*D^Y0Nm|V?Ja0MPjJ<;7Nt9huNacSUA+l3 zwKqXW)3)ow<#6i=eJQ$bZMB+{1(EBGUrIju$C9Jf0$qF(-T=TkT#)(QzZS<_iR#1T zMTT{6)uZt}yI~Wnx%-pNGK=*^;II5p+-rWGI(7~uy0It^0DvU@`&@uRns}FpyU{DwwQ8fcf zDL5eTy8WUQJi(xx3oRd!-b%jvIFOBYlp?>7FylmUM-SnS(gWIy^&oz_e+Id z37lua04E1FHWtQzRt+4u3#L`tJ?a`+JP=;<4a%ey{@h?MY4M?bmMmMqEV1YIwOG$q ze=9AvZ@&APMMtat9eQ!I`O@5jr1%nX(`QgeT(}^&P)JQ2>mz5fvgueIs!ILXK-4F6 z^0g8BvlX>z7OYc?;;Y|xk>y1>Oae?Vc-FkKi6JXtFT`T`@_TQO<|Qq+BC7%S@4qzr z!2QF9%zC_T-RNFb3jn}HICMIspvm5Z7wWfS2>Q9k4@&UQ@&p*KF8V93DtJW9$3(R~ zLK97-At5fT4Z2j349|b2IDj<~%GX;AdLjcs1BnL@KAUe#=r$iWAn=h?@*@u#ZiTlOE7fO&(Q0~c!#rp! zON*=dx$0#6_F<40$l7}6CjRKoYzmXXAV(l4Z-u`_^)n)(d}*mwHs9%K*bV>MKb|5Y p8_NI5<23#|4(0#WO0)X@!7g-lprGB?3{<58Bt_*!DuoQb{TEoW&O-nI literal 0 HcmV?d00001 diff --git a/kubernetes/templates/torchserve.yaml b/kubernetes/templates/torchserve.yaml index 7dde346928..f87e98104f 100644 --- a/kubernetes/templates/torchserve.yaml +++ b/kubernetes/templates/torchserve.yaml @@ -5,6 +5,9 @@ metadata: name: torchserve labels: app: torchserve + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8082' spec: ports: - name: preds @@ -13,6 +16,9 @@ spec: - name: mdl port: {{ .Values.torchserve.management_port }} targetPort: ts-management + - name: metrics + port: {{ .Values.torchserve.metrics_port }} + targetPort: ts-metrics type: LoadBalancer selector: app: torchserve @@ -46,6 +52,8 @@ spec: containerPort: {{ .Values.torchserve.inference_port }} - name: ts-management containerPort: {{ .Values.torchserve.management_port }} + - name: ts-metrics + containerPort: {{ .Values.torchserve.metrics_port }} imagePullPolicy: IfNotPresent volumeMounts: - mountPath: {{ .Values.torchserve.pvd_mount }} diff --git a/kubernetes/values.yaml b/kubernetes/values.yaml index 8e427cc122..8a82b4d085 100644 --- a/kubernetes/values.yaml +++ b/kubernetes/values.yaml @@ -7,6 +7,7 @@ namespace: torchserve torchserve: management_port: 8081 inference_port: 8080 + metrics_port: 8082 pvd_mount: /home/model-server/shared/ n_gpu: 1 n_cpu: 1