From 1d4e8deead0e3a07daf903c2d29d53bd10d59f7b Mon Sep 17 00:00:00 2001 From: Jakub Date: Mon, 12 Aug 2019 17:01:11 +0200 Subject: [PATCH] Sacred (#44) * added Sacred integration via observer * update docs in `get_filepaths` --- docs/_static/images/sacred_neptuneML.png | Bin 0 -> 11735 bytes docs/conf.py | 7 +- docs/examples/examples_index.rst | 1 + docs/examples/observer_sacred.ipynb | 150 ++++++++++++++++++++++ docs/index.rst | 8 +- neptunecontrib/api/utils.py | 2 +- neptunecontrib/monitoring/lightgbm.py | 2 +- neptunecontrib/monitoring/sacred.py | 157 +++++++++++++++++++++++ pylintrc | 2 +- requirements.txt | 1 + setup.py | 2 +- 11 files changed, 324 insertions(+), 8 deletions(-) create mode 100644 docs/_static/images/sacred_neptuneML.png create mode 100644 docs/examples/observer_sacred.ipynb create mode 100644 neptunecontrib/monitoring/sacred.py diff --git a/docs/_static/images/sacred_neptuneML.png b/docs/_static/images/sacred_neptuneML.png new file mode 100644 index 0000000000000000000000000000000000000000..f7a9c87ace87c91f0946c38139092e26080893f1 GIT binary patch literal 11735 zcmeHt_fu1Qylrd;6%j=Nk#p;zgIP^_pRpcH{X=)DAKA_;_} z2odQ72t9`)Aq0rD1PJAB&%HD6-Ffd1cyH$Zkg_M)v%mYh*Qc$u-k2Keah$z+76bxu z=s&n;4gwvw1-`RR{{g())<2C0KF&OTU>gJiv2*=?9UGdC-UosH1nJ+qYZ;!sI?X=i zYH>hVR~C_nUv4(yOPBvCWbj?=YTU(6^ks978i!a9(d%&mHw*Zh6yikEXSMu_Mcl5P zd;d;kUg*-iU1#x&#iKpH{)iR#+(w2Y>oyWi$v%jjsUacMqAA&}B+aCwqy4?=#%Sc< z`@5SIeqcS8%O@hyz~{*IuHzuk>nEb@zVt1Gp#G%GVEXXLS%ebU(9D{Ul(TVX$DaY>&ClvkLIoK;TaVDj=p!>SZ2stP=pEk?i% z!v9{}bfhFs@`G|QXT_b0oaP9`cI;@`f9*zW$=S-H7u6$Ye%y3t`qknLN673IcYj~- zv>l;{!7id==igf-Z1uVu;|Wvl4lUu#U#F>SFCnRw+pfQUhhOlEfm1V=I)#&!_|ftq z+Cv}fU-RTOc|lURs@_ri=G&OnF8kQ?Fq45=_rF}n{M4^?Z73JlT*G9@tqnPK&c$0R zcNW06Rs7(t^VI_EPafxU{|Sv|j6I59i?P<*`lcSG`wm)Mz3k` z=2zKv2Jxc)=#yh7t~x_IxHmgsd)+pX>lK|g!BsPzzEwUgY4BqgWE+kzU;{P|6K$-d zD()Cu)|pT#O1bB~atx{Rtu8~sM@tae^~37KPym?~KSzaQ6IYRr%-WqfXi{z4AqOb+ zQu0X-p@uTP=8jm7^#o1(7EC#}MQr|v0xA4ebR;!T+0BUW6Yr?YF%NHMt5Uxaa?F8pVDy6!$>|YFN>IsS7o)V z!Ph&wZEuX;|^Qksqb_nx2OkjDtCG|!<;Wj&peTphbk`zY&^ve`%7GyIi0eCEC8@dIJY%kK zxxEM1uHKR)@ZL$kyE52z>Mv~6N8CF}gIuwz2*R@ORQdpShrJq)(h#bW<#^sHtk{b* zrAqdCU8o6cI^xWSDzjf$rPmwuz%=dhOP~JM+yXluJvOfHF&!MB$ zK~7YB+z(k4Ms zOfoQ^nDRQF=R9a4mZvej_~k}f%|MZa=f9@Thx* z<$L=YI!TYN(E(s&@x=#J!3_xYlyeECNmTVubc7V)_@7Q5b%jOOi(K!156;%-O7*I| zlW!5053P7ddp*6${z%g``WV(!*x0I(=>UU zS}NqRCA=^oOeyua3uVgFSH+8tyBHXqKyjTh@OWk&o;^My5mz@D(b44$B|TZJ{RNho z$qL$=3%L?6H`8PD79J-f*q@`Ql%rWf8^cq8Oap3T)8UP0Gde1MDXs{GKGiN2>K2U? zw-t4Y6l7?|E@?D827wyT1A@`6Qg`y1N0k+4FAqH{@G0;(yB3hagQ~iAK9qOyTfN0? z43j|sa)xWChX5_bVLh1p#(&?{4(BRY$<0i*6Q#QwFPwCCMWwc|m413p1IJ()m%<<^ zyZ?PJi76uMR*9#ckXxQ)y~q;3l=?19I_qr7xpv;Qbd$>=N$v0A9MUwzwxhC*yQ>GdU+fyS-n#eZoE=^adsHv*;grhOa614b7IcVG3bmqtNr)gDgm(-vOammt?sg`LqZXUDy ziB4KS5OiHg`Enzlg@jT|EHl_BIqksdX#B@r$EnU#)9EPY^x__^MwLm9X+zAumIbrNB%yU`!7&*zJ*z@)svNj zqqtGncIY~?N|$aGuDf=uoO9Pa68!jnJZPJ1!zRvk_1s2(bFOKl7FH?*AE`M z&eK~q6PwpN`BTQ$3Bvws%@mwh!LS9ZT4|`z;ckBn%22LV3E}hc z%O8tuT02h>^9iabp?Jg09Z>(8l66%|=Tby<9B`1R=`=-P&M2(cQBLk@h~3fEpln6> zbEOVwTI35(@@w)U265bh{p|ygleTtAX{ylI&W-r6+1XdZe$kBj+{`d?lqy1r)~iZZ zl0XNWD@>2A*l6=6vgQ!|@V6bjH4$%DA5V)oP^i5&poK|)x><8g68Ld{tPNkS{htj5 z$<1<-^(q(R}pq{jbTB0H_DQH!$~Yb+fuA^FJgU(VUJH zw065pT6jNpiDzXz=hPglY8}lW&3nf4IUwF+C0XwL5ZoG3E zwj~sJ^yQj7>BGxtahD%;Z52P)zG@w=C%G|J#yJS@!Hlu??3lw{@SgDad|6cc>x?PO9qU|kLmZZC|9I1m_UB5&!lQ%WXX z5N@bc`T);q1C<}5w$&)f2It1Y(+5UNvV_C35A0-rELEJMl)v{dL0Ahf@1dw21nIWULrL(Pg__xuM1y0fOo&IY>E=8~1nb93@|{zzKYzm`QcIlvx2JwTQR++y?QNj zxxDmhqKH)0$7g&BEYlbeYVR-BG+Nta5FkhnSyAW@$;x>p9DQq-31lC& zOuyl05sgnTAS-jWNls*+`d$^c?GbBL;hj7E4ywV@*irY~sL`Gb`Rlt(t)tz3x4vw( zuI6Cj`ytg%@C4;#(x)-!_=RS)WC-kNp(_RXt>KSVcBdWfFy{(7J(l0^fg^5zqr0~- zNfWj5eb{pOqMpEe(OUdEes8(5YqM?tr(74Rx$R)pU*8C}#l|U=-BPknjr6W*4Xryb z8U@vD((KwB#6T|Av_TdEu_0)oU>S*6kh)ZVk1M!|w0~MA7r%0Wxo2 zksI6{x9}e*g9!f&f@4(sIj8KK?5U+M=PfXqW;<$-W3i%_zo2KNe2{qo<$}6qCY1{D zWZNyrgo#zz;iN-7&5if5{oU{+@P{hW1cHJ_3S37DkZU*RyaFRepw^jZE{SI#>8+}7 zq;7T%wO;!9L#oDT&u_JrIkWhAKGD$bK`LMOC6n8ajBm}J3|eg@q0${A%4cueVrSP& zviwle%Ve+0>TFUQ9nVsU^lO?9RkSEElCNpmh~raqGaAYs2^I_gz{MvfN8agF%_etu z@H+VzYpgbimMr{ebnG)9DlS?bgC{9F9d4xa^Yq}jcv*Bg{7!+|Vg@{Uv;1OpH$9q( zD+)tJ9?+aryXD$i=3ynXKoVxWntmBXj_jUNXL4`uP12QGtkOv2Xwp9UQonLYgX&B) z>22n29?-bIizQVwtP}_|nS=WYF(tPI5FqS(kesG$SlCeD8Ccdcd7+uE8}KTR4agP|P| zrv5%L_s}JJGm9K-Vz`)~dt~Wcyh;6olpnlGEHzPWnu+ipD%xHuOj3!L-HX1i>C~v3 zCIRil14xk=k>X7-(|2aD%ung7I$3Q?$<-PVnWZelwYqNE#lKH*8gc1&Sqay_DNAkw zGp7`r*TXIz9JtPXdo%N5YPY7MiKWqh<LCqD$}aYQ{a@>{X@vHN7kT zcAuMf`-eKMsy_Lq9N0nY1zzwGHG}IoBw#wQTLu!AW_>N_?_+GDt=O$!Xp1B%%;&F> z+m%i-X%CBUj+=AJi+SO_4Ul`((%ww>oXl*J9=S2OYKvLy*|XY2o-xW&k-7EdN8;ud z(Zr-8GGgfpzU`Hyjn`%VyX}CG`p>Qcrn8_~mn15>%j!ewov1xxuCe4j&Aq!;I_8Zb z7pfD?x9<_*p&ZGTfoX#Aqq{q6ugaF=x$q{DbC>i-9(1Kv=$BMFr<8a~vrr|Qbrj1+ zBTK$I8m^AV-bcP3c-Emc?dH#1Qp%fyD4}ik`uQ{I2hp%#-(|i<18o_|pw~fqaetYe zTPdWzK?p)DvO^X(!Z~mMnmM|AG{@WaV8XS?a}RnyQNU9Z$UMv|HISLk1b)hDd`WE- z)zHa6X!%zS{T@_@*cdC5N^3htis8hBe>%R%!c2DQGd2B$vM7#|r{S2GMcw+90(OhR ziZc*CN3HfFZqZ1Z9t=6xA2`FOV+Q#G}{%gAVKXDQ{@mOPy z>PtiJKTcYPf0ep3A|a8#dPBQJG53(L$sy0n)5=>3WN#w$?$F$#Ow-g5jyN#%%x^G4 z5{zd})5X~_(4XeIJ;5?p!_dUwRUS*#aG4v{uDaZUP|t<$*OQdGWpO@9o8w+1K>Ilc z5BL-cK3ukLGsuZqk9S%z6CAu;AgLqXm~*37c7fC|qtGo)oL9zux38h*Prsy~!yM~` zdcz>LM~BQdT$?p?als;Y&vK)ml7s9!cU9>|`RF*jETW`i!_#g+7AVm+b$tdFZy5)6 zN^*NMmTBtS!xp$oOUBfM=&wogp~|lP*gBBh6%m-pKV8Sq7GoLypT^y~f>`Kffr3*T zY=Xh>5I{P+s!88--0NQZYOnwCnaA?4nRH4fBpp_8;(q+?SykI;g{HHYBWl_PiZId; zGIVek9PE|)J5&&K?eWL$94SS*?4NwEaciGRSaT|CyoYjDfLU0z$8v1uSZEC!l%zhs zQColJwf73vHz|^4PhK{DB3(^afYDy8`01c6EV8`VW(!XK{K_t868e@g^VUG9 zG0S-M+7KZ$VNjR-TL{rtEKfX5srit{ZZi!ct)zQXYi}?K>;DanHYUn9{^=xhv(5rx z@I+axj0Kq}*PaZ#rV&-)JzNlK_ACwES1+TOIt<=jNa+eLOY&}d7`hTN2@5A1CcL>l zWx0L5tAcI2T5<-R?sV#GIhW`As4CA-??Nvx&p8Imb-pV%gZA2NIaYnZvHLn`oNgRl zW|5Dgxhz`e6)`gvzz`$ zeR}9_nuxWov3^$)Q0;_pCpYoD)xw)%R)`TED7r!aA#U|kkP!LMLSz4}dv!|k?G%GO z2g-0j1ICD*RZkcoL$cUYUEWlf7`%XKvc~(x)4s0M!YD?ns1RyDj{WJOs7u&P12ybw zkhad}$sSx>DZ8Fu?IWEk$ytd00n^!MTjuF92Dk@^mmUdd_A)I?wg!?KN{&I(eyD(Y z3*8EUrU+?X?iDI+3v}DTNiTGiqRRBtFAggepu(_a&<=OaO+JFuT9C>mRTxfLS&VAn zM)Yf0?_78fkxac+{>;5HoC^P(SO{U4H3(g0jcgjpT=OX$R1$k`aC>`c)mV#+q_yBo z*|8U6b8NRS*5M!c3PtL_p40XH)5yBzixO7QkLi$de{(cO)Nw zC^=2^&VO5oEu-ui!%~$>{6E6JVcLJx`ATi{6)!ug=y&7ROfiBl$KmO_{0;RXn%M9< zTk)WUnY5B%@woh|S4m<|5~eU2@A zRYizvto)c6H! z5oKP=o-p~%8Rm^QIMua#x5w?<|w+hYhyu2h?>`Fl7d{vxpctsEkOQR`6 zXB>#3;z5EU#8AxiGuL^HXJ=@sV2~b0V=7yorY~v7XYf(nui+19)zz~QQ-<2k$=}qt zxOZOa!_}Sb!e^uK)moi(Uqpl<+`L%7>tT*XKYsY!!Pw9=d?;i^QQRFG^W*!)bsjni zQXR(Y7S_hyu{~d9zcvNX3i@?v#fQ0|(%-e|%rMK= zR%L;j?i|?veP-Z1p(gUD)EB+Ss}8{(-2wz=)JRC!?Du>_G4I5_+=I8x%wojiZVaw; zkQMLBV>|ZNA-a5F=B0FaP3gc%-7_ORH~~~cRQTRF?^$3jDH*dH5s3murk*0FqY{yv zkne++dYXA*Su2-^l^#7P@Hp$eArGv=y}tqu^s1}e7;+zx-J5Y@ywZ{N@$ zY~k_D*3&Sr;{)fgM~+jC_I;|{#V31YBbwspIRWIx zp6phcoC1dF6avsnX9xe;6fihE*G$QwLF!g)hV!F<8I!(tH?8E<_Fg2mxspLqUm#aB_-oQi-JVv6p3uWe`Idwq>4Q;Y*C8_Dtg-2j-;0+gUSlbAS(JhA zH!jjmc?Ii#fpOkaI9zY%O5*M5(VwKY!;qm>F#TYBo|YRnL!E);;|&cmwuHp2BFc`e zP{{>A=Y1Qo6iZmXT_c`X@t~M+C8F10xlVAQy6tegX0g09=j@@Q_M9eyoF0XVwqtzK z@HDUk#(*CSkI4R)E|C3N09to_#LB*U_I62Sa`DV`-mmF0U6UTSd5#>FS5`Nm_ogxRtut%`6OSyZqq!OljgmgiEF4 zGOQ8k=)^fmSF)NNky2UGGT*e0_BYXluqZhM{EY3+8o$#XYf$Hn$0(nB_WRqzy_!fP z(v>Jm!;$5PQA~WWd!%7&|JL-nu9=J&8O~^^@EQ#) z7mTb{VLVw*)`iyYU9(*Y7JTo*@mML?$fUohoJ%=+e`oD%O37hx?j3MLkg0e6aE=I0 zx{yR-%!;bC7~R)*sI)F+`x7;e?1B8){_z2@8xBYG+3w?W}V>>W6@!` zYVIBd{TAHr3*J)IZ*y7|2nI7ciJn+1U&(>?PiYTo(uOYA39$)0ev+vk;?+(eI*>92 zasFuQy;dXc<9;Fs#Z`zJMBoKezamtB3bheXq7o}Th7MPvIt!_K%-*$%NZj9~$4vA(H) zFu=VRl{Ph`SLfZlv84Ixl|bNwjYN7RMFTujergA+)+i9YGus8JeRYm6IXet!5T_|- z3T|!bt&E9ohnRzHx21u64l?`(iREiVASspLO}fST47`HlufdZ1K37j3FISH3L2Ib; zkzVpTW^7$igtd0Aga9FfnC9n9?HQ;2W&&_YkFb5NZ4`iOP}wz3O{D^3TToKwtpMi@ zy9c}xq>v8@IA_pIN)^BL;J=TY9$~ZuD{UfeT^oZY;u1*7-HJt z)%Rpq@RwHY-sTW5U>u10?FG;YeLPNLO!{kWb~_!`&0cBEM?b~(&r&p*&1*pwlGpit zKHk-tH=n*>Yw0@3Q2}kn9v!k$gv0GSx(Z!<07`@&fDToi2gnrk)Y63euYbPiYv=cS z>E%XvHVXiWc5&FkWHq{@(YRZEvq!P_DVxw3m?xz|eK&N<^Q~R_q8Ip)XO)-9qf4R* zBVmTu#h?ZQ_YxExWoR}6bb4e|=h z5?X$r=32^C)5@?5ntLOGBARG{GvkaTY$~hj3=}v%NJOERm5X1k<(BcoSU1cNz{Hw@ zWbJZQ^{Pxt>(@n+YvR2n@@W2#a%ln+ciu6>26lp-I(eNU8(&bj*ULWX8(Bh#OlzJqR`e0SuzX1&I3W?@=UYz@LW;_8qtw#5^7#Eeiy+F> zJQ>*xf|F{7G1KKoVgD*X(sbz%o%I9)UNEQewBgISB_L&JiMI@VAn&ezU~&I%e#!rq zlMqQ(oqUCeyVgkb2Pot|wK}o{z>2!WnI`2PZk0|8{;Y@dX-0+*)#)6at6i?B6z7C? zhEDo6x#*Q8`CI=9wAo%uYz|rrh&LaiR~p63_V44GkrF;L3a6#cusMXd^{ed$}8rhLp(SSw7DA z-Clpx#q0M)EsQ$X+cUrvpx9(rABK!TX!B4^ zti6A$a(S?8f2L6U1ZSDd%ZPyKU?N1{$Rda_rC!sQzinIKsUov)eQnO5FL>YegKde3 zY{j|ddIb5Y_uCJ6e=+-lZ7V=CS&_G5MIUgap7uJ(vU}`t)f8yUl)+SWe1bP5J-Ies z4*wYkeQ}^c^DqCSL>;pR@x+!q-5&2sk;n>|@7(q{S1FmU0vY4Eh!2YW28*Aa?F@c& zRE&NO?}T`GvA0b<71$`Y)P$(>>J~CxYHz?5SvL$bEWgp=UWew1{=n2#os^-IfsLXI2_sJ zR6ed&_=gP0r&U#y4Hd~!$4`v|yo2DYP^O(?L&*b;(IM{|ONjaQ4gX$K|8Xm>2@%i! zYT7_*UhBbIN&8jpdFac*883p>=$0UrM|-)7%&&gFKebxJq|8qgYE{G={F zhq(e@C&1)8=Ms;BmTTas_kevVI63h_-B5fLjQu?Ap zEkHxfly^Jo8~-oW2v^g29JC~??gg=W8y+eHk?NzyHxd7px3A1+p!ZclPeK_rh?=ba zw^RL*^S^{Q!z7>boL3952E>s0-|h_*N{T-WLU1M9tIPlkxFcbfygi)V+;w91_fX@z zr1`>?!zuF5m^+pRXF6U_a~66%ccf(~+say2F3Wei<+>-qbX%bCr<3Z!f<9|VM9!wN zaMT}V$;nkL;RtC2sC7`@yRI%tA%GmA6%oi``IASwIc}z9&CjY|2UUA5u;I;d6 zho4U~Ja0UBT@ptb9fOKvKgTE6(ra*a^V}d0&K23MeGu09;J2Cyg})8xF+QXpqw}i;-&Bk~fCft0 zbxYTM%=|ofwTf_jVDw-0w&O~?)>_=|+@_$$){o9rEm(kchqNfLAAbTlk2JkObejOo z4wE^^VpeNKu&;v48_|n8Q63z_JUC%DmC47xii=qqS@XuZK7P%c(e+;gyHRHLzCP!7 zJNst5(Z4Fo7g79vCZi_Q&B9L;-&X3ot-v$Hta-WSQO0zC(r%w-7c!&;eP0-E<>1G+ zS(zJMGplI#SIm)4QWn+1SW#H$_Ki2IN_%XJPMb4-SsSbO86CFqpC$NsW~bDA)>O-> z`Z%TFmJ4fADa1Dfv?J^5C@sI@_>;rN=#M&net=+H=j2LJ-qA70cJHOuW1G82d!Q$p zMV~;+UtD6gwoL;IVo(_2Ln#SCGW?0kpjkmd|IJ;~Yk*RD;F{!5mZQ4d?|OF#KG^H# zbeR0rG5&2O$CDB&A*E6hbThypmz*P!^-+*fSf9M`J=!>uw3HlXoZ!7&G~d%#n52)9 zI6&n{-8GW1(s36%;WA8wq1Ci`f+VkLw3n2pxLGhi#QQfQXa zZfEv3Q>{NNMvJkzCu=!v;hYu)k&kLy0{?c6d^G9I(0pYmNP_Lofo4jJ+O15tp4Z$| zAKXF}i{8<`zMcipUI@z_9k$BLjIc<^1 zP*jn}{k45~TOD8hf;zEE*#<8+)0D*ms0REfVl7j)N;1b$Cndq1G`^E)_S;ne$c_MP zpVbct;2DzzRH?Ki9e_>I>PodR(6L{gUdHQ2t8k|itZL< zX0jCEQrJ#uD1B&%95CY@YRBD619BGg`iN8XTiUJrBxKzi8^=H%sTphdG1Hu65UC-L z>y*q4<^+TfD`|l3^x@`Q1r&|{74w?pTAdo7+gTTi=P`m&fK4Ic)E(tz{AmpwNDy6I zhnPafifZff&ab+%G^_iiBN(7K>)W*vzU7u179YvS(Ml{n3x34W9=d- zxsy?cO09&oA@2{JK}XM%#JuZ Hyper parameter comparison Log model diagnostics + Integrate with Sacred Monitor lightGBM training Monitor fast.ai training Log matplotlib charts to neptune diff --git a/docs/examples/observer_sacred.ipynb b/docs/examples/observer_sacred.ipynb new file mode 100644 index 0000000..1a32045 --- /dev/null +++ b/docs/examples/observer_sacred.ipynb @@ -0,0 +1,150 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Log Sacred experiments to neptune\n", + "\n", + "![sacred neptune.ml integration](../_static/images/sacred_neptuneML.png)\n", + "\n", + "## Create Sacred experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from numpy.random import permutation\n", + "from sklearn import svm, datasets\n", + "from sacred import Experiment\n", + "\n", + "ex = Experiment('iris_rbf_svm')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Add Neptune observer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from neptunecontrib.monitoring.sacred_integration import NeptuneObserver\n", + "ex.observers.append(NeptuneObserver(project_name='jakub-czakon/examples'))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup config and run for your experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "@ex.config\n", + "def cfg():\n", + " C = 1.0\n", + " gamma = 0.7\n", + "\n", + "@ex.automain\n", + "def run(C, gamma, _run):\n", + " iris = datasets.load_iris()\n", + " per = permutation(iris.target.size)\n", + " iris.data = iris.data[per]\n", + " iris.target = iris.target[per]\n", + " clf = svm.SVC(C, 'rbf', gamma=gamma)\n", + " clf.fit(iris.data[:90],\n", + " iris.target[:90])\n", + " return clf.score(iris.data[90:],\n", + " iris.target[90:])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Go to Neptune app and observe your experiment\n", + "Now you can watch your Sacred model training in neptune!\n", + "\n", + "For example, you can check [this experiment](https://ui.neptune.ml/jakub-czakon/examples/e/EX-263)\n", + " \n", + "![image](https://gist.githubusercontent.com/jakubczakon/f754769a39ea6b8fa9728ede49b9165c/raw/ae86f7321113327602be89c6ed3ac9d618ffdb4c/sacred_observer.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Full script" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from numpy.random import permutation\n", + "from sklearn import svm, datasets\n", + "from sacred import Experiment\n", + "\n", + "from neptunecontrib.monitoring.sacred import NeptuneObserver\n", + "\n", + "ex = Experiment('iris_rbf_svm')\n", + "ex.observers.append(NeptuneObserver(project_name='jakub-czakon/examples'))\n", + "\n", + "@ex.config\n", + "def cfg():\n", + " C = 1.0\n", + " gamma = 0.7\n", + "\n", + "@ex.automain\n", + "def run(C, gamma, _run):\n", + "\n", + " iris = datasets.load_iris()\n", + " per = permutation(iris.target.size)\n", + " iris.data = iris.data[per]\n", + " iris.target = iris.target[per]\n", + " clf = svm.SVC(C, 'rbf', gamma=gamma)\n", + " clf.fit(iris.data[:90],\n", + " iris.target[:90])\n", + " return clf.score(iris.data[90:],\n", + " iris.target[90:])" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/index.rst b/docs/index.rst index 28067f3..f358631 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ neptune-contrib: open-source contributions to Neptune.ml =========================================== This library is a collection of helpers and extensions that make working -with `Neptune website`_ more effective and better. It is build on top of neptune-client +with `Neptune app`_ more effective and better. It is build on top of neptune-client and gives you option to do things like: * interactive visualizations of experiment runs or hyperparameters * running hyper parameter sweeps in scikit-optimize, hyperopt or any other tool you like @@ -14,6 +14,10 @@ Enjoy the following integrations: .. image:: _static/images/fastai_neptuneML.png :target: _static/images/fastai_neptuneML.png :alt: fastai neptune.ml integration + +.. image:: _static/images/sacred_neptuneML.png + :target: _static/images/sacred_neptuneML.png + :alt: Sacred neptune.ml integration .. image:: _static/images/LightGBM_neptuneML.png :target: _static/images/LightGBM_neptuneML.png @@ -86,7 +90,7 @@ Indices and tables .. _GitHub: https://github.com/neptune-ml/neptune-contrib .. _Git Issues: https://github.com/neptune-ml/neptune-contrib/issues .. _Git Feature Request: https://github.com/neptune-ml/neptune-contrib/issues -.. _Neptune website: https://neptune.ml/ +.. _Neptune app: https://neptune.ml/ .. _Neptune community forum: https://community.neptune.ml/ .. _Github projects: https://github.com/neptune-ml/neptune-contrib/projects .. _Neptune community spectrum: https://spectrum.chat/neptune-community?tab=posts diff --git a/neptunecontrib/api/utils.py b/neptunecontrib/api/utils.py index 87bdd58..ae188c8 100644 --- a/neptunecontrib/api/utils.py +++ b/neptunecontrib/api/utils.py @@ -197,7 +197,7 @@ def strip_prefices(columns, prefices): def get_filepaths(dirpath='.', extensions=None): - """Filters leaderboard columns to get the system column names. + """Creates a list of all the files with selected extensions. Args: dirpath(str): Folder from which all files with given extensions should be added to list. diff --git a/neptunecontrib/monitoring/lightgbm.py b/neptunecontrib/monitoring/lightgbm.py index 82fcde9..7769d3d 100644 --- a/neptunecontrib/monitoring/lightgbm.py +++ b/neptunecontrib/monitoring/lightgbm.py @@ -26,7 +26,7 @@ def neptune_monitor(experiment=None, prefix=''): `train_multiclass_logloss` and `valid_multiclass_logloss`. Args: - ctx(`neptune.Context`): Neptune context. + experiment(`neptune.experiments.Experiment`): Neptune experiment. prefix(str): Prefix that should be added before the `metric_name` and `valid_name` before logging to the appropriate channel. diff --git a/neptunecontrib/monitoring/sacred.py b/neptunecontrib/monitoring/sacred.py new file mode 100644 index 0000000..7a87df1 --- /dev/null +++ b/neptunecontrib/monitoring/sacred.py @@ -0,0 +1,157 @@ +# +# Copyright (c) 2019, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import collections +import os + +import neptune +from sacred.dependencies import get_digest +from sacred.observers import RunObserver + + +class NeptuneObserver(RunObserver): + """Logs sacred experiment data to Neptune. + + Sacred observer that logs experiment metadata to neptune.ml app. + The experiment data can be accessed and shared via web UI or experiment API. + Check Neptune docs for more information https://docs.neptune.ml. + + Args: + project_name(str): project name in Neptune app + api_token(str): Neptune API token. If it is kept in the NEPTUNE_API_TOKEN environment + variable leave None here. + base_dir(str): base directory from which you run your code. + source_extensions(list(str)): list of extensions that Neptune should treat as source files + extensions and send. + + Examples: + Create sacred experiment:: + + from numpy.random import permutation + from sklearn import svm, datasets + from sacred import Experiment + + ex = Experiment('iris_rbf_svm') + + Add Neptune observer:: + + from neptunecontrib.monitoring.sacred import NeptuneObserver + ex.observers.append(NeptuneObserver(api_token='YOUR_LONG_API_TOKEN', + project_name='USER_NAME/PROJECT_NAME')) + + Run experiment:: + + @ex.config + def cfg(): + C = 1.0 + gamma = 0.7 + + @ex.automain + def run(C, gamma, _run): + iris = datasets.load_iris() + per = permutation(iris.target.size) + iris.data = iris.data[per] + iris.target = iris.target[per] + clf = svm.SVC(C, 'rbf', gamma=gamma) + clf.fit(iris.data[:90], + iris.target[:90]) + return clf.score(iris.data[90:], + iris.target[90:]) + + Go to the app and see the experiment. For example, https://ui.neptune.ml/jakub-czakon/examples/e/EX-263 + """ + + def __init__(self, project_name, api_token=None, base_dir='.', source_extensions=None): + neptune.init(project_qualified_name=project_name, api_token=api_token) + self.resources = {} + self.base_dir = base_dir + if source_extensions: + self.source_extensions = source_extensions + else: + self.source_extensions = ['.py', '.R', '.cpp', '.yaml', '.yml'] + + def started_event(self, ex_info, command, host_info, start_time, config, meta_info, _id): + + neptune.create_experiment(name=ex_info['name'], + params=_flatten_dict(config), + upload_source_files=_get_filepaths(dirpath=self.base_dir, + extensions=self.source_extensions), + properties={'mainfile': ex_info['mainfile'], + 'dependencies': str(ex_info['dependencies']), + 'sacred_id': str(_id), + **_str_dict_values(host_info), + **_str_dict_values(_flatten_dict(meta_info))}) + + def completed_event(self, stop_time, result): + if result: + neptune.log_metric('result', result) + neptune.stop() + + def interrupted_event(self, interrupt_time, status): + neptune.stop() + + def failed_event(self, fail_time, fail_trace): + neptune.stop() + + def artifact_event(self, name, filename, metadata=None, content_type=None): + neptune.log_artifact(filename) + + def resource_event(self, filename): + if filename not in self.resources: + new_prefix = self._create_new_prefix() + self.resources[filename] = new_prefix + md5 = get_digest(filename) + + neptune.set_property('{}data_path'.format(new_prefix), filename) + neptune.set_property('{}data_version'.format(new_prefix), md5) + + def log_metrics(self, metrics_by_name, info): + for metric_name, metric_ptr in metrics_by_name.items(): + for step, value in zip(metric_ptr["steps"], metric_ptr["values"]): + neptune.log_metric(metric_name, x=step, y=value) + + def _create_new_prefix(self): + existing_prefixes = self.resources.values() + if existing_prefixes: + prefix_ids = [int(prefix.replace('resource', '')) for prefix in existing_prefixes] + new_prefix = 'resource{}'.format(max(prefix_ids) + 1) + else: + new_prefix = 'resource0' + return new_prefix + + +def _get_filepaths(dirpath, extensions): + files = [] + for r, _, f in os.walk(dirpath): + for file in f: + if any(file.endswith(ext) for ext in extensions): + files.append(os.path.join(r, file)) + return files + + +def _flatten_dict(d, parent_key='', sep='_'): + items = [] + for k, v in d.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, collections.MutableMapping): + items.extend(_flatten_dict(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def _str_dict_values(d): + return {k: str(v) for k, v in d.items()} diff --git a/pylintrc b/pylintrc index e6dc4f6..3408fa5 100644 --- a/pylintrc +++ b/pylintrc @@ -41,7 +41,7 @@ load-plugins=pylintfileheader # W0511 Allow TODO/FIXME comments. # W0703 Allow too broad except clause (Exception). # I0011 Do not show Locally disabled warnings in report -disable=R,C0103,C0111,W0401,W0511,W0614,W0703,I0011 +disable=R,C0103,C0111,W0401,W0511,W0614,W0703,I0011,W0613 # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option diff --git a/requirements.txt b/requirements.txt index e36ac96..c2bde92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ scikit-optimize==0.5.2 retrying==1.3.3 hyperopt==0.1.1 python-telegram-bot==11.1.0 +sacred==0.7.5 scikit-plot==0.3.7 seaborn==0.9.0 vega diff --git a/setup.py b/setup.py index 781bffa..51c0f5c 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ def main(): requirements = [r.strip() for r in f] setup( name='neptune-contrib', - version='0.9.2', + version='0.10.1', description='Neptune Python library contributions', author='neptune.ml', author_email='contact@neptune.ml',