From 96ec18a81106a3f1584158f5ccc2e5f0e3e2a102 Mon Sep 17 00:00:00 2001 From: damienkilgannon Date: Tue, 31 Jan 2017 08:35:52 +0000 Subject: [PATCH 01/14] initial commit ui feature --- .gitignore | 5 +- ui/index.html | 76 ++++++++++++++++++++++ ui/js/app.js | 28 +++++++++ ui/js/controllers.js | 33 ++++++++++ ui/js/services.js | 11 ++++ ui/logo.png | Bin 0 -> 8799 bytes ui/pages/crawlers.html | 5 ++ ui/pages/debug.html | 5 ++ ui/pages/kafka.html | 5 ++ ui/pages/overview.html | 50 +++++++++++++++ ui/pages/redis.html | 5 ++ ui/stats/stats.json | 126 +++++++++++++++++++++++++++++++++++++ ui/stats/stats_schema.json | 33 ++++++++++ ui/stats_api.py | 18 ++++++ ui/ui_kafka_agent.py | 76 ++++++++++++++++++++++ 15 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 ui/index.html create mode 100644 ui/js/app.js create mode 100644 ui/js/controllers.js create mode 100644 ui/js/services.js create mode 100644 ui/logo.png create mode 100644 ui/pages/crawlers.html create mode 100644 ui/pages/debug.html create mode 100644 ui/pages/kafka.html create mode 100644 ui/pages/overview.html create mode 100644 ui/pages/redis.html create mode 100644 ui/stats/stats.json create mode 100644 ui/stats/stats_schema.json create mode 100644 ui/stats_api.py create mode 100644 ui/ui_kafka_agent.py diff --git a/.gitignore b/.gitignore index d4ca88ee..82d55d87 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,7 @@ docker-compose.prod.yml docker-compose.override.yml # backup files -*.bak \ No newline at end of file +*.bak + +# IDE stuff +.idea \ No newline at end of file diff --git a/ui/index.html b/ui/index.html new file mode 100644 index 00000000..622c4621 --- /dev/null +++ b/ui/index.html @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+
+ +
+ + + + + + diff --git a/ui/js/app.js b/ui/js/app.js new file mode 100644 index 00000000..f99759e9 --- /dev/null +++ b/ui/js/app.js @@ -0,0 +1,28 @@ +angular.module('uiApp', ['ngRoute', 'ngResource','uiApp.controllers','uiApp.services']); + +angular.module('uiApp').config(function($routeProvider) { + $routeProvider + .when('/', { + templateUrl: 'pages/overview.html', + controller: 'mainController' + }) + .when('/kafka', { + templateUrl: 'pages/kafka.html', + controller: 'kafkaController' + }) + .when('/crawlers', { + templateUrl: 'pages/crawlers.html', + controller: 'crawlersController' + }) + .when('/redis', { + templateUrl: 'pages/redis.html', + controller: 'redisController' + }) + .when('/debug', { + templateUrl: 'pages/debug.html', + controller: 'debugController' + }) + .otherwise({ + redirectTo: '/' + }); +}); \ No newline at end of file diff --git a/ui/js/controllers.js b/ui/js/controllers.js new file mode 100644 index 00000000..b073b03d --- /dev/null +++ b/ui/js/controllers.js @@ -0,0 +1,33 @@ +angular.module('uiApp.controllers',[]).controller('tabsController', function($scope) { + $scope.tabs = [ + { link : '#/', label : 'Overview' }, + { link : '#/kafka', label : 'Kafka' }, + { link : '#/redis', label : 'Redis' }, + { link : '#/crawlers', label : 'Crawlers' }, + { link : '#/debug', label : 'Debug' } + + ]; + + $scope.selectedTab = $scope.tabs[0]; + $scope.setSelectedTab = function(tab) { + $scope.selectedTab = tab; + } + + $scope.tabClass = function(tab) { + if ($scope.selectedTab == tab) { + return "active"; + } else { + return ""; + } + } +}).controller('mainController', function($scope) { + $scope.message = 'Overview!'; +}).controller('kafkaController', function($scope) { + $scope.message = 'Kafka...'; +}).controller('redisController', function($scope) { + $scope.message = 'Redis...'; +}).controller('crawlersController', function($scope) { + $scope.message = 'Crawler...'; +}).controller('debugController', function($scope) { + $scope.message = 'Debug...'; +}); \ No newline at end of file diff --git a/ui/js/services.js b/ui/js/services.js new file mode 100644 index 00000000..beaa66d8 --- /dev/null +++ b/ui/js/services.js @@ -0,0 +1,11 @@ +angular.module('uiApp.services',[]).factory('jsonStats',function($resource){ + return $resource('http://localhost:5000/getStats',{},{ + update: { + method: 'GET' + } + }); +}).service('popupService',function($window){ + this.showPopup=function(message){ + return $window.confirm(message); + } +}); \ No newline at end of file diff --git a/ui/logo.png b/ui/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..71faaf639435153db15175a24a91776dfce9a62a GIT binary patch literal 8799 zcmZ{KWl$T=^LGLPf)k*)1Oj|RTO5i@aF+rtv`{2yad)T1Apu(4wWSR$#k~Rqr?^A0 z;8MKUpWn;p#WOd1JGVD`H)p$hpV>HlT{UtNCK3PuK(3*#{PdpN-_tWN;e9=7AA5Aq zK=ue71OV_gh4k8r;Jyt-sz22M00MacfbeJl;NQI{d=mih5d;9XEdc|jrLefYuP2o@M8wGg@?!4?2>QuBW<>Q|LW%won8OV2ua}yZGP&?e{cbb63 z3rI*FvDUQMKcGFZgYcF6)FNLd@~23#D1LH1k|rYSY|-uUG2Iwp0SYmNKLvc_D{xvT z!w8l2n0y+x@~I41q~Uwa=D72M(9E`ny5)^&LRRf!XH_S6=Qm|-Ceil0UaOTeyuOHs zSf|PPraaW45kLSq%#k)hVM6X)d(4Axj_#e`#nedy@p{7L@jFx!dA*e*d58#soU^a} z3h4xwGsh?-z*)5rtmremg>Wy~;UHdcB)aoF0-=x{D?ewRx$5fwcm&eelWhF^SL|me zgN({E2|@zs=Xl}Gc@9?Kp$sF}()r84{08dG7?xYbf%C^#HZ%2dvR8n4lKF53Lj~n& zU;1x!V!X!KPofVM=Vtye;4cTf@;rV&_+4({Oyi=x;83Qz#^T_1AXDV$T)XUH>YL9| z`zkI`+C1jt7@Zc~jC{W>B0pL8jHx2?7|WmCR<6xu^g;=`w z-!OKh?{6`L_s4{1yo&xIf`X(bCMZqsM;0&hU$wZpY`1>XlYxUHy~4u^!_O*{a-LwE z$C{JG|9m8V7pKh0YCh*$BH3cSHbUEz{@T$OmaK<~1gV`mQc)HW64BL}!nvdbpZ}~MywZ63DE$m5@#9rN&6M@> zlV$uMd3F=fJVhcxoubppXfUoYyR`)EX`u6jQ(LomdewnzkP;{!l%L_c*edzx8Rl>| zS9FB*0&GH@OWf+}MMlV0FWDDSP$FYa><*0&m;}b)JqN_Vu(HhRPq@>4N7p%5suQqc zs{B+|IBAzAyzghPe0TLhQ8a-2!Q6K|%UuiG_6b_awDA{}6SF9;$uBc!)J2}m1uwYl z532tR#NTn@?=Z-snTSsvtr^hOuKNi~ft`OS2$#4UCAkEmhD~A#TC2342Ys-42xtbl zgzH5}R!)x5O^18B;|f`p-DxfNQ)-y*qH4>;5BXZUALp#nw8*6;Jc=wAbkhA7nmj&( zA4(ET>~EgXEGjuV`x~M~vp^b5V5Y?NTEODve(tTCLH?d0Y~Q<(NV9j~3-7n5S_+U6 zppaGKU|Y)XZA#z0poqFoA04a7+!NNfrFIbO0SuR*>PTBSO}OwfquRxb0j5|XHut6& zdZTB?JoTc!4#Mbvq4h&9pBYXF`i{x|F@EMzZb1xU;j*=xGXEBCrCd&Gf#pbO#mPuP$e(n z`t!>M9+52VR%ks#Q4I!*b5>LICm;P(P#8G&034U}Who>+^HL&m=*~&>AA>&u14-}( z_`OPaa0w9JKXZl)XGYViq=;woqW@4mb&>nUdN&RBuWTFcie@5Xk}np>$DwsAZeljB zW76JwKR%QZaH*)RnHub*U0ZWg?H#cK|KTolY+HgBsMl@ChOwwb)e-lP0R^dKg)p!<$E%k_b&T4EkVkK4i(UI$=+&vP#pmy*{@_ zps(_xhSMW8a-ahWBt@l%g;UKuwaM;={&#?n;X0v-?SXr~l zC0D(XD_x3|S@Cmfq;~9^!_J#IM6=*s7YOw_J|l^U4p8?m=SXV+O?bE4<8B0mTc90Y z5_s$w@yOn;2V$Cp6z1Lpx6wvtOVKFNFK-OhyrOb&I<&#ILe&@suI_72u8=nLT9XQE z0;h8fX5}aA9c@-YK&7%-6hf$6g6}h^Jpr`hjT}t_n>{89CBCL8s%^(AyNIeL;qe1_ zIF`nSlf&LZ1NAH^K|TH0A$6k^2``q>ZCiQ?;P=*`-57*4V>xy-7j@-^ zzw!6<;f@9g}(+YwBeA=utN)ou;)~~2#<=% z*f4akbv$(SueB@{(-rUEqH|anNzsgJ(mz0R(e%0&a$l*t^~iV85Z=1U4*G6?XI`JY zBqJ&N+o^Np^mm8Lyy&KDfJZ35fWA2{_COzBq=$~0aSJzQo_x@4nD z3_5&wp~tG}HE>wswZUYtcmMpXZ{D6(CZ>*&NlE3#uAn;bybNB4rT(rW{Y$#1f9ou9 z*q%2wlX+M+xE|btXVj2oe~{vX)8@xiV#;UT+s9It#`&35Md>{#6_b72`1~nQ&T}8& zA{&$cm7FusM>6^3Nu+hBY-&+~N=ZoCl6Lx<_6oQ^nbZjSdu+3tURTfbLczl7vpt-( zK2)Rikj@skLkK;$Y){d}I&2L;&-N0$>E^^3IdGSSO^>WSaLjQc9>qS7h{R8k>uovo zqkI->dfNp-P~}=pO{>CV5iTCOECjZ|NHfrf|B%JM_S~xZB0lf;lRIg7add2A83#6E zbK}DuD4v`D|WzO^nY~kjT zhT$F9i^Smim~8vO#e8ywUfYV@k39wSGVSYwz`D)35lkKy$S0v8A=N2NM zU^==dPD1s_nEHcc$xr=1+Y1;IS5Q;>gjQmIS?Bcanix|z4rV0%IM*&fA3a?;oFz`s zDWUjfmP0CznPD^QYQ^Cb#M})77b6SVW8Dweg3QtTTY1*H6e63=!d}$E?@#j*^!o+y zmjqe^{|1j5&sC)oyg@8qjpGwbMboQz3}vt;I^R70@xIhrK8>*hmv3$%{jp6grJc0` zdP0zT7z%flq|f?B(CLTjzJP!9bIcjjCp&*K`tjU3zuxy&oi=$sC3x@va_gz9f2qB)^+bn%|3N(#Ir8`wzFbn#x<}>y%ZKB8 zBtk z^?%|tAT65SwfLc>ZSVQZ7@ts)!4SOfEoP!mJcf?X;b#o5asR<^pOhR+hIwD$NwJhDab&Ks%Q z4Q>=CIgwMn1=?Uz^Ko@1FJJgBmmIhhG-cpU<*d|uc)_fnl_q)(uz2%v!9!kHw>rd& zI++?WdiM>ciFbyTANZOWXl*sq8>MBe?ObSo9Cs*EH8ExV%FOb?$F@J?+;URQBu_cx zK1Xa+kx39};DD7@)&w;-(|#qy3D5Y+?_|?+%~k?l1zt@K64g2^_@=0Z2F-}Dw_dK~ zDsu`6f=y^aw}9c}(6`zjQc!b2SVXmQsJ&C7E5d~=w#(NPP>yk(2o$Pdt7GYjjWW1d zCBoYER24KC1@qU;X@SMcsa%qol7kiWY&^?3MSW;B|Eoj5H>GlLk6X^hR7ZF_DIq5} z0K~e1!b9+Nl6gY>nPlg2Dn3y|7**rsF}0TLa0?T+2s`%T7TK%3lA!bQVVjNcmXE+| zT!ZWH#NjEU@eky6f|bpW_35LS`f{1l)Kp2ijL$iaePzUsO?Y9mQa>1MiXiRNp(2o? z(gfoE{#7uky=)@KPt=ywgEF0X{SaFVn5zgff9~zqnf-N>=vb*|E=!YD)-%_`Qm0J0 zftLqkBJxe%tzo?!tfVg+_E5a)WutH2ym1-bI)UdLXbd0k6QtT&q}y)y=c=1Eh%34Y z(9G$eQZ&f-IAYHWB{lS@pUM13#e+Yr47ZoiBx_H3x`hw(`c!+x0Jvr?sxnX)`KUj; zstY@j1$Qr6X@eYl_bShX2_;|aN8h@E^poqZuJ8!nRsdl+4X+JH*IU#Ebw!<;9H$*| z9YP03K`lPJKH_NKrTn8`H6R6(Gf9Ibj z^7{^k)8w2O4Hw`S!{!c)hN!{;b60PZCy96nHz)jU`T49J-+}VPjn7_nUkp9 zWzHjrgE72vI6o(hM=mNj7N4V@u(~--liHTB6AghQEjuq}{EotH!3|=Y!T%_Ne1sVJHhCOn3Q!Co>1fX%G zg)K;|Uvg37+1$eVP>Na8adsT~fynsPj-rnr3>rZFGlej)j}Lv7B)^_jzPS7X3He6c zrH%}r%P>`#?lpBRSV9N}U7qnVV;1N@;%@&QHg*r&3~ftq#u5;vFyy^Bye^qraaorX z-bGbx*!+wrV#L<6M~rMlU}wio%ZHdC&}DvaGGPJWY)>P5WCQ)QqPf?C3D-;ml+SYx zKL{Eo1ni4g&O57L|6-h}tCT-{hA zhuC=Z?Q<+^l_57RzI$G^wj+WbM}8+>-p3=9 zdlx`#bV#D}xgA$iKh;WeKf@SxQ3XUjC-q6sOSrBi(D8d6WTUpODX8Ky7WcF4f{T|Z#C>6z$u#(rAA3Jc+ zgRO0zUzRFuduKzbwlo1ZHx5ZMGe;2D()yk!x$_x(X<)gGImM{Q1C3i<3WRg)h?UOldEaPYGo!P z%Z@DNX}l{sMGdIeZnPEI==|L7H>CEW909EGm|d@A_ooK7v>-lsm9E6#+RuYbY2=FMQ0gY=9h{g+bu&G8Zj|`)LA40OP&n=ambNyNWO5n_b&Z?TjBr-*jIHm zeTf-Z0oz)@?_!H7?sO+t)Tf8m-HMRVT8MKB2ywoHSqjo-WjymudI07b7PD4?Bz)@$ zLu~LmYTwO+L#0R8{PfW~b~NdGSb918jz=n4c&PU}h+7Z&HGv_aZgN3Fu?y4Jk}J8Y z0c%<$k{dy&Bze&vEbKl({EE6)hA`y_p+Yc6Q3*QkN9+Q%#^2xA-2p`7Ej4UK2piYJ zbY}^oeKB{gf)(XP@3oIC(nIAZ7#%g3B>jk}pOr9tBKYewJ-c6jUWJ;eQlJXPw~D#~ zDISk6$IP!oLra{gjx{y=Szs8IY_G(tA@I|A3f$3rIRj#O%5FTGg#arfZfH zjx7dd#;vF^hw7b+wk|7DkJm#qRe$f!l!q1=D^7#|Jn4J&#`2I|2V7n#rA_EKX#H}S zX_>yevhwGF>-iP1&d99;gN|=qjC4NGQBO5H1aJ45S%QKR1Vb4eK#~MSqA0@1Ypv(w zF~+t7doD{lNJa_pv$A3)5Rd*vFiqxKDK^QX*!A5@;wV9V=4}MT?o{yzD}b_i5kD}x09QPS z$qoD}w9mc9E(IfJ4qrY}j^0(6aj0l|gtE9t3eRY;0w>T!B+qP{Ud?a`s+%o#`A+R@ zs-M9a;fpLKWt*F=N?67}iNyxiA;s1Ytr3+-lS!l;SZC&s`nS+hOQ-oaO%k>h3^WK&v48)zjmw9|kY*uB?lRloNYW&@ zyvk{Ij{fYLoDJuh^3yA-*G{a-&Vhbwk91X27vBj&gX6y}$`53Eyb0k)*p0Xd_Z3!N z6WhpLF^jzC=%@om#D0dWJt65*=0gi6BW9e$GlFD?lb>_Y2D~RWq{jZr|BA{e3}E7Q zO8nQk^Y%njns6^lnE0hW-~yz{wHVISEDrTLN8julA?FKFCu}L&6HaF>#Ala*IjgU2 z@;5kb{6O(;Y)b71h_zKVQ9DsJ#F2nL<0=IxYw7FGL7b<38%nQrG`h<))VztzbXv3BH8mF3y}%1sn4 z83239{x#)8XI&8sCsZ)wyD9T#6);Bhl*}*Sr6U-ekMAHwg3psrk}?IRkF?NR%>xXL zgti#9BGEQA-y69v!};bxBEoU&YhZORldNW6bBPU>xX!-gwWWuqt1;w^u0PJv(Jgrv z9f;WzKzw&bNoH9m!`~1tG*D5O;r9l|=GDVKUwOx#3MlPne((<7FL4)-DUL9ZqPw{c zxIIRc*Y^Bx&EQxcL)vNMAC?wJmw6?x20Wy5 zh>_7{>P&Z$g$Gg~Fb^~?XgoAfQrHWVHtJ&Nx4MO`nj7>X^E z2=ziaX2;-7Tc~Wb0M{pOBL)UUui8l!?F}KC*%8{k=HL}wwd8Ap^lE17urd9!#%b{A z0LF_d4ibqk(Dut1S0NZ2aLy9(4o z*DTjSg#JHVk0tbuX^6foS{PjLju-5OIJS2_FCs zRKDs`U$z`r64WURXXOU25@g*+dVy)(Y>c13UfkOa1H8I=Nm<2kw39@np4x&O6-^|| zPsIL&?_MCJ@p}uI=ZOw|@tRH8)m8KtuMJO78EbeWSaQhTB{>To5s0VxqCC;j4=OD; zADge}p%cz>`yK3Bow$%M!Y7NCm*E6o-#z3oxbvnh!ga!NutC|O{RZ)E zdRw--O7{h$jgTCuiPh2Js<|5(wO%CTX|r+tU8T zsmvY4gVp)deeT_e77_$b33a18z4Hh1qxH>z33ne|I@4--p|{qgDz{%A6nwGgNaX){ z&hO;=RdSyE2xQ^bSkqgUw`EDLaz1}Z<-}alcN{0d+&n@TLiv~6Vy1Y$RgFVOf;Oui z&;|Rc-`8afTT*V{lKJs@alxQ!JO|6wZj%@<-@%tbLn*xVu=;rMD(K}CObm20g{7fD z`vI3*sIP~enQ6c^?+)WQ^zVvz(a1*Z{4!+j_atG~$Dh9^4+!9ot{D~GVUXRMuQ%83 zpdx$N^&MNy-44}@AhxXD9jfDhB?Bj-SGQbONiFBErSn(OR+uL~lpoLMx0BcXy)&am zDIU_~9q%xZPv%Zd^l$EP=J#&0yDEBVnpJu$94B> zqvUC&+bGs?y8s^k#l#?&PNR;}y=6tFF56q7>w~rac~f(U=z`H?#Qx zK&)Krz5KQ%l<3Fghqk>sgW31KUDJP{qTSGjp#D>G<(<1Lj*B2^5-J4Go;-GbH{q$s z4Y6NH8CoNl{j$)pvZPzgp)q?v+fU%v+2PzyMyzCy+GO+Cz>M+TR>N0cLp7XNg5S`^ zW1}vR&=wB7?xM^co~-^uZ4Q@%4c&gz&o!^wc)lcrC020JQ*=Io%t@>lUhDe#Z*bb> z!QMZJm2dsdnsUzCn8TKgH>j8{rVew=PwtH9*KR;WY%b}O$dULkN9TK9SK0LJ28ur) zw_o$`R(SRy(7yWHx9l{LZ!h|33g$1{(Ck}p@Y>JLE|soA@vK5hyMI2|uHw+K4ZX*& zcR&Yl+KeQQ$N1w|=L79(?~u2!*M29IbnlD82W6v0cMw(jt26J05G}`Ys&#o&h*L;bbede*B%b)Lr3+TC&%!|_A*7OD7@ zQzBZ4@R%t%mNN>E5jOoUHROiEBtJ}OW6|50#p zvvITw`2Q>DE^;T|D=_|V217SHZ$B$fTL8kw&Ds{K;bLWP`_$ITCcvZD_J3I)01Xvg KWsIU_ +

Crawlers Page

+ +

{{ message }}

+ \ No newline at end of file diff --git a/ui/pages/debug.html b/ui/pages/debug.html new file mode 100644 index 00000000..b2f0f90f --- /dev/null +++ b/ui/pages/debug.html @@ -0,0 +1,5 @@ +
+

Debug

+ +

{{ message }}

+
\ No newline at end of file diff --git a/ui/pages/kafka.html b/ui/pages/kafka.html new file mode 100644 index 00000000..b431f429 --- /dev/null +++ b/ui/pages/kafka.html @@ -0,0 +1,5 @@ +
+

Home Page

+ +

{{ message }}

+
\ No newline at end of file diff --git a/ui/pages/overview.html b/ui/pages/overview.html new file mode 100644 index 00000000..826630ed --- /dev/null +++ b/ui/pages/overview.html @@ -0,0 +1,50 @@ +
+
+
+
+
+

Spiders

+
+
+

Total number of spiders: {{ message }}

+

Total unique spiders:

+

Total number of machines:

+
+
+
+
+
+
+

Crawl Queue

+
+
+

Status: {{ message }}

+

Queue backlog:

+
+
+
+
+
+
+

Kafka

+
+
+

Status: {{ message }}

+

Total number of requests:

+

Total failed requests:

+
+
+
+
+
+

Submit a job to Scrapy-Cluster

+
+ URL: + + AppID: + + Expires: + +
+
+
\ No newline at end of file diff --git a/ui/pages/redis.html b/ui/pages/redis.html new file mode 100644 index 00000000..c448e690 --- /dev/null +++ b/ui/pages/redis.html @@ -0,0 +1,5 @@ +
+

Contact Page

+ +

{{ message }}

+
\ No newline at end of file diff --git a/ui/stats/stats.json b/ui/stats/stats.json new file mode 100644 index 00000000..e01c0b62 --- /dev/null +++ b/ui/stats/stats.json @@ -0,0 +1,126 @@ +{ + "kafka-monitor": { + "fail": { + "604800": 4, + "lifetime": 7, + "43200": 4, + "86400": 4, + "21600": 4 + }, + "total": { + "900": 3, + "3600": 9, + "604800": 35, + "43200": 30, + "lifetime": 58, + "86400": 35, + "21600": 30 + }, + "plugins": { + "ActionHandler": { + "604800": 11, + "lifetime": 17, + "43200": 11, + "86400": 11, + "21600": 11 + }, + "ScraperHandler": { + "604800": 10, + "lifetime": 18, + "43200": 5, + "86400": 10, + "21600": 5 + }, + "StatsHandler": { + "900": 3, + "3600": 9, + "604800": 10, + "43200": 10, + "lifetime": 17, + "86400": 10, + "21600": 10 + } + } + }, + "stats": "all", + "uuid": "hij3", + "redis-monitor": { + "total": { + "900": 3, + "3600": 9, + "604800": 23, + "43200": 23, + "lifetime": 36, + "86400": 23, + "21600": 23 + }, + "plugins": { + "ExpireMonitor": { + "604800": 2, + "lifetime": 2, + "43200": 2, + "86400": 2, + "21600": 2 + }, + "StopMonitor": { + "604800": 5, + "lifetime": 6, + "43200": 5, + "86400": 5, + "21600": 5 + }, + "StatsMonitor": { + "900": 3, + "3600": 9, + "604800": 10, + "43200": 10, + "lifetime": 17, + "86400": 10, + "21600": 10 + }, + "InfoMonitor": { + "604800": 6, + "lifetime": 11, + "43200": 6, + "86400": 6, + "21600": 6 + } + } + }, + "appid": "testapp", + "server_time": 1452105176, + "crawler": { + "spiders": { + "unique_spider_count": 1, + "total_spider_count": 2, + "link": { + "count": 2, + "200": { + "604800": 44, + "lifetime": 61, + "43200": 39, + "86400": 44, + "21600": 39 + }, + "504": { + "lifetime": 4 + } + } + }, + "machines": { + "count": 1, + "Madisons-MacBook-Pro-2.local": { + "200": { + "604800": 44, + "lifetime": 61, + "43200": 39, + "86400": 44, + "21600": 39 + }, + "504": { + "lifetime": 4 + } + } + } + } +} \ No newline at end of file diff --git a/ui/stats/stats_schema.json b/ui/stats/stats_schema.json new file mode 100644 index 00000000..0b9bb9d3 --- /dev/null +++ b/ui/stats/stats_schema.json @@ -0,0 +1,33 @@ +{ + "type": "object", + "properties": { + "uuid": { + "type": "string", + "minLength": 1, + "maxLength": 40 + }, + "stats": { + "type": "string", + "enum": [ + "kafka-monitor", + "redis-monitor", + "crawler", + "spider", + "machine", + "queue", + "all" + ], + "default": "all" + }, + "appid": { + "type": "string", + "minLength": 3, + "maxLength": 40 + } + }, + "required": [ + "uuid", + "appid" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/ui/stats_api.py b/ui/stats_api.py new file mode 100644 index 00000000..b75e7f3e --- /dev/null +++ b/ui/stats_api.py @@ -0,0 +1,18 @@ +from flask import Flask +from flask_restful import Resource, Api +import json + +app = Flask(__name__) +api = Api(app) + + +class Stats(Resource): + def get(self): + with open("stats/stats.json", 'rb') as stats_cache: + stats = json.load(stats_cache) + return stats + +api.add_resource(Stats, '/getStats') + +if __name__ == '__main__': + app.run(debug=True) diff --git a/ui/ui_kafka_agent.py b/ui/ui_kafka_agent.py new file mode 100644 index 00000000..ab929554 --- /dev/null +++ b/ui/ui_kafka_agent.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +from kafka import KafkaConsumer, KafkaProducer +from jsonschema import ValidationError +from jsonschema import Draft4Validator, validators +import threading +import time +import json + + +def extend_with_default(validator_class): + + validate_properties = validator_class.VALIDATORS["properties"] + + def set_defaults(validator, properties, instance, schema): + for error in validate_properties( + validator, properties, instance, schema, + ): + yield error + + for property, subschema in list(properties.items()): + if "default" in subschema: + instance.setdefault(property, subschema["default"]) + + return validators.extend( + validator_class, {"properties": set_defaults}, + ) + + +class Producer(threading.Thread): + daemon = True + + def run(self): + producer = KafkaProducer(bootstrap_servers='localhost:9092') + + while True: + producer.send('my-topic', b"{'appid': 'testapp', 'uuid': 'hij3', 'stats': 'all'}") + time.sleep(1) + + +class Consumer(threading.Thread): + daemon = True + + def run(self): + consumer = KafkaConsumer(bootstrap_servers='localhost:9092', + auto_offset_reset='earliest') + consumer.subscribe(['my-topic']) + + validator = extend_with_default(Draft4Validator) + + for message in consumer: + json_message = json.loads(message.value) + valid = False + try: + validator("stats_schema").validate(json_message) + valid = True + except ValidationError: + pass + + if valid: + with open("stats/data.json", 'w') as stats_cache: + json.dump(json_message['stats'], stats_cache) + + +def main(): + threads = [ + Producer(), + Consumer() + ] + + for t in threads: + t.start() + + time.sleep(10) + +if __name__ == "__main__": + main() \ No newline at end of file From 41a431cb849bcdfaa33b54d8a296f10c1e7b594a Mon Sep 17 00:00:00 2001 From: damienkilgannon Date: Tue, 31 Jan 2017 21:13:09 +0000 Subject: [PATCH 02/14] updated ui-service.py --- ui/js/app.js | 28 ---- ui/js/controllers.js | 33 ----- ui/js/services.js | 11 -- ui/pages/redis.html | 5 - ui/requirements.txt | 0 ui/settings.py | 17 +++ ui/static/css/style.css | 27 ++++ ui/{ => static/img}/logo.png | Bin ui/static/js/app.js | 59 ++++++++ ui/static/partials/about.html | 6 + ui/{pages => static/partials}/crawlers.html | 0 ui/static/partials/index.html | 11 ++ ui/{pages => static/partials}/kafka.html | 2 +- ui/{pages => static/partials}/overview.html | 0 .../debug.html => static/partials/redis.html} | 2 +- ui/stats/stats.json | 126 ------------------ ui/stats/stats_schema.json | 33 ----- ui/stats_api.py | 18 --- ui/templates/index.html | 38 ++++++ ui/ui_kafka_agent.py | 76 ----------- ui/ui_service.py | 122 +++++++++++++++++ 21 files changed, 282 insertions(+), 332 deletions(-) delete mode 100644 ui/js/app.js delete mode 100644 ui/js/controllers.js delete mode 100644 ui/js/services.js delete mode 100644 ui/pages/redis.html create mode 100644 ui/requirements.txt create mode 100644 ui/settings.py create mode 100644 ui/static/css/style.css rename ui/{ => static/img}/logo.png (100%) create mode 100644 ui/static/js/app.js create mode 100644 ui/static/partials/about.html rename ui/{pages => static/partials}/crawlers.html (100%) create mode 100644 ui/static/partials/index.html rename ui/{pages => static/partials}/kafka.html (75%) rename ui/{pages => static/partials}/overview.html (100%) rename ui/{pages/debug.html => static/partials/redis.html} (75%) delete mode 100644 ui/stats/stats.json delete mode 100644 ui/stats/stats_schema.json delete mode 100644 ui/stats_api.py create mode 100644 ui/templates/index.html delete mode 100644 ui/ui_kafka_agent.py create mode 100644 ui/ui_service.py diff --git a/ui/js/app.js b/ui/js/app.js deleted file mode 100644 index f99759e9..00000000 --- a/ui/js/app.js +++ /dev/null @@ -1,28 +0,0 @@ -angular.module('uiApp', ['ngRoute', 'ngResource','uiApp.controllers','uiApp.services']); - -angular.module('uiApp').config(function($routeProvider) { - $routeProvider - .when('/', { - templateUrl: 'pages/overview.html', - controller: 'mainController' - }) - .when('/kafka', { - templateUrl: 'pages/kafka.html', - controller: 'kafkaController' - }) - .when('/crawlers', { - templateUrl: 'pages/crawlers.html', - controller: 'crawlersController' - }) - .when('/redis', { - templateUrl: 'pages/redis.html', - controller: 'redisController' - }) - .when('/debug', { - templateUrl: 'pages/debug.html', - controller: 'debugController' - }) - .otherwise({ - redirectTo: '/' - }); -}); \ No newline at end of file diff --git a/ui/js/controllers.js b/ui/js/controllers.js deleted file mode 100644 index b073b03d..00000000 --- a/ui/js/controllers.js +++ /dev/null @@ -1,33 +0,0 @@ -angular.module('uiApp.controllers',[]).controller('tabsController', function($scope) { - $scope.tabs = [ - { link : '#/', label : 'Overview' }, - { link : '#/kafka', label : 'Kafka' }, - { link : '#/redis', label : 'Redis' }, - { link : '#/crawlers', label : 'Crawlers' }, - { link : '#/debug', label : 'Debug' } - - ]; - - $scope.selectedTab = $scope.tabs[0]; - $scope.setSelectedTab = function(tab) { - $scope.selectedTab = tab; - } - - $scope.tabClass = function(tab) { - if ($scope.selectedTab == tab) { - return "active"; - } else { - return ""; - } - } -}).controller('mainController', function($scope) { - $scope.message = 'Overview!'; -}).controller('kafkaController', function($scope) { - $scope.message = 'Kafka...'; -}).controller('redisController', function($scope) { - $scope.message = 'Redis...'; -}).controller('crawlersController', function($scope) { - $scope.message = 'Crawler...'; -}).controller('debugController', function($scope) { - $scope.message = 'Debug...'; -}); \ No newline at end of file diff --git a/ui/js/services.js b/ui/js/services.js deleted file mode 100644 index beaa66d8..00000000 --- a/ui/js/services.js +++ /dev/null @@ -1,11 +0,0 @@ -angular.module('uiApp.services',[]).factory('jsonStats',function($resource){ - return $resource('http://localhost:5000/getStats',{},{ - update: { - method: 'GET' - } - }); -}).service('popupService',function($window){ - this.showPopup=function(message){ - return $window.confirm(message); - } -}); \ No newline at end of file diff --git a/ui/pages/redis.html b/ui/pages/redis.html deleted file mode 100644 index c448e690..00000000 --- a/ui/pages/redis.html +++ /dev/null @@ -1,5 +0,0 @@ -
-

Contact Page

- -

{{ message }}

-
\ No newline at end of file diff --git a/ui/requirements.txt b/ui/requirements.txt new file mode 100644 index 00000000..e69de29b diff --git a/ui/settings.py b/ui/settings.py new file mode 100644 index 00000000..cf6b92d9 --- /dev/null +++ b/ui/settings.py @@ -0,0 +1,17 @@ +# Flask configuration +FLASK_LOGGING_ENABLED = True +FLASK_PORT = 5000 + +# logging setup +LOGGER_NAME = 'ui_service' +LOG_DIR = 'logs' +LOG_FILE = 'ui_service.log' +LOG_MAX_BYTES = 10 * 1024 * 1024 +LOG_BACKUPS = 5 +LOG_STDOUT = True +LOG_JSON = False +LOG_LEVEL = 'INFO' + +# internal configuration +SLEEP_TIME = 5 +WAIT_FOR_RESPONSE_TIME = 5 diff --git a/ui/static/css/style.css b/ui/static/css/style.css new file mode 100644 index 00000000..b669a1b4 --- /dev/null +++ b/ui/static/css/style.css @@ -0,0 +1,27 @@ +.btn-lg, +.input-lg, +.input-group-lg>.form-control, +.input-group-lg>.input-group-addon, +.input-group-lg>.input-group-btn>.btn +{ + line-height: 1.33; + height: 50px; +} +.navbar-brand +{ + font-family: 'Lato', sans-serif; + color:black; + font-size: 30px; + margin: 20px; +} +.btn +{ + margin: 60px; +} +.body-content +{ + max-width:1200px; + margin-left:auto; + margin-right:auto; + font-family:Arial; +} \ No newline at end of file diff --git a/ui/logo.png b/ui/static/img/logo.png similarity index 100% rename from ui/logo.png rename to ui/static/img/logo.png diff --git a/ui/static/js/app.js b/ui/static/js/app.js new file mode 100644 index 00000000..5e4e9155 --- /dev/null +++ b/ui/static/js/app.js @@ -0,0 +1,59 @@ +'use strict'; // See note about 'use strict'; below + +var myApp = angular.module('myApp', [ + 'ngRoute', +]); + +myApp.config(['$routeProvider', + function($routeProvider) { + $routeProvider. + when('/', { + templateUrl: '/static/partials/overview.html', + controller: 'mainController' + }). + when('/kafka', { + templateUrl: '/static/partials/kafka.html', + controller: 'kafkaController' + }). + when('/crawlers', { + templateUrl: '/static/partials/crawlers.html', + controller: 'crawlersController' + }). + when('/redis', { + templateUrl: '/static/partials/redis.html', + controller: 'redisController' + }). + otherwise({ + redirectTo: '/' + }); + }]); + +myApp.controller('tabsController', ['$scope', function($scope) { + $scope.tabs = [ + { link : '#/', label : 'Overview' }, + { link : '#/kafka', label : 'Kafka' }, + { link : '#/redis', label : 'Redis' }, + { link : '#/crawlers', label : 'Crawlers' } + ]; + + $scope.selectedTab = $scope.tabs[0]; + $scope.setSelectedTab = function(tab) { + $scope.selectedTab = tab; + } + + $scope.tabClass = function(tab) { + if ($scope.selectedTab == tab) { + return "active"; + } else { + return ""; + } + } +}]).controller('mainController', function($scope) { + $scope.message = 'Overview!'; +}).controller('kafkaController', function($scope) { + $scope.message = 'Kafka...'; +}).controller('redisController', function($scope) { + $scope.message = 'Redis...'; +}).controller('crawlersController', function($scope) { + $scope.message = 'Crawler...'; +}); diff --git a/ui/static/partials/about.html b/ui/static/partials/about.html new file mode 100644 index 00000000..46c19ad3 --- /dev/null +++ b/ui/static/partials/about.html @@ -0,0 +1,6 @@ +

About

+ +

Hmm, not much here.

+

Best get back to the home page.

+ +

Tip: going to an incorrect URL will take you to the homepage, Try it!

\ No newline at end of file diff --git a/ui/pages/crawlers.html b/ui/static/partials/crawlers.html similarity index 100% rename from ui/pages/crawlers.html rename to ui/static/partials/crawlers.html diff --git a/ui/static/partials/index.html b/ui/static/partials/index.html new file mode 100644 index 00000000..280f425f --- /dev/null +++ b/ui/static/partials/index.html @@ -0,0 +1,11 @@ +

Home Page

+ +

You should head over to the about page.

+ +

Here's an Angular Demo:

+ +
+ +
+

Hello {{ yourName }}

+
\ No newline at end of file diff --git a/ui/pages/kafka.html b/ui/static/partials/kafka.html similarity index 75% rename from ui/pages/kafka.html rename to ui/static/partials/kafka.html index b431f429..d2fa5993 100644 --- a/ui/pages/kafka.html +++ b/ui/static/partials/kafka.html @@ -1,5 +1,5 @@
-

Home Page

+

Kafka Page

{{ message }}

\ No newline at end of file diff --git a/ui/pages/overview.html b/ui/static/partials/overview.html similarity index 100% rename from ui/pages/overview.html rename to ui/static/partials/overview.html diff --git a/ui/pages/debug.html b/ui/static/partials/redis.html similarity index 75% rename from ui/pages/debug.html rename to ui/static/partials/redis.html index b2f0f90f..b835ae57 100644 --- a/ui/pages/debug.html +++ b/ui/static/partials/redis.html @@ -1,5 +1,5 @@
-

Debug

+

Redis Page

{{ message }}

\ No newline at end of file diff --git a/ui/stats/stats.json b/ui/stats/stats.json deleted file mode 100644 index e01c0b62..00000000 --- a/ui/stats/stats.json +++ /dev/null @@ -1,126 +0,0 @@ -{ - "kafka-monitor": { - "fail": { - "604800": 4, - "lifetime": 7, - "43200": 4, - "86400": 4, - "21600": 4 - }, - "total": { - "900": 3, - "3600": 9, - "604800": 35, - "43200": 30, - "lifetime": 58, - "86400": 35, - "21600": 30 - }, - "plugins": { - "ActionHandler": { - "604800": 11, - "lifetime": 17, - "43200": 11, - "86400": 11, - "21600": 11 - }, - "ScraperHandler": { - "604800": 10, - "lifetime": 18, - "43200": 5, - "86400": 10, - "21600": 5 - }, - "StatsHandler": { - "900": 3, - "3600": 9, - "604800": 10, - "43200": 10, - "lifetime": 17, - "86400": 10, - "21600": 10 - } - } - }, - "stats": "all", - "uuid": "hij3", - "redis-monitor": { - "total": { - "900": 3, - "3600": 9, - "604800": 23, - "43200": 23, - "lifetime": 36, - "86400": 23, - "21600": 23 - }, - "plugins": { - "ExpireMonitor": { - "604800": 2, - "lifetime": 2, - "43200": 2, - "86400": 2, - "21600": 2 - }, - "StopMonitor": { - "604800": 5, - "lifetime": 6, - "43200": 5, - "86400": 5, - "21600": 5 - }, - "StatsMonitor": { - "900": 3, - "3600": 9, - "604800": 10, - "43200": 10, - "lifetime": 17, - "86400": 10, - "21600": 10 - }, - "InfoMonitor": { - "604800": 6, - "lifetime": 11, - "43200": 6, - "86400": 6, - "21600": 6 - } - } - }, - "appid": "testapp", - "server_time": 1452105176, - "crawler": { - "spiders": { - "unique_spider_count": 1, - "total_spider_count": 2, - "link": { - "count": 2, - "200": { - "604800": 44, - "lifetime": 61, - "43200": 39, - "86400": 44, - "21600": 39 - }, - "504": { - "lifetime": 4 - } - } - }, - "machines": { - "count": 1, - "Madisons-MacBook-Pro-2.local": { - "200": { - "604800": 44, - "lifetime": 61, - "43200": 39, - "86400": 44, - "21600": 39 - }, - "504": { - "lifetime": 4 - } - } - } - } -} \ No newline at end of file diff --git a/ui/stats/stats_schema.json b/ui/stats/stats_schema.json deleted file mode 100644 index 0b9bb9d3..00000000 --- a/ui/stats/stats_schema.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "uuid": { - "type": "string", - "minLength": 1, - "maxLength": 40 - }, - "stats": { - "type": "string", - "enum": [ - "kafka-monitor", - "redis-monitor", - "crawler", - "spider", - "machine", - "queue", - "all" - ], - "default": "all" - }, - "appid": { - "type": "string", - "minLength": 3, - "maxLength": 40 - } - }, - "required": [ - "uuid", - "appid" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/ui/stats_api.py b/ui/stats_api.py deleted file mode 100644 index b75e7f3e..00000000 --- a/ui/stats_api.py +++ /dev/null @@ -1,18 +0,0 @@ -from flask import Flask -from flask_restful import Resource, Api -import json - -app = Flask(__name__) -api = Api(app) - - -class Stats(Resource): - def get(self): - with open("stats/stats.json", 'rb') as stats_cache: - stats = json.load(stats_cache) - return stats - -api.add_resource(Stats, '/getStats') - -if __name__ == '__main__': - app.run(debug=True) diff --git a/ui/templates/index.html b/ui/templates/index.html new file mode 100644 index 00000000..865d4b64 --- /dev/null +++ b/ui/templates/index.html @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + +
+ +
+
+ +
+

...

+
+ + + \ No newline at end of file diff --git a/ui/ui_kafka_agent.py b/ui/ui_kafka_agent.py deleted file mode 100644 index ab929554..00000000 --- a/ui/ui_kafka_agent.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python -from kafka import KafkaConsumer, KafkaProducer -from jsonschema import ValidationError -from jsonschema import Draft4Validator, validators -import threading -import time -import json - - -def extend_with_default(validator_class): - - validate_properties = validator_class.VALIDATORS["properties"] - - def set_defaults(validator, properties, instance, schema): - for error in validate_properties( - validator, properties, instance, schema, - ): - yield error - - for property, subschema in list(properties.items()): - if "default" in subschema: - instance.setdefault(property, subschema["default"]) - - return validators.extend( - validator_class, {"properties": set_defaults}, - ) - - -class Producer(threading.Thread): - daemon = True - - def run(self): - producer = KafkaProducer(bootstrap_servers='localhost:9092') - - while True: - producer.send('my-topic', b"{'appid': 'testapp', 'uuid': 'hij3', 'stats': 'all'}") - time.sleep(1) - - -class Consumer(threading.Thread): - daemon = True - - def run(self): - consumer = KafkaConsumer(bootstrap_servers='localhost:9092', - auto_offset_reset='earliest') - consumer.subscribe(['my-topic']) - - validator = extend_with_default(Draft4Validator) - - for message in consumer: - json_message = json.loads(message.value) - valid = False - try: - validator("stats_schema").validate(json_message) - valid = True - except ValidationError: - pass - - if valid: - with open("stats/data.json", 'w') as stats_cache: - json.dump(json_message['stats'], stats_cache) - - -def main(): - threads = [ - Producer(), - Consumer() - ] - - for t in threads: - t.start() - - time.sleep(10) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/ui/ui_service.py b/ui/ui_service.py new file mode 100644 index 00000000..25219adc --- /dev/null +++ b/ui/ui_service.py @@ -0,0 +1,122 @@ +import argparse +from functools import wraps +from flask import Flask, request, send_file +import time +import logging + +from scutils.log_factory import LogFactory +from scutils.settings_wrapper import SettingsWrapper + + +# Route Decorators -------------------- + +def log_call(call_name): + """Log the API call to the logger.""" + def decorator(f): + @wraps(f) + def wrapper(*args, **kw): + instance = args[0] + instance.logger.info(call_name, {"content": request.get_json()}) + return f(*args, **kw) + return wrapper + return decorator + + +class UIService(object): + + closed = False + start_time = 0 + + def __init__(self, settings_name): + """ + @param settings_name: the local settings file name + """ + self.settings_name = settings_name + self.wrapper = SettingsWrapper() + self.logger = None + self.app = Flask(__name__) + + def setup(self, level=None, log_file=None, json=None): + """ + Load everything up. Note that any arg here will override both + default and custom settings + + @param level: the log level + @param log_file: boolean t/f whether to log to a file, else stdout + @param json: boolean t/f whether to write the logs in json + """ + self.settings = self.wrapper.load(self.settings_name) + + my_level = level if level else self.settings['LOG_LEVEL'] + # negate because logger wants True for std out + my_output = not log_file if log_file else self.settings['LOG_STDOUT'] + my_json = json if json else self.settings['LOG_JSON'] + self.logger = LogFactory.get_instance(json=my_json, stdout=my_output, + level=my_level, + name=self.settings['LOGGER_NAME'], + dir=self.settings['LOG_DIR'], + file=self.settings['LOG_FILE'], + bytes=self.settings['LOG_MAX_BYTES'], + backups=self.settings['LOG_BACKUPS']) + + self._decorate_routes() + + self.start_time = time.time() + + # disable flask logger + if self.settings['FLASK_LOGGING_ENABLED'] == False: + log = logging.getLogger('werkzeug') + log.disabled = True + + def run(self): + """Main flask run loop""" + self.logger.info("Running main flask method on port " + str(self.settings['FLASK_PORT'])) + self.app.run(host='0.0.0.0', port=self.settings['FLASK_PORT']) + + def close(self): + """ + Cleans up anything from the process + """ + self.logger.info("Closing Rest Service") + self.closed = True + + # Routes -------------------- + + def _decorate_routes(self): + """ + Decorates the routes to use within the flask app + """ + self.logger.debug("Decorating routes") + self.app.add_url_rule('/', 'index', self.index, methods=['GET']) + + @log_call('\'index\' endpoint called') + def index(self): + return send_file("templates/index.html") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Rest Service: Used for interacting and feeding Kafka' + ' requests to the cluster and returning data back\n') + + parser.add_argument('-s', '--settings', action='store', required=False, + help="The settings file to read from", default="localsettings.py") + parser.add_argument('-ll', '--log-level', action='store', required=False, + help="The log level", default=None, + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']) + parser.add_argument('-lf', '--log-file', action='store_const', + required=False, const=True, default=None, + help='Log the output to the file specified in settings.py. Otherwise logs to stdout') + parser.add_argument('-lj', '--log-json', action='store_const', + required=False, const=True, default=None, + help="Log the data in JSON format") + + args = vars(parser.parse_args()) + + ui_service = UIService(args['settings']) + ui_service.setup(level=args['log_level'], log_file=args['log_file'], json=args['log_json']) + + try: + ui_service.run() + finally: + ui_service.close() From 9f36a86d114d6ffdc68effac1f04a30fbbf11a2b Mon Sep 17 00:00:00 2001 From: damienkilgannon Date: Tue, 31 Jan 2017 21:40:37 +0000 Subject: [PATCH 03/14] added rest api call --- ui/index.html | 76 ----------------------------------- ui/static/js/app.js | 6 ++- ui/static/partials/index.html | 11 ----- 3 files changed, 5 insertions(+), 88 deletions(-) delete mode 100644 ui/index.html delete mode 100644 ui/static/partials/index.html diff --git a/ui/index.html b/ui/index.html deleted file mode 100644 index 622c4621..00000000 --- a/ui/index.html +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- -
-
- -
- -
-

...

-
- - - - diff --git a/ui/static/js/app.js b/ui/static/js/app.js index 5e4e9155..5f09aa54 100644 --- a/ui/static/js/app.js +++ b/ui/static/js/app.js @@ -48,7 +48,11 @@ myApp.controller('tabsController', ['$scope', function($scope) { return ""; } } -}]).controller('mainController', function($scope) { +}]).controller('mainController', function($scope, $http) { + $http.get('http://0.0.0.0:5343/'). + then(function(response) { + $scope.greeting = response.data; + }); $scope.message = 'Overview!'; }).controller('kafkaController', function($scope) { $scope.message = 'Kafka...'; diff --git a/ui/static/partials/index.html b/ui/static/partials/index.html deleted file mode 100644 index 280f425f..00000000 --- a/ui/static/partials/index.html +++ /dev/null @@ -1,11 +0,0 @@ -

Home Page

- -

You should head over to the about page.

- -

Here's an Angular Demo:

- -
- -
-

Hello {{ yourName }}

-
\ No newline at end of file From ca08293ca3a20a46d0a538b97d261c9ff27625a4 Mon Sep 17 00:00:00 2001 From: damienkilgannon Date: Wed, 1 Feb 2017 00:29:20 +0000 Subject: [PATCH 04/14] support for CORS --- rest/requirements.txt | 3 ++- rest/rest_service.py | 2 ++ ui/static/js/app.js | 13 +++++++++---- ui/static/partials/overview.html | 6 ++++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/rest/requirements.txt b/rest/requirements.txt index d7b7d31b..4fe1eaef 100644 --- a/rest/requirements.txt +++ b/rest/requirements.txt @@ -20,4 +20,5 @@ scutils==1.1.0 six==1.10.0 testfixtures==4.11.0 ujson==1.35 -Werkzeug==0.11.11 \ No newline at end of file +Werkzeug==0.11.11 +Flask-CORS diff --git a/rest/rest_service.py b/rest/rest_service.py index 6387385e..82deae4e 100644 --- a/rest/rest_service.py +++ b/rest/rest_service.py @@ -1,6 +1,7 @@ import argparse from functools import wraps from flask import (Flask, jsonify, request) +from flask_cors import CORS from werkzeug.exceptions import BadRequest from copy import deepcopy import sys @@ -142,6 +143,7 @@ def __init__(self, settings_name): self.wrapper = SettingsWrapper() self.logger = None self.app = Flask(__name__) + CORS(self.app) self.kafka_connected = False self.redis_connected = False self.my_uuid = str(uuid.uuid4()).split('-')[4] diff --git a/ui/static/js/app.js b/ui/static/js/app.js index 5f09aa54..87f42260 100644 --- a/ui/static/js/app.js +++ b/ui/static/js/app.js @@ -49,10 +49,15 @@ myApp.controller('tabsController', ['$scope', function($scope) { } } }]).controller('mainController', function($scope, $http) { - $http.get('http://0.0.0.0:5343/'). - then(function(response) { - $scope.greeting = response.data; - }); + $scope.loadstatus=function(){ + $http.get('http://192.168.33.99:5343/') + .success(function(response){ + $scope.data=response; + }) + .error(function(){ + alert("An unexpected error occurred!"); + }); + } $scope.message = 'Overview!'; }).controller('kafkaController', function($scope) { $scope.message = 'Kafka...'; diff --git a/ui/static/partials/overview.html b/ui/static/partials/overview.html index 826630ed..a564890c 100644 --- a/ui/static/partials/overview.html +++ b/ui/static/partials/overview.html @@ -47,4 +47,10 @@

Submit a job to Scrapy-Cluster

+ + +
+ +

The ID is {{data}}

+

The content is {{data}}

\ No newline at end of file From 842e84572d75e564f56c4de0f0e32f2296ab05ad Mon Sep 17 00:00:00 2001 From: damienkilgannon Date: Thu, 2 Feb 2017 00:22:30 +0000 Subject: [PATCH 05/14] simple ui --- ui/static/css/style.css | 16 ------- ui/static/js/app.js | 34 +++++++++++++- ui/static/partials/overview.html | 77 +++++++++++++++++++------------- ui/ui_service.py | 2 +- 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/ui/static/css/style.css b/ui/static/css/style.css index b669a1b4..037566f0 100644 --- a/ui/static/css/style.css +++ b/ui/static/css/style.css @@ -1,12 +1,3 @@ -.btn-lg, -.input-lg, -.input-group-lg>.form-control, -.input-group-lg>.input-group-addon, -.input-group-lg>.input-group-btn>.btn -{ - line-height: 1.33; - height: 50px; -} .navbar-brand { font-family: 'Lato', sans-serif; @@ -18,10 +9,3 @@ { margin: 60px; } -.body-content -{ - max-width:1200px; - margin-left:auto; - margin-right:auto; - font-family:Arial; -} \ No newline at end of file diff --git a/ui/static/js/app.js b/ui/static/js/app.js index 87f42260..67c85863 100644 --- a/ui/static/js/app.js +++ b/ui/static/js/app.js @@ -1,4 +1,4 @@ -'use strict'; // See note about 'use strict'; below +'use strict'; var myApp = angular.module('myApp', [ 'ngRoute', @@ -58,7 +58,37 @@ myApp.controller('tabsController', ['$scope', function($scope) { alert("An unexpected error occurred!"); }); } - $scope.message = 'Overview!'; + + // create a blank object to handle form data. + $scope.request = {}; + + // calling our submit function. + $scope.submitForm = function() { + + var reqObj = { + url : $scope.request.url, + appid : "uiservice", + crawlid : $scope.request.crawlid, + maxdepth : $scope.request.maxdepth, + priority : $scope.request.priority, + }; + + // Posting data to php file + $http({ + method : 'POST', + url : 'http://192.168.33.99:5343/feed', + data : angular.toJson(reqObj), //forms user object + headers : {'Content-Type': 'application/json'} + }) + .success(function(data) { + if (data.errors) { + $scope.error = data.errors; + } else { + $scope.message = data.message; + } + }); + }; + }).controller('kafkaController', function($scope) { $scope.message = 'Kafka...'; }).controller('redisController', function($scope) { diff --git a/ui/static/partials/overview.html b/ui/static/partials/overview.html index a564890c..cf358d72 100644 --- a/ui/static/partials/overview.html +++ b/ui/static/partials/overview.html @@ -1,56 +1,71 @@ -
-
+
+
+
+
Scrapy Cluster status
+
+
-
+
-

Spiders

+

Crawl Queue

-

Total number of spiders: {{ message }}

-

Total unique spiders:

-

Total number of machines:

+

Status: {{ data.redis_connected == true ? 'OK' : 'FAULT' }}

-
+
-

Crawl Queue

+

Kafka

-

Status: {{ message }}

-

Queue backlog:

+

Status: {{ data.kafka_connected == true ? 'OK' : 'FAULT' }}

-
+
-

Kafka

+

Spiders

-

Status: {{ message }}

-

Total number of requests:

-

Total failed requests:

+

Total number of spiders:

-
-

Submit a job to Scrapy-Cluster

-
- URL: - - AppID: - - Expires: - +
+
+
+
+
Feed a crawl request to Scrapy Cluster
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
- -
- -

The ID is {{data}}

-

The content is {{data}}

-
\ No newline at end of file diff --git a/ui/ui_service.py b/ui/ui_service.py index 25219adc..80101b87 100644 --- a/ui/ui_service.py +++ b/ui/ui_service.py @@ -77,7 +77,7 @@ def close(self): """ Cleans up anything from the process """ - self.logger.info("Closing Rest Service") + self.logger.info("Closing UI Service") self.closed = True # Routes -------------------- From 97f23299f81a9f0916f0d320afbf887cfde5388c Mon Sep 17 00:00:00 2001 From: damienkilgannon Date: Thu, 2 Feb 2017 01:50:46 +0000 Subject: [PATCH 06/14] mvp --- ui/settings.py | 2 ++ ui/static/angular_settings.js | 3 +++ ui/static/js/app.js | 13 ++++++------- ui/static/partials/overview.html | 4 ++-- ui/templates/index.html | 3 ++- ui/ui_service.py | 4 +++- 6 files changed, 18 insertions(+), 11 deletions(-) create mode 100644 ui/static/angular_settings.js diff --git a/ui/settings.py b/ui/settings.py index cf6b92d9..41944f98 100644 --- a/ui/settings.py +++ b/ui/settings.py @@ -15,3 +15,5 @@ # internal configuration SLEEP_TIME = 5 WAIT_FOR_RESPONSE_TIME = 5 + +# Angular settings are dir /static diff --git a/ui/static/angular_settings.js b/ui/static/angular_settings.js new file mode 100644 index 00000000..237ecb16 --- /dev/null +++ b/ui/static/angular_settings.js @@ -0,0 +1,3 @@ +uiApp.constant('REST_CONFIG', { + url: 'http://localhost:5343/', +}); \ No newline at end of file diff --git a/ui/static/js/app.js b/ui/static/js/app.js index 67c85863..da063734 100644 --- a/ui/static/js/app.js +++ b/ui/static/js/app.js @@ -1,10 +1,10 @@ 'use strict'; -var myApp = angular.module('myApp', [ +var uiApp = angular.module('uiApp', [ 'ngRoute', ]); -myApp.config(['$routeProvider', +uiApp.config(['$routeProvider', function($routeProvider) { $routeProvider. when('/', { @@ -28,7 +28,7 @@ myApp.config(['$routeProvider', }); }]); -myApp.controller('tabsController', ['$scope', function($scope) { +uiApp.controller('tabsController', ['$scope', function($scope) { $scope.tabs = [ { link : '#/', label : 'Overview' }, { link : '#/kafka', label : 'Kafka' }, @@ -48,9 +48,9 @@ myApp.controller('tabsController', ['$scope', function($scope) { return ""; } } -}]).controller('mainController', function($scope, $http) { +}]).controller('mainController', function($scope, $http, REST_CONFIG) { $scope.loadstatus=function(){ - $http.get('http://192.168.33.99:5343/') + $http.get(REST_CONFIG.url) .success(function(response){ $scope.data=response; }) @@ -64,7 +64,6 @@ myApp.controller('tabsController', ['$scope', function($scope) { // calling our submit function. $scope.submitForm = function() { - var reqObj = { url : $scope.request.url, appid : "uiservice", @@ -76,7 +75,7 @@ myApp.controller('tabsController', ['$scope', function($scope) { // Posting data to php file $http({ method : 'POST', - url : 'http://192.168.33.99:5343/feed', + url : REST_CONFIG.url + '/feed', data : angular.toJson(reqObj), //forms user object headers : {'Content-Type': 'application/json'} }) diff --git a/ui/static/partials/overview.html b/ui/static/partials/overview.html index cf358d72..df4fdf6b 100644 --- a/ui/static/partials/overview.html +++ b/ui/static/partials/overview.html @@ -51,12 +51,12 @@

Spiders

- +
- +
diff --git a/ui/templates/index.html b/ui/templates/index.html index 865d4b64..9e3e573e 100644 --- a/ui/templates/index.html +++ b/ui/templates/index.html @@ -6,11 +6,12 @@ + - +