UI service MVP

.gitignore

@@ -56,4 +56,7 @@ docker-compose.prod.yml
 docker-compose.override.yml
 
 # backup files
-*.bak
+*.bak
+
+# IDE stuff
+.idea

requirements.txt

@@ -1,33 +1,46 @@
-# this file is deprecated and will be removed
-
+appdirs==1.4.0
+attrs==16.3.0
 cffi==1.7.0
 characteristic==14.3.0
+click==6.7
 ConcurrentLogHandler==0.9.1
 coverage==4.0.3
 cryptography==1.5
 cssselect==0.9.2
 enum34==1.1.6
 Flask==0.11
+Flask-Cors==3.0.2
+Flask-Triangle==0.5.4
 funcsigs==1.0.2
+functools32==3.2.3.post2
 future==0.15.2
 idna==2.1
 ipaddress==1.0.16
+itsdangerous==0.24
+Jinja2==2.9.5
 jsonschema==2.5.1
 kafka-python==1.3.1
-kazoo>=2.2.1
+kazoo==2.2.1
 lxml==3.6.4
+MarkupSafe==0.23
 mock==2.0.0
 nose==1.3.7
+packaging==16.8
+parsel==1.1.0
 pbr==1.10.0
 pyasn1==0.1.9
 pyasn1-modules==0.0.8
 pycparser==2.14
+PyDispatcher==2.0.5
 pyOpenSSL==16.0.0
+pyparsing==2.1.10
 python-json-logger==0.1.4
 python-redis-lock==3.1.0
 PyYAML==3.12
 queuelib==1.4.2
 redis==2.10.5
+requests==2.13.0
+requests-file==1.4.1
 retrying==1.3.3
 Scrapy==1.2.0
 scutils==1.1
@@ -38,5 +51,5 @@ tldextract==2.0.1
 Twisted==16.4.0
 ujson==1.35
 w3lib==1.16.0
-wheel==0.29.0
+Werkzeug==0.11.15
 zope.interface==4.2.0

rest/requirements.txt

@@ -20,4 +20,5 @@ scutils==1.1.0
 six==1.10.0
 testfixtures==4.11.0
 ujson==1.35
-Werkzeug==0.11.11
+Werkzeug==0.11.11
+Flask-Cors==3.0.2

rest/rest_service.py

@@ -1,6 +1,7 @@
 import argparse
 from functools import wraps
 from flask import (Flask, jsonify, request)
+from flask_cors import CORS
 from werkzeug.exceptions import BadRequest
 from copy import deepcopy
 import sys
@@ -142,6 +143,7 @@ def __init__(self, settings_name):
         self.wrapper = SettingsWrapper()
         self.logger = None
         self.app = Flask(__name__)
+        CORS(self.app)
         self.kafka_connected = False
         self.redis_connected = False
         self.my_uuid = str(uuid.uuid4()).split('-')[4]

run_online_tests.sh

@@ -7,6 +7,8 @@
 HOST='localhost'
 PORT=6379
 
+export PYTHONPATH='.'
+
 if [ $# -ne 2 ]
   then
     echo "---- Running utils online test with localhost 6379"

ui/requirements.txt

@@ -0,0 +1,25 @@
+appdirs==1.4.0
+click==6.7
+ConcurrentLogHandler==0.9.1
+Flask==0.11
+Flask-Triangle==0.5.4
+funcsigs==1.0.2
+functools32==3.2.3.post2
+future==0.15.2
+itsdangerous==0.24
+Jinja2==2.9.5
+jsonschema==2.5.1
+kazoo==2.2.1
+MarkupSafe==0.23
+mock==2.0.0
+packaging==16.8
+pbr==1.10.0
+pyparsing==2.1.10
+python-json-logger==0.1.4
+redis==2.10.5
+scutils==1.2.0.dev6
+six==1.10.0
+testfixtures==4.10.0
+ujson==1.35
+Werkzeug==0.11.15
+Flask-Cors==3.0.2

ui/settings.py

@@ -0,0 +1,19 @@
+# Flask configuration
+FLASK_LOGGING_ENABLED = True
+FLASK_PORT = 5000
+
+# logging setup
+LOGGER_NAME = 'ui_service'
+LOG_DIR = 'logs'
+LOG_FILE = 'ui_service.log'
+LOG_MAX_BYTES = 10 * 1024 * 1024
+LOG_BACKUPS = 5
+LOG_STDOUT = True
+LOG_JSON = False
+LOG_LEVEL = 'INFO'
+
+# internal configuration
+SLEEP_TIME = 5
+WAIT_FOR_RESPONSE_TIME = 5
+
+# Angular settings are dir /static

ui/static/angular_settings.js

@@ -0,0 +1,3 @@
+uiApp.constant('REST_CONFIG', {
+  url: 'http://localhost:5343/',
+});

ui/static/css/style.css

@@ -0,0 +1,11 @@
+.navbar-brand
+{
+  font-family: 'Lato', sans-serif;
+  color:black;
+  font-size: 30px;
+  margin: 20px;
+}
+.btn
+{
+  margin: 60px;
+}

ui/static/js/app.js

@@ -0,0 +1,97 @@
+'use strict';
+
+var uiApp = angular.module('uiApp', [
+ 'ngRoute',
+]);
+
+uiApp.config(['$routeProvider',
+     function($routeProvider) {
+        $routeProvider.
+        when('/', {
+			templateUrl: '/static/partials/overview.html',
+			controller: 'mainController'
+		}).
+		when('/kafka', {
+			templateUrl: '/static/partials/kafka.html',
+			controller: 'kafkaController'
+		}).
+		when('/crawlers', {
+			templateUrl: '/static/partials/crawlers.html',
+			controller: 'crawlersController'
+		}).
+		when('/redis', {
+			templateUrl: '/static/partials/redis.html',
+			controller: 'redisController'
+		}).
+		otherwise({
+			redirectTo: '/'
+		});
+    }]);
+
+uiApp.controller('tabsController', ['$scope', function($scope) {
+  $scope.tabs = [
+      { link : '#/', label : 'Overview' },
+      { link : '#/kafka', label : 'Kafka' },
+      { link : '#/redis', label : 'Redis' },
+      { link : '#/crawlers', label : 'Crawlers' }
+    ];
+
+  $scope.selectedTab = $scope.tabs[0];
+  $scope.setSelectedTab = function(tab) {
+    $scope.selectedTab = tab;
+  }
+
+  $scope.tabClass = function(tab) {
+    if ($scope.selectedTab == tab) {
+      return "active";
+    } else {
+      return "";
+    }
+  }
+}]).controller('mainController', function($scope, $http, REST_CONFIG) {
+     $scope.loadstatus=function(){
+         $http.get(REST_CONFIG.url)
+         .success(function(response){
+              $scope.data=response;
+         })
+         .error(function(){
+              alert("An unexpected error occurred!");
+         });
+     }
+
+    // create a blank object to handle form data.
+    $scope.request = {};
+
+    // calling our submit function.
+    $scope.submitForm = function() {
+    var reqObj = {
+            url : $scope.request.url,
+            appid : "uiservice",
+            crawlid : $scope.request.crawlid,
+            maxdepth : $scope.request.maxdepth,
+            priority : $scope.request.priority,
+    };
+
+    // Posting data to php file
+    $http({
+      method  : 'POST',
+      url     : REST_CONFIG.url + '/feed',
+      data    : angular.toJson(reqObj), //forms user object
+      headers : {'Content-Type': 'application/json'}
+     })
+     .success(function(data) {
+        if (data.errors) {
+          $scope.error = data.errors;
+        } else {
+          $scope.message = data.message;
+        }
+      });
+    };
+
+}).controller('kafkaController', function($scope) {
+    $scope.message = 'Kafka...';
+}).controller('redisController', function($scope) {
+    $scope.message = 'Redis...';
+}).controller('crawlersController', function($scope) {
+    $scope.message = 'Crawler...';
+});

ui/static/partials/about.html

@@ -0,0 +1,6 @@
+<h1>About</h1>
+
+<p>Hmm, not much here.</p>
+<p>Best get back to the <a href="/#/">home page</a>.</p>
+
+<p>Tip: going to an incorrect URL will take you to the homepage, <a href="/#/404">Try it!</a></p>

ui/static/partials/crawlers.html

@@ -0,0 +1,5 @@
+<div class="jumbotron text-center">
+	<h1>Crawlers Page</h1>
+
+	<p>{{ message }}</p>
+</div>

ui/static/partials/kafka.html

@@ -0,0 +1,5 @@
+<div class="jumbotron text-center">
+	<h1>Kafka Page</h1>
+
+	<p>{{ message }}</p>
+</div>

ui/static/partials/overview.html

@@ -0,0 +1,71 @@
+<div class="col-xs-12" style="height:20px;"></div>
+<div class="panel-group">
+  <div class="panel panel-default">
+    <div class="panel-heading">Scrapy Cluster status</div>
+    <div class="panel-body">
+		<div class="row" ng-controller="mainController" ng-init="loadstatus()">
+		<div class="col-md-4">
+			<div ng-attr-class="{{ data.redis_connected && 'panel panel-success' || 'panel panel-danger' }}">
+				<div class="panel-heading">
+					<h3 class="panel-title">Crawl Queue</h3>
+				</div>
+				<div class="panel-body">
+					<p style="font-size:70%;">Status: {{ data.redis_connected == true ? 'OK' : 'FAULT' }} <br/></p>
+				</div>
+			</div>
+		</div>
+		<div class="col-md-4">
+			<div ng-attr-class="{{ data.kafka_connected && 'panel panel-success' || 'panel panel-danger' }}">
+				<div class="panel-heading">
+					<h3 class="panel-title">Kafka</h3>
+				</div>
+				<div class="panel-body">
+					<p style="font-size:70%;">Status: {{ data.kafka_connected == true ? 'OK' : 'FAULT' }} <br/></p>
+				</div>
+			</div>
+		</div>
+		<div class="col-md-4">
+			<div ng-attr-class="{{ data.kafka_connected + data.redis_connected ? 'panel panel-success' : 'panel panel-danger' }}">
+				<div class="panel-heading">
+					<h3 class="panel-title">Spiders</h3>
+				</div>
+				<div class="panel-body">
+					<p style="font-size:70%;">Total number of spiders: <br/></p>
+				</div>
+			</div>
+		</div>
+	</div>
+	</div>
+  </div>
+  <div class="col-xs-12" style="height:20px;"></div>
+  <div class="panel panel-default">
+    <div class="panel-heading">Feed a crawl request to Scrapy Cluster</div>
+    <div class="panel-body">
+		<div class="container" ng-controller="mainController">
+		<div class="col-md-10">
+			<!--<div class="page-header"><h2></h2></div>-->
+			<!-- FORM -->
+			<form name="userForm" ng-submit="submitForm()">
+			<div class="form-group">
+				<label>URL:</label>
+				<input type="text" name="url" class="form-control" ng-model="request.url">
+			</div>
+			<div class="form-group">
+				<label>CrawlID:</label>
+				<input type="text" name="crawlid" class="form-control" ng-model="request.crawlid">
+			</div>
+			<div class="form-group">
+				<label>Depth:</label>
+				<input type="text" name="depth" class="form-control" ng-model="request.depth">
+			</div>
+			<div class="form-group">
+				<label>Priority:</label>
+				<input type="text" name="priority" class="form-control" ng-model="request.priority">
+			</div>
+			<button type="submit" class="btn btn-primary">Submit</button>
+			</form>
+		</div>
+	</div>
+	</div>
+  </div>
+</div>

ui/static/partials/redis.html

@@ -0,0 +1,5 @@
+<div class="jumbotron text-center">
+	<h1>Redis Page</h1>
+
+	<p>{{ message }}</p>
+</div>