diff --git a/.gitignore b/.gitignore index d4ca88ee..82d55d87 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,7 @@ docker-compose.prod.yml docker-compose.override.yml # backup files -*.bak \ No newline at end of file +*.bak + +# IDE stuff +.idea \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index cff3264f..4c3cf0f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ coverage==4.3.4 # Updated from 4.0.3 cryptography==1.8.1 # Updated from 1.5 cssselect==1.0.1 # Updated from 0.9.2 enum34==1.1.6 +scutils==1.2.0dev7 funcsigs==1.0.2 future==0.16.0 # Updated from 0.15.2 idna==2.5 # Updated from 2.1 @@ -41,7 +42,6 @@ redis==2.10.5 requests-file==1.4.1 # Updated from 1.4 requests==2.13.0 # Updated from 2.11.1 retrying==1.3.3 -scutils==1.2.0dev7 service-identity==16.0.0 six==1.10.0 testfixtures==4.13.5 # Updated from 4.11.0 @@ -49,4 +49,4 @@ tldextract==2.0.2 # Updated from 2.0.1 ujson==1.35 w3lib==1.17.0 # Updated from 1.16.0 zope.interface==4.3.3 # Updated from 4.2.0 -# Generated with piprot 0.9.7 \ No newline at end of file +# Generated with piprot 0.9.7 diff --git a/rest/requirements.txt b/rest/requirements.txt index 832efe48..685eb2c2 100644 --- a/rest/requirements.txt +++ b/rest/requirements.txt @@ -20,5 +20,6 @@ scutils==1.2.0dev7 six==1.10.0 testfixtures==4.13.5 # Updated from 4.11.0 ujson==1.35 +Flask-Cors==3.0.2 Werkzeug==0.12.1 # Updated from 0.11.11 -# Generated with piprot 0.9.7 \ No newline at end of file +# Generated with piprot 0.9.7 diff --git a/rest/rest_service.py b/rest/rest_service.py index 06ddeb3b..858dc1ac 100644 --- a/rest/rest_service.py +++ b/rest/rest_service.py @@ -1,6 +1,6 @@ import argparse from functools import wraps -from flask import (Flask, jsonify, request) +from flask import (Flask, jsonify, request, current_app, make_response) from werkzeug.exceptions import BadRequest from copy import deepcopy import sys @@ -8,6 +8,8 @@ import os from retrying import retry from threading import Thread +from datetime import timedelta +from functools import update_wrapper import time import traceback import uuid @@ -35,6 +37,38 @@ # Route Decorators -------------------- + +def crossdomain(origin='*', max_age=21600, attach_to_all=True, automatic_options=True): + + if not isinstance(origin, basestring): + origin = ', '.join(origin) + if isinstance(max_age, timedelta): + max_age = max_age.total_seconds() + + options_resp = current_app.make_default_options_response() + return options_resp.headers['allow'] + + def decorator(f): + def wrapped_function(*args, **kwargs): + if automatic_options and request.method == 'OPTIONS': + resp = current_app.make_default_options_response() + else: + resp = make_response(f(*args, **kwargs)) + if not attach_to_all and request.method != 'OPTIONS': + return resp + + h = resp.headers + + h['Access-Control-Allow-Origin'] = origin + h['Access-Control-Max-Age'] = str(max_age) + + return resp + + f.provide_automatic_options = False + return update_wrapper(wrapped_function, f) + return decorator + + def log_call(call_name): """Log the API call to the logger.""" def decorator(f): @@ -582,12 +616,14 @@ def _decorate_routes(self): self.app.add_url_rule('/poll', 'poll', self.poll, methods=['POST']) + @crossdomain(origin='*') @log_call('Non-existant route called') @error_catch def catch(self, path): return self._create_ret_object(self.FAILURE, None, True, self.DOES_NOT_EXIST), 404 + @crossdomain(origin='*') @log_call('\'index\' endpoint called') @error_catch def index(self): @@ -601,6 +637,7 @@ def index(self): return data + @crossdomain(origin='*') @validate_json @log_call('\'feed\' endpoint called') @error_catch @@ -654,6 +691,7 @@ def feed(self): return self._create_ret_object(self.FAILURE, None, True, "Unable to connect to Kafka"), 500 + @crossdomain(origin='*') @validate_json @validate_schema('poll') @log_call('\'poll\' endpoint called') diff --git a/rest/tests/test_rest_service.py b/rest/tests/test_rest_service.py index efc15e52..76fc1e20 100644 --- a/rest/tests/test_rest_service.py +++ b/rest/tests/test_rest_service.py @@ -509,7 +509,7 @@ def test_index(self): "my_id": 'a908', "node_health": 'RED' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) def test_feed(self): @@ -528,9 +528,9 @@ def test_feed(self): }, u'status': u'FAILURE' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 500) + self.assertEquals(results.status_code, 500) # connected self.rest_service.kafka_connected = True @@ -548,9 +548,9 @@ def test_feed(self): }, u'status': u'FAILURE' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 500) + self.assertEquals(results.status_code, 500) # test no uuid self.rest_service._feed_to_kafka = MagicMock(return_value=True) @@ -562,9 +562,9 @@ def test_feed(self): u'error': None, u'status': u'SUCCESS' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 200) + self.assertEquals(results.status_code, 200) # test with uuid, got response time_list = [0, 1, 2, 3, 4, 5] @@ -584,9 +584,9 @@ def fancy_get_time(): u'error': None, u'status': u'SUCCESS' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 200) + self.assertEquals(results.status_code, 200) self.assertFalse(self.rest_service.uuids.has_key('key')) # test with uuid, no response @@ -603,9 +603,9 @@ def fancy_get_time2(): u'error': None, u'status': u'SUCCESS' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 200) + self.assertEquals(results.status_code, 200) self.assertTrue(self.rest_service.uuids.has_key('key')) self.assertEquals(self.rest_service.uuids['key'], 'poll') @@ -629,9 +629,9 @@ def test_poll(self): }, u'status': u'FAILURE' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 500) + self.assertEquals(results.status_code, 500) # test connected found poll key self.rest_service.redis_conn = MagicMock() @@ -645,9 +645,9 @@ def test_poll(self): u'error': None, u'status': u'SUCCESS' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 200) + self.assertEquals(results.status_code, 200) # test connected didnt find poll key self.rest_service.redis_conn.get = MagicMock(return_value=None) @@ -662,9 +662,9 @@ def test_poll(self): }, u'status': u'FAILURE' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 404) + self.assertEquals(results.status_code, 404) # test connection error self.rest_service._spawn_redis_connection_thread = MagicMock() @@ -684,9 +684,9 @@ def test_poll(self): }, u'status': u'FAILURE' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 500) + self.assertEquals(results.status_code, 500) # test value error self.rest_service.logger.warning = MagicMock() @@ -704,9 +704,8 @@ def test_poll(self): }, u'status': u'FAILURE' } - data = json.loads(results[0].data) + data = json.loads(results.data) self.assertEquals(data, d) - self.assertEquals(results[1], 500) + self.assertEquals(results.status_code, 500) self.rest_service.validator = orig - diff --git a/run_online_tests.sh b/run_online_tests.sh index 36c04c17..c61df2a9 100755 --- a/run_online_tests.sh +++ b/run_online_tests.sh @@ -7,6 +7,8 @@ HOST='localhost' PORT=6379 +export PYTHONPATH='.' + if [ $# -ne 2 ] then echo "---- Running utils online test with localhost 6379" diff --git a/ui/requirements.txt b/ui/requirements.txt new file mode 100644 index 00000000..a72d32b2 --- /dev/null +++ b/ui/requirements.txt @@ -0,0 +1,24 @@ +appdirs==1.4.0 +click==6.7 +ConcurrentLogHandler==0.9.1 +Flask==0.11 +Flask-Triangle==0.5.4 +funcsigs==1.0.2 +functools32==3.2.3.post2 +future==0.15.2 +itsdangerous==0.24 +Jinja2==2.9.5 +jsonschema==2.5.1 +kazoo==2.2.1 +MarkupSafe==0.23 +mock==2.0.0 +packaging==16.8 +pbr==1.10.0 +pyparsing==2.1.10 +python-json-logger==0.1.4 +redis==2.10.5 +scutils==1.2.0.dev6 +six==1.10.0 +testfixtures==4.10.0 +ujson==1.35 +Werkzeug==0.11.15 diff --git a/ui/settings.py b/ui/settings.py new file mode 100644 index 00000000..41944f98 --- /dev/null +++ b/ui/settings.py @@ -0,0 +1,19 @@ +# Flask configuration +FLASK_LOGGING_ENABLED = True +FLASK_PORT = 5000 + +# logging setup +LOGGER_NAME = 'ui_service' +LOG_DIR = 'logs' +LOG_FILE = 'ui_service.log' +LOG_MAX_BYTES = 10 * 1024 * 1024 +LOG_BACKUPS = 5 +LOG_STDOUT = True +LOG_JSON = False +LOG_LEVEL = 'INFO' + +# internal configuration +SLEEP_TIME = 5 +WAIT_FOR_RESPONSE_TIME = 5 + +# Angular settings are dir /static diff --git a/ui/static/angular_settings.js b/ui/static/angular_settings.js new file mode 100644 index 00000000..237ecb16 --- /dev/null +++ b/ui/static/angular_settings.js @@ -0,0 +1,3 @@ +uiApp.constant('REST_CONFIG', { + url: 'http://localhost:5343/', +}); \ No newline at end of file diff --git a/ui/static/css/style.css b/ui/static/css/style.css new file mode 100644 index 00000000..037566f0 --- /dev/null +++ b/ui/static/css/style.css @@ -0,0 +1,11 @@ +.navbar-brand +{ + font-family: 'Lato', sans-serif; + color:black; + font-size: 30px; + margin: 20px; +} +.btn +{ + margin: 60px; +} diff --git a/ui/static/img/logo.png b/ui/static/img/logo.png new file mode 100644 index 00000000..71faaf63 Binary files /dev/null and b/ui/static/img/logo.png differ diff --git a/ui/static/js/app.js b/ui/static/js/app.js new file mode 100644 index 00000000..da063734 --- /dev/null +++ b/ui/static/js/app.js @@ -0,0 +1,97 @@ +'use strict'; + +var uiApp = angular.module('uiApp', [ + 'ngRoute', +]); + +uiApp.config(['$routeProvider', + function($routeProvider) { + $routeProvider. + when('/', { + templateUrl: '/static/partials/overview.html', + controller: 'mainController' + }). + when('/kafka', { + templateUrl: '/static/partials/kafka.html', + controller: 'kafkaController' + }). + when('/crawlers', { + templateUrl: '/static/partials/crawlers.html', + controller: 'crawlersController' + }). + when('/redis', { + templateUrl: '/static/partials/redis.html', + controller: 'redisController' + }). + otherwise({ + redirectTo: '/' + }); + }]); + +uiApp.controller('tabsController', ['$scope', function($scope) { + $scope.tabs = [ + { link : '#/', label : 'Overview' }, + { link : '#/kafka', label : 'Kafka' }, + { link : '#/redis', label : 'Redis' }, + { link : '#/crawlers', label : 'Crawlers' } + ]; + + $scope.selectedTab = $scope.tabs[0]; + $scope.setSelectedTab = function(tab) { + $scope.selectedTab = tab; + } + + $scope.tabClass = function(tab) { + if ($scope.selectedTab == tab) { + return "active"; + } else { + return ""; + } + } +}]).controller('mainController', function($scope, $http, REST_CONFIG) { + $scope.loadstatus=function(){ + $http.get(REST_CONFIG.url) + .success(function(response){ + $scope.data=response; + }) + .error(function(){ + alert("An unexpected error occurred!"); + }); + } + + // create a blank object to handle form data. + $scope.request = {}; + + // calling our submit function. + $scope.submitForm = function() { + var reqObj = { + url : $scope.request.url, + appid : "uiservice", + crawlid : $scope.request.crawlid, + maxdepth : $scope.request.maxdepth, + priority : $scope.request.priority, + }; + + // Posting data to php file + $http({ + method : 'POST', + url : REST_CONFIG.url + '/feed', + data : angular.toJson(reqObj), //forms user object + headers : {'Content-Type': 'application/json'} + }) + .success(function(data) { + if (data.errors) { + $scope.error = data.errors; + } else { + $scope.message = data.message; + } + }); + }; + +}).controller('kafkaController', function($scope) { + $scope.message = 'Kafka...'; +}).controller('redisController', function($scope) { + $scope.message = 'Redis...'; +}).controller('crawlersController', function($scope) { + $scope.message = 'Crawler...'; +}); diff --git a/ui/static/partials/about.html b/ui/static/partials/about.html new file mode 100644 index 00000000..46c19ad3 --- /dev/null +++ b/ui/static/partials/about.html @@ -0,0 +1,6 @@ +

About

+ +

Hmm, not much here.

+

Best get back to the home page.

+ +

Tip: going to an incorrect URL will take you to the homepage, Try it!

\ No newline at end of file diff --git a/ui/static/partials/crawlers.html b/ui/static/partials/crawlers.html new file mode 100644 index 00000000..eff16654 --- /dev/null +++ b/ui/static/partials/crawlers.html @@ -0,0 +1,5 @@ +
+

Crawlers Page

+ +

{{ message }}

+
\ No newline at end of file diff --git a/ui/static/partials/kafka.html b/ui/static/partials/kafka.html new file mode 100644 index 00000000..d2fa5993 --- /dev/null +++ b/ui/static/partials/kafka.html @@ -0,0 +1,5 @@ +
+

Kafka Page

+ +

{{ message }}

+
\ No newline at end of file diff --git a/ui/static/partials/overview.html b/ui/static/partials/overview.html new file mode 100644 index 00000000..df4fdf6b --- /dev/null +++ b/ui/static/partials/overview.html @@ -0,0 +1,71 @@ +
+
+
+
Scrapy Cluster status
+
+
+
+
+
+

Crawl Queue

+
+
+

Status: {{ data.redis_connected == true ? 'OK' : 'FAULT' }}

+
+
+
+
+
+
+

Kafka

+
+
+

Status: {{ data.kafka_connected == true ? 'OK' : 'FAULT' }}

+
+
+
+
+
+
+

Spiders

+
+
+

Total number of spiders:

+
+
+
+
+
+
+
+
+
Feed a crawl request to Scrapy Cluster
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ +
+
+
+
+
+
diff --git a/ui/static/partials/redis.html b/ui/static/partials/redis.html new file mode 100644 index 00000000..b835ae57 --- /dev/null +++ b/ui/static/partials/redis.html @@ -0,0 +1,5 @@ +
+

Redis Page

+ +

{{ message }}

+
\ No newline at end of file diff --git a/ui/templates/index.html b/ui/templates/index.html new file mode 100644 index 00000000..9e3e573e --- /dev/null +++ b/ui/templates/index.html @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + +
+ +
+
+ + + + + \ No newline at end of file diff --git a/ui/ui_service.py b/ui/ui_service.py new file mode 100644 index 00000000..b6aec56d --- /dev/null +++ b/ui/ui_service.py @@ -0,0 +1,124 @@ +import argparse +from functools import wraps +from flask import Flask, request, send_file +from flask_triangle import Triangle +import time +import logging + +from scutils.log_factory import LogFactory +from scutils.settings_wrapper import SettingsWrapper + + +# Route Decorators -------------------- + +def log_call(call_name): + """Log the API call to the logger.""" + def decorator(f): + @wraps(f) + def wrapper(*args, **kw): + instance = args[0] + instance.logger.info(call_name, {"content": request.get_json()}) + return f(*args, **kw) + return wrapper + return decorator + + +class UIService(object): + + closed = False + start_time = 0 + + def __init__(self, settings_name): + """ + @param settings_name: the local settings file name + """ + self.settings_name = settings_name + self.wrapper = SettingsWrapper() + self.logger = None + self.app = Flask(__name__) + Triangle(self.app) + + def setup(self, level=None, log_file=None, json=None): + """ + Load everything up. Note that any arg here will override both + default and custom settings + + @param level: the log level + @param log_file: boolean t/f whether to log to a file, else stdout + @param json: boolean t/f whether to write the logs in json + """ + self.settings = self.wrapper.load(self.settings_name) + + my_level = level if level else self.settings['LOG_LEVEL'] + # negate because logger wants True for std out + my_output = not log_file if log_file else self.settings['LOG_STDOUT'] + my_json = json if json else self.settings['LOG_JSON'] + self.logger = LogFactory.get_instance(json=my_json, stdout=my_output, + level=my_level, + name=self.settings['LOGGER_NAME'], + dir=self.settings['LOG_DIR'], + file=self.settings['LOG_FILE'], + bytes=self.settings['LOG_MAX_BYTES'], + backups=self.settings['LOG_BACKUPS']) + + self._decorate_routes() + + self.start_time = time.time() + + # disable flask logger + if self.settings['FLASK_LOGGING_ENABLED'] == False: + log = logging.getLogger('werkzeug') + log.disabled = True + + def run(self): + """Main flask run loop""" + self.logger.info("Running main flask method on port " + str(self.settings['FLASK_PORT'])) + self.app.run(host='0.0.0.0', port=self.settings['FLASK_PORT']) + + def close(self): + """ + Cleans up anything from the process + """ + self.logger.info("Closing UI Service") + self.closed = True + + # Routes -------------------- + + def _decorate_routes(self): + """ + Decorates the routes to use within the flask app + """ + self.logger.debug("Decorating routes") + self.app.add_url_rule('/', 'index', self.index, methods=['GET']) + + @log_call('\'index\' endpoint called') + def index(self): + return send_file("templates/index.html") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Rest Service: Used for interacting and feeding Kafka' + ' requests to the cluster and returning data back\n') + + parser.add_argument('-s', '--settings', action='store', required=False, + help="The settings file to read from", default="localsettings.py") + parser.add_argument('-ll', '--log-level', action='store', required=False, + help="The log level", default=None, + choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']) + parser.add_argument('-lf', '--log-file', action='store_const', + required=False, const=True, default=None, + help='Log the output to the file specified in settings.py. Otherwise logs to stdout') + parser.add_argument('-lj', '--log-json', action='store_const', + required=False, const=True, default=None, + help="Log the data in JSON format") + + args = vars(parser.parse_args()) + + ui_service = UIService(args['settings']) + ui_service.setup(level=args['log_level'], log_file=args['log_file'], json=args['log_json']) + + try: + ui_service.run() + finally: + ui_service.close()