From 6689d277914422f45639025667693baa5fb7560e Mon Sep 17 00:00:00 2001 From: CarltonS13 Date: Tue, 2 Jul 2019 15:21:44 -0700 Subject: [PATCH 1/3] Implemented a file connector based of local to work with S3 via the smart open package and boto3 for reading in multiple keys in a bucket. --- cranial/messaging/file.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cranial/messaging/file.py b/cranial/messaging/file.py index 47dfc8d..5519b96 100644 --- a/cranial/messaging/file.py +++ b/cranial/messaging/file.py @@ -6,6 +6,8 @@ from cranial.messaging import base from cranial.common import logger +from cranial.connectors import FileConnector +# from cranial.connectors import file log = logger.get() @@ -53,12 +55,16 @@ def send(self, address=None, message='', endpoint=None, serde=json, # make sure the path exists for actual local files. if d != '' and '://' not in endpoint: os.makedirs(d, exist_ok=True) - self.logfiles[endpoint] = open(endpoint, - 'ab' if append else 'wb') + # todo pass to Instantiate File Connector with address and call get + # with endpoint. + # If given a path, break it up into address and endpoint + # self.logfiles[endpoint] = open(endpoint, 'ab' if append else 'wb') + self.logfiles[endpoint] = FileConnector(endpoint) - bytes_written = self.logfiles[endpoint].write( - message + '\n'.encode('utf-8')) - if bytes_written > 0: + # todo instead of write use put and pass in path + bytes_written = self.logfiles[endpoint].put( + message + '\n'.encode('utf-8'), append=append) + if bytes_written is True: return message else: raise Exception("Couldn't write to destination.") From c41bb44bc27d42e131eed8ea8ac77919e47b58fb Mon Sep 17 00:00:00 2001 From: CarltonS13 Date: Tue, 23 Jul 2019 17:02:20 -0700 Subject: [PATCH 2/3] Initial version of generate command Initial version of service type under generate Created cookiecutter service template to be used by generate service --- .../cli/service_template/cookiecutter.json | 7 +++++++ .../hooks/post_gen_project.sh | 4 ++++ .../service_template/hooks/pre_gen_project.py | 12 +++++++++++ ...name|lower|replace(' ', '-') }}_service.py | 11 ++++++++++ cranial/generate.py | 21 +++++++++++++++++++ 5 files changed, 55 insertions(+) create mode 100644 cranial/cli/service_template/cookiecutter.json create mode 100644 cranial/cli/service_template/hooks/post_gen_project.sh create mode 100644 cranial/cli/service_template/hooks/pre_gen_project.py create mode 100644 cranial/cli/service_template/{{cookiecutter.project_name}}/{{ cookiecutter.project_name|lower|replace(' ', '-') }}_service.py create mode 100644 cranial/generate.py diff --git a/cranial/cli/service_template/cookiecutter.json b/cranial/cli/service_template/cookiecutter.json new file mode 100644 index 0000000..53371f6 --- /dev/null +++ b/cranial/cli/service_template/cookiecutter.json @@ -0,0 +1,7 @@ +{ + "project_name": "my_project", + "listner": ["zmq","file","stdin","postgresql","kafka","db"], + "serde": "json", + "module": "", + "function": "" +} diff --git a/cranial/cli/service_template/hooks/post_gen_project.sh b/cranial/cli/service_template/hooks/post_gen_project.sh new file mode 100644 index 0000000..4e797db --- /dev/null +++ b/cranial/cli/service_template/hooks/post_gen_project.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +cp -rf . .. +rm -rf -- "$(pwd -P)" \ No newline at end of file diff --git a/cranial/cli/service_template/hooks/pre_gen_project.py b/cranial/cli/service_template/hooks/pre_gen_project.py new file mode 100644 index 0000000..0cee9b4 --- /dev/null +++ b/cranial/cli/service_template/hooks/pre_gen_project.py @@ -0,0 +1,12 @@ +import sys +import importlib + +# Todo check if function exists + +module= '{{ cookiecutter.module }}' + +try: + mod = importlib.import_module(module) +except Exception as e: + print('Could not import module.', 'Error: ', e) + sys.exit(1) \ No newline at end of file diff --git a/cranial/cli/service_template/{{cookiecutter.project_name}}/{{ cookiecutter.project_name|lower|replace(' ', '-') }}_service.py b/cranial/cli/service_template/{{cookiecutter.project_name}}/{{ cookiecutter.project_name|lower|replace(' ', '-') }}_service.py new file mode 100644 index 0000000..8e96c6e --- /dev/null +++ b/cranial/cli/service_template/{{cookiecutter.project_name}}/{{ cookiecutter.project_name|lower|replace(' ', '-') }}_service.py @@ -0,0 +1,11 @@ +from cranial.listeners.{{cookiecutter.listner}} import Listener +import {{cookiecutter.module}} +import {{cookiecutter.serde}} as serde + +listener = Listener() +while True: + data = listener.recv() # type: bytes + data = serde.loads(data) # type: dict + resp = {{cookiecutter.module}}.{{cookiecutter.function}}(data) + resp = serde.dumps(resp).encode('utf-8') + listener.resp(resp) \ No newline at end of file diff --git a/cranial/generate.py b/cranial/generate.py new file mode 100644 index 0000000..177c897 --- /dev/null +++ b/cranial/generate.py @@ -0,0 +1,21 @@ +#! /usr/bin/python3 +""" +Usage: + generate [] + + Possible types are: + service: generate a receiving listener with a custom module and function + which processes bytes from the specified listener +""" + +from docopt import docopt +from cookiecutter.main import cookiecutter +import os + +opts = docopt(__doc__) + +if opts.get('') == 'service': + # todo better way to get to the template + cookiecutter(os.getcwd() + '/cranial/cli/service_template') +else: + print('Only types that can be generated are: service') From fda3d83a83677a293d75b7b071f70c04428a2a0d Mon Sep 17 00:00:00 2001 From: CarltonS13 Date: Wed, 24 Jul 2019 10:59:31 -0700 Subject: [PATCH 3/3] Fixed pre-hood for service generator to allow for importing relative modules --- cranial/cli/service_template/hooks/pre_gen_project.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cranial/cli/service_template/hooks/pre_gen_project.py b/cranial/cli/service_template/hooks/pre_gen_project.py index 0cee9b4..f03090f 100644 --- a/cranial/cli/service_template/hooks/pre_gen_project.py +++ b/cranial/cli/service_template/hooks/pre_gen_project.py @@ -1,12 +1,11 @@ +import os import sys -import importlib -# Todo check if function exists - -module= '{{ cookiecutter.module }}' +p = os.path.abspath('..') +sys.path.append(p) try: - mod = importlib.import_module(module) + import {{ cookiecutter.module }} except Exception as e: - print('Could not import module.', 'Error: ', e) + print('Could not find module. Is module in the current directory?', 'Error: ', e) sys.exit(1) \ No newline at end of file