From e61e14ded4aef6ff0720f734ff2eb7868189618f Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 6 Mar 2019 15:33:38 +1100 Subject: [PATCH 01/47] NEW: Predict Endpoint --- surround/client.1.py | 23 +++++++++ surround/init-stage-with-data/README.md | 5 ++ surround/init-stage-with-data/config.yaml | 3 ++ .../data/HelloSurround.txt | 1 + .../init-stage-with-data/data/HelloWorld.txt | 1 + surround/init-stage-with-data/main.py | 49 +++++++++++++++++++ surround/test.py | 35 +++++++++++++ 7 files changed, 117 insertions(+) create mode 100644 surround/client.1.py create mode 100644 surround/init-stage-with-data/README.md create mode 100644 surround/init-stage-with-data/config.yaml create mode 100644 surround/init-stage-with-data/data/HelloSurround.txt create mode 100644 surround/init-stage-with-data/data/HelloWorld.txt create mode 100644 surround/init-stage-with-data/main.py create mode 100644 surround/test.py diff --git a/surround/client.1.py b/surround/client.1.py new file mode 100644 index 00000000..3cd4c008 --- /dev/null +++ b/surround/client.1.py @@ -0,0 +1,23 @@ +import tornado.httpclient +import json +from tornado.escape import json_decode, json_encode +from tornado import gen +import tornado.options + + +@tornado.gen.coroutine +def json_fetch(http_client, body): + response = yield http_client.fetch("http://localhost:8889/predict", method='POST', body=body) + raise gen.Return(response) + +@tornado.gen.coroutine +def request(): + body = '{"test_json": "ok"}' + http_client = tornado.httpclient.AsyncHTTPClient() + http_response = yield json_fetch(http_client, body) + print(http_response.body) + + +if __name__ == "__main__": + tornado.options.parse_command_line() + tornado.ioloop.IOLoop.instance().run_sync(request) diff --git a/surround/init-stage-with-data/README.md b/surround/init-stage-with-data/README.md new file mode 100644 index 00000000..ec90a867 --- /dev/null +++ b/surround/init-stage-with-data/README.md @@ -0,0 +1,5 @@ +# Initialise stage with some data example +## Run +```bash +python3 main.py +``` diff --git a/surround/init-stage-with-data/config.yaml b/surround/init-stage-with-data/config.yaml new file mode 100644 index 00000000..8449b79b --- /dev/null +++ b/surround/init-stage-with-data/config.yaml @@ -0,0 +1,3 @@ +surround: + path_to_HelloSurround: temp/init-stage-with-data/data/HelloSurround.txt + path_to_HelloWorld: temp/init-stage-with-data/data/HelloWorld.txt diff --git a/surround/init-stage-with-data/data/HelloSurround.txt b/surround/init-stage-with-data/data/HelloSurround.txt new file mode 100644 index 00000000..5b0cd1af --- /dev/null +++ b/surround/init-stage-with-data/data/HelloSurround.txt @@ -0,0 +1 @@ +HelloSurround \ No newline at end of file diff --git a/surround/init-stage-with-data/data/HelloWorld.txt b/surround/init-stage-with-data/data/HelloWorld.txt new file mode 100644 index 00000000..8970971f --- /dev/null +++ b/surround/init-stage-with-data/data/HelloWorld.txt @@ -0,0 +1 @@ +HelloWorld \ No newline at end of file diff --git a/surround/init-stage-with-data/main.py b/surround/init-stage-with-data/main.py new file mode 100644 index 00000000..259bf73c --- /dev/null +++ b/surround/init-stage-with-data/main.py @@ -0,0 +1,49 @@ +import logging + +import tornado.ioloop + +from surround import Stage, SurroundData, Surround, Config, test + +class HelloSurround(Stage): + + def __init__(self): + self.data = None + + def init_stage(self, config): + file_ = open(config.get_path("surround.path_to_HelloSurround"), "r") + self.data = file_.read() + + def operate(self, surround_data, config): + print(self.data) + +class HelloWorld(Stage): + + def __init__(self): + self.data = None + + def init_stage(self, config): + file_ = open(config.get_path("surround.path_to_HelloWorld"), "r") + self.data = file_.read() + + def operate(self, surround_data, config): + print(self.data) + +class BasicData(SurroundData): + text = None + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + surround = Surround([HelloSurround(), HelloWorld()]) + surround_config = Config() + surround_config.read_config_files(["temp/init-stage-with-data/config.yaml"]) + surround.set_config(surround_config) + surround.init_stages() + # surround.process(BasicData()) + + test.Predict.surround = surround + test.Predict.data = BasicData() + + app = test.make_app() + app.listen(8889) + tornado.ioloop.IOLoop.current().start() + diff --git a/surround/test.py b/surround/test.py new file mode 100644 index 00000000..e06c9906 --- /dev/null +++ b/surround/test.py @@ -0,0 +1,35 @@ +import tornado.ioloop +import tornado.web +from tornado.escape import json_decode, json_encode + + +class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + +class Predict(tornado.web.RequestHandler): + surround = None + data = None + + def get(self): + self.write("hi") + + def post(self): + print(json_decode(self.request.body)) + print(Predict.surround) + Predict.surround.process(Predict.data) + print(Predict.data.text) + + + self.write("response") + +def make_app(): + return tornado.web.Application([ + (r"/", MainHandler), + (r"/predict", Predict), + ]) + +if __name__ == "__main__": + app = make_app() + app.listen(8888) + tornado.ioloop.IOLoop.current().start() From a50201fceaf731efe848a4c5faf4042d176a4284 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 8 Mar 2019 10:02:15 +1100 Subject: [PATCH 02/47] FIX: Path to config and other files --- surround/init-stage-with-data/config.yaml | 4 ++-- surround/init-stage-with-data/main.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/surround/init-stage-with-data/config.yaml b/surround/init-stage-with-data/config.yaml index 8449b79b..0134115a 100644 --- a/surround/init-stage-with-data/config.yaml +++ b/surround/init-stage-with-data/config.yaml @@ -1,3 +1,3 @@ surround: - path_to_HelloSurround: temp/init-stage-with-data/data/HelloSurround.txt - path_to_HelloWorld: temp/init-stage-with-data/data/HelloWorld.txt + path_to_HelloSurround: surround/init-stage-with-data/data/HelloSurround.txt + path_to_HelloWorld: surround/init-stage-with-data/data/HelloWorld.txt diff --git a/surround/init-stage-with-data/main.py b/surround/init-stage-with-data/main.py index 259bf73c..c0dfca56 100644 --- a/surround/init-stage-with-data/main.py +++ b/surround/init-stage-with-data/main.py @@ -35,7 +35,7 @@ class BasicData(SurroundData): logging.basicConfig(level=logging.INFO) surround = Surround([HelloSurround(), HelloWorld()]) surround_config = Config() - surround_config.read_config_files(["temp/init-stage-with-data/config.yaml"]) + surround_config.read_config_files(["surround/init-stage-with-data/config.yaml"]) surround.set_config(surround_config) surround.init_stages() # surround.process(BasicData()) From 3642f3578108ffc5b4b85e76e130c0b5f1d00f56 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 10:39:51 +1100 Subject: [PATCH 03/47] NEW: Run tasks on web endpoint --- setup.py | 2 +- surround/cli.py | 16 ++++++++++++++-- surround/runner/__init__.py | 0 surround/runner/web/__init__.py | 0 surround/runner/web/api.py | 20 ++++++++++++++++++++ 5 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 surround/runner/__init__.py create mode 100644 surround/runner/web/__init__.py create mode 100644 surround/runner/web/api.py diff --git a/setup.py b/setup.py index 7b27b94c..bf15d6f1 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ author='Scott Barnett', author_email='scott.barnett@deakin.edu.au', include_package_data=True, - packages=['surround', 'templates', 'surround.remote'], + packages=['surround', 'templates', 'surround.remote', 'surround.runner', 'surround.runner.web'], test_suite='surround.tests', entry_points={ 'console_scripts': [ diff --git a/surround/cli.py b/surround/cli.py index 84b7e29b..15d978b1 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -3,9 +3,11 @@ import sys import logging import subprocess +import tornado.ioloop from .remote import cli as remote_cli from .linter import Linter +from .runner.web import api PROJECTS = { "new" : { @@ -112,6 +114,7 @@ def parse_run_args(args): } path = args.path + web = args.web errors, warnings = Linter().check_project(deploy, path) if errors: print("Invalid Surround project") @@ -123,8 +126,16 @@ def parse_run_args(args): task = args.task else: task = 'list' - run_process = subprocess.Popen(['python3', '-m', 'doit', task], cwd=path) - run_process.wait() + + if web: + api.Predict.task = task + api.Predict.path = path + app = api.make_app() + app.listen(8889) + tornado.ioloop.IOLoop.current().start() + else: + run_process = subprocess.Popen(['python3', '-m', 'doit', task], cwd=path) + run_process.wait() def parse_tutorial_args(args): new_dir = os.path.join(args.path, "tutorial") @@ -197,6 +208,7 @@ def main(): run_parser = sub_parser.add_parser('run', help="Run a Surround project task, witout an argument all tasks will be shown") run_parser.add_argument('task', help="Task defined in a Surround project dodo.py file.", nargs='?') run_parser.add_argument('path', type=lambda x: is_valid_dir(parser, x), help="Path to a Surround project", nargs='?', default="./") + run_parser.add_argument('-w', '--web', help="Web Runner", action='store_true') linter_parser = sub_parser.add_parser('lint', help="Run the Surround linter") linter_group = linter_parser.add_mutually_exclusive_group(required=False) diff --git a/surround/runner/__init__.py b/surround/runner/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/surround/runner/web/__init__.py b/surround/runner/web/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py new file mode 100644 index 00000000..931eab80 --- /dev/null +++ b/surround/runner/web/api.py @@ -0,0 +1,20 @@ +import subprocess +import tornado.ioloop +import tornado.web +from tornado.escape import json_decode, json_encode + +__author__ = 'Akshat Bajaj' +__date__ = '2019/03/08' + +class Predict(tornado.web.RequestHandler): + task = None + path = None + + def post(self): + subprocess.Popen(['python3', '-m', 'doit', Predict.task], cwd=Predict.path) + self.write("Task executed successfully") + +def make_app(): + return tornado.web.Application([ + (r"/predict", Predict), + ]) From afdae23a7e1871343ab5fb712af30697cee2c0d3 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 10:47:43 +1100 Subject: [PATCH 04/47] FIX: Move client to web package --- surround/{client.1.py => runner/web/send_request.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename surround/{client.1.py => runner/web/send_request.py} (100%) diff --git a/surround/client.1.py b/surround/runner/web/send_request.py similarity index 100% rename from surround/client.1.py rename to surround/runner/web/send_request.py From 4c405e6ce1aadfe99c0a9273f4a67b62e710afd3 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 10:56:06 +1100 Subject: [PATCH 05/47] FIX: Remove unused imports from send_request --- surround/runner/web/send_request.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/surround/runner/web/send_request.py b/surround/runner/web/send_request.py index 3cd4c008..a49a1ff6 100644 --- a/surround/runner/web/send_request.py +++ b/surround/runner/web/send_request.py @@ -1,23 +1,19 @@ import tornado.httpclient -import json -from tornado.escape import json_decode, json_encode from tornado import gen import tornado.options - -@tornado.gen.coroutine +@tornado.gen.coroutine def json_fetch(http_client, body): response = yield http_client.fetch("http://localhost:8889/predict", method='POST', body=body) raise gen.Return(response) @tornado.gen.coroutine def request(): - body = '{"test_json": "ok"}' + body = '{"endpoint": "predict"}' http_client = tornado.httpclient.AsyncHTTPClient() http_response = yield json_fetch(http_client, body) print(http_response.body) - if __name__ == "__main__": tornado.options.parse_command_line() tornado.ioloop.IOLoop.instance().run_sync(request) From fcd0b42cece34681b99e024814fe953cf084386b Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 10:57:08 +1100 Subject: [PATCH 06/47] FIX: Remove init stage example --- surround/init-stage-with-data/README.md | 5 -- surround/init-stage-with-data/config.yaml | 3 -- .../data/HelloSurround.txt | 1 - .../init-stage-with-data/data/HelloWorld.txt | 1 - surround/init-stage-with-data/main.py | 49 ------------------- 5 files changed, 59 deletions(-) delete mode 100644 surround/init-stage-with-data/README.md delete mode 100644 surround/init-stage-with-data/config.yaml delete mode 100644 surround/init-stage-with-data/data/HelloSurround.txt delete mode 100644 surround/init-stage-with-data/data/HelloWorld.txt delete mode 100644 surround/init-stage-with-data/main.py diff --git a/surround/init-stage-with-data/README.md b/surround/init-stage-with-data/README.md deleted file mode 100644 index ec90a867..00000000 --- a/surround/init-stage-with-data/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Initialise stage with some data example -## Run -```bash -python3 main.py -``` diff --git a/surround/init-stage-with-data/config.yaml b/surround/init-stage-with-data/config.yaml deleted file mode 100644 index 0134115a..00000000 --- a/surround/init-stage-with-data/config.yaml +++ /dev/null @@ -1,3 +0,0 @@ -surround: - path_to_HelloSurround: surround/init-stage-with-data/data/HelloSurround.txt - path_to_HelloWorld: surround/init-stage-with-data/data/HelloWorld.txt diff --git a/surround/init-stage-with-data/data/HelloSurround.txt b/surround/init-stage-with-data/data/HelloSurround.txt deleted file mode 100644 index 5b0cd1af..00000000 --- a/surround/init-stage-with-data/data/HelloSurround.txt +++ /dev/null @@ -1 +0,0 @@ -HelloSurround \ No newline at end of file diff --git a/surround/init-stage-with-data/data/HelloWorld.txt b/surround/init-stage-with-data/data/HelloWorld.txt deleted file mode 100644 index 8970971f..00000000 --- a/surround/init-stage-with-data/data/HelloWorld.txt +++ /dev/null @@ -1 +0,0 @@ -HelloWorld \ No newline at end of file diff --git a/surround/init-stage-with-data/main.py b/surround/init-stage-with-data/main.py deleted file mode 100644 index c0dfca56..00000000 --- a/surround/init-stage-with-data/main.py +++ /dev/null @@ -1,49 +0,0 @@ -import logging - -import tornado.ioloop - -from surround import Stage, SurroundData, Surround, Config, test - -class HelloSurround(Stage): - - def __init__(self): - self.data = None - - def init_stage(self, config): - file_ = open(config.get_path("surround.path_to_HelloSurround"), "r") - self.data = file_.read() - - def operate(self, surround_data, config): - print(self.data) - -class HelloWorld(Stage): - - def __init__(self): - self.data = None - - def init_stage(self, config): - file_ = open(config.get_path("surround.path_to_HelloWorld"), "r") - self.data = file_.read() - - def operate(self, surround_data, config): - print(self.data) - -class BasicData(SurroundData): - text = None - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO) - surround = Surround([HelloSurround(), HelloWorld()]) - surround_config = Config() - surround_config.read_config_files(["surround/init-stage-with-data/config.yaml"]) - surround.set_config(surround_config) - surround.init_stages() - # surround.process(BasicData()) - - test.Predict.surround = surround - test.Predict.data = BasicData() - - app = test.make_app() - app.listen(8889) - tornado.ioloop.IOLoop.current().start() - From 08b037d4830a09237f4d6fd12f68308e57952bb2 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 10:59:12 +1100 Subject: [PATCH 07/47] FIX: Remove test.py --- surround/test.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 surround/test.py diff --git a/surround/test.py b/surround/test.py deleted file mode 100644 index e06c9906..00000000 --- a/surround/test.py +++ /dev/null @@ -1,35 +0,0 @@ -import tornado.ioloop -import tornado.web -from tornado.escape import json_decode, json_encode - - -class MainHandler(tornado.web.RequestHandler): - def get(self): - self.write("Hello, world") - -class Predict(tornado.web.RequestHandler): - surround = None - data = None - - def get(self): - self.write("hi") - - def post(self): - print(json_decode(self.request.body)) - print(Predict.surround) - Predict.surround.process(Predict.data) - print(Predict.data.text) - - - self.write("response") - -def make_app(): - return tornado.web.Application([ - (r"/", MainHandler), - (r"/predict", Predict), - ]) - -if __name__ == "__main__": - app = make_app() - app.listen(8888) - tornado.ioloop.IOLoop.current().start() From e4fb76ab5222814e8fe2c39f4e018ac98091f955 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 11:05:31 +1100 Subject: [PATCH 08/47] FIX: Remove unused imports from api --- surround/runner/web/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 931eab80..b3b04a13 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -1,7 +1,6 @@ import subprocess import tornado.ioloop import tornado.web -from tornado.escape import json_decode, json_encode __author__ = 'Akshat Bajaj' __date__ = '2019/03/08' From 69b1cb1d3f49a548d9e612f040878d2f76c65149 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 11 Mar 2019 11:13:03 +1100 Subject: [PATCH 09/47] FIX: CI errors --- .circleci/config.yml | 2 ++ pylintrc | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c67d449f..6de8bbec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -23,6 +23,8 @@ jobs: sudo pip3 install pylint==2.3.0 sudo pip3 install Flask==1.0.2 sudo pip3 install gunicorn==19.9.0 + sudo pip3 install tornado==6.0.1 + pylint setup.py find surround/ -iname "*.py" | xargs pylint diff --git a/pylintrc b/pylintrc index 509081f3..8592bda8 100644 --- a/pylintrc +++ b/pylintrc @@ -7,4 +7,5 @@ disable = missing-docstring, no-self-use, inconsistent-return-statements, line-too-long, - duplicate-code \ No newline at end of file + duplicate-code, + abstract-method \ No newline at end of file From ed1aad4c8450101f844849430b53d8f5cf4c0fd3 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 13 Mar 2019 15:46:22 +1100 Subject: [PATCH 10/47] FIX: Use wrapper class to run as web --- surround/cli.py | 8 +++++--- surround/runner/web/api.py | 13 +++++++------ surround/runner/wrapper.py | 9 +++++++++ 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 surround/runner/wrapper.py diff --git a/surround/cli.py b/surround/cli.py index 15d978b1..086383dd 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -3,6 +3,7 @@ import sys import logging import subprocess +import importlib.util import tornado.ioloop from .remote import cli as remote_cli @@ -128,9 +129,10 @@ def parse_run_args(args): task = 'list' if web: - api.Predict.task = task - api.Predict.path = path - app = api.make_app() + spec = importlib.util.spec_from_file_location("stages", os.getcwd() + "/" + os.path.basename(os.getcwd()) + "/" + "stages.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + app = api.make_app(module.WebWrapper()) app.listen(8889) tornado.ioloop.IOLoop.current().start() else: diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index b3b04a13..d26622fc 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -1,4 +1,3 @@ -import subprocess import tornado.ioloop import tornado.web @@ -6,14 +5,16 @@ __date__ = '2019/03/08' class Predict(tornado.web.RequestHandler): - task = None - path = None + def initialize(self, wrapper): + self.wrapper = wrapper def post(self): - subprocess.Popen(['python3', '-m', 'doit', Predict.task], cwd=Predict.path) + self.wrapper.run() self.write("Task executed successfully") -def make_app(): +def make_app(wrapper_object): + predict_init_args = dict(wrapper=wrapper_object) + return tornado.web.Application([ - (r"/predict", Predict), + (r"/predict", Predict, predict_init_args), ]) diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py new file mode 100644 index 00000000..72429c77 --- /dev/null +++ b/surround/runner/wrapper.py @@ -0,0 +1,9 @@ +from abc import ABC, abstractmethod + +class Wrapper(ABC): + def __init__(self, surround): + self.surround = surround + + @abstractmethod + def run(self): + pass From a0fae49c06f4765de1c89bf0732ee2d5bd807b44 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 13 Mar 2019 15:47:22 +1100 Subject: [PATCH 11/47] FIX: Add attribute-defined-outside-init to pylintrc --- pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pylintrc b/pylintrc index 8592bda8..afcb19ea 100644 --- a/pylintrc +++ b/pylintrc @@ -8,4 +8,5 @@ disable = missing-docstring, inconsistent-return-statements, line-too-long, duplicate-code, - abstract-method \ No newline at end of file + abstract-method, + attribute-defined-outside-init \ No newline at end of file From df1d985d5e003a68f2406903c84195c13e75ea0f Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 13 Mar 2019 16:46:23 +1100 Subject: [PATCH 12/47] NEW: Add hello_world_web example --- examples/hello_world_web/.gitignore | 1 + .../hello_world_web/.surround/config.yaml | 2 ++ examples/hello_world_web/README.md | 3 +++ examples/hello_world_web/data/.gitignore | 4 ++++ examples/hello_world_web/docs/.gitignore | 4 ++++ examples/hello_world_web/dodo.py | 7 +++++++ .../hello_world_web/__main__.py | 11 ++++++++++ .../hello_world_web/config.yaml | 2 ++ .../hello_world_web/hello_world_web/stages.py | 21 +++++++++++++++++++ examples/hello_world_web/main.py | 0 examples/hello_world_web/models/.gitignore | 4 ++++ examples/hello_world_web/notebooks/.gitignore | 4 ++++ examples/hello_world_web/output/.gitignore | 4 ++++ examples/hello_world_web/requirements.txt | 1 + examples/hello_world_web/scripts/.gitignore | 4 ++++ examples/hello_world_web/spikes/.gitignore | 4 ++++ examples/hello_world_web/tests/.gitignore | 4 ++++ pylintrc | 3 ++- 18 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 examples/hello_world_web/.gitignore create mode 100644 examples/hello_world_web/.surround/config.yaml create mode 100644 examples/hello_world_web/README.md create mode 100644 examples/hello_world_web/data/.gitignore create mode 100644 examples/hello_world_web/docs/.gitignore create mode 100644 examples/hello_world_web/dodo.py create mode 100644 examples/hello_world_web/hello_world_web/__main__.py create mode 100644 examples/hello_world_web/hello_world_web/config.yaml create mode 100644 examples/hello_world_web/hello_world_web/stages.py create mode 100644 examples/hello_world_web/main.py create mode 100644 examples/hello_world_web/models/.gitignore create mode 100644 examples/hello_world_web/notebooks/.gitignore create mode 100644 examples/hello_world_web/output/.gitignore create mode 100644 examples/hello_world_web/requirements.txt create mode 100644 examples/hello_world_web/scripts/.gitignore create mode 100644 examples/hello_world_web/spikes/.gitignore create mode 100644 examples/hello_world_web/tests/.gitignore diff --git a/examples/hello_world_web/.gitignore b/examples/hello_world_web/.gitignore new file mode 100644 index 00000000..667e9b39 --- /dev/null +++ b/examples/hello_world_web/.gitignore @@ -0,0 +1 @@ +.doit.db \ No newline at end of file diff --git a/examples/hello_world_web/.surround/config.yaml b/examples/hello_world_web/.surround/config.yaml new file mode 100644 index 00000000..c14e00c3 --- /dev/null +++ b/examples/hello_world_web/.surround/config.yaml @@ -0,0 +1,2 @@ +project-info: + project-name: hello_world_web \ No newline at end of file diff --git a/examples/hello_world_web/README.md b/examples/hello_world_web/README.md new file mode 100644 index 00000000..e9d38ff7 --- /dev/null +++ b/examples/hello_world_web/README.md @@ -0,0 +1,3 @@ +# Run project +`cd examples/hello_world_web` +`surround run dev` diff --git a/examples/hello_world_web/data/.gitignore b/examples/hello_world_web/data/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/data/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/docs/.gitignore b/examples/hello_world_web/docs/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/docs/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/dodo.py b/examples/hello_world_web/dodo.py new file mode 100644 index 00000000..95cee4b6 --- /dev/null +++ b/examples/hello_world_web/dodo.py @@ -0,0 +1,7 @@ +DOIT_CONFIG = {'verbosity':2} + +def task_dev(): + """Run the main task for the project""" + return { + 'actions': ["python3 -m hello_world_web"], + } diff --git a/examples/hello_world_web/hello_world_web/__main__.py b/examples/hello_world_web/hello_world_web/__main__.py new file mode 100644 index 00000000..dd5c31c9 --- /dev/null +++ b/examples/hello_world_web/hello_world_web/__main__.py @@ -0,0 +1,11 @@ +import logging +from .stages import WebWrapper + +logging.basicConfig(level=logging.INFO) + +def main(): + wrapper = WebWrapper() + wrapper.run() + +if __name__ == "__main__": + main() diff --git a/examples/hello_world_web/hello_world_web/config.yaml b/examples/hello_world_web/hello_world_web/config.yaml new file mode 100644 index 00000000..5f95ee15 --- /dev/null +++ b/examples/hello_world_web/hello_world_web/config.yaml @@ -0,0 +1,2 @@ +output: + text: Hello World \ No newline at end of file diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello_world_web/hello_world_web/stages.py new file mode 100644 index 00000000..02221758 --- /dev/null +++ b/examples/hello_world_web/hello_world_web/stages.py @@ -0,0 +1,21 @@ +from surround import Stage, SurroundData +from surround.runner.wrapper import Wrapper +from surround import Surround + +class HelloStage(Stage): + def operate(self, surround_data, config): + surround_data.text = "hello" + +class BasicData(SurroundData): + text = None + +class WebWrapper(Wrapper): + def __init__(self): + surround = Surround([HelloStage()]) + self.config = surround.config + super().__init__(surround) + + def run(self): + data = BasicData() + self.surround.process(data) + print(data.text) diff --git a/examples/hello_world_web/main.py b/examples/hello_world_web/main.py new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello_world_web/models/.gitignore b/examples/hello_world_web/models/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/models/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/notebooks/.gitignore b/examples/hello_world_web/notebooks/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/notebooks/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/output/.gitignore b/examples/hello_world_web/output/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/output/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/requirements.txt b/examples/hello_world_web/requirements.txt new file mode 100644 index 00000000..5ca9d1d4 --- /dev/null +++ b/examples/hello_world_web/requirements.txt @@ -0,0 +1 @@ +surround==0.0.2 \ No newline at end of file diff --git a/examples/hello_world_web/scripts/.gitignore b/examples/hello_world_web/scripts/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/scripts/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/spikes/.gitignore b/examples/hello_world_web/spikes/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/spikes/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/examples/hello_world_web/tests/.gitignore b/examples/hello_world_web/tests/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/examples/hello_world_web/tests/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/pylintrc b/pylintrc index afcb19ea..7b1e4c17 100644 --- a/pylintrc +++ b/pylintrc @@ -9,4 +9,5 @@ disable = missing-docstring, line-too-long, duplicate-code, abstract-method, - attribute-defined-outside-init \ No newline at end of file + attribute-defined-outside-init, + relative-beyond-top-level \ No newline at end of file From 0fe033e642c3cd66acf914714a7a5fe45537ede6 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 13 Mar 2019 17:13:19 +1100 Subject: [PATCH 13/47] FIX: Change port to 8888 and print details --- surround/cli.py | 7 +++++-- surround/runner/web/send_request.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/surround/cli.py b/surround/cli.py index 086383dd..49346e94 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -122,7 +122,6 @@ def parse_run_args(args): for e in errors + warnings: print(e) if not errors: - print("Project tasks:") if args.task: task = args.task else: @@ -133,9 +132,13 @@ def parse_run_args(args): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) app = api.make_app(module.WebWrapper()) - app.listen(8889) + app.listen(8888) + print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888") + print("Available endpoints:") + print("* POST /predict # Send JSON data to the Surround pipeline") tornado.ioloop.IOLoop.current().start() else: + print("Project tasks:") run_process = subprocess.Popen(['python3', '-m', 'doit', task], cwd=path) run_process.wait() diff --git a/surround/runner/web/send_request.py b/surround/runner/web/send_request.py index a49a1ff6..66c13609 100644 --- a/surround/runner/web/send_request.py +++ b/surround/runner/web/send_request.py @@ -4,7 +4,7 @@ @tornado.gen.coroutine def json_fetch(http_client, body): - response = yield http_client.fetch("http://localhost:8889/predict", method='POST', body=body) + response = yield http_client.fetch("http://localhost:8888/predict", method='POST', body=body) raise gen.Return(response) @tornado.gen.coroutine From 37862de06177b69fda3bfc5b34721e52343dabb3 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 13 Mar 2019 17:24:44 +1100 Subject: [PATCH 14/47] NEW: Add health check endpoint --- surround/cli.py | 1 + surround/runner/web/api.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/surround/cli.py b/surround/cli.py index 49346e94..90e3697a 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -135,6 +135,7 @@ def parse_run_args(args): app.listen(8888) print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888") print("Available endpoints:") + print("* GET / # Health check") print("* POST /predict # Send JSON data to the Surround pipeline") tornado.ioloop.IOLoop.current().start() else: diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index d26622fc..f5ddce0f 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -4,6 +4,10 @@ __author__ = 'Akshat Bajaj' __date__ = '2019/03/08' +class HealthCheck(tornado.web.RequestHandler): + def get(self): + self.write("OK!") + class Predict(tornado.web.RequestHandler): def initialize(self, wrapper): self.wrapper = wrapper @@ -16,5 +20,6 @@ def make_app(wrapper_object): predict_init_args = dict(wrapper=wrapper_object) return tornado.web.Application([ + (r"/", HealthCheck), (r"/predict", Predict, predict_init_args), ]) From 19c33d567a7152ab3150613f02df24b8cf08b19a Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Wed, 13 Mar 2019 17:27:27 +1100 Subject: [PATCH 15/47] FIX: Remove author and date from api --- surround/runner/web/api.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index f5ddce0f..495878e1 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -1,9 +1,6 @@ import tornado.ioloop import tornado.web -__author__ = 'Akshat Bajaj' -__date__ = '2019/03/08' - class HealthCheck(tornado.web.RequestHandler): def get(self): self.write("OK!") From b8bc5343638b6c3d184a27d201792ab090ec5f15 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 10:14:25 +1100 Subject: [PATCH 16/47] ENHANCE: Ask for class inherited from Wrapper --- pylintrc | 3 ++- surround/cli.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pylintrc b/pylintrc index 7b1e4c17..c0a42b72 100644 --- a/pylintrc +++ b/pylintrc @@ -10,4 +10,5 @@ disable = missing-docstring, duplicate-code, abstract-method, attribute-defined-outside-init, - relative-beyond-top-level \ No newline at end of file + relative-beyond-top-level, + eval-used \ No newline at end of file diff --git a/surround/cli.py b/surround/cli.py index 90e3697a..acfda01c 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -116,6 +116,7 @@ def parse_run_args(args): path = args.path web = args.web + classname = args.classname errors, warnings = Linter().check_project(deploy, path) if errors: print("Invalid Surround project") @@ -131,7 +132,10 @@ def parse_run_args(args): spec = importlib.util.spec_from_file_location("stages", os.getcwd() + "/" + os.path.basename(os.getcwd()) + "/" + "stages.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - app = api.make_app(module.WebWrapper()) + + obj = eval("module" + "." + str(classname) + "()") + app = api.make_app(obj) + app.listen(8888) print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888") print("Available endpoints:") @@ -215,6 +219,7 @@ def main(): run_parser.add_argument('task', help="Task defined in a Surround project dodo.py file.", nargs='?') run_parser.add_argument('path', type=lambda x: is_valid_dir(parser, x), help="Path to a Surround project", nargs='?', default="./") run_parser.add_argument('-w', '--web', help="Web Runner", action='store_true') + run_parser.add_argument('-c', '--classname', help="Name of the class inherited from Wrapper") linter_parser = sub_parser.add_parser('lint', help="Run the Surround linter") linter_group = linter_parser.add_mutually_exclusive_group(required=False) From d316e197a4818803988e0ffca16efb1c8066f715 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 10:36:02 +1100 Subject: [PATCH 17/47] FIX: Match available endpoints output horizontally --- surround/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/surround/cli.py b/surround/cli.py index acfda01c..5d5b9702 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -139,7 +139,7 @@ def parse_run_args(args): app.listen(8888) print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888") print("Available endpoints:") - print("* GET / # Health check") + print("* GET / # Health check") print("* POST /predict # Send JSON data to the Surround pipeline") tornado.ioloop.IOLoop.current().start() else: From 9a6c2a7a9f2225fca573967f9f086ba75ef9baba Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 11:30:26 +1100 Subject: [PATCH 18/47] NEW: Update health check to print more info --- surround/runner/web/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 495878e1..6ba2c64a 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -1,9 +1,13 @@ import tornado.ioloop import tornado.web +import pkg_resources class HealthCheck(tornado.web.RequestHandler): def get(self): - self.write("OK!") + self.write(dict( + app="Surround Server", + version=pkg_resources.get_distribution("surround").version + )) class Predict(tornado.web.RequestHandler): def initialize(self, wrapper): From acb9132ee9f8c1b0f22c1b5249a57dcddd232267 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 11:34:50 +1100 Subject: [PATCH 19/47] ENHANCE: Add uptime to health check endpoint --- surround/runner/web/api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 6ba2c64a..11eeea48 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -1,12 +1,17 @@ +import datetime import tornado.ioloop import tornado.web import pkg_resources +# Get time for uptime calculation. +START_TIME = datetime.datetime.now() + class HealthCheck(tornado.web.RequestHandler): def get(self): self.write(dict( app="Surround Server", - version=pkg_resources.get_distribution("surround").version + version=pkg_resources.get_distribution("surround").version, + uptime=str(datetime.datetime.now() - START_TIME) )) class Predict(tornado.web.RequestHandler): From 5ffb63186f5e26fe2b496657bc06ac5eedb771f5 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 13:46:15 +1100 Subject: [PATCH 20/47] NEW: Add upload endpoint --- examples/hello_world_web/template.html | 15 +++++++++++++++ surround/cli.py | 5 +++-- surround/runner/web/api.py | 12 ++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 examples/hello_world_web/template.html diff --git a/examples/hello_world_web/template.html b/examples/hello_world_web/template.html new file mode 100644 index 00000000..74740589 --- /dev/null +++ b/examples/hello_world_web/template.html @@ -0,0 +1,15 @@ + + + + Upload Form + + +

Select & Upload

+
+ File: +
+
+ +
+ + \ No newline at end of file diff --git a/surround/cli.py b/surround/cli.py index 5d5b9702..24e0e39a 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -139,8 +139,9 @@ def parse_run_args(args): app.listen(8888) print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888") print("Available endpoints:") - print("* GET / # Health check") - print("* POST /predict # Send JSON data to the Surround pipeline") + print("* GET / # Health check") + print("* GET /upload # Upload data") + print("* POST /predict # Send data to the Surround pipeline") tornado.ioloop.IOLoop.current().start() else: print("Project tasks:") diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 11eeea48..b3175d77 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -1,4 +1,5 @@ import datetime +import os import tornado.ioloop import tornado.web import pkg_resources @@ -14,11 +15,21 @@ def get(self): uptime=str(datetime.datetime.now() - START_TIME) )) +class Upload(tornado.web.RequestHandler): + def get_template_path(self): + return os.getcwd() + + def get(self): + self.render("template.html") + class Predict(tornado.web.RequestHandler): def initialize(self, wrapper): self.wrapper = wrapper def post(self): + fileinfo = self.request.files['data'][0] + print(fileinfo) + self.wrapper.run() self.write("Task executed successfully") @@ -27,5 +38,6 @@ def make_app(wrapper_object): return tornado.web.Application([ (r"/", HealthCheck), + (r"/upload", Upload), (r"/predict", Predict, predict_init_args), ]) From 3c562feb18a2e13a93141f233030076797a08f54 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 16:52:59 +1100 Subject: [PATCH 21/47] NEW: Handle uploaded data from web --- examples/hello_world_web/hello_world_web/stages.py | 13 ++++++++++--- surround/runner/web/api.py | 4 +--- surround/runner/wrapper.py | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello_world_web/hello_world_web/stages.py index 02221758..9ccd79c1 100644 --- a/examples/hello_world_web/hello_world_web/stages.py +++ b/examples/hello_world_web/hello_world_web/stages.py @@ -9,13 +9,20 @@ def operate(self, surround_data, config): class BasicData(SurroundData): text = None + def __init__(self, uploaded_data): + self.uploaded_data = uploaded_data + +class RotateImage(Stage): + def operate(self, surround_data, config): + print(surround_data.uploaded_data) + class WebWrapper(Wrapper): def __init__(self): - surround = Surround([HelloStage()]) + surround = Surround([HelloStage(), RotateImage()]) self.config = surround.config super().__init__(surround) - def run(self): - data = BasicData() + def run(self, uploaded_data): + data = BasicData(uploaded_data) self.surround.process(data) print(data.text) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index b3175d77..32349732 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -28,9 +28,7 @@ def initialize(self, wrapper): def post(self): fileinfo = self.request.files['data'][0] - print(fileinfo) - - self.wrapper.run() + self.wrapper.run(fileinfo['body']) self.write("Task executed successfully") def make_app(wrapper_object): diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py index 72429c77..5e8b07dd 100644 --- a/surround/runner/wrapper.py +++ b/surround/runner/wrapper.py @@ -5,5 +5,5 @@ def __init__(self, surround): self.surround = surround @abstractmethod - def run(self): + def run(self, uploaded_data): pass From cd95c835f3fa6089ed903fb7d63f15b1bf5e04bc Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 16:56:59 +1100 Subject: [PATCH 22/47] ENHANCE: Call init stages from wrapper --- surround/runner/wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py index 5e8b07dd..f47dd9fb 100644 --- a/surround/runner/wrapper.py +++ b/surround/runner/wrapper.py @@ -3,6 +3,7 @@ class Wrapper(ABC): def __init__(self, surround): self.surround = surround + self.surround.init_stages() @abstractmethod def run(self, uploaded_data): From 91cc43d83c7b7b1b04db948415d56f59de0ea469 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 17:05:02 +1100 Subject: [PATCH 23/47] FIX: Remove classname arg and use web arg --- surround/cli.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/surround/cli.py b/surround/cli.py index 24e0e39a..9ba5cfe1 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -115,8 +115,7 @@ def parse_run_args(args): } path = args.path - web = args.web - classname = args.classname + classname = args.web errors, warnings = Linter().check_project(deploy, path) if errors: print("Invalid Surround project") @@ -128,7 +127,7 @@ def parse_run_args(args): else: task = 'list' - if web: + if classname: spec = importlib.util.spec_from_file_location("stages", os.getcwd() + "/" + os.path.basename(os.getcwd()) + "/" + "stages.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -219,8 +218,8 @@ def main(): run_parser = sub_parser.add_parser('run', help="Run a Surround project task, witout an argument all tasks will be shown") run_parser.add_argument('task', help="Task defined in a Surround project dodo.py file.", nargs='?') run_parser.add_argument('path', type=lambda x: is_valid_dir(parser, x), help="Path to a Surround project", nargs='?', default="./") - run_parser.add_argument('-w', '--web', help="Web Runner", action='store_true') - run_parser.add_argument('-c', '--classname', help="Name of the class inherited from Wrapper") + run_parser.add_argument('-w', '--web', help="Name of the class inherited from Wrapper") + # run_parser.add_argument('-c', '--classname', help="Name of the class inherited from Wrapper") linter_parser = sub_parser.add_parser('lint', help="Run the Surround linter") linter_group = linter_parser.add_mutually_exclusive_group(required=False) From 01ad80720980ae1afc8591d08395b28069cb11df Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 17:07:35 +1100 Subject: [PATCH 24/47] FIX: Replace WebWrapper with PipelineWrapper in the example --- examples/hello_world_web/hello_world_web/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello_world_web/hello_world_web/stages.py index 9ccd79c1..b919b2d2 100644 --- a/examples/hello_world_web/hello_world_web/stages.py +++ b/examples/hello_world_web/hello_world_web/stages.py @@ -16,7 +16,7 @@ class RotateImage(Stage): def operate(self, surround_data, config): print(surround_data.uploaded_data) -class WebWrapper(Wrapper): +class PipelineWrapper(Wrapper): def __init__(self): surround = Surround([HelloStage(), RotateImage()]) self.config = surround.config From 4f8991b03d0af2fa0f01bacf910f6e56e21f93a4 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Thu, 14 Mar 2019 18:01:54 +1100 Subject: [PATCH 25/47] ENHANCE: Add type of uploaded object --- examples/hello_world_web/hello_world_web/stages.py | 3 ++- surround/runner/web/api.py | 7 +++++-- surround/runner/wrapper.py | 3 ++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello_world_web/hello_world_web/stages.py index b919b2d2..453073a1 100644 --- a/examples/hello_world_web/hello_world_web/stages.py +++ b/examples/hello_world_web/hello_world_web/stages.py @@ -19,8 +19,9 @@ def operate(self, surround_data, config): class PipelineWrapper(Wrapper): def __init__(self): surround = Surround([HelloStage(), RotateImage()]) + type_of_uploaded_object = "image" self.config = surround.config - super().__init__(surround) + super().__init__(surround, type_of_uploaded_object) def run(self, uploaded_data): data = BasicData(uploaded_data) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 32349732..f8052f66 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -27,8 +27,11 @@ def initialize(self, wrapper): self.wrapper = wrapper def post(self): - fileinfo = self.request.files['data'][0] - self.wrapper.run(fileinfo['body']) + if self.wrapper.type_of_uploaded_object == "image": + fileinfo = self.request.files['data'][0] + self.wrapper.run(fileinfo['body']) + else: + self.wrapper.run(None) self.write("Task executed successfully") def make_app(wrapper_object): diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py index f47dd9fb..0ed325c8 100644 --- a/surround/runner/wrapper.py +++ b/surround/runner/wrapper.py @@ -1,8 +1,9 @@ from abc import ABC, abstractmethod class Wrapper(ABC): - def __init__(self, surround): + def __init__(self, surround, type_of_uploaded_object): self.surround = surround + self.type_of_uploaded_object = type_of_uploaded_object self.surround.init_stages() @abstractmethod From 6c689999102bafd4992f603a18debd09bccbc792 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 10:14:50 +1100 Subject: [PATCH 26/47] ENHANCE: Validate type of uploaded object --- .../hello_world_web/hello_world_web/stages.py | 7 ++++--- pylintrc | 3 ++- surround/runner/wrapper.py | 18 +++++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello_world_web/hello_world_web/stages.py index 453073a1..feb2ef8f 100644 --- a/examples/hello_world_web/hello_world_web/stages.py +++ b/examples/hello_world_web/hello_world_web/stages.py @@ -24,6 +24,7 @@ def __init__(self): super().__init__(surround, type_of_uploaded_object) def run(self, uploaded_data): - data = BasicData(uploaded_data) - self.surround.process(data) - print(data.text) + if super().run(uploaded_data): + data = BasicData(uploaded_data) + self.surround.process(data) + print(data.text) diff --git a/pylintrc b/pylintrc index c0a42b72..f222b33a 100644 --- a/pylintrc +++ b/pylintrc @@ -11,4 +11,5 @@ disable = missing-docstring, abstract-method, attribute-defined-outside-init, relative-beyond-top-level, - eval-used \ No newline at end of file + eval-used, + unused-argument \ No newline at end of file diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py index 0ed325c8..ed65244d 100644 --- a/surround/runner/wrapper.py +++ b/surround/runner/wrapper.py @@ -1,11 +1,19 @@ -from abc import ABC, abstractmethod - -class Wrapper(ABC): +class Wrapper(): def __init__(self, surround, type_of_uploaded_object): self.surround = surround self.type_of_uploaded_object = type_of_uploaded_object self.surround.init_stages() - @abstractmethod def run(self, uploaded_data): - pass + return self.validate + + def validate(self): + return self.validate_type_of_uploaded_object() + + def validate_type_of_uploaded_object(self): + allowed_types = ['JSON', 'image'] + for type_ in allowed_types: + if self.type_of_uploaded_object == type_: + return True + print("Selected type not allowed") + return False From ec734d869eda346bdfaddb2d85e162aec2e4eb94 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 11:23:03 +1100 Subject: [PATCH 27/47] ENHANCE: Add process to streamline run --- examples/hello_world_web/hello_world_web/stages.py | 7 +++---- surround/runner/web/api.py | 4 ++-- surround/runner/wrapper.py | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello_world_web/hello_world_web/stages.py index feb2ef8f..453073a1 100644 --- a/examples/hello_world_web/hello_world_web/stages.py +++ b/examples/hello_world_web/hello_world_web/stages.py @@ -24,7 +24,6 @@ def __init__(self): super().__init__(surround, type_of_uploaded_object) def run(self, uploaded_data): - if super().run(uploaded_data): - data = BasicData(uploaded_data) - self.surround.process(data) - print(data.text) + data = BasicData(uploaded_data) + self.surround.process(data) + print(data.text) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index f8052f66..8544591a 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -29,9 +29,9 @@ def initialize(self, wrapper): def post(self): if self.wrapper.type_of_uploaded_object == "image": fileinfo = self.request.files['data'][0] - self.wrapper.run(fileinfo['body']) + self.wrapper.process(fileinfo['body']) else: - self.wrapper.run(None) + self.wrapper.process(None) self.write("Task executed successfully") def make_app(wrapper_object): diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py index ed65244d..d9d4e711 100644 --- a/surround/runner/wrapper.py +++ b/surround/runner/wrapper.py @@ -1,3 +1,5 @@ +import sys + class Wrapper(): def __init__(self, surround, type_of_uploaded_object): self.surround = surround @@ -5,7 +7,8 @@ def __init__(self, surround, type_of_uploaded_object): self.surround.init_stages() def run(self, uploaded_data): - return self.validate + if self.validate() is False: + sys.exit() def validate(self): return self.validate_type_of_uploaded_object() @@ -15,5 +18,12 @@ def validate_type_of_uploaded_object(self): for type_ in allowed_types: if self.type_of_uploaded_object == type_: return True - print("Selected type not allowed") + print("error: selected upload type not allowed") + print("Choose from: ") + for type_ in allowed_types: + print(type_) return False + + def process(self, uploaded_data): + Wrapper.run(self, uploaded_data) + self.run(uploaded_data) From 9ec9ea4d660d50a8c6c4931eccb43baf2e12d414 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 11:46:30 +1100 Subject: [PATCH 28/47] ENHANCE: Set default type as json --- surround/runner/wrapper.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py index d9d4e711..e305936d 100644 --- a/surround/runner/wrapper.py +++ b/surround/runner/wrapper.py @@ -1,9 +1,12 @@ import sys class Wrapper(): - def __init__(self, surround, type_of_uploaded_object): + def __init__(self, surround, type_of_uploaded_object=None): self.surround = surround - self.type_of_uploaded_object = type_of_uploaded_object + if type_of_uploaded_object: + self.type_of_uploaded_object = type_of_uploaded_object + else: + self.type_of_uploaded_object = "json" self.surround.init_stages() def run(self, uploaded_data): @@ -14,7 +17,7 @@ def validate(self): return self.validate_type_of_uploaded_object() def validate_type_of_uploaded_object(self): - allowed_types = ['JSON', 'image'] + allowed_types = ['json', 'image'] for type_ in allowed_types: if self.type_of_uploaded_object == type_: return True From 3b869ad78ad13ac534d46e423a8dc4dd53f0d259 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 11:49:14 +1100 Subject: [PATCH 29/47] QUALITY: Remove extra comment about classname arg --- surround/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/surround/cli.py b/surround/cli.py index 9ba5cfe1..42747174 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -219,7 +219,6 @@ def main(): run_parser.add_argument('task', help="Task defined in a Surround project dodo.py file.", nargs='?') run_parser.add_argument('path', type=lambda x: is_valid_dir(parser, x), help="Path to a Surround project", nargs='?', default="./") run_parser.add_argument('-w', '--web', help="Name of the class inherited from Wrapper") - # run_parser.add_argument('-c', '--classname', help="Name of the class inherited from Wrapper") linter_parser = sub_parser.add_parser('lint', help="Run the Surround linter") linter_group = linter_parser.add_mutually_exclusive_group(required=False) From 2175f1ae11ad903d6513e4f288d80d182499bf38 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 14:17:08 +1100 Subject: [PATCH 30/47] ENHANCE: Class inherited from wrapper can be anywhere --- pylintrc | 1 - surround/cli.py | 67 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/pylintrc b/pylintrc index f222b33a..6c814f3b 100644 --- a/pylintrc +++ b/pylintrc @@ -11,5 +11,4 @@ disable = missing-docstring, abstract-method, attribute-defined-outside-init, relative-beyond-top-level, - eval-used, unused-argument \ No newline at end of file diff --git a/surround/cli.py b/surround/cli.py index 42747174..724f59a8 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -1,9 +1,9 @@ import argparse import os import sys +import inspect import logging import subprocess -import importlib.util import tornado.ioloop from .remote import cli as remote_cli @@ -89,6 +89,51 @@ def is_valid_name(aparser, arg): else: return arg +def load_modules_from_path(path): + """Import all modules from the given directory + + :param path: Path to the directory + :type path: string + """ + # Check and fix the path + if path[-1:] != '/': + path += '/' + + # Get a list of files in the directory, if the directory exists + if not os.path.exists(path): + raise OSError("Directory does not exist: %s" % path) + + # Add path to the system path + sys.path.append(path) + # Load all the files in path + for f in os.listdir(path): + # Ignore anything that isn't a .py file + if len(f) > 3 and f[-3:] == '.py': + modname = f[:-3] + # Import the module + __import__(modname, globals(), locals(), ['*']) + +def load_class_from_name(modulename, classname): + """Import class from given module + + :param modulename: Name of the module + :type modulename: string + :param classname: Name of the class + :type classname: string + """ + + # Import the module + __import__(modulename, globals(), locals(), ['*']) + + # Get the class + cls = getattr(sys.modules[modulename], classname) + + # Check cls + if not inspect.isclass(cls): + raise TypeError("%s is not a class" % classname) + + return cls + def parse_lint_args(args): linter = Linter() if args.list: @@ -128,13 +173,23 @@ def parse_run_args(args): task = 'list' if classname: - spec = importlib.util.spec_from_file_location("stages", os.getcwd() + "/" + os.path.basename(os.getcwd()) + "/" + "stages.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + obj = None + loaded_class = None + load_modules_from_path(os.getcwd() + "/" + os.path.basename(os.getcwd())) + for file_ in os.listdir(os.getcwd() + "/" + os.path.basename(os.getcwd())): + if file_.endswith(".py"): + modulename = os.path.splitext(file_)[0] + if hasattr(sys.modules[modulename], classname): + print("has") + loaded_class = load_class_from_name(modulename, classname) + obj = loaded_class() + break - obj = eval("module" + "." + str(classname) + "()") - app = api.make_app(obj) + if obj is None: + print("error: " + classname + " not found") + return + app = api.make_app(obj) app.listen(8888) print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888") print("Available endpoints:") From fa5ce655fb6abd1a4219ea342e7574bcf34e19d0 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 14:27:38 +1100 Subject: [PATCH 31/47] ENHANCE: Use os.path.join for building paths in cli web --- surround/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/surround/cli.py b/surround/cli.py index 724f59a8..2fa599f2 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -175,8 +175,9 @@ def parse_run_args(args): if classname: obj = None loaded_class = None - load_modules_from_path(os.getcwd() + "/" + os.path.basename(os.getcwd())) - for file_ in os.listdir(os.getcwd() + "/" + os.path.basename(os.getcwd())): + path_to_modules = os.path.join(os.getcwd(), os.path.basename(os.getcwd())) + load_modules_from_path(path_to_modules) + for file_ in os.listdir(path_to_modules): if file_.endswith(".py"): modulename = os.path.splitext(file_)[0] if hasattr(sys.modules[modulename], classname): From 250828d8c1da65797fdde76a68250faae6f3325e Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 14:39:59 +1100 Subject: [PATCH 32/47] QUALITY: Remove pointless print statement from cli web --- surround/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/surround/cli.py b/surround/cli.py index 2fa599f2..8b8f04ca 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -181,7 +181,6 @@ def parse_run_args(args): if file_.endswith(".py"): modulename = os.path.splitext(file_)[0] if hasattr(sys.modules[modulename], classname): - print("has") loaded_class = load_class_from_name(modulename, classname) obj = loaded_class() break From 0225857cdb2673059f5b5e6de2d960ecdeacf3b4 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 14:42:27 +1100 Subject: [PATCH 33/47] FIX: Follow examples naming convention --- examples/{hello_world_web => hello-world-web}/.gitignore | 0 .../{hello_world_web => hello-world-web}/.surround/config.yaml | 0 examples/{hello_world_web => hello-world-web}/README.md | 0 examples/{hello_world_web => hello-world-web}/data/.gitignore | 0 examples/{hello_world_web => hello-world-web}/docs/.gitignore | 0 examples/{hello_world_web => hello-world-web}/dodo.py | 0 .../hello-world-web}/__main__.py | 0 .../hello-world-web}/config.yaml | 0 .../hello_world_web => hello-world-web/hello-world-web}/stages.py | 0 examples/{hello_world_web => hello-world-web}/main.py | 0 examples/{hello_world_web => hello-world-web}/models/.gitignore | 0 .../{hello_world_web => hello-world-web}/notebooks/.gitignore | 0 examples/{hello_world_web => hello-world-web}/output/.gitignore | 0 examples/{hello_world_web => hello-world-web}/requirements.txt | 0 examples/{hello_world_web => hello-world-web}/scripts/.gitignore | 0 examples/{hello_world_web => hello-world-web}/spikes/.gitignore | 0 examples/{hello_world_web => hello-world-web}/template.html | 0 examples/{hello_world_web => hello-world-web}/tests/.gitignore | 0 18 files changed, 0 insertions(+), 0 deletions(-) rename examples/{hello_world_web => hello-world-web}/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/.surround/config.yaml (100%) rename examples/{hello_world_web => hello-world-web}/README.md (100%) rename examples/{hello_world_web => hello-world-web}/data/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/docs/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/dodo.py (100%) rename examples/{hello_world_web/hello_world_web => hello-world-web/hello-world-web}/__main__.py (100%) rename examples/{hello_world_web/hello_world_web => hello-world-web/hello-world-web}/config.yaml (100%) rename examples/{hello_world_web/hello_world_web => hello-world-web/hello-world-web}/stages.py (100%) rename examples/{hello_world_web => hello-world-web}/main.py (100%) rename examples/{hello_world_web => hello-world-web}/models/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/notebooks/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/output/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/requirements.txt (100%) rename examples/{hello_world_web => hello-world-web}/scripts/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/spikes/.gitignore (100%) rename examples/{hello_world_web => hello-world-web}/template.html (100%) rename examples/{hello_world_web => hello-world-web}/tests/.gitignore (100%) diff --git a/examples/hello_world_web/.gitignore b/examples/hello-world-web/.gitignore similarity index 100% rename from examples/hello_world_web/.gitignore rename to examples/hello-world-web/.gitignore diff --git a/examples/hello_world_web/.surround/config.yaml b/examples/hello-world-web/.surround/config.yaml similarity index 100% rename from examples/hello_world_web/.surround/config.yaml rename to examples/hello-world-web/.surround/config.yaml diff --git a/examples/hello_world_web/README.md b/examples/hello-world-web/README.md similarity index 100% rename from examples/hello_world_web/README.md rename to examples/hello-world-web/README.md diff --git a/examples/hello_world_web/data/.gitignore b/examples/hello-world-web/data/.gitignore similarity index 100% rename from examples/hello_world_web/data/.gitignore rename to examples/hello-world-web/data/.gitignore diff --git a/examples/hello_world_web/docs/.gitignore b/examples/hello-world-web/docs/.gitignore similarity index 100% rename from examples/hello_world_web/docs/.gitignore rename to examples/hello-world-web/docs/.gitignore diff --git a/examples/hello_world_web/dodo.py b/examples/hello-world-web/dodo.py similarity index 100% rename from examples/hello_world_web/dodo.py rename to examples/hello-world-web/dodo.py diff --git a/examples/hello_world_web/hello_world_web/__main__.py b/examples/hello-world-web/hello-world-web/__main__.py similarity index 100% rename from examples/hello_world_web/hello_world_web/__main__.py rename to examples/hello-world-web/hello-world-web/__main__.py diff --git a/examples/hello_world_web/hello_world_web/config.yaml b/examples/hello-world-web/hello-world-web/config.yaml similarity index 100% rename from examples/hello_world_web/hello_world_web/config.yaml rename to examples/hello-world-web/hello-world-web/config.yaml diff --git a/examples/hello_world_web/hello_world_web/stages.py b/examples/hello-world-web/hello-world-web/stages.py similarity index 100% rename from examples/hello_world_web/hello_world_web/stages.py rename to examples/hello-world-web/hello-world-web/stages.py diff --git a/examples/hello_world_web/main.py b/examples/hello-world-web/main.py similarity index 100% rename from examples/hello_world_web/main.py rename to examples/hello-world-web/main.py diff --git a/examples/hello_world_web/models/.gitignore b/examples/hello-world-web/models/.gitignore similarity index 100% rename from examples/hello_world_web/models/.gitignore rename to examples/hello-world-web/models/.gitignore diff --git a/examples/hello_world_web/notebooks/.gitignore b/examples/hello-world-web/notebooks/.gitignore similarity index 100% rename from examples/hello_world_web/notebooks/.gitignore rename to examples/hello-world-web/notebooks/.gitignore diff --git a/examples/hello_world_web/output/.gitignore b/examples/hello-world-web/output/.gitignore similarity index 100% rename from examples/hello_world_web/output/.gitignore rename to examples/hello-world-web/output/.gitignore diff --git a/examples/hello_world_web/requirements.txt b/examples/hello-world-web/requirements.txt similarity index 100% rename from examples/hello_world_web/requirements.txt rename to examples/hello-world-web/requirements.txt diff --git a/examples/hello_world_web/scripts/.gitignore b/examples/hello-world-web/scripts/.gitignore similarity index 100% rename from examples/hello_world_web/scripts/.gitignore rename to examples/hello-world-web/scripts/.gitignore diff --git a/examples/hello_world_web/spikes/.gitignore b/examples/hello-world-web/spikes/.gitignore similarity index 100% rename from examples/hello_world_web/spikes/.gitignore rename to examples/hello-world-web/spikes/.gitignore diff --git a/examples/hello_world_web/template.html b/examples/hello-world-web/template.html similarity index 100% rename from examples/hello_world_web/template.html rename to examples/hello-world-web/template.html diff --git a/examples/hello_world_web/tests/.gitignore b/examples/hello-world-web/tests/.gitignore similarity index 100% rename from examples/hello_world_web/tests/.gitignore rename to examples/hello-world-web/tests/.gitignore From c432be7137e9f31ff2521c00e64531ee52ddf35f Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 14:51:24 +1100 Subject: [PATCH 34/47] FIX: Dodo and classname in example --- examples/hello-world-web/dodo.py | 2 +- examples/hello-world-web/hello-world-web/__main__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/hello-world-web/dodo.py b/examples/hello-world-web/dodo.py index 95cee4b6..197d5ea7 100644 --- a/examples/hello-world-web/dodo.py +++ b/examples/hello-world-web/dodo.py @@ -3,5 +3,5 @@ def task_dev(): """Run the main task for the project""" return { - 'actions': ["python3 -m hello_world_web"], + 'actions': ["python3 -m hello-world-web"], } diff --git a/examples/hello-world-web/hello-world-web/__main__.py b/examples/hello-world-web/hello-world-web/__main__.py index dd5c31c9..2cce2189 100644 --- a/examples/hello-world-web/hello-world-web/__main__.py +++ b/examples/hello-world-web/hello-world-web/__main__.py @@ -1,11 +1,11 @@ import logging -from .stages import WebWrapper +from .stages import PipelineWrapper logging.basicConfig(level=logging.INFO) def main(): - wrapper = WebWrapper() - wrapper.run() + wrapper = PipelineWrapper() + wrapper.run(None) if __name__ == "__main__": main() From 93c10030f395878ea426c0975cb2fa89a2df61dc Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 15:02:44 +1100 Subject: [PATCH 35/47] FIX: Move Wrapper class inside surround module --- .../hello-world-web/hello-world-web/stages.py | 3 +- surround/__init__.py | 2 +- surround/runner/wrapper.py | 32 ------------------- surround/surround.py | 31 ++++++++++++++++++ 4 files changed, 33 insertions(+), 35 deletions(-) delete mode 100644 surround/runner/wrapper.py diff --git a/examples/hello-world-web/hello-world-web/stages.py b/examples/hello-world-web/hello-world-web/stages.py index 453073a1..f100f01a 100644 --- a/examples/hello-world-web/hello-world-web/stages.py +++ b/examples/hello-world-web/hello-world-web/stages.py @@ -1,5 +1,4 @@ -from surround import Stage, SurroundData -from surround.runner.wrapper import Wrapper +from surround import Stage, SurroundData, Wrapper from surround import Surround class HelloStage(Stage): diff --git a/surround/__init__.py b/surround/__init__.py index 2d5d6f08..737671df 100644 --- a/surround/__init__.py +++ b/surround/__init__.py @@ -1,3 +1,3 @@ -from .surround import Surround, SurroundData +from .surround import Surround, SurroundData, Wrapper from .stage import Stage from .config import Config diff --git a/surround/runner/wrapper.py b/surround/runner/wrapper.py deleted file mode 100644 index e305936d..00000000 --- a/surround/runner/wrapper.py +++ /dev/null @@ -1,32 +0,0 @@ -import sys - -class Wrapper(): - def __init__(self, surround, type_of_uploaded_object=None): - self.surround = surround - if type_of_uploaded_object: - self.type_of_uploaded_object = type_of_uploaded_object - else: - self.type_of_uploaded_object = "json" - self.surround.init_stages() - - def run(self, uploaded_data): - if self.validate() is False: - sys.exit() - - def validate(self): - return self.validate_type_of_uploaded_object() - - def validate_type_of_uploaded_object(self): - allowed_types = ['json', 'image'] - for type_ in allowed_types: - if self.type_of_uploaded_object == type_: - return True - print("error: selected upload type not allowed") - print("Choose from: ") - for type_ in allowed_types: - print(type_) - return False - - def process(self, uploaded_data): - Wrapper.run(self, uploaded_data) - self.run(uploaded_data) diff --git a/surround/surround.py b/surround/surround.py index 66523c25..9b40d05c 100644 --- a/surround/surround.py +++ b/surround/surround.py @@ -115,3 +115,34 @@ def process(self, surround_data): LOGGER.exception("Failed processing Surround") surround_data.thaw() + +class Wrapper(): + def __init__(self, surround, type_of_uploaded_object=None): + self.surround = surround + if type_of_uploaded_object: + self.type_of_uploaded_object = type_of_uploaded_object + else: + self.type_of_uploaded_object = "json" + self.surround.init_stages() + + def run(self, uploaded_data): + if self.validate() is False: + sys.exit() + + def validate(self): + return self.validate_type_of_uploaded_object() + + def validate_type_of_uploaded_object(self): + allowed_types = ['json', 'image'] + for type_ in allowed_types: + if self.type_of_uploaded_object == type_: + return True + print("error: selected upload type not allowed") + print("Choose from: ") + for type_ in allowed_types: + print(type_) + return False + + def process(self, uploaded_data): + Wrapper.run(self, uploaded_data) + self.run(uploaded_data) From 46433845f94b4522e87a51da6e2ff5e84e23fd16 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Fri, 15 Mar 2019 17:06:51 +1100 Subject: [PATCH 36/47] ENHANCE: Class inherited from Wrapper should be inside module wrapper --- .../hello-world-web/__main__.py | 2 +- .../hello-world-web/hello-world-web/stages.py | 15 +------------- .../hello-world-web/wrapper.py | 14 +++++++++++++ surround/cli.py | 20 ++++++++++--------- 4 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 examples/hello-world-web/hello-world-web/wrapper.py diff --git a/examples/hello-world-web/hello-world-web/__main__.py b/examples/hello-world-web/hello-world-web/__main__.py index 2cce2189..07ec7408 100644 --- a/examples/hello-world-web/hello-world-web/__main__.py +++ b/examples/hello-world-web/hello-world-web/__main__.py @@ -1,5 +1,5 @@ import logging -from .stages import PipelineWrapper +from .wrapper import PipelineWrapper logging.basicConfig(level=logging.INFO) diff --git a/examples/hello-world-web/hello-world-web/stages.py b/examples/hello-world-web/hello-world-web/stages.py index f100f01a..f52f3634 100644 --- a/examples/hello-world-web/hello-world-web/stages.py +++ b/examples/hello-world-web/hello-world-web/stages.py @@ -1,5 +1,4 @@ -from surround import Stage, SurroundData, Wrapper -from surround import Surround +from surround import Stage, SurroundData class HelloStage(Stage): def operate(self, surround_data, config): @@ -14,15 +13,3 @@ def __init__(self, uploaded_data): class RotateImage(Stage): def operate(self, surround_data, config): print(surround_data.uploaded_data) - -class PipelineWrapper(Wrapper): - def __init__(self): - surround = Surround([HelloStage(), RotateImage()]) - type_of_uploaded_object = "image" - self.config = surround.config - super().__init__(surround, type_of_uploaded_object) - - def run(self, uploaded_data): - data = BasicData(uploaded_data) - self.surround.process(data) - print(data.text) diff --git a/examples/hello-world-web/hello-world-web/wrapper.py b/examples/hello-world-web/hello-world-web/wrapper.py new file mode 100644 index 00000000..b9709bbd --- /dev/null +++ b/examples/hello-world-web/hello-world-web/wrapper.py @@ -0,0 +1,14 @@ +from surround import Surround, Wrapper +from stages import HelloStage, RotateImage, BasicData + +class PipelineWrapper(Wrapper): + def __init__(self): + surround = Surround([HelloStage(), RotateImage()]) + type_of_uploaded_object = "image" + self.config = surround.config + super().__init__(surround, type_of_uploaded_object) + + def run(self, uploaded_data): + data = BasicData(uploaded_data) + self.surround.process(data) + print(data.text) diff --git a/surround/cli.py b/surround/cli.py index 8b8f04ca..860c97e6 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -105,13 +105,15 @@ def load_modules_from_path(path): # Add path to the system path sys.path.append(path) - # Load all the files in path - for f in os.listdir(path): - # Ignore anything that isn't a .py file - if len(f) > 3 and f[-3:] == '.py': - modname = f[:-3] - # Import the module - __import__(modname, globals(), locals(), ['*']) + + # Another possibility + # Load all the files, check: https://github.com/dstil/surround/pull/68/commits/2175f1ae11ad903d6513e4f288d80d182499bf38 + + # For now, just load the wrapper.py + modname = "wrapper" + + # Import the module + __import__(modname, globals(), locals(), ['*']) def load_class_from_name(modulename, classname): """Import class from given module @@ -180,13 +182,13 @@ def parse_run_args(args): for file_ in os.listdir(path_to_modules): if file_.endswith(".py"): modulename = os.path.splitext(file_)[0] - if hasattr(sys.modules[modulename], classname): + if modulename == "wrapper" and hasattr(sys.modules[modulename], classname): loaded_class = load_class_from_name(modulename, classname) obj = loaded_class() break if obj is None: - print("error: " + classname + " not found") + print("error: " + classname + " not found in the module wrapper") return app = api.make_app(obj) From 26dfdc9e666b3255a5707b6506e9bd8beb25ec3a Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 18 Mar 2019 15:05:41 +1100 Subject: [PATCH 37/47] ENHANCE: Add wrapper to project generated by init --- .../hello-world-web/__init__.py | 4 ++++ .../{template.html => upload.html} | 0 surround/cli.py | 3 +++ surround/runner/web/api.py | 2 +- templates/new/init.py.txt | 4 ++++ templates/new/main.py.txt | 18 +++--------------- templates/new/stages.py.txt | 10 +++++----- templates/new/upload.html.txt | 15 +++++++++++++++ templates/new/wrapper.py.txt | 15 +++++++++++++++ 9 files changed, 50 insertions(+), 21 deletions(-) create mode 100644 examples/hello-world-web/hello-world-web/__init__.py rename examples/hello-world-web/{template.html => upload.html} (100%) create mode 100644 templates/new/init.py.txt create mode 100644 templates/new/upload.html.txt create mode 100644 templates/new/wrapper.py.txt diff --git a/examples/hello-world-web/hello-world-web/__init__.py b/examples/hello-world-web/hello-world-web/__init__.py new file mode 100644 index 00000000..cc2c489b --- /dev/null +++ b/examples/hello-world-web/hello-world-web/__init__.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) diff --git a/examples/hello-world-web/template.html b/examples/hello-world-web/upload.html similarity index 100% rename from examples/hello-world-web/template.html rename to examples/hello-world-web/upload.html diff --git a/surround/cli.py b/surround/cli.py index 860c97e6..988eef68 100644 --- a/surround/cli.py +++ b/surround/cli.py @@ -34,6 +34,9 @@ ("README.md", "README.md.txt", False), ("{project_name}/stages.py", "stages.py.txt", True), ("{project_name}/__main__.py", "main.py.txt", True), + ("{project_name}/__init__.py", "init.py.txt", True), + ("{project_name}/wrapper.py", "wrapper.py.txt", True), + ("upload.html", "upload.html.txt", False), ("dodo.py", "dodo.py.txt", False) ] } diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 8544591a..671fcf65 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -20,7 +20,7 @@ def get_template_path(self): return os.getcwd() def get(self): - self.render("template.html") + self.render("upload.html") class Predict(tornado.web.RequestHandler): def initialize(self, wrapper): diff --git a/templates/new/init.py.txt b/templates/new/init.py.txt new file mode 100644 index 00000000..cc2c489b --- /dev/null +++ b/templates/new/init.py.txt @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) diff --git a/templates/new/main.py.txt b/templates/new/main.py.txt index a13af397..07ec7408 100644 --- a/templates/new/main.py.txt +++ b/templates/new/main.py.txt @@ -1,23 +1,11 @@ import logging -import argparse -import os -from surround import Surround -from .stages import ValidateData, {project_name}Data +from .wrapper import PipelineWrapper logging.basicConfig(level=logging.INFO) def main(): - surround = Surround([ValidateData()], __name__) - config = surround.config - - # TODO: Read data from input directory here - # os.path.join(config["data_path"], "your file here") - - data = {project_name}Data("data") - surround.process(data) - with open(os.path.abspath(os.path.join(config["output_path"], "output.txt")), 'w') as f: - f.write(data.output_data) - logging.info(data.output_data) + wrapper = PipelineWrapper() + wrapper.run(None) if __name__ == "__main__": main() diff --git a/templates/new/stages.py.txt b/templates/new/stages.py.txt index bfb76cb2..da83f5b2 100644 --- a/templates/new/stages.py.txt +++ b/templates/new/stages.py.txt @@ -1,13 +1,13 @@ -import logging from surround import Stage, SurroundData class {project_name}Data(SurroundData): input_data = None output_data = None - def __init__(self, input_data): - self.input_data = input_data + def __init__(self, uploaded_data): + self.uploaded_data = uploaded_data class ValidateData(Stage): - def operate(self, data, config): - data.output_data = "TODO: Validate input data assumptions here" + def operate(self, surround_data, config): + surround_data.output_data = "TODO: Validate input data assumptions here" + diff --git a/templates/new/upload.html.txt b/templates/new/upload.html.txt new file mode 100644 index 00000000..74740589 --- /dev/null +++ b/templates/new/upload.html.txt @@ -0,0 +1,15 @@ + + + + Upload Form + + +

Select & Upload

+
+ File: +
+
+ +
+ + \ No newline at end of file diff --git a/templates/new/wrapper.py.txt b/templates/new/wrapper.py.txt new file mode 100644 index 00000000..1c64b841 --- /dev/null +++ b/templates/new/wrapper.py.txt @@ -0,0 +1,15 @@ +from surround import Surround, Wrapper +from stages import ValidateData, {project_name}Data + +class PipelineWrapper(Wrapper): + def __init__(self): + surround = Surround([ValidateData()]) + type_of_uploaded_object = "image" + self.config = surround.config + super().__init__(surround, type_of_uploaded_object) + + def run(self, uploaded_data): + data = {project_name}Data(uploaded_data) + self.surround.process(data) + print(data.uploaded_data) + print(data.output_data) From b7f09d7e1711ac4e36203a983da645a70f22d478 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 18 Mar 2019 17:33:32 +1100 Subject: [PATCH 38/47] FIX: Write output data in surround init --- templates/new/wrapper.py.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/templates/new/wrapper.py.txt b/templates/new/wrapper.py.txt index 1c64b841..33a7680a 100644 --- a/templates/new/wrapper.py.txt +++ b/templates/new/wrapper.py.txt @@ -1,9 +1,10 @@ +import os from surround import Surround, Wrapper from stages import ValidateData, {project_name}Data class PipelineWrapper(Wrapper): def __init__(self): - surround = Surround([ValidateData()]) + surround = Surround([ValidateData()], __name__) type_of_uploaded_object = "image" self.config = surround.config super().__init__(surround, type_of_uploaded_object) @@ -11,5 +12,9 @@ class PipelineWrapper(Wrapper): def run(self, uploaded_data): data = {project_name}Data(uploaded_data) self.surround.process(data) + + with open(os.path.abspath(os.path.join(self.config["output_path"], "output.txt")), 'w') as f: + f.write(data.output_data) + print(data.uploaded_data) print(data.output_data) From 77d20a9de485d7ac68acd019dceddf794a9d7ab1 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 18 Mar 2019 17:39:08 +1100 Subject: [PATCH 39/47] FIX: Create empty wrapper file for tutorial --- templates/tutorial/init.py.txt | 4 ++++ templates/tutorial/upload.html.txt | 15 +++++++++++++++ templates/tutorial/wrapper.py.txt | 0 3 files changed, 19 insertions(+) create mode 100644 templates/tutorial/init.py.txt create mode 100644 templates/tutorial/upload.html.txt create mode 100644 templates/tutorial/wrapper.py.txt diff --git a/templates/tutorial/init.py.txt b/templates/tutorial/init.py.txt new file mode 100644 index 00000000..cc2c489b --- /dev/null +++ b/templates/tutorial/init.py.txt @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) diff --git a/templates/tutorial/upload.html.txt b/templates/tutorial/upload.html.txt new file mode 100644 index 00000000..74740589 --- /dev/null +++ b/templates/tutorial/upload.html.txt @@ -0,0 +1,15 @@ + + + + Upload Form + + +

Select & Upload

+
+ File: +
+
+ +
+ + \ No newline at end of file diff --git a/templates/tutorial/wrapper.py.txt b/templates/tutorial/wrapper.py.txt new file mode 100644 index 00000000..e69de29b From 10348be8720b7439dd79f0a15ef6f0e2d1f388cd Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Mon, 18 Mar 2019 17:52:24 +1100 Subject: [PATCH 40/47] FIX: Replace .gitignore with .gitkeep in example --- examples/hello-world-web/data/.gitignore | 4 ---- examples/hello-world-web/data/.gitkeep | 0 examples/hello-world-web/docs/.gitignore | 4 ---- examples/hello-world-web/docs/.gitkeep | 0 examples/hello-world-web/models/.gitignore | 4 ---- examples/hello-world-web/models/.gitkeep | 0 examples/hello-world-web/notebooks/.gitignore | 4 ---- examples/hello-world-web/notebooks/.gitkeep | 0 examples/hello-world-web/output/.gitignore | 4 ---- examples/hello-world-web/output/.gitkeep | 0 examples/hello-world-web/scripts/.gitignore | 4 ---- examples/hello-world-web/scripts/.gitkeep | 0 examples/hello-world-web/spikes/.gitignore | 4 ---- examples/hello-world-web/spikes/.gitkeep | 0 examples/hello-world-web/tests/.gitignore | 4 ---- examples/hello-world-web/tests/.gitkeep | 0 16 files changed, 32 deletions(-) delete mode 100644 examples/hello-world-web/data/.gitignore create mode 100644 examples/hello-world-web/data/.gitkeep delete mode 100644 examples/hello-world-web/docs/.gitignore create mode 100644 examples/hello-world-web/docs/.gitkeep delete mode 100644 examples/hello-world-web/models/.gitignore create mode 100644 examples/hello-world-web/models/.gitkeep delete mode 100644 examples/hello-world-web/notebooks/.gitignore create mode 100644 examples/hello-world-web/notebooks/.gitkeep delete mode 100644 examples/hello-world-web/output/.gitignore create mode 100644 examples/hello-world-web/output/.gitkeep delete mode 100644 examples/hello-world-web/scripts/.gitignore create mode 100644 examples/hello-world-web/scripts/.gitkeep delete mode 100644 examples/hello-world-web/spikes/.gitignore create mode 100644 examples/hello-world-web/spikes/.gitkeep delete mode 100644 examples/hello-world-web/tests/.gitignore create mode 100644 examples/hello-world-web/tests/.gitkeep diff --git a/examples/hello-world-web/data/.gitignore b/examples/hello-world-web/data/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/data/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/data/.gitkeep b/examples/hello-world-web/data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/docs/.gitignore b/examples/hello-world-web/docs/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/docs/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/docs/.gitkeep b/examples/hello-world-web/docs/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/models/.gitignore b/examples/hello-world-web/models/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/models/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/models/.gitkeep b/examples/hello-world-web/models/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/notebooks/.gitignore b/examples/hello-world-web/notebooks/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/notebooks/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/notebooks/.gitkeep b/examples/hello-world-web/notebooks/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/output/.gitignore b/examples/hello-world-web/output/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/output/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/output/.gitkeep b/examples/hello-world-web/output/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/scripts/.gitignore b/examples/hello-world-web/scripts/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/scripts/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/scripts/.gitkeep b/examples/hello-world-web/scripts/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/spikes/.gitignore b/examples/hello-world-web/spikes/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/spikes/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/spikes/.gitkeep b/examples/hello-world-web/spikes/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/examples/hello-world-web/tests/.gitignore b/examples/hello-world-web/tests/.gitignore deleted file mode 100644 index 5e7d2734..00000000 --- a/examples/hello-world-web/tests/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore everything in this directory -* -# Except this file -!.gitignore diff --git a/examples/hello-world-web/tests/.gitkeep b/examples/hello-world-web/tests/.gitkeep new file mode 100644 index 00000000..e69de29b From 8854cb67216a7a309a979c71d1443f7f2cab9388 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 09:43:46 +1100 Subject: [PATCH 41/47] ENHANCE: Use enum class for allowed types --- surround/surround.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/surround/surround.py b/surround/surround.py index 9b40d05c..d853cfa6 100644 --- a/surround/surround.py +++ b/surround/surround.py @@ -5,6 +5,7 @@ import logging import sys import os +from enum import Enum from datetime import datetime from abc import ABC from .stage import Stage @@ -116,6 +117,10 @@ def process(self, surround_data): surround_data.thaw() +class AllowedTypes(Enum): + JSON = "json" + IMAGE = "image" + class Wrapper(): def __init__(self, surround, type_of_uploaded_object=None): self.surround = surround @@ -133,13 +138,12 @@ def validate(self): return self.validate_type_of_uploaded_object() def validate_type_of_uploaded_object(self): - allowed_types = ['json', 'image'] - for type_ in allowed_types: - if self.type_of_uploaded_object == type_: + for type_ in AllowedTypes: + if self.type_of_uploaded_object == type_.value: return True print("error: selected upload type not allowed") print("Choose from: ") - for type_ in allowed_types: + for type_ in AllowedTypes: print(type_) return False From bc27cebb179e371531f9c4f022635e9ae3f85d19 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 09:59:09 +1100 Subject: [PATCH 42/47] ENHANCE: Use enum class var instead of value --- surround/__init__.py | 2 +- surround/runner/web/api.py | 3 ++- surround/surround.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/surround/__init__.py b/surround/__init__.py index 737671df..5ec339d3 100644 --- a/surround/__init__.py +++ b/surround/__init__.py @@ -1,3 +1,3 @@ -from .surround import Surround, SurroundData, Wrapper +from .surround import Surround, SurroundData, Wrapper, AllowedTypes from .stage import Stage from .config import Config diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 671fcf65..16f082c0 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -3,6 +3,7 @@ import tornado.ioloop import tornado.web import pkg_resources +from surround import AllowedTypes # Get time for uptime calculation. START_TIME = datetime.datetime.now() @@ -27,7 +28,7 @@ def initialize(self, wrapper): self.wrapper = wrapper def post(self): - if self.wrapper.type_of_uploaded_object == "image": + if self.wrapper.type_of_uploaded_object == AllowedTypes.IMAGE: fileinfo = self.request.files['data'][0] self.wrapper.process(fileinfo['body']) else: diff --git a/surround/surround.py b/surround/surround.py index d853cfa6..ba5988b0 100644 --- a/surround/surround.py +++ b/surround/surround.py @@ -127,7 +127,7 @@ def __init__(self, surround, type_of_uploaded_object=None): if type_of_uploaded_object: self.type_of_uploaded_object = type_of_uploaded_object else: - self.type_of_uploaded_object = "json" + self.type_of_uploaded_object = AllowedTypes.JSON self.surround.init_stages() def run(self, uploaded_data): @@ -139,7 +139,7 @@ def validate(self): def validate_type_of_uploaded_object(self): for type_ in AllowedTypes: - if self.type_of_uploaded_object == type_.value: + if self.type_of_uploaded_object == type_: return True print("error: selected upload type not allowed") print("Choose from: ") From dd60089652903f712ddf9cf2445a1699e14cba61 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 10:05:19 +1100 Subject: [PATCH 43/47] FIX: Make hello-world-web example to use Enum and remove print --- examples/hello-world-web/hello-world-web/stages.py | 3 ++- examples/hello-world-web/hello-world-web/wrapper.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/hello-world-web/hello-world-web/stages.py b/examples/hello-world-web/hello-world-web/stages.py index f52f3634..98eb425c 100644 --- a/examples/hello-world-web/hello-world-web/stages.py +++ b/examples/hello-world-web/hello-world-web/stages.py @@ -12,4 +12,5 @@ def __init__(self, uploaded_data): class RotateImage(Stage): def operate(self, surround_data, config): - print(surround_data.uploaded_data) + """Add operate code + """ diff --git a/examples/hello-world-web/hello-world-web/wrapper.py b/examples/hello-world-web/hello-world-web/wrapper.py index b9709bbd..7e3b93d1 100644 --- a/examples/hello-world-web/hello-world-web/wrapper.py +++ b/examples/hello-world-web/hello-world-web/wrapper.py @@ -1,14 +1,13 @@ -from surround import Surround, Wrapper +from surround import Surround, Wrapper, AllowedTypes from stages import HelloStage, RotateImage, BasicData class PipelineWrapper(Wrapper): def __init__(self): surround = Surround([HelloStage(), RotateImage()]) - type_of_uploaded_object = "image" + type_of_uploaded_object = AllowedTypes.IMAGE self.config = surround.config super().__init__(surround, type_of_uploaded_object) def run(self, uploaded_data): data = BasicData(uploaded_data) self.surround.process(data) - print(data.text) From 30bc4306a6930940d47052d8a25ca04e9be8d662 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 10:11:38 +1100 Subject: [PATCH 44/47] FIX: Use enum in template and remove print statements --- templates/new/stages.py.txt | 6 ++---- templates/new/wrapper.py.txt | 7 ++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/templates/new/stages.py.txt b/templates/new/stages.py.txt index da83f5b2..1034746f 100644 --- a/templates/new/stages.py.txt +++ b/templates/new/stages.py.txt @@ -1,13 +1,11 @@ from surround import Stage, SurroundData class {project_name}Data(SurroundData): - input_data = None output_data = None - def __init__(self, uploaded_data): - self.uploaded_data = uploaded_data + def __init__(self, input_data): + self.input_data = input_data class ValidateData(Stage): def operate(self, surround_data, config): surround_data.output_data = "TODO: Validate input data assumptions here" - diff --git a/templates/new/wrapper.py.txt b/templates/new/wrapper.py.txt index 33a7680a..26deb2c4 100644 --- a/templates/new/wrapper.py.txt +++ b/templates/new/wrapper.py.txt @@ -1,11 +1,11 @@ import os -from surround import Surround, Wrapper +from surround import Surround, Wrapper, AllowedTypes from stages import ValidateData, {project_name}Data class PipelineWrapper(Wrapper): def __init__(self): surround = Surround([ValidateData()], __name__) - type_of_uploaded_object = "image" + type_of_uploaded_object = AllowedTypes.IMAGE self.config = surround.config super().__init__(surround, type_of_uploaded_object) @@ -15,6 +15,3 @@ class PipelineWrapper(Wrapper): with open(os.path.abspath(os.path.join(self.config["output_path"], "output.txt")), 'w') as f: f.write(data.output_data) - - print(data.uploaded_data) - print(data.output_data) From 75568fe72bf7612555a3031eae1d816f097c48af Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 10:47:04 +1100 Subject: [PATCH 45/47] ENHANCE: Add info statements to endpoints --- surround/runner/web/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py index 16f082c0..e9388c10 100644 --- a/surround/runner/web/api.py +++ b/surround/runner/web/api.py @@ -10,30 +10,36 @@ class HealthCheck(tornado.web.RequestHandler): def get(self): + print("info: started get request at /") self.write(dict( app="Surround Server", version=pkg_resources.get_distribution("surround").version, uptime=str(datetime.datetime.now() - START_TIME) )) + print("info: finished get request at /") class Upload(tornado.web.RequestHandler): def get_template_path(self): return os.getcwd() def get(self): + print("info: started get request at /upload") self.render("upload.html") + print("info: finished get request at /upload") class Predict(tornado.web.RequestHandler): def initialize(self, wrapper): self.wrapper = wrapper def post(self): + print("info: started post request at /predict") if self.wrapper.type_of_uploaded_object == AllowedTypes.IMAGE: fileinfo = self.request.files['data'][0] self.wrapper.process(fileinfo['body']) else: self.wrapper.process(None) self.write("Task executed successfully") + print("info: finished post request at /predict") def make_app(wrapper_object): predict_init_args = dict(wrapper=wrapper_object) From a0eeb2974bf261af1a8dcf370d0c0611f6518064 Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 10:58:50 +1100 Subject: [PATCH 46/47] FIX: Remove send_request.py --- surround/runner/web/send_request.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 surround/runner/web/send_request.py diff --git a/surround/runner/web/send_request.py b/surround/runner/web/send_request.py deleted file mode 100644 index 66c13609..00000000 --- a/surround/runner/web/send_request.py +++ /dev/null @@ -1,19 +0,0 @@ -import tornado.httpclient -from tornado import gen -import tornado.options - -@tornado.gen.coroutine -def json_fetch(http_client, body): - response = yield http_client.fetch("http://localhost:8888/predict", method='POST', body=body) - raise gen.Return(response) - -@tornado.gen.coroutine -def request(): - body = '{"endpoint": "predict"}' - http_client = tornado.httpclient.AsyncHTTPClient() - http_response = yield json_fetch(http_client, body) - print(http_response.body) - -if __name__ == "__main__": - tornado.options.parse_command_line() - tornado.ioloop.IOLoop.instance().run_sync(request) From 72979a08226d1efc0f6c64475b115f08e2b96b1c Mon Sep 17 00:00:00 2001 From: AkshatBajaj Date: Tue, 19 Mar 2019 11:04:55 +1100 Subject: [PATCH 47/47] FIX: Replace uploaded data with input data --- examples/hello-world-web/hello-world-web/stages.py | 4 ++-- examples/hello-world-web/hello-world-web/wrapper.py | 4 ++-- surround/surround.py | 8 ++++---- templates/new/wrapper.py.txt | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/hello-world-web/hello-world-web/stages.py b/examples/hello-world-web/hello-world-web/stages.py index 98eb425c..3350749d 100644 --- a/examples/hello-world-web/hello-world-web/stages.py +++ b/examples/hello-world-web/hello-world-web/stages.py @@ -7,8 +7,8 @@ def operate(self, surround_data, config): class BasicData(SurroundData): text = None - def __init__(self, uploaded_data): - self.uploaded_data = uploaded_data + def __init__(self, input_data): + self.input_data = input_data class RotateImage(Stage): def operate(self, surround_data, config): diff --git a/examples/hello-world-web/hello-world-web/wrapper.py b/examples/hello-world-web/hello-world-web/wrapper.py index 7e3b93d1..ccc0b989 100644 --- a/examples/hello-world-web/hello-world-web/wrapper.py +++ b/examples/hello-world-web/hello-world-web/wrapper.py @@ -8,6 +8,6 @@ def __init__(self): self.config = surround.config super().__init__(surround, type_of_uploaded_object) - def run(self, uploaded_data): - data = BasicData(uploaded_data) + def run(self, input_data): + data = BasicData(input_data) self.surround.process(data) diff --git a/surround/surround.py b/surround/surround.py index ba5988b0..efa8f2ee 100644 --- a/surround/surround.py +++ b/surround/surround.py @@ -130,7 +130,7 @@ def __init__(self, surround, type_of_uploaded_object=None): self.type_of_uploaded_object = AllowedTypes.JSON self.surround.init_stages() - def run(self, uploaded_data): + def run(self, input_data): if self.validate() is False: sys.exit() @@ -147,6 +147,6 @@ def validate_type_of_uploaded_object(self): print(type_) return False - def process(self, uploaded_data): - Wrapper.run(self, uploaded_data) - self.run(uploaded_data) + def process(self, input_data): + Wrapper.run(self, input_data) + self.run(input_data) diff --git a/templates/new/wrapper.py.txt b/templates/new/wrapper.py.txt index 26deb2c4..d9866442 100644 --- a/templates/new/wrapper.py.txt +++ b/templates/new/wrapper.py.txt @@ -9,8 +9,8 @@ class PipelineWrapper(Wrapper): self.config = surround.config super().__init__(surround, type_of_uploaded_object) - def run(self, uploaded_data): - data = {project_name}Data(uploaded_data) + def run(self, input_data): + data = {project_name}Data(input_data) self.surround.process(data) with open(os.path.abspath(os.path.join(self.config["output_path"], "output.txt")), 'w') as f: