Merge pull request #68 from AkshatBajaj/feature/web-adapter

NEW: Predict Endpoint
a2i2 · Mar 19, 2019 · 4bc7272 · 4bc7272
2 parents 81bfbdb + 72979a0
commit 4bc7272
Show file tree

Hide file tree

Showing 37 changed files with 315 additions and 25 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -23,6 +23,8 @@ jobs:
             sudo pip3 install pylint==2.3.0
             sudo pip3 install Flask==1.0.2
             sudo pip3 install gunicorn==19.9.0
+            sudo pip3 install tornado==6.0.1
+
             pylint setup.py
 
             find surround/ -iname "*.py" | xargs pylint

diff --git a/examples/hello-world-web/.gitignore b/examples/hello-world-web/.gitignore
@@ -0,0 +1 @@
+.doit.db
diff --git a/examples/hello-world-web/.surround/config.yaml b/examples/hello-world-web/.surround/config.yaml
@@ -0,0 +1,2 @@
+project-info:
+  project-name: hello_world_web
diff --git a/examples/hello-world-web/README.md b/examples/hello-world-web/README.md
@@ -0,0 +1,3 @@
+# Run project
+`cd examples/hello_world_web`
+`surround run dev`
diff --git a/examples/hello-world-web/data/.gitkeep b/examples/hello-world-web/data/.gitkeep
diff --git a/examples/hello-world-web/docs/.gitkeep b/examples/hello-world-web/docs/.gitkeep
diff --git a/examples/hello-world-web/dodo.py b/examples/hello-world-web/dodo.py
@@ -0,0 +1,7 @@
+DOIT_CONFIG = {'verbosity':2}
+
+def task_dev():
+    """Run the main task for the project"""
+    return {
+        'actions': ["python3 -m hello-world-web"],
+    }
diff --git a/examples/hello-world-web/hello-world-web/__init__.py b/examples/hello-world-web/hello-world-web/__init__.py
@@ -0,0 +1,4 @@
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
diff --git a/examples/hello-world-web/hello-world-web/__main__.py b/examples/hello-world-web/hello-world-web/__main__.py
@@ -0,0 +1,11 @@
+import logging
+from .wrapper import PipelineWrapper
+
+logging.basicConfig(level=logging.INFO)
+
+def main():
+    wrapper = PipelineWrapper()
+    wrapper.run(None)
+
+if __name__ == "__main__":
+    main()
diff --git a/examples/hello-world-web/hello-world-web/config.yaml b/examples/hello-world-web/hello-world-web/config.yaml
@@ -0,0 +1,2 @@
+output:
+  text: Hello World
diff --git a/examples/hello-world-web/hello-world-web/stages.py b/examples/hello-world-web/hello-world-web/stages.py
@@ -0,0 +1,16 @@
+from surround import Stage, SurroundData
+
+class HelloStage(Stage):
+    def operate(self, surround_data, config):
+        surround_data.text = "hello"
+
+class BasicData(SurroundData):
+    text = None
+
+    def __init__(self, input_data):
+        self.input_data = input_data
+
+class RotateImage(Stage):
+    def operate(self, surround_data, config):
+        """Add operate code
+        """
diff --git a/examples/hello-world-web/hello-world-web/wrapper.py b/examples/hello-world-web/hello-world-web/wrapper.py
@@ -0,0 +1,13 @@
+from surround import Surround, Wrapper, AllowedTypes
+from stages import HelloStage, RotateImage, BasicData
+
+class PipelineWrapper(Wrapper):
+    def __init__(self):
+        surround = Surround([HelloStage(), RotateImage()])
+        type_of_uploaded_object = AllowedTypes.IMAGE
+        self.config = surround.config
+        super().__init__(surround, type_of_uploaded_object)
+
+    def run(self, input_data):
+        data = BasicData(input_data)
+        self.surround.process(data)
diff --git a/examples/hello-world-web/main.py b/examples/hello-world-web/main.py
diff --git a/examples/hello-world-web/models/.gitkeep b/examples/hello-world-web/models/.gitkeep
diff --git a/examples/hello-world-web/notebooks/.gitkeep b/examples/hello-world-web/notebooks/.gitkeep
diff --git a/examples/hello-world-web/output/.gitkeep b/examples/hello-world-web/output/.gitkeep
diff --git a/examples/hello-world-web/requirements.txt b/examples/hello-world-web/requirements.txt
@@ -0,0 +1 @@
+surround==0.0.2
diff --git a/examples/hello-world-web/scripts/.gitkeep b/examples/hello-world-web/scripts/.gitkeep
diff --git a/examples/hello-world-web/spikes/.gitkeep b/examples/hello-world-web/spikes/.gitkeep
diff --git a/examples/hello-world-web/tests/.gitkeep b/examples/hello-world-web/tests/.gitkeep
diff --git a/examples/hello-world-web/upload.html b/examples/hello-world-web/upload.html
@@ -0,0 +1,15 @@
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+    <title>Upload Form</title>
+  </head>
+  <body>
+    <p><h1>Select & Upload</h1></p>
+    <form enctype="multipart/form-data" action="/predict" method="post">
+      File: <input type="file" name="data" />
+      <br />
+      <br />
+      <input type="submit" value="upload" />
+    </form>
+  </body>
+</html>
diff --git a/pylintrc b/pylintrc
@@ -7,4 +7,8 @@ disable = missing-docstring,
     no-self-use,
     inconsistent-return-statements,
     line-too-long,
-    duplicate-code
+    duplicate-code,
+    abstract-method,
+    attribute-defined-outside-init,
+    relative-beyond-top-level,
+    unused-argument
diff --git a/setup.py b/setup.py
@@ -17,7 +17,7 @@
       author='Scott Barnett',
       author_email='scott.barnett@deakin.edu.au',
       include_package_data=True,
-      packages=['surround', 'templates', 'surround.remote'],
+      packages=['surround', 'templates', 'surround.remote', 'surround.runner', 'surround.runner.web'],
       test_suite='surround.tests',
       entry_points={
           'console_scripts': [

diff --git a/surround/__init__.py b/surround/__init__.py
@@ -1,3 +1,3 @@
-from .surround import Surround, SurroundData
+from .surround import Surround, SurroundData, Wrapper, AllowedTypes
 from .stage import Stage
 from .config import Config
diff --git a/surround/cli.py b/surround/cli.py
@@ -1,11 +1,14 @@
 import argparse
 import os
 import sys
+import inspect
 import logging
 import subprocess
+import tornado.ioloop
 
 from .remote import cli as remote_cli
 from .linter import Linter
+from .runner.web import api
 
 PROJECTS = {
     "new" : {
@@ -31,6 +34,9 @@
             ("README.md", "README.md.txt", False),
             ("{project_name}/stages.py", "stages.py.txt", True),
             ("{project_name}/__main__.py", "main.py.txt", True),
+            ("{project_name}/__init__.py", "init.py.txt", True),
+            ("{project_name}/wrapper.py", "wrapper.py.txt", True),
+            ("upload.html", "upload.html.txt", False),
             ("dodo.py", "dodo.py.txt", False)
         ]
     }
@@ -86,6 +92,53 @@ def is_valid_name(aparser, arg):
     else:
         return arg
 
+def load_modules_from_path(path):
+    """Import all modules from the given directory
+
+    :param path: Path to the directory
+    :type path: string
+    """
+    # Check and fix the path
+    if path[-1:] != '/':
+        path += '/'
+
+    # Get a list of files in the directory, if the directory exists
+    if not os.path.exists(path):
+        raise OSError("Directory does not exist: %s" % path)
+
+    # Add path to the system path
+    sys.path.append(path)
+
+    # Another possibility
+    # Load all the files, check: https://github.com/dstil/surround/pull/68/commits/2175f1ae11ad903d6513e4f288d80d182499bf38
+
+    # For now, just load the wrapper.py
+    modname = "wrapper"
+
+    # Import the module
+    __import__(modname, globals(), locals(), ['*'])
+
+def load_class_from_name(modulename, classname):
+    """Import class from given module
+
+    :param modulename: Name of the module
+    :type modulename: string
+    :param classname: Name of the class
+    :type classname: string
+    """
+
+    # Import the module
+    __import__(modulename, globals(), locals(), ['*'])
+
+    # Get the class
+    cls = getattr(sys.modules[modulename], classname)
+
+    # Check cls
+    if not inspect.isclass(cls):
+        raise TypeError("%s is not a class" % classname)
+
+    return cls
+
 def parse_lint_args(args):
     linter = Linter()
     if args.list:
@@ -112,19 +165,47 @@ def parse_run_args(args):
     }
 
     path = args.path
+    classname = args.web
     errors, warnings = Linter().check_project(deploy, path)
     if errors:
         print("Invalid Surround project")
     for e in errors + warnings:
         print(e)
     if not errors:
-        print("Project tasks:")
         if args.task:
             task = args.task
         else:
             task = 'list'
-        run_process = subprocess.Popen(['python3', '-m', 'doit', task], cwd=path)
-        run_process.wait()
+
+        if classname:
+            obj = None
+            loaded_class = None
+            path_to_modules = os.path.join(os.getcwd(), os.path.basename(os.getcwd()))
+            load_modules_from_path(path_to_modules)
+            for file_ in os.listdir(path_to_modules):
+                if file_.endswith(".py"):
+                    modulename = os.path.splitext(file_)[0]
+                    if modulename == "wrapper" and hasattr(sys.modules[modulename], classname):
+                        loaded_class = load_class_from_name(modulename, classname)
+                        obj = loaded_class()
+                        break
+
+            if obj is None:
+                print("error: " + classname + " not found in the module wrapper")
+                return
+
+            app = api.make_app(obj)
+            app.listen(8888)
+            print(os.path.basename(os.getcwd()) + " is running on http://localhost:8888")
+            print("Available endpoints:")
+            print("* GET  /                 # Health check")
+            print("* GET  /upload           # Upload data")
+            print("* POST /predict          # Send data to the Surround pipeline")
+            tornado.ioloop.IOLoop.current().start()
+        else:
+            print("Project tasks:")
+            run_process = subprocess.Popen(['python3', '-m', 'doit', task], cwd=path)
+            run_process.wait()
 
 def parse_tutorial_args(args):
     new_dir = os.path.join(args.path, "tutorial")
@@ -197,6 +278,7 @@ def main():
     run_parser = sub_parser.add_parser('run', help="Run a Surround project task, witout an argument all tasks will be shown")
     run_parser.add_argument('task', help="Task defined in a Surround project dodo.py file.", nargs='?')
     run_parser.add_argument('path', type=lambda x: is_valid_dir(parser, x), help="Path to a Surround project", nargs='?', default="./")
+    run_parser.add_argument('-w', '--web', help="Name of the class inherited from Wrapper")
 
     linter_parser = sub_parser.add_parser('lint', help="Run the Surround linter")
     linter_group = linter_parser.add_mutually_exclusive_group(required=False)

diff --git a/surround/runner/__init__.py b/surround/runner/__init__.py
diff --git a/surround/runner/web/__init__.py b/surround/runner/web/__init__.py
diff --git a/surround/runner/web/api.py b/surround/runner/web/api.py
@@ -0,0 +1,51 @@
+import datetime
+import os
+import tornado.ioloop
+import tornado.web
+import pkg_resources
+from surround import AllowedTypes
+
+# Get time for uptime calculation.
+START_TIME = datetime.datetime.now()
+
+class HealthCheck(tornado.web.RequestHandler):
+    def get(self):
+        print("info: started get request at /")
+        self.write(dict(
+            app="Surround Server",
+            version=pkg_resources.get_distribution("surround").version,
+            uptime=str(datetime.datetime.now() - START_TIME)
+        ))
+        print("info: finished get request at /")
+
+class Upload(tornado.web.RequestHandler):
+    def get_template_path(self):
+        return os.getcwd()
+
+    def get(self):
+        print("info: started get request at /upload")
+        self.render("upload.html")
+        print("info: finished get request at /upload")
+
+class Predict(tornado.web.RequestHandler):
+    def initialize(self, wrapper):
+        self.wrapper = wrapper
+
+    def post(self):
+        print("info: started post request at /predict")
+        if self.wrapper.type_of_uploaded_object == AllowedTypes.IMAGE:
+            fileinfo = self.request.files['data'][0]
+            self.wrapper.process(fileinfo['body'])
+        else:
+            self.wrapper.process(None)
+        self.write("Task executed successfully")
+        print("info: finished post request at /predict")
+
+def make_app(wrapper_object):
+    predict_init_args = dict(wrapper=wrapper_object)
+
+    return tornado.web.Application([
+        (r"/", HealthCheck),
+        (r"/upload", Upload),
+        (r"/predict", Predict, predict_init_args),
+    ])
diff --git a/surround/surround.py b/surround/surround.py
@@ -5,6 +5,7 @@
 import logging
 import sys
 import os
+from enum import Enum
 from datetime import datetime
 from abc import ABC
 from .stage import Stage
@@ -115,3 +116,37 @@ def process(self, surround_data):
             LOGGER.exception("Failed processing Surround")
 
         surround_data.thaw()
+
+class AllowedTypes(Enum):
+    JSON = "json"
+    IMAGE = "image"
+
+class Wrapper():
+    def __init__(self, surround, type_of_uploaded_object=None):
+        self.surround = surround
+        if type_of_uploaded_object:
+            self.type_of_uploaded_object = type_of_uploaded_object
+        else:
+            self.type_of_uploaded_object = AllowedTypes.JSON
+        self.surround.init_stages()
+
+    def run(self, input_data):
+        if self.validate() is False:
+            sys.exit()
+
+    def validate(self):
+        return self.validate_type_of_uploaded_object()
+
+    def validate_type_of_uploaded_object(self):
+        for type_ in AllowedTypes:
+            if self.type_of_uploaded_object == type_:
+                return True
+        print("error: selected upload type not allowed")
+        print("Choose from: ")
+        for type_ in AllowedTypes:
+            print(type_)
+        return False
+
+    def process(self, input_data):
+        Wrapper.run(self, input_data)
+        self.run(input_data)
diff --git a/templates/new/init.py.txt b/templates/new/init.py.txt
@@ -0,0 +1,4 @@
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
diff --git a/templates/new/main.py.txt b/templates/new/main.py.txt
@@ -1,23 +1,11 @@
 import logging
-import argparse
-import os
-from surround import Surround
-from .stages import ValidateData, {project_name}Data
+from .wrapper import PipelineWrapper
 
 logging.basicConfig(level=logging.INFO)
 
 def main():
-    surround = Surround([ValidateData()], __name__)
-    config = surround.config
-
-    # TODO: Read data from input directory here
-    # os.path.join(config["data_path"], "your file here")
-
-    data = {project_name}Data("data")
-    surround.process(data)
-    with open(os.path.abspath(os.path.join(config["output_path"], "output.txt")), 'w') as f:
-        f.write(data.output_data)
-    logging.info(data.output_data)
+    wrapper = PipelineWrapper()
+    wrapper.run(None)
 
 if __name__ == "__main__":
     main()