renamed lock to freeze, adjusted architecture of model data service

Author: zeryx

adk/ADK.py

@@ -2,49 +2,54 @@
 import json
 import os
 import sys
+import Algorithmia
 from adk.io import create_exception, format_data, format_response
-from adk.manifest.manifest import Manifest
+from adk.manifest.modeldata import ModelData
 
 
 class ADK(object):
-    def __init__(self, apply_func, load_func=None, client=None, manifest_path="model_manifest.json.lock"):
+    def __init__(self, apply_func, load_func=None, client=None):
         """
         Creates the adk object
         :param apply_func: A required function that can have an arity of 1-2, depending on if loading occurs
         :param load_func: An optional supplier function used if load time events are required, if a model manifest is provided;
         the function may have a single `manifest` parameter to interact with the model manifest, otherwise must have no parameters.
         :param client: A Algorithmia Client instance that might be user defined,
          and is used for interacting with a model manifest file; if defined.
-        :param manifest_path: A development / testing facing variable used to set the name and path
         """
         self.FIFO_PATH = "/tmp/algoout"
+
+        if client:
+            self.client = client
+        else:
+            self.client = Algorithmia.client()
+
         apply_args, _, _, _, _, _, _ = inspect.getfullargspec(apply_func)
         self.apply_arity = len(apply_args)
         if load_func:
             load_args, _, _, _, _, _, _ = inspect.getfullargspec(load_func)
             self.load_arity = len(load_args)
-            if self.load_arity > 2:
-                raise Exception("load function may either have no parameters, or one parameter providing the manifest "
-                                "state.")
+            if self.load_arity != 1:
+                raise Exception("load function expects 1 parameter to be used to store algorithm state")
             self.load_func = load_func
         else:
             self.load_func = None
-        if len(apply_args) > 2 or len(apply_args) == 0:
-            raise Exception("apply function may have between 1 and 2 parameters, not {}".format(len(apply_args)))
         self.apply_func = apply_func
         self.is_local = not os.path.exists(self.FIFO_PATH)
         self.load_result = None
         self.loading_exception = None
-        self.manifest = Manifest(client, manifest_path)
+        self.manifest_path = "model_manifest.json.freeze"
+        self.model_data = self.init_manifest(self.manifest_path)
+
+    def init_manifest(self, path):
+        return ModelData(self.client, path)
 
     def load(self):
         try:
-            if self.manifest.available():
-                self.manifest.initialize()
-                if self.load_func and self.load_arity == 1:
-                    self.load_result = self.load_func(self.manifest)
-            elif self.load_func:
-                self.load_result = self.load_func()
+            if self.model_data.available():
+                self.model_data.initialize()
+            if self.load_func:
+                self.load_result = self.load_func(self.model_data)
         except Exception as e:
             self.loading_exception = e
         finally:

adk/manifest/manifest.py renamed to adk/manifest/modeldata.py

@@ -4,12 +4,14 @@
 from adk.manifest.classes import FileData
 
 
-class Manifest(object):
+class ModelData(object):
     def __init__(self, client, model_manifest_path):
         self.manifest_lock_path = model_manifest_path
         self.manifest_data = get_manifest(self.manifest_lock_path)
         self.client = client
         self.models = {}
+        self.user_data = {}
+        self.system_data = {}
 
     def available(self):
         if self.manifest_data:

examples/pytorch_image_classification/model_manifest.json.freeze

@@ -0,0 +1 @@
+{"required_files": [{"name": "squeezenet", "source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth", "fail_on_tamper": true, "metadata": {"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"}, "md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"}, {"name": "labels", "source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json", "fail_on_tamper": true, "metadata": {"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"}, "md5_checksum": "c2c37ea517e94d9795004a39431a14cb"}], "optional_files": [{"name": "mobilenet", "source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth", "fail_on_tamper": false, "metadata": {"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"}}], "timestamp": "1633450866.985464", "lock_checksum": "24f5eca888d87661ca6fc08042e40cb7"}

tests/AdkTest.py

@@ -0,0 +1,7 @@
+from adk import ADK
+
+
+class ADKTest(ADK):
+    def __init__(self, apply_func, load_func=None, client=None, manifest_path="model_manifest.json.freeze"):
+        super(ADKTest, self).__init__(apply_func, load_func, client)
+        self.model_data = self.init_manifest(manifest_path)

tests/adk_algorithms.py

@@ -14,45 +14,39 @@ def apply_binary(input):
     return bytes("hello " + input, encoding='utf8')
 
 
-def apply_input_or_context(input, globals=None):
-    if isinstance(globals, dict):
-        return globals
+def apply_input_or_context(input, model_data=None):
+    if model_data:
+        return model_data.user_data
     else:
         return "hello " + input
 
 
-def apply_successful_manifest_parsing(input, result):
-    if result:
+def apply_successful_manifest_parsing(input, model_data):
+    if model_data:
         return "all model files were successfully loaded"
     else:
         return "model files were not loaded correctly"
 
 
 # -- Loading functions --- #
-def loading_text():
-    context = dict()
-    context['message'] = 'This message was loaded prior to runtime'
-    return context
+def loading_text(modelData):
+    modelData.user_data['message'] = 'This message was loaded prior to runtime'
+    return modelData
 
 
-def loading_exception():
+def loading_exception(modelData):
     raise Exception("This exception was thrown in loading")
 
 
-def loading_file_from_algorithmia():
-    context = dict()
-    client = Algorithmia.client()
-    context['data_url'] = 'data://demo/collection/somefile.json'
-    context['data'] = client.file(context['data_url']).getJson()
-    return context
+def loading_file_from_algorithmia(modelData):
+    modelData.user_data['data_url'] = 'data://demo/collection/somefile.json'
+    modelData.user_data['data'] = modelData.client.file(modelData.user_data['data_url']).getJson()
+    return modelData
 
 
-def loading_with_manifest(manifest):
-    squeezenet_path = manifest.get_model("squeezenet")
-    labels_path = manifest.get_model("labels")
+def loading_with_manifest(modelData):
+    modelData.user_data["squeezenet"] = modelData.get_model("squeezenet")
+    modelData.user_data['labels'] = modelData.get_model("labels")
     # optional model
-    mobilenet_path = manifest.get_model("mobilenet")
-    if os.path.exists(squeezenet_path) and os.path.exists(labels_path) and os.path.exists(mobilenet_path):
-        return True
-    else:
-        return False
+    modelData.user_data['mobilenet'] = modelData.get_model("mobilenet")
+    return modelData

tests/manifests/bad_model_manifest.json.lock renamed to tests/manifests/bad_model_manifest.json.freeze

@@ -1,7 +1,7 @@
 import json
 import os
 import unittest
-from adk import ADK
+from tests.AdkTest import ADKTest
 from tests.adk_algorithms import *
 
 
@@ -14,21 +14,24 @@ def setUp(self):
         except:
             pass
 
-    def execute_example(self, input, apply, load=lambda: None):
-        algo = ADK(apply, load)
+    def execute_example(self, input, apply, load=None):
+        if load:
+            algo = ADKTest(apply, load)
+        else:
+            algo = ADKTest(apply)
         output = []
         algo.init(input, pprint=lambda x: output.append(x))
         return output[0]
 
-    def execute_manifest_example(self, input, apply, load, manifest_path="manifests/good_model_manifest.json.lock"):
+    def execute_manifest_example(self, input, apply, load, manifest_path="manifests/good_model_manifest.json.freeze"):
         client = Algorithmia.client()
-        algo = ADK(apply, load, manifest_path=manifest_path, client=client)
+        algo = ADKTest(apply, load, manifest_path=manifest_path, client=client)
         output = []
         algo.init(input, pprint=lambda x: output.append(x))
         return output[0]
 
     def execute_without_load(self, input, apply):
-        algo = ADK(apply)
+        algo = ADKTest(apply)
         output = []
         algo.init(input, pprint=lambda x: output.append(x))
         return output[0]
@@ -128,7 +131,7 @@ def test_manifest_file_success(self):
         actual_output = json.loads(self.execute_manifest_example(input, apply_successful_manifest_parsing,
                                                                  loading_with_manifest,
                                                                  manifest_path="tests/manifests/good_model_manifest"
-                                                                               ".json.lock"))
+                                                                               ".json.freeze"))
         self.assertEqual(expected_output, actual_output)
 
     def test_manifest_file_tampered(self):
@@ -142,7 +145,7 @@ def test_manifest_file_tampered(self):
         actual_output = json.loads(self.execute_manifest_example(input, apply_successful_manifest_parsing,
                                                                  loading_with_manifest,
                                                                  manifest_path="tests/manifests/bad_model_manifest"
-                                                                               ".json.lock"))
+                                                                               ".json.freeze"))
         self.assertEqual(expected_output, actual_output)
 
 

examples/pytorch_image_classification/model_manifest.json.lock renamed to tests/manifests/good_model_manifest.json.freeze

@@ -2,7 +2,7 @@
 import json
 import unittest
 import os
-from adk import ADK
+from tests.AdkTest import ADKTest
 import base64
 from tests.adk_algorithms import *
 
@@ -43,30 +43,32 @@ def open_pipe(self):
         if os.name == "posix":
             self.fifo_pipe = os.open(self.fifo_pipe_path, os.O_RDONLY | os.O_NONBLOCK)
 
-    def execute_example(self, input, apply, load=lambda: None):
+    def execute_example(self, input, apply, load=None):
         self.open_pipe()
-        algo = ADK(apply, load)
+        algo = ADKTest(apply, load)
         sys.stdin = input
         algo.init()
         output = self.read_in()
         return output
 
     def execute_without_load(self, input, apply):
         self.open_pipe()
-        algo = ADK(apply)
+        algo = ADKTest(apply)
         sys.stdin = input
         algo.init()
         output = self.read_in()
         return output
 
-    def execute_example_local(self, input, apply, load=lambda: None):
-        algo = ADK(apply, load)
-        output = algo.init(input, pprint=lambda x: x)
+    def execute_manifest_example(self, input, apply, load, manifest_path):
+        client = Algorithmia.client()
+        self.open_pipe()
+        algo = ADKTest(apply, load, manifest_path=manifest_path, client=client)
+        sys.stdin = input
+        algo.init()
+        output = self.read_in()
         return output
 
-
-
-# ----- Tests ----- #
+    # ----- Tests ----- #
 
     def test_basic(self):
         input = {'content_type': 'json', 'data': 'Algorithmia'}
@@ -92,7 +94,6 @@ def test_basic_2(self):
         actual_output = self.execute_without_load(input, apply_basic)
         self.assertEqual(expected_output, actual_output)
 
-
     def test_algorithm_loading_basic(self):
         input = {'content_type': 'json', 'data': 'ignore me'}
         expected_output = {'metadata':
@@ -160,6 +161,21 @@ def test_binary_data(self):
         actual_output = self.execute_without_load(input, apply_binary)
         self.assertEqual(expected_output, actual_output)
 
+    def test_manifest_file_success(self):
+        input = {'content_type': 'json', 'data': 'Algorithmia'}
+        expected_output = {'metadata':
+            {
+                'content_type': 'text'
+            },
+            'result': "all model files were successfully loaded"
+        }
+        input = [str(json.dumps(input))]
+        actual_output = self.execute_manifest_example(input, apply_successful_manifest_parsing,
+                                                      loading_with_manifest,
+                                                      manifest_path="tests/manifests/good_model_manifest"
+                                                                    ".json.freeze")
+        self.assertEqual(expected_output, actual_output)
+
 
 def run_test():
-    unittest.main()
+    unittest.main()