add: daily backup

Author: cfhamlet

src/os_3m_engine/__init__.py

@@ -1,9 +1,13 @@
 import sys
+from .engine import create
 
-__all__ = ['__version__', 'version_info']
+__all__ = ['__version__', 'version_info', 'create']
 
 import pkgutil
 __version__ = pkgutil.get_data(__package__, 'VERSION').decode('ascii').strip()
 version_info = tuple(int(v) if v.isdigit() else v
                      for v in __version__.split('.'))
+
+
 del pkgutil
+del sys

src/os_3m_engine/backend.py

@@ -1,4 +1,4 @@
-from .common import Queue
+from .common import Configurable, Queue
 from .driver import Driver
 from .othread import Othread
 
@@ -43,3 +43,7 @@ def run(self):
         if hasattr(self.runtime_context, 'transport_thread'):
             self.runtime_context.transport_thread.stop()
 
+
+class Backend(Configurable):
+    def process(self):
+        raise NotImplementedError

src/os_3m_engine/common.py

@@ -1,5 +1,6 @@
 
 import sys
+
 _PY3 = sys.version_info[0] == 3
 
 if _PY3:

src/os_3m_engine/engine.py

@@ -3,61 +3,27 @@
 import sys
 import threading
 import time
-from .common import Configurable, RuntimeContext, Queue
-from .frontend import FrontendThread
-from .transport import TransportThread
-from .backend import BackendThread
-from .othread import MultithreadManager
-from .utils import getatters
 
+from .backend import BackendThread, BackendDriver
+from .common import Configurable, Queue, RuntimeContext
+from .frontend import FrontendThread, FrontendDriver
+from .transport import TransportThread, TransportDriver, BridgeDriver
+from .utils import getatters, getlastattr
+from .othread import OthreadManager
 
-class MultithreadEngine(Configurable):
 
-    def __init__(self, config):
-        super(MultithreadEngine, self).__init__(config)
-        self.__ensure()
+class Engine(Configurable):
+
+    def __init__(self, config, runtime_context):
+        super(Engine, self).__init__(config)
         self.__start_lock = threading.Lock()
         self.__started = False
         self.__stopped = False
-        self._runtime_context = RuntimeContext()
-
-    def __ensure(self):
-        engine_config = self.config.engine
-        assert any([hasattr(engine_config, x)
-                    for x in ('transport', 'backend')])
-
-    def __setup(self):
-        runtime_context = self._runtime_context
-        engine_config = self.config.engine
-
-        runtime_context.frontend_thread_queue = Queue.Queue(
-            engine_config.frontend.queue_size)
-
-        runtime_context.frontend_thread = MultithreadManager(
-            self.config, runtime_context,
-            FrontendThread, engine_config.frontend.thread_num,
-            engine_config.frontend.driver_cls)
-        runtime_context.frontend_thread.setDaemon(True)
-
-        if hasattr(engine_config, 'transport'):
-
-            if hasattr(engine_config, 'backend'):
-                runtime_context.backend_thread_queue = Queue.Queue(
-                    engine_config.backend.queue_size)
-
-            runtime_context.transport_thread = MultithreadManager(
-                self.config, runtime_context,
-                TransportThread, engine_config.transport.thread_num,
-                engine_config.transport.driver_cls)
+        self._runtime_context = runtime_context
 
-        if hasattr(engine_config, 'backend'):
-            runtime_context.backend_thread = MultithreadManager(
-                self.config, runtime_context,
-                BackendThread, engine_config.backend.thread_num,
-                engine_config.backend.driver_cls)
-
-    def __start(self):
+    def start(self):
 
+        self.__acquire_start_lock(False, 'Can not start twice')
         runtime_context = self._runtime_context
         m = getatters(runtime_context, [
                       'backend_thread',
@@ -87,11 +53,6 @@ def __acquire_start_lock(self, started, err_msg):
             finally:
                 self.__start_lock.release()
 
-    def start(self):
-        self.__acquire_start_lock(False, 'Can not start twice')
-        self.__setup()
-        self.__start()
-
     def __stop(self):
         runtime_context = self._runtime_context
         runtime_context.frontend_thread.stop()
@@ -106,3 +67,58 @@ def stop(self):
         if not self.__stopped:
             self.__stop()
         self.__start_lock.release()
+
+
+class DEFAULT_FRONTEND_CONFIG(object):
+    thread_num = 1
+    driver_cls = FrontendDriver
+
+
+class DEFAULT_TRANSPORT_CONFIG(object):
+    thread_num = 10
+
+
+class DEFAULT_BACKEND_CONFIG(object):
+    thread_num = 
+
+
+def create(frontend_config=DEFAULT_FRONTEND_CONFIG,
+           transport_config=DEFAULT_TRANSPORT_CONFIG,
+           backend_config=DEFAULT_BACKEND_CONFIG,
+           app_config=None, runtime_context=None):
+
+    if frontend_config is None:
+        raise ValueError('No frontend config')
+
+    if transport_config is None and backend_config is None:
+        raise ValueError('Config at least one of transport/backend')
+
+    runtime_context = runtime_context if runtime_context is not None else RuntimeContext()
+
+    runtime_context.frontend_thread_queue = Queue.Queue(
+        engine_config.frontend.queue_size)
+
+    runtime_context.frontend_thread = OthreadManager(
+        app_config, runtime_context,
+        FrontendThread, engine_config.frontend.thread_num,
+        engine_config.frontend.driver_cls)
+    runtime_context.frontend_thread.setDaemon(True)
+
+    if hasattr(engine_config, 'transport'):
+
+        if hasattr(engine_config, 'backend'):
+            runtime_context.backend_thread_queue = Queue.Queue(
+                engine_config.backend.queue_size)
+
+        runtime_context.transport_thread = OthreadManager(
+            app_config, runtime_context,
+            TransportThread, engine_config.transport.thread_num,
+            engine_config.transport.driver_cls)
+
+    if hasattr(engine_config, 'backend'):
+        runtime_context.backend_thread = OthreadManager(
+            app_config, runtime_context,
+            BackendThread, engine_config.backend.thread_num,
+            engine_config.backend.driver_cls)
+
+    return Engine(engine_config, runtime_context)

src/os_3m_engine/frontend.py

@@ -1,5 +1,5 @@
+from .common import Configurable, Queue
 from .driver import Driver
-from .common import Queue
 from .othread import Othread
 
 
@@ -38,3 +38,8 @@ def run(self):
             self.runtime_context.transport_thread.stop()
         elif hasattr(self.runtime_context, 'backend_thread'):
             self.runtime_context.backend_thread.stop()
+
+
+class Frontend(Configurable):
+    def generate(self):
+        raise NotImplementedError

src/os_3m_engine/othread.py

@@ -1,5 +1,6 @@
 import logging
 import threading
+
 from .common import Configurable
 
 
@@ -36,9 +37,9 @@ def stop(self):
         self._stopping = True
 
 
-class MultithreadManager(Configurable):
+class OthreadManager(Configurable):
     def __init__(self, config, runtime_context, thread_cls, thread_num, driver_cls):
-        super(MultithreadManager, self).__init__(config)
+        super(OthreadManager, self).__init__(config)
         started = threading.Event()
         self._started = started
         self._threads = [thread_cls(config, started, driver_cls, name='%s.%d' % (

src/os_3m_engine/transport.py

@@ -1,4 +1,4 @@
-from .common import Queue
+from .common import Configurable, Queue
 from .driver import Driver
 from .othread import Othread
 
@@ -57,3 +57,8 @@ def run(self):
         self.runtime_context.frontend_thread.stop()
         if hasattr(self.runtime_context, 'backend_thread'):
             self.runtime_context.backend_thread.stop()
+
+
+class Transport(Configurable):
+    def transport(self):
+        raise NotImplementedError

src/os_3m_engine/utils.py

@@ -1,3 +1,13 @@
 
 def getatters(obj, attrs):
     return [getattr(obj, attr) for attr in attrs if hasattr(obj, attr)]
+
+
+def getlastattr(attr, *objs):
+    for obj in reversed(objs):
+        if obj is None:
+            continue
+        elif hasattr(obj, attr):
+            return getattr(obj, attr)
+
+    raise AttributeError("no attribute '%s'" % attr)