Skip to content

Commit c98afcf

Browse files
committed
test: engine launcher utils
1 parent a2dd753 commit c98afcf

File tree

12 files changed

+206
-34
lines changed

12 files changed

+206
-34
lines changed

.coveragerc

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
[run]
22
parallel = true
33
branch = true
4-
source = os_m3_engine
4+
source = os_3m_engine
55

66
[paths]
77
source =
8-
src/os_m3_engine
9-
.tox/*/lib/python*/site-packages/os_m3_engine
8+
src/os_3m_engine
9+
.tox/*/lib/python*/site-packages/os_3m_engine

MANIFEST.in

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
include LICENSE
22
include README.md
33
include MANIFEST.in
4+
include requirements.txt
45
include src/os_3m_engine/VERSION
5-
recursive-include tests *.py
6+
graft src
7+
graft tests
8+
global-exclude __pycache__
9+
global-exclude *.py[co]
610

setup.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ def read(*filenames, **kwargs):
2525
author='Ozzy',
2626
author_email='cfhamlet@gmail.com',
2727
url='https://github.com/cfhamlet/os-3m-engine',
28+
install_requires=open('requirements.txt').read().split('\n'),
2829
zip_safe=False,
2930
classifiers=[
3031
'Development Status :: 2 - Pre-Alpha',
@@ -36,4 +37,5 @@ def read(*filenames, **kwargs):
3637
'Programming Language :: Python :: 3',
3738
'Programming Language :: Python :: 3.6',
3839
'Programming Language :: Python :: Implementation :: CPython',
40+
'Programming Language :: Python :: Implementation :: PyPy',
3941
])

src/os_3m_engine/core/engine.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,12 @@ def __init__(self, config, runtime_context):
1818
self.__stopped = False
1919
self._runtime_context = runtime_context
2020

21+
def started(self):
22+
return self.__started
23+
24+
def stopped(self):
25+
return self.__stopped
26+
2127
def start(self):
2228

2329
self.__acquire_start_lock(False, 'Can not start twice')
@@ -33,7 +39,7 @@ def start(self):
3339
self.__start_lock.release()
3440

3541
while not any([x.stopped() for x in m]):
36-
time.sleep(0.1)
42+
time.sleep(self.config.main_loop_wait)
3743

3844
m = getatters(runtime_context, [
3945
'transport_thread',
@@ -58,6 +64,7 @@ def __stop(self):
5864
elif hasattr(runtime_context, 'backend_thread'):
5965
runtime_context.backend_thread.stop()
6066
self.__stopped = True
67+
self.__started = False
6168

6269
def stop(self):
6370
self.__acquire_start_lock(True, 'Can stop before start')

src/os_3m_engine/core/othread.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ def __init__(self, engine_config, runtime_context, component_factory):
5959
name='%s-%d' % (thread_cls.__name__, i))
6060
for i in range(0, engine_config.thread_num)]
6161

62+
@property
63+
def thread_num(self):
64+
return len(self._threads)
65+
6266
def start(self):
6367
while not self._start_trigger.wait(0.1):
6468
if self._start_trigger.is_set():

src/os_3m_engine/launcher.py

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,14 @@
1515
TRANSPORT_ENGINE_CONFIG = Config.create(
1616
thread_cls='os_3m_engine.core.transport.TransportThread',
1717
thread_num=3,
18-
driver_cls='os_3m_engine.core.transport.BridgeDriver',
18+
driver_cls='os_3m_engine.core.transport.TransportDriver',
1919
component_factory_cls='os_3m_engine.common.ComponentFactory',
2020
)
21+
22+
TRANSPORT_BRIDGE_ENGINE_CONFIG = Config.create()
23+
TRANSPORT_BRIDGE_ENGINE_CONFIG.update(TRANSPORT_ENGINE_CONFIG)
24+
TRANSPORT_BRIDGE_ENGINE_CONFIG.driver_cls = 'os_3m_engine.core.transport.BridgeDriver'
25+
2126
BACKEND_ENGINE_CONFIG = Config.create(
2227
thread_cls='os_3m_engine.core.backend.BackendThread',
2328
thread_num=1,
@@ -47,7 +52,7 @@ def create(frontend_cls='os_3m_engine.ootb.StdinFrontend',
4752
app_config=None,
4853
engine_config=ENGINE_CONFIG,
4954
frontend_engine_config=FRONTEND_ENGINE_CONFIG,
50-
transport_engine_config=TRANSPORT_ENGINE_CONFIG,
55+
transport_engine_config=TRANSPORT_BRIDGE_ENGINE_CONFIG,
5156
backend_engine_config=BACKEND_ENGINE_CONFIG,
5257
runtime_context=None):
5358

@@ -57,20 +62,12 @@ def create(frontend_cls='os_3m_engine.ootb.StdinFrontend',
5762
if transport_cls is None and backend_cls is None:
5863
raise ValueError('Spiecify at least one of transport_cls/backend_cls')
5964

60-
engine_config = combine_from_default_config(ENGINE_CONFIG, engine_config)
65+
runtime_context = runtime_context if runtime_context is not None else RuntimeContext()
6166

67+
# init frontend
6268
frontend_engine_config = combine_from_default_config(
6369
FRONTEND_ENGINE_CONFIG, frontend_engine_config)
6470

65-
transport_engine_config = combine_from_default_config(
66-
TRANSPORT_ENGINE_CONFIG, transport_engine_config)
67-
68-
backend_engine_config = combine_from_default_config(
69-
BACKEND_ENGINE_CONFIG, backend_engine_config)
70-
71-
runtime_context = runtime_context if runtime_context is not None else RuntimeContext()
72-
73-
# init frontend
7471
runtime_context.frontend_thread_queue = Queue.Queue(
7572
frontend_engine_config.queue_size)
7673

@@ -82,6 +79,18 @@ def create(frontend_cls='os_3m_engine.ootb.StdinFrontend',
8279
runtime_context.frontend_thread.setDaemon(True)
8380

8481
# init transport
82+
default_transport_engine_config = TRANSPORT_BRIDGE_ENGINE_CONFIG \
83+
if backend_cls is not None else TRANSPORT_ENGINE_CONFIG
84+
85+
if transport_engine_config in (TRANSPORT_ENGINE_CONFIG, TRANSPORT_BRIDGE_ENGINE_CONFIG):
86+
transport_engine_config = None
87+
88+
transport_engine_config = combine_from_default_config(
89+
default_transport_engine_config, transport_engine_config)
90+
91+
backend_engine_config = combine_from_default_config(
92+
BACKEND_ENGINE_CONFIG, backend_engine_config)
93+
8594
if transport_cls:
8695
if backend_cls:
8796
runtime_context.backend_thread_queue = Queue.Queue(
@@ -101,4 +110,5 @@ def create(frontend_cls='os_3m_engine.ootb.StdinFrontend',
101110
runtime_context.backend_thread = OthreadManager(
102111
backend_engine_config, runtime_context, backend_factory)
103112

113+
engine_config = combine_from_default_config(ENGINE_CONFIG, engine_config)
104114
return Engine(engine_config, runtime_context)

src/os_3m_engine/utils.py

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,22 @@
1-
from importlib import import_module
21
import inspect
2+
from importlib import import_module
33

44

55
def getatters(obj, attrs):
66
return [getattr(obj, attr) for attr in attrs if hasattr(obj, attr)]
77

88

9-
def getlastattr(attr, *objs):
10-
for obj in reversed(objs):
11-
if obj is None:
12-
continue
13-
elif hasattr(obj, attr):
14-
return getattr(obj, attr)
15-
16-
raise AttributeError("no attribute '%s'" % attr)
17-
18-
19-
def load_class(class_path, base_class, include_base_class=False):
9+
def load_class(class_path, base_class=None, include_base_class=False):
2010
module_path, class_name = class_path.rsplit('.', 1)
2111
_mod = import_module(module_path)
12+
if not hasattr(_mod, class_name):
13+
return None
2214
_cls = getattr(_mod, class_name)
23-
if inspect.isclass(_cls) and \
24-
issubclass(_cls, base_class) and \
25-
(include_base_class or _cls != base_class):
15+
if not inspect.isclass(_cls):
16+
return None
17+
elif base_class is None:
18+
return _cls
19+
elif issubclass(_cls, base_class) and (include_base_class or _cls != base_class):
2620
return _cls
27-
return None
21+
else:
22+
return None

tests/misc.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import time
2+
from os_3m_engine.core.backend import Backend
3+
from os_3m_engine.core.frontend import Frontend
4+
from os_3m_engine.core.transport import Transport
5+
6+
DATA = 'Hello World!'
7+
8+
9+
class SleepFrontend(Frontend):
10+
def produce(self):
11+
time.sleep(100)
12+
yield DATA
13+
14+
15+
class OneRecordFrontend(Frontend):
16+
def produce(self):
17+
assert 1 == 1
18+
yield DATA
19+
20+
21+
class ConfirmTransport(Transport):
22+
def transport(self, data):
23+
assert data == DATA
24+
return data
25+
26+
27+
class ConfirmBackend(Backend):
28+
def process(self, data):
29+
assert data == DATA

tests/test_engine.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import threading
2+
import time
3+
4+
import pytest
5+
6+
from os_3m_engine.launcher import create
7+
8+
9+
@pytest.fixture(scope='function')
10+
def config():
11+
from os_config import Config
12+
return Config.create()
13+
14+
15+
def test_engine_001(config):
16+
17+
engine = create(
18+
frontend_cls='misc.OneRecordFrontend',
19+
transport_cls='misc.ConfirmTransport',
20+
backend_cls='misc.ConfirmBackend',
21+
)
22+
engine.start()
23+
24+
25+
def test_engine_002(config):
26+
engine = create(
27+
frontend_cls='misc.SleepFrontend',
28+
transport_cls='misc.ConfirmTransport',
29+
backend_cls='misc.ConfirmBackend',
30+
)
31+
t1 = threading.Thread(target=engine.start)
32+
t1.start()
33+
34+
while not engine.started():
35+
time.sleep(0.1)
36+
37+
engine.stop()
38+
t1.join()
39+
assert engine.stopped()

tests/test_launcher.py

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import pytest
2+
3+
from os_3m_engine.core.engine import RuntimeContext
4+
from os_3m_engine.launcher import create
5+
6+
7+
@pytest.fixture(scope='function')
8+
def config():
9+
from os_config import Config
10+
return Config.create()
11+
12+
13+
def test_create_001():
14+
runtime_context = RuntimeContext()
15+
_ = create(runtime_context=runtime_context)
16+
for attr in ['frontend_thread_queue', 'backend_thread_queue',
17+
'frontend_thread', 'backend_thread', 'transport_thread']:
18+
assert hasattr(runtime_context, attr)
19+
20+
21+
def test_create_002():
22+
with pytest.raises(ValueError):
23+
create(frontend_cls=None)
24+
25+
26+
def test_create_003(config):
27+
config.queue_size = 1000
28+
config.thread_num = 10
29+
runtime_context = RuntimeContext()
30+
_ = create(frontend_engine_config=config,
31+
backend_engine_config=config,
32+
runtime_context=runtime_context)
33+
assert runtime_context.frontend_thread_queue.maxsize == config.queue_size
34+
assert runtime_context.frontend_thread.thread_num == config.thread_num
35+
assert runtime_context.backend_thread_queue.maxsize == config.queue_size
36+
assert runtime_context.backend_thread.thread_num == config.thread_num
37+
38+
39+
def test_create_004(config):
40+
with pytest.raises(ImportError):
41+
create('not_exist.What')
42+
43+
44+
def test_create_005(config):
45+
runtime_context = RuntimeContext()
46+
_ = create(transport_cls=None, runtime_context=runtime_context)
47+
for attr in ['thransport_thread', 'backend_thread_queue']:
48+
assert not hasattr(runtime_context, attr)
49+
50+
runtime_context = RuntimeContext()
51+
_ = create(backend_cls=None, runtime_context=runtime_context)
52+
for attr in ['backend_thread', 'backend_thread_queue']:
53+
assert not hasattr(runtime_context, attr)

0 commit comments

Comments
 (0)