Skip to content

Commit 3a92a5a

Browse files
DevAloneDevAlone
committed
Merge branch 'dockerizing'
2 parents 11f4904 + 30ddb94 commit 3a92a5a

File tree

10 files changed

+197
-121
lines changed

10 files changed

+197
-121
lines changed

‎Dockerfile

Lines changed: 17 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,22 @@
1-
FROM ubuntu
1+
FROM python:3.7-slim
22

3-
ENV DEBIAN_FRONTEND noninteractive
4-
RUN apt-get update && apt-get install -y postgresql python3 python3-pip wget unzip sudo
3+
RUN apt update && apt install -y wget > /dev/null && rm -rf /var/lib/apt/lists/* \
4+
&& rm /bin/sh && ln -s /bin/bash /bin/sh \
5+
&& groupadd -r user && useradd --create-home --no-log-init -r -g user user \
6+
&& mkdir /proxy_py && chown user:user /proxy_py
57

6-
USER postgres
7-
RUN /etc/init.d/postgresql start && \
8-
createuser proxy_py && \
9-
createdb proxy_py && \
10-
psql -c "alter user proxy_py with encrypted password 'proxy_py';" && \
11-
psql -c "grant all privileges on database proxy_py to proxy_py;" && \
12-
cat /etc/postgresql/10/main/pg_hba.conf | \
13-
sed -e "s/local all all peer/local all all md5/" | \
14-
tee /etc/postgresql/10/main/pg_hba.conf && \
15-
/etc/init.d/postgresql stop
8+
WORKDIR /proxy_py
9+
USER user
10+
ARG VERSION=master
1611

17-
USER root
18-
RUN wget https://github.com/DevAlone/proxy_py/archive/v2.2.zip -O master.zip 2> /dev/null
19-
RUN unzip master.zip; rm master.zip; mv proxy_py-* proxy_py; cd proxy_py
20-
WORKDIR proxy_py
21-
RUN pip3 install -r requirements.txt
22-
RUN cp config_examples/settings.py /proxy_py/proxy_py/
12+
RUN wget https://github.com/DevAlone/proxy_py/archive/$VERSION.tar.gz -O sources.tar.gz 2> /dev/null \
13+
&& tar -xf sources.tar.gz && rm sources.tar.gz \
14+
&& mv proxy_py-*/.[!.]* ./ && mv proxy_py-*/* ./ \
15+
&& rmdir proxy_py-* \
16+
&& python3 -m venv env
17+
18+
RUN cp config_examples/settings.py proxy_py/settings.py
19+
RUN source ./env/bin/activate && pip3 install -r requirements.txt --no-cache-dir
2320

2421
EXPOSE 55555
25-
CMD /etc/init.d/postgresql start; python3 main.py
22+
CMD ./run.sh

‎build_docker.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
if [ $# -eq 0 ]
3+
then
4+
tag='latest'
5+
git checkout master || exit 1
6+
else
7+
tag=$1
8+
git checkout $tag || exit 1
9+
fi
10+
11+
docker build -t DevAlone/proxy_py:$tag .
12+
docker push DevAlone/proxy_py:$tag

‎config_examples/docker_settings.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
from ._settings import *
2+
3+
DEBUG = False
4+
5+
# override db settings here
6+
DATABASE_CONNECTION_KWARGS['host'] = 'localhost'
7+
DATABASE_CONNECTION_KWARGS['database'] = 'proxy_py'
8+
DATABASE_CONNECTION_KWARGS['user'] = 'proxy_py'
9+
DATABASE_CONNECTION_KWARGS['password'] = 'proxy_py'
10+
11+
PROXY_PROVIDER_SERVER_ADDRESS = {
12+
'HOST': '0.0.0.0',
13+
'PORT': 55555,
14+
}

‎config_examples/settings.py

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,3 @@
11
from ._settings import *
22

3-
DEBUG = False
4-
5-
DATABASE_CONNECTION_KWARGS['database'] = 'proxy_py'
6-
DATABASE_CONNECTION_KWARGS['user'] = 'proxy_py'
7-
DATABASE_CONNECTION_KWARGS['password'] = 'proxy_py'
8-
9-
PROXY_PROVIDER_SERVER_ADDRESS = {
10-
'HOST': '0.0.0.0',
11-
'PORT': 55555,
12-
}
3+
# it should be empty to let environment variables be the most important ones

‎docker-compose.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
version: '3'
2+
3+
services:
4+
db:
5+
image: "postgres:12"
6+
restart: always
7+
environment:
8+
- POSTGRES_USER=proxy_py
9+
- POSTGRES_PASSWORD=proxy_py
10+
- POSTGRES_DB=proxy_py
11+
volumes:
12+
- db_data:/var/lib/postgresql/data
13+
core:
14+
build: .
15+
ports:
16+
- "55555:55555"
17+
environment:
18+
- PROXY_PY_PROXY_PROVIDER_SERVER_ADDRESS={'HOST':'0.0.0.0','PORT':55555}
19+
- PROXY_PY_DATABASE_CONNECTION_KWARGS={'host':'db','database':'proxy_py','user':'proxy_py','password':'proxy_py'}
20+
depends_on:
21+
- db
22+
23+
volumes:
24+
db_data:

‎main.py

Lines changed: 53 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,19 @@
11
#!/usr/bin/env python3
2-
import os
32

4-
import uvloop
3+
# should be called before everything else
4+
def init_uvloop():
5+
import uvloop
6+
uvloop.install()
57

6-
uvloop.install()
8+
init_uvloop()
79

10+
import os
811
from proxy_py import settings
912
from processor import Processor
1013
from server.proxy_provider_server import ProxyProviderServer
1114
from statistics import statistics
1215
from checkers.base_checker import BaseChecker
1316
from tools import test_collector
14-
1517
import materialized_view_updater
1618
import asyncio
1719
import logging
@@ -20,7 +22,6 @@
2022
import sys
2123

2224

23-
2425
test_collector_path = None
2526
main_logger = None
2627

@@ -37,11 +38,13 @@ def str_to_bool(value):
3738
raise argparse.ArgumentTypeError("Boolean value expected.")
3839

3940
cmd_parser = argparse.ArgumentParser()
40-
cmd_parser.add_argument("--debug", type=str_to_bool, help="override settings' debug value")
41+
cmd_parser.add_argument("--debug", type=str_to_bool,
42+
help="override settings' debug value")
4143
cmd_parser.add_argument(
42-
"--proxy-checking-timeout", type=float, help="override settings' proxy checking timeout"
43-
)
44-
cmd_parser.add_argument("--test-collector", help="test collector with a given path")
44+
"--proxy-checking-timeout", type=float, help="override settings' proxy checking timeout"
45+
)
46+
cmd_parser.add_argument(
47+
"--test-collector", help="test collector with a given path")
4548

4649
args = cmd_parser.parse_args()
4750

@@ -61,36 +64,22 @@ def prepare_loggers():
6164
global main_logger
6265

6366
asyncio_logger = logging.getLogger("asyncio")
64-
asyncio_logger_file_handler = logging.FileHandler("logs/asyncio.log")
65-
asyncio_logger_file_handler.setLevel(logging.DEBUG)
66-
asyncio_logger_file_handler.setFormatter(
67-
logging.Formatter(
68-
"%(levelname)s ~ %(asctime)s ~ %(funcName)30s() - %(message)s"
69-
)
67+
asyncio_logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
68+
asyncio_logger_handler = logging.StreamHandler(sys.stdout)
69+
asyncio_logger_handler.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
70+
asyncio_logger_handler.setFormatter(
71+
logging.Formatter(settings.LOG_FORMAT_STRING)
7072
)
71-
asyncio_logger.addHandler(asyncio_logger_file_handler)
72-
73-
if settings.DEBUG:
74-
asyncio.get_event_loop().set_debug(True)
75-
76-
asyncio_logger.setLevel(logging.DEBUG)
73+
asyncio_logger.addHandler(asyncio_logger_handler)
74+
asyncio.get_event_loop().set_debug(settings.DEBUG)
7775

7876
main_logger = logging.getLogger("proxy_py/main")
79-
80-
if settings.DEBUG:
81-
main_logger.setLevel(logging.DEBUG)
82-
else:
83-
main_logger.setLevel(logging.INFO)
84-
85-
logger_file_handler = logging.FileHandler("logs/main.log")
86-
logger_file_handler.setLevel(logging.DEBUG)
87-
logger_file_handler.setFormatter(
88-
logging.Formatter(
89-
"%(levelname)s ~ %(asctime)s ~ %(funcName)30s() ~ %(message)s"
90-
)
91-
)
92-
93-
main_logger.addHandler(logger_file_handler)
77+
main_logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
78+
logger_handler = logging.StreamHandler(sys.stdout)
79+
logger_handler.setFormatter(logging.Formatter(settings.LOG_FORMAT_STRING))
80+
logger_handler.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
81+
82+
main_logger.addHandler(logger_handler)
9483

9584

9685
async def core():
@@ -108,7 +97,7 @@ async def core():
10897
proxy_processor.worker(),
10998
statistics.worker(),
11099
materialized_view_updater.worker(),
111-
])
100+
])
112101
BaseChecker.clean()
113102
return code
114103
except KeyboardInterrupt:
@@ -123,13 +112,29 @@ async def core():
123112

124113
def server():
125114
proxy_provider_server = ProxyProviderServer(
126-
settings.PROXY_PROVIDER_SERVER_ADDRESS['HOST'],
127-
settings.PROXY_PROVIDER_SERVER_ADDRESS['PORT'],
128-
)
115+
settings.PROXY_PROVIDER_SERVER_ADDRESS['HOST'],
116+
settings.PROXY_PROVIDER_SERVER_ADDRESS['PORT'],
117+
)
129118

130119
return proxy_provider_server.start(asyncio.get_event_loop())
131120

132121

122+
def print_help():
123+
print("""Usage: ./main.py [COMMAND] [OPTION]...
124+
Runs proxy_py
125+
126+
The commands are:
127+
128+
"core" - runs core part of the project (proxies parsing and processing)
129+
"server" - runs server for providing API
130+
"" - runs both
131+
132+
use ./main.py COMMAND --help to get more information
133+
134+
Project's page: https://github.com/DevAlone/proxy_py
135+
""")
136+
137+
133138
def main():
134139
if len(sys.argv) < 2:
135140
# run default configuration
@@ -143,10 +148,14 @@ def main():
143148

144149
command = sys.argv[1].strip()
145150
sys.argv = sys.argv[1:]
146-
return {
147-
'core': lambda: asyncio.get_event_loop().run_until_complete(core()),
148-
'server': server,
149-
}[command]()
151+
try:
152+
return {
153+
'core': lambda: asyncio.get_event_loop().run_until_complete(core()),
154+
'server': server,
155+
}[command]()
156+
except KeyError:
157+
print_help()
158+
return 0
150159

151160

152161
if __name__ == "__main__":

‎processor.py

Lines changed: 11 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,7 @@
99
import time
1010
import logging
1111
import peewee
12-
13-
14-
15-
LOGGERS_FORMAT = "%(levelname)s ~ %(asctime)s ~ %(funcName)30s() ~ %(message)s"
12+
import sys
1613

1714

1815
class Processor:
@@ -31,29 +28,22 @@ def get_instance():
3128

3229
def __init__(self):
3330
self.logger = logging.getLogger("proxy_py/processor")
31+
self.logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
3432

35-
if settings.DEBUG:
36-
self.logger.setLevel(logging.DEBUG)
37-
else:
38-
self.logger.setLevel(logging.INFO)
39-
40-
logger_file_handler = logging.FileHandler("logs/processor.log")
41-
logger_file_handler.setLevel(logging.DEBUG)
42-
logger_file_handler.setFormatter(logging.Formatter(LOGGERS_FORMAT))
33+
logger_handler = logging.StreamHandler(sys.stdout)
34+
logger_handler.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
35+
logger_handler.setFormatter(logging.Formatter(settings.LOG_FORMAT_STRING))
4336

44-
self.logger.addHandler(logger_file_handler)
37+
self.logger.addHandler(logger_handler)
4538

4639
self.collectors_logger = logging.getLogger("proxy_py/collectors")
47-
if settings.DEBUG:
48-
self.collectors_logger.setLevel(logging.DEBUG)
49-
else:
50-
self.collectors_logger.setLevel(logging.INFO)
40+
self.collectors_logger.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
5141

52-
collectors_logger_file_handler = logging.FileHandler("logs/collectors.log")
53-
collectors_logger_file_handler.setLevel(logging.DEBUG)
54-
logger_file_handler.setFormatter(logging.Formatter(LOGGERS_FORMAT))
42+
collectors_logger_handler = logging.StreamHandler(sys.stdout)
43+
collectors_logger_handler.setLevel(logging.DEBUG if settings.DEBUG else logging.INFO)
44+
collectors_logger_handler.setFormatter(logging.Formatter(settings.LOG_FORMAT_STRING))
5545

56-
self.collectors_logger.addHandler(collectors_logger_file_handler)
46+
self.collectors_logger.addHandler(collectors_logger_handler)
5747

5848
self.logger.debug("processor initialization...")
5949

0 commit comments

Comments
 (0)