Skip to content

Commit 6d44f80

Browse files
authored
Update loguru and deploy log config (#517)
* Update loguru config * Update config * fix log level * Update deploy log config
1 parent 9602e28 commit 6d44f80

File tree

9 files changed

+53
-56
lines changed

9 files changed

+53
-56
lines changed

backend/backend.dockerfile

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,9 @@ RUN apt-get update \
1919

2020
ENV TZ="Asia/Shanghai"
2121

22-
RUN mkdir -p /var/log/fastapi_server \
23-
&& mkdir -p /var/log/supervisor \
24-
&& mkdir -p /etc/supervisor/conf.d
22+
RUN mkdir -p /var/log/fastapi_server
2523

26-
COPY deploy/backend/supervisor.conf /etc/supervisor/supervisord.conf
24+
COPY deploy/backend/supervisord.conf /etc/supervisor/supervisord.conf
2725

2826
COPY deploy/backend/fastapi_server.conf /etc/supervisor/conf.d/
2927

backend/celery.dockerfile

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,9 @@ RUN apt-get update \
1919

2020
ENV TZ="Asia/Shanghai"
2121

22-
RUN mkdir -p /var/log/celery \
23-
&& mkdir -p /var/log/supervisor \
24-
&& mkdir -p /etc/supervisor/conf.d
22+
RUN mkdir -p /var/log/celery
2523

26-
COPY deploy/backend/supervisor.conf /etc/supervisor/supervisord.conf
24+
COPY deploy/backend/supervisord.conf /etc/supervisor/supervisord.conf
2725

2826
COPY deploy/backend/celery.conf /etc/supervisor/conf.d/
2927

backend/common/log.py

Lines changed: 31 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,7 @@
33
import inspect
44
import logging
55
import os
6-
7-
from sys import stderr, stdout
6+
import sys
87

98
from asgi_correlation_id import correlation_id
109
from loguru import logger
@@ -37,12 +36,12 @@ def emit(self, record: logging.LogRecord):
3736

3837
def setup_logging():
3938
"""
40-
From https://pawamoy.github.io/posts/unify-logging-for-a-gunicorn-uvicorn-app/
39+
From https://github.com/benoitc/gunicorn/issues/1572#issuecomment-638391953
4140
https://github.com/pawamoy/pawamoy.github.io/issues/17
4241
"""
43-
# Intercept everything at the root logger
42+
# Set the logging handler and level
4443
logging.root.handlers = [InterceptHandler()]
45-
logging.root.setLevel(settings.LOG_ROOT_LEVEL)
44+
logging.root.setLevel('NOTSET')
4645

4746
# Remove all log handlers and propagate to root logger
4847
for name in logging.root.manager.loggerDict.keys():
@@ -55,70 +54,65 @@ def setup_logging():
5554
# Debug log handlers
5655
# logging.debug(f'{logging.getLogger(name)}, {logging.getLogger(name).propagate}')
5756

58-
# Remove every other logger's handlers
59-
logger.remove()
60-
61-
# Define the correlation_id filter function
62-
# https://github.com/snok/asgi-correlation-id?tab=readme-ov-file#configure-logging
57+
# Define the correlation_id default filter function
6358
# https://github.com/snok/asgi-correlation-id/issues/7
64-
def correlation_id_filter(record) -> bool:
59+
def correlation_id_filter(record):
6560
cid = correlation_id.get(settings.LOG_CID_DEFAULT_VALUE)
6661
record['correlation_id'] = cid[: settings.LOG_CID_UUID_LENGTH]
67-
return True
62+
return record
63+
64+
# Remove default loguru logger
65+
logger.remove()
6866

69-
# Configure loguru logger before starts logging
67+
# Set the loguru default handlers
7068
logger.configure(
7169
handlers=[
7270
{
73-
'sink': stdout,
74-
'level': settings.LOG_STDOUT_LEVEL,
75-
'filter': lambda record: correlation_id_filter(record) and record['level'].no <= 25,
76-
'format': settings.LOG_STD_FORMAT,
77-
},
78-
{
79-
'sink': stderr,
80-
'level': settings.LOG_STDERR_LEVEL,
81-
'filter': lambda record: correlation_id_filter(record) and record['level'].no >= 30,
71+
'sink': sys.stdout,
72+
'filter': lambda record: correlation_id_filter(record),
8273
'format': settings.LOG_STD_FORMAT,
83-
},
74+
}
8475
]
8576
)
8677

8778

88-
def set_customize_logfile():
79+
def set_custom_logfile():
8980
log_path = path_conf.LOG_DIR
9081
if not os.path.exists(log_path):
9182
os.mkdir(log_path)
9283

9384
# log files
94-
log_stdout_file = os.path.join(log_path, settings.LOG_STDOUT_FILENAME)
95-
log_stderr_file = os.path.join(log_path, settings.LOG_STDERR_FILENAME)
85+
log_access_file = os.path.join(log_path, settings.LOG_ACCESS_FILENAME)
86+
log_error_file = os.path.join(log_path, settings.LOG_ERROR_FILENAME)
9687

97-
# loguru logger: https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add
88+
# set loguru logger default config
89+
# https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.add
9890
log_config = {
99-
'rotation': '10 MB',
100-
'retention': '15 days',
101-
'compression': 'tar.gz',
102-
'enqueue': True,
10391
'format': settings.LOG_FILE_FORMAT,
92+
'enqueue': True,
93+
'rotation': '5 MB',
94+
'retention': '7 days',
95+
'compression': 'tar.gz',
10496
}
10597

10698
# stdout file
10799
logger.add(
108-
str(log_stdout_file),
109-
level=settings.LOG_STDOUT_LEVEL,
110-
**log_config,
100+
str(log_access_file),
101+
level=settings.LOG_ACCESS_LEVEL,
102+
filter=lambda record: record['level'].no <= 25,
111103
backtrace=False,
112104
diagnose=False,
105+
**log_config,
113106
)
114107

115108
# stderr file
116109
logger.add(
117-
str(log_stderr_file),
118-
level=settings.LOG_STDERR_LEVEL,
119-
**log_config,
110+
str(log_error_file),
111+
level=settings.LOG_ERROR_LEVEL,
112+
filter=lambda record: record['level'].no >= 30,
120113
backtrace=True,
121114
diagnose=True,
115+
**log_config,
122116
)
123117

124118

backend/core/conf.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,6 @@ class Settings(BaseSettings):
9494
COOKIE_REFRESH_TOKEN_EXPIRE_SECONDS: int = TOKEN_REFRESH_EXPIRE_SECONDS
9595

9696
# Log
97-
LOG_ROOT_LEVEL: str = 'NOTSET'
9897
LOG_STD_FORMAT: str = (
9998
'<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</> | <lvl>{level: <8}</> | '
10099
'<cyan> {correlation_id} </> | <lvl>{message}</>'
@@ -105,10 +104,10 @@ class Settings(BaseSettings):
105104
)
106105
LOG_CID_DEFAULT_VALUE: str = '-'
107106
LOG_CID_UUID_LENGTH: int = 32 # must <= 32
108-
LOG_STDOUT_LEVEL: str = 'INFO'
109-
LOG_STDERR_LEVEL: str = 'ERROR'
110-
LOG_STDOUT_FILENAME: str = 'fba_access.log'
111-
LOG_STDERR_FILENAME: str = 'fba_error.log'
107+
LOG_ACCESS_LEVEL: str = 'INFO'
108+
LOG_ERROR_LEVEL: str = 'ERROR'
109+
LOG_ACCESS_FILENAME: str = 'fba_access.log'
110+
LOG_ERROR_FILENAME: str = 'fba_error.log'
112111

113112
# Middleware
114113
MIDDLEWARE_CORS: bool = True

backend/core/registrar.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from starlette.staticfiles import StaticFiles
1515

1616
from backend.common.exception.exception_handler import register_exception
17-
from backend.common.log import set_customize_logfile, setup_logging
17+
from backend.common.log import set_custom_logfile, setup_logging
1818
from backend.core.conf import settings
1919
from backend.core.path_conf import STATIC_DIR, UPLOAD_DIR
2020
from backend.database.db import create_table
@@ -99,7 +99,7 @@ def register_logger() -> None:
9999
:return:
100100
"""
101101
setup_logging()
102-
set_customize_logfile()
102+
set_custom_logfile()
103103

104104

105105
def register_static_file(app: FastAPI):

deploy/backend/celery.conf

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ autorestart=true
77
startretries=5
88
redirect_stderr=true
99
stdout_logfile=/var/log/celery/fba_celery_worker.log
10+
stdout_logfile_maxbytes=5MB
11+
stdout_logfile_backups=5
1012

1113
[program:celery_beat]
1214
directory=/fba/backend
@@ -17,6 +19,8 @@ autorestart=true
1719
startretries=5
1820
redirect_stderr=true
1921
stdout_logfile=/var/log/celery/fba_celery_beat.log
22+
stdout_logfile_maxbytes=5MB
23+
stdout_logfile_backups=5
2024

2125
[program:celery_flower]
2226
directory=/fba/backend
@@ -27,3 +31,5 @@ autorestart=true
2731
startretries=5
2832
redirect_stderr=true
2933
stdout_logfile=/var/log/celery/fba_celery_flower.log
34+
stdout_logfile_maxbytes=5MB
35+
stdout_logfile_backups=5

deploy/backend/fastapi_server.conf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,5 @@ autorestart=true
77
startretries=5
88
redirect_stderr=true
99
stdout_logfile=/var/log/fastapi_server/fba_server.log
10+
stdout_logfile_maxbytes=5MB
11+
stdout_logfile_backups=5

deploy/backend/gunicorn.conf.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@
2626
worker_connections = 2000
2727

2828
# 设置进程文件目录
29-
pidfile = '/fba/gunicorn.pid'
29+
pidfile = '/var/run/gunicorn.pid'
3030

31-
# 设置访问日志和错误信息日志路径
32-
accesslog = '/var/log/fastapi_server/gunicorn_access.log'
33-
errorlog = '/var/log/fastapi_server/gunicorn_error.log'
31+
# 设置访问日志和错误信息日志
32+
accesslog = '-'
33+
errorlog = '-'
3434

3535
# 设置这个值为true 才会把打印信息记录到错误日志里
3636
capture_output = True
File renamed without changes.

0 commit comments

Comments
 (0)