Skip to content

Commit 29da1a3

Browse files
devsetgodevsetgo
authored
Merge pull request #429 from devsetgo/dev
Adding Resiliency to Logging Config
2 parents f536620 + 24ba712 commit 29da1a3

File tree

15 files changed

+608
-141
lines changed

15 files changed

+608
-141
lines changed

‎.bumpversion.cfg

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[bumpversion]
2-
current_version = 0.13.0
2+
current_version = 0.13.1
33
commit = False
44
tag = False
55
parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-(?P<release>[a-z]+)(?P<num>\d+))?

‎.github/workflows/autofill_pullrequest.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,5 @@ jobs:
1919
with:
2020
github_token: ${{ secrets.GITHUB_TOKEN }}
2121
openai_api_key: ${{ secrets.OPENAI_API_KEY }}
22-
max_tokens: 4000
23-
openai_model: gpt-4
22+
max_tokens: 16384
23+
openai_model: gpt-4o-mini

‎coverage-badge.svg

Lines changed: 1 addition & 1 deletion
Loading

‎coverage.xml

Lines changed: 40 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
<?xml version="1.0" ?>
2-
<coverage version="7.5.2" timestamp="1716752378688" lines-valid="660" lines-covered="660" line-rate="1" branches-covered="0" branches-valid="0" branch-rate="0" complexity="0">
3-
<!-- Generated by coverage.py: https://coverage.readthedocs.io/en/7.5.2 -->
2+
<coverage version="7.6.0" timestamp="1721521996459" lines-valid="675" lines-covered="675" line-rate="1" branches-covered="0" branches-valid="0" branch-rate="0" complexity="0">
3+
<!-- Generated by coverage.py: https://coverage.readthedocs.io/en/7.6.0 -->
44
<!-- Based on https://raw.githubusercontent.com/cobertura/web/master/htdocs/xml/coverage-04.dtd -->
55
<sources>
66
<source>/github/workspace</source>
@@ -624,35 +624,50 @@
624624
<class name="logging_config.py" filename="dsg_lib/common_functions/logging_config.py" complexity="0" line-rate="1" branch-rate="0">
625625
<methods/>
626626
<lines>
627-
<line number="39" hits="1"/>
628-
<line number="40" hits="1"/>
627+
<line number="33" hits="1"/>
628+
<line number="34" hits="1"/>
629+
<line number="35" hits="1"/>
630+
<line number="36" hits="1"/>
631+
<line number="38" hits="1"/>
629632
<line number="41" hits="1"/>
630-
<line number="43" hits="1"/>
631-
<line number="46" hits="1"/>
632-
<line number="99" hits="1"/>
633633
<line number="103" hits="1"/>
634-
<line number="110" hits="1"/>
635-
<line number="116" hits="1"/>
636-
<line number="117" hits="1"/>
637-
<line number="118" hits="1"/>
634+
<line number="107" hits="1"/>
635+
<line number="114" hits="1"/>
638636
<line number="121" hits="1"/>
639637
<line number="122" hits="1"/>
640-
<line number="125" hits="1"/>
638+
<line number="123" hits="1"/>
641639
<line number="126" hits="1"/>
642-
<line number="129" hits="1"/>
643-
<line number="132" hits="1"/>
644-
<line number="133" hits="1"/>
645-
<line number="136" hits="1"/>
646-
<line number="139" hits="1"/>
647-
<line number="152" hits="1"/>
648-
<line number="179" hits="1"/>
649-
<line number="181" hits="1"/>
650-
<line number="182" hits="1"/>
651-
<line number="187" hits="1"/>
652-
<line number="198" hits="1"/>
653-
<line number="201" hits="1"/>
654-
<line number="202" hits="1"/>
640+
<line number="127" hits="1"/>
641+
<line number="130" hits="1"/>
642+
<line number="131" hits="1"/>
643+
<line number="134" hits="1"/>
644+
<line number="137" hits="1"/>
645+
<line number="138" hits="1"/>
646+
<line number="141" hits="1"/>
647+
<line number="144" hits="1"/>
648+
<line number="157" hits="1"/>
649+
<line number="159" hits="1"/>
650+
<line number="186" hits="1"/>
651+
<line number="188" hits="1"/>
652+
<line number="189" hits="1"/>
653+
<line number="194" hits="1"/>
655654
<line number="205" hits="1"/>
655+
<line number="207" hits="1"/>
656+
<line number="208" hits="1"/>
657+
<line number="211" hits="1"/>
658+
<line number="214" hits="1"/>
659+
<line number="217" hits="1"/>
660+
<line number="231" hits="1"/>
661+
<line number="232" hits="1"/>
662+
<line number="233" hits="1"/>
663+
<line number="234" hits="1"/>
664+
<line number="249" hits="1"/>
665+
<line number="251" hits="1"/>
666+
<line number="254" hits="1"/>
667+
<line number="256" hits="1"/>
668+
<line number="257" hits="1"/>
669+
<line number="259" hits="1"/>
670+
<line number="260" hits="1"/>
656671
</lines>
657672
</class>
658673
<class name="patterns.py" filename="dsg_lib/common_functions/patterns.py" complexity="0" line-rate="1" branch-rate="0">

‎dsg_lib/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# -*- coding: utf-8 -*-
22

3-
__version__ = '0.13.0'
3+
__version__ = '0.13.1'

‎dsg_lib/common_functions/logging_config.py

Lines changed: 81 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,22 @@
11
# -*- coding: utf-8 -*-
22
"""
3-
This module provides a function to configure and set up a logger using the loguru package.
3+
This module provides a comprehensive logging setup using the loguru library, facilitating easy logging management for Python applications. The `config_log` function, central to this module, allows for extensive customization of logging behavior. It supports specifying the logging directory, log file name, logging level, and controls for log rotation, retention, and formatting among other features. Additionally, it offers advanced options like backtrace and diagnose for in-depth debugging, and the ability to append the application name to the log file for clearer identification.
44
5-
The `config_log` function takes several optional parameters to customize the logger's behavior,
6-
including the logging directory, log name, logging level, log rotation size, log retention period,
7-
and more. It also provides an option to append the application name to the log file name.
5+
Usage example:
86
9-
Example:
107
```python
118
from dsg_lib.common_functions.logging_config import config_log
129
1310
config_log(
14-
logging_directory='logs', # Directory where logs will be stored
15-
log_name='log', # Name of the log file (extension will be added automatically set v0.12.2)
16-
logging_level='DEBUG', # Logging level
17-
log_rotation='100 MB', # Log rotation size
18-
log_retention='30 days', # Log retention period
19-
log_backtrace=True, # Enable backtrace
20-
log_format="<green>{time:YYYY-MM-DD HH:mm:ss.SSSSSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>", # Log format
21-
log_serializer=False, # Disable log serialization
22-
log_diagnose=True, # Enable diagnose
23-
app_name='my_app', # Application name
24-
append_app_name=True # Append application name to the log file name
11+
logging_directory='logs', # Directory for storing logs
12+
log_name='log', # Base name for log files
13+
logging_level='DEBUG', # Minimum logging level
14+
log_rotation='100 MB', # Size threshold for log rotation
15+
log_retention='30 days', # Duration to retain old log files
16+
enqueue=True, # Enqueue log messages
2517
)
2618
19+
# Example log messages
2720
logger.debug("This is a debug message")
2821
logger.info("This is an info message")
2922
logger.error("This is an error message")
@@ -32,10 +25,12 @@
3225
```
3326
3427
Author: Mike Ryan
35-
Date: 2024/05/16
28+
DateCreated: 2021/07/16
29+
DateUpdated: 2024/07/24
30+
3631
License: MIT
3732
"""
38-
33+
import time
3934
import logging
4035
from pathlib import Path
4136
from uuid import uuid4
@@ -55,6 +50,9 @@ def config_log(
5550
log_diagnose: bool = False,
5651
app_name: str = None,
5752
append_app_name: bool = False,
53+
enqueue: bool = True,
54+
intercept_standard_logging: bool = True,
55+
file_sink: bool = True,
5856
):
5957
"""
6058
Configures and sets up a logger using the loguru package.
@@ -71,6 +69,9 @@ def config_log(
7169
- log_diagnose (bool): Whether to enable diagnose. Default is False.
7270
- app_name (str): The application name. Default is None.
7371
- append_app_name (bool): Whether to append the application name to the log file name. Default is False.
72+
- enqueue (bool): Whether to enqueue log messages. Default is True.
73+
- intercept_standard_logging (bool): Whether to intercept standard logging. Default is True.
74+
- file_sink (bool): Whether to use a file sink. Default is True.
7475
7576
Raises:
7677
- ValueError: If the provided logging level is not valid.
@@ -83,14 +84,17 @@ def config_log(
8384
logging_directory='logs',
8485
log_name='app.log',
8586
logging_level='DEBUG',
86-
log_rotation='500 MB',
87+
log_rotation='100 MB',
8788
log_retention='10 days',
8889
log_backtrace=True,
8990
log_format="<green>{time:YYYY-MM-DD HH:mm:ss.SSSSSS}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
9091
log_serializer=False,
9192
log_diagnose=True,
9293
app_name='my_app',
93-
append_app_name=True
94+
append_app_name=True,
95+
enqueue=True,
96+
intercept_standard_logging=True,
97+
file_sink=True
9498
)
9599
```
96100
"""
@@ -108,6 +112,7 @@ def config_log(
108112
log_format = '<green>{time:YYYY-MM-DD HH:mm:ss.SSSSSS}</green> | <level>{level: <8}</level> | <cyan> {name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>' # pragma: no cover
109113

110114
if log_serializer is True:
115+
log_format = '{message}' # pragma: no cover
111116
log_name = f'{log_name}.json' # pragma: no cover
112117
else:
113118
log_name = f'{log_name}.log' # pragma: no cover
@@ -140,7 +145,7 @@ def config_log(
140145
log_path,
141146
level=logging_level.upper(),
142147
format=log_format,
143-
enqueue=True,
148+
enqueue=enqueue,
144149
backtrace=log_backtrace,
145150
rotation=log_rotation,
146151
retention=log_retention,
@@ -149,6 +154,8 @@ def config_log(
149154
diagnose=log_diagnose,
150155
)
151156

157+
basic_config_handlers:list = []
158+
152159
class InterceptHandler(logging.Handler):
153160
"""
154161
Interceptor for standard logging.
@@ -194,12 +201,60 @@ def emit(self, record):
194201
level, record.getMessage()
195202
) # pragma: no cover
196203

197-
# Configure standard logging to use interceptor handler
198-
logging.basicConfig(handlers=[InterceptHandler()], level=logging_level.upper())
199204

200-
# Add interceptor handler to all existing loggers
201-
for name in logging.root.manager.loggerDict:
202-
logging.getLogger(name).addHandler(InterceptHandler())
205+
if intercept_standard_logging:
206+
# Add interceptor handler to all existing loggers
207+
for name in logging.root.manager.loggerDict:
208+
logging.getLogger(name).addHandler(InterceptHandler())
209+
210+
# Add interceptor handler to the root logger
211+
basic_config_handlers.append(InterceptHandler())
203212

204213
# Set the root logger's level to the lowest level possible
205214
logging.getLogger().setLevel(logging.NOTSET)
215+
216+
217+
class ResilientFileSink:
218+
"""
219+
A file sink designed for resilience, capable of retrying write operations.
220+
221+
This class implements a resilient file writing mechanism that attempts to write messages to a file, retrying the operation a specified number of times if it fails. This is particularly useful in scenarios where write operations might intermittently fail due to temporary issues such as file system locks or networked file system delays.
222+
223+
Attributes:
224+
path (str): The path to the file where messages will be written.
225+
max_retries (int): The maximum number of retry attempts for a failed write operation.
226+
retry_delay (float): The delay between retry attempts, in seconds.
227+
228+
Methods:
229+
write(message): Attempts to write a message to the file, retrying on failure up to `max_retries` times.
230+
"""
231+
def __init__(self, path, max_retries=5, retry_delay=0.1):
232+
self.path = path
233+
self.max_retries = max_retries
234+
self.retry_delay = retry_delay
235+
236+
def write(self, message): # pragma: no cover
237+
for attempt in range(self.max_retries):
238+
try:
239+
with open(self.path, 'a') as file:
240+
file.write(str(message))
241+
break # Successfully written, break the loop
242+
except FileNotFoundError:
243+
if attempt < self.max_retries - 1:
244+
time.sleep(self.retry_delay) # Wait before retrying
245+
else:
246+
raise # Reraise if max retries exceeded
247+
248+
249+
if file_sink:
250+
# Create an instance of ResilientFileSink
251+
resilient_sink = ResilientFileSink(str(log_path))
252+
253+
# Configure the logger to use the ResilientFileSink
254+
basic_config_handlers.append(resilient_sink)
255+
256+
if intercept_standard_logging:
257+
basic_config_handlers.append(InterceptHandler())
258+
259+
if len(basic_config_handlers) > 0:
260+
logging.basicConfig(handlers=basic_config_handlers, level=logging_level.upper())

0 commit comments

Comments
 (0)