From 1392cddec92056d62fd5f0645463ea4a8181e5ec Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 17:15:26 -0500 Subject: [PATCH 1/9] Update adafruit_logging.py --- adafruit_logging.py | 63 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index d146496..3015ae0 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -210,11 +210,69 @@ class FileHandler(StreamHandler): :param str filename: The filename of the log file :param str mode: Whether to write ('w') or append ('a'); default is to append + :param int LogFileSizeLimit: The max allowable size of the log file in bytes. """ - def __init__(self, filename: str, mode: str = "a") -> None: + def __init__(self, filename: str, mode: str = "a", LogFileSizeLimit: int = None) -> None: # pylint: disable=consider-using-with - super().__init__(open(filename, mode=mode)) + if mode is 'r': + raise ValueError("Can't write to a read only file") + + self._LogFileName = filename + self._WriteMode = mode + self._OldFilePostfix = "_old" #TODO: Make this settable by the user? + self._LogFileSizeLimit = LogFileSizeLimit + + #Here we are assuming that if there is a period in the filename, the stuff after the period + #is the extension of the file. It is possible that is not the case, but probably unlikely. + if '.' in filename: + [basefilename, extension] = filename.rsplit(".", 1) + self._OldLogFileName = basefilename + self._OldFilePostfix + "." + extension + else: + basefilename = filename + self._OldLogFileName = basefilename + self._OldFilePostfix + + super().__init__(open(self._LogFileName, mode=self._WriteMode)) + self.CheckLogSize() + + def CheckLogSize(self) -> None: + if self._LogFileSizeLimit is None: + #No log limit set + return + + #Close the log file. Probably needed if we want to delete/rename files. + self.close() + + #Get the size of the log file. + try: + LogFileSize = os.stat(self._LogFileName)[6] + except OSError as e: + if e.args[0] == 2: + #Log file does not exsist. This is okay. + LogFileSize = None + else: + raise e + + #Get the size of the old log file. + try: + OldLogFileSize = os.stat(self._OldLogFileName)[6] + except OSError as e: + if e.args[0] == 2: + #Log file does not exsist. This is okay. + OldLogFileSize = None + else: + raise e + + #This checks if the log file is too big. If so, it deletes the old log file and renames the + #log file to the old log filename. + if LogFileSize > self._LogFileSizeLimit: + print("Rotating Logs") + if OldLogFileSize is not None: + os.remove(self._OldLogFileName) + os.rename(self._LogFileName, self._OldLogFileName) + + #Reopen the file. + self.stream = open(self._LogFileName, mode=self._WriteMode) def close(self) -> None: """Closes the file""" @@ -233,6 +291,7 @@ def emit(self, record: LogRecord) -> None: :param record: The record (message object) to be logged """ + self.CheckLogSize() self.stream.write(self.format(record)) From 4d824c70289a7d192cba12727882c30785e9a853 Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 20:33:45 -0500 Subject: [PATCH 2/9] Update the docs --- adafruit_logging.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index 3015ae0..57ecd26 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -206,7 +206,10 @@ def emit(self, record: LogRecord) -> None: class FileHandler(StreamHandler): """File handler for working with log files off of the microcontroller (like - an SD card) + an SD card). This handler implements a very simple log rotating system. If LogFileSizeLimit + is set, the handler will check to see if the log file is larger than the given limit. If the + log file is larger than the limit, it will rename it to the filename with _old appended. If + a file with _old already exsists, it will be deleted. :param str filename: The filename of the log file :param str mode: Whether to write ('w') or append ('a'); default is to append From 7b02107aac0a28bfd6b433fb3a912e33591afbea Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 20:55:35 -0500 Subject: [PATCH 3/9] Updated from pylint and black --- adafruit_logging.py | 53 +++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index 57ecd26..3055202 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -61,6 +61,7 @@ from collections import namedtuple try: + # pylint: disable=deprecated-class from typing import Optional, Hashable from typing_extensions import Protocol @@ -216,65 +217,69 @@ class FileHandler(StreamHandler): :param int LogFileSizeLimit: The max allowable size of the log file in bytes. """ - def __init__(self, filename: str, mode: str = "a", LogFileSizeLimit: int = None) -> None: - # pylint: disable=consider-using-with - if mode is 'r': + def __init__( + self, filename: str, mode: str = "a", LogFileSizeLimit: int = None + ) -> None: + if mode == "r": raise ValueError("Can't write to a read only file") - + self._LogFileName = filename self._WriteMode = mode - self._OldFilePostfix = "_old" #TODO: Make this settable by the user? + self._OldFilePostfix = "_old" # TODO: Make this settable by the user? self._LogFileSizeLimit = LogFileSizeLimit - - #Here we are assuming that if there is a period in the filename, the stuff after the period - #is the extension of the file. It is possible that is not the case, but probably unlikely. - if '.' in filename: + + # Here we are assuming that if there is a period in the filename, the stuff after the period + # is the extension of the file. It is possible that is not the case, but probably unlikely. + if "." in filename: [basefilename, extension] = filename.rsplit(".", 1) self._OldLogFileName = basefilename + self._OldFilePostfix + "." + extension else: basefilename = filename self._OldLogFileName = basefilename + self._OldFilePostfix + # pylint: disable=consider-using-with super().__init__(open(self._LogFileName, mode=self._WriteMode)) self.CheckLogSize() - + def CheckLogSize(self) -> None: + """Check the size of the log file and rotate if needed.""" if self._LogFileSizeLimit is None: - #No log limit set + # No log limit set return - - #Close the log file. Probably needed if we want to delete/rename files. + + # Close the log file. Probably needed if we want to delete/rename files. self.close() - - #Get the size of the log file. + + # Get the size of the log file. try: LogFileSize = os.stat(self._LogFileName)[6] except OSError as e: if e.args[0] == 2: - #Log file does not exsist. This is okay. + # Log file does not exsist. This is okay. LogFileSize = None else: raise e - - #Get the size of the old log file. + + # Get the size of the old log file. try: OldLogFileSize = os.stat(self._OldLogFileName)[6] except OSError as e: if e.args[0] == 2: - #Log file does not exsist. This is okay. + # Log file does not exsist. This is okay. OldLogFileSize = None else: raise e - - #This checks if the log file is too big. If so, it deletes the old log file and renames the - #log file to the old log filename. + + # This checks if the log file is too big. If so, it deletes the old log file and renames the + # log file to the old log filename. if LogFileSize > self._LogFileSizeLimit: print("Rotating Logs") if OldLogFileSize is not None: os.remove(self._OldLogFileName) os.rename(self._LogFileName, self._OldLogFileName) - - #Reopen the file. + + # Reopen the file. + # pylint: disable=consider-using-with self.stream = open(self._LogFileName, mode=self._WriteMode) def close(self) -> None: From 216afff33afc20171e00eeb38dc60d5f91561a8d Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 21:14:23 -0500 Subject: [PATCH 4/9] Add the postfix as a settable variable --- adafruit_logging.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index 3055202..64758c9 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -215,17 +215,22 @@ class FileHandler(StreamHandler): :param str filename: The filename of the log file :param str mode: Whether to write ('w') or append ('a'); default is to append :param int LogFileSizeLimit: The max allowable size of the log file in bytes. + :param str OldFilePostfix: What to append to the filename for the old log file. """ def __init__( - self, filename: str, mode: str = "a", LogFileSizeLimit: int = None + self, + filename: str, + mode: str = "a", + LogFileSizeLimit: int = None, + OldFilePostfix: str = "_old", ) -> None: if mode == "r": raise ValueError("Can't write to a read only file") self._LogFileName = filename self._WriteMode = mode - self._OldFilePostfix = "_old" # TODO: Make this settable by the user? + self._OldFilePostfix = OldFilePostfix self._LogFileSizeLimit = LogFileSizeLimit # Here we are assuming that if there is a period in the filename, the stuff after the period From 8153ddcfce1ab7426cd0eb41b7c61750520ad265 Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 21:44:22 -0500 Subject: [PATCH 5/9] Add missing import and remove some debug strings. --- adafruit_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index 64758c9..86a9327 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -58,6 +58,7 @@ import time import sys +import os from collections import namedtuple try: @@ -278,7 +279,6 @@ def CheckLogSize(self) -> None: # This checks if the log file is too big. If so, it deletes the old log file and renames the # log file to the old log filename. if LogFileSize > self._LogFileSizeLimit: - print("Rotating Logs") if OldLogFileSize is not None: os.remove(self._OldLogFileName) os.rename(self._LogFileName, self._OldLogFileName) From d0691ceb0f85a0d8154a94d5944dd723a7cecb40 Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 21:57:41 -0500 Subject: [PATCH 6/9] Update the help text --- adafruit_logging.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index 86a9327..6eb7c5c 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -207,16 +207,19 @@ def emit(self, record: LogRecord) -> None: class FileHandler(StreamHandler): - """File handler for working with log files off of the microcontroller (like - an SD card). This handler implements a very simple log rotating system. If LogFileSizeLimit - is set, the handler will check to see if the log file is larger than the given limit. If the - log file is larger than the limit, it will rename it to the filename with _old appended. If - a file with _old already exsists, it will be deleted. + """File handler for working with log files off of the microcontroller (like an SD card). + This handler implements a very simple log rotating system. If LogFileSizeLimit is set, the + handler will check to see if the log file is larger than the given limit. If the log file is + larger than the limit, it is renamed and a new file is started for log entries. The old log + file is renamed with the OldFilePostfix variable appended to the name. If another file exsists + with this old file name, it will be deleted. Because there are two log files. The max size of + the log files is two times the limit set by LogFileSizeLimit. :param str filename: The filename of the log file :param str mode: Whether to write ('w') or append ('a'); default is to append :param int LogFileSizeLimit: The max allowable size of the log file in bytes. - :param str OldFilePostfix: What to append to the filename for the old log file. + :param str OldFilePostfix: What to append to the filename for the old log file. Defaults + to '_old'. """ def __init__( From f5178594307f181ae9cdc49b85f0f03adf2afe5f Mon Sep 17 00:00:00 2001 From: Pat Date: Sun, 28 Apr 2024 21:59:56 -0500 Subject: [PATCH 7/9] Add author info. --- adafruit_logging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/adafruit_logging.py b/adafruit_logging.py index 6eb7c5c..beec4dd 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -1,4 +1,5 @@ # SPDX-FileCopyrightText: 2019 Dave Astels for Adafruit Industries +# SPDX-FileCopyrightText: 2024 Pat Satyshur # # SPDX-License-Identifier: MIT From 767fda40387608857f49c2b8bbee2d43c6af0c98 Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 16 May 2024 23:02:25 -0500 Subject: [PATCH 8/9] Update adafruit_logging.py Update the code to make a subclass of filehandler that implements some of the RotatingFileHandler from: https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler --- adafruit_logging.py | 171 ++++++++++++++++++++++++++------------------ 1 file changed, 101 insertions(+), 70 deletions(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index beec4dd..42d87d9 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -208,60 +208,118 @@ def emit(self, record: LogRecord) -> None: class FileHandler(StreamHandler): - """File handler for working with log files off of the microcontroller (like an SD card). - This handler implements a very simple log rotating system. If LogFileSizeLimit is set, the - handler will check to see if the log file is larger than the given limit. If the log file is - larger than the limit, it is renamed and a new file is started for log entries. The old log - file is renamed with the OldFilePostfix variable appended to the name. If another file exsists - with this old file name, it will be deleted. Because there are two log files. The max size of - the log files is two times the limit set by LogFileSizeLimit. + """File handler for working with log files off of the microcontroller (like + an SD card) :param str filename: The filename of the log file :param str mode: Whether to write ('w') or append ('a'); default is to append - :param int LogFileSizeLimit: The max allowable size of the log file in bytes. - :param str OldFilePostfix: What to append to the filename for the old log file. Defaults - to '_old'. + """ + + def __init__(self, filename: str, mode: str = "a") -> None: + # pylint: disable=consider-using-with + if mode == "r": + raise ValueError("Can't write to a read only file") + super().__init__(open(filename, mode=mode)) + + def close(self) -> None: + """Closes the file""" + self.stream.flush() + self.stream.close() + + def format(self, record: LogRecord) -> str: + """Generate a string to log + + :param record: The record (message object) to be logged + """ + return super().format(record) + "\r\n" + + def emit(self, record: LogRecord) -> None: + """Generate the message and write it to the file. + + :param record: The record (message object) to be logged + """ + self.stream.write(self.format(record)) + + +class RotatingFileHandler(FileHandler): + """File handler for writing log files to flash memory or external memory such as an SD card. + This handler implements a very simple log rotating system similar to the python function of the + same name (https://docs.python.org/3/library/logging.handlers.html#rotatingfilehandler) + + If maxBytes is set, the handler will check to see if the log file is larger than the given + limit. If the log file is larger than the limit, it is renamed and a new file is started. + The old log file will be renamed with a numerical appendix '.1', '.2', etc... The variable + backupCount controls how many old log files to keep. For example, if the filename is 'log.txt' + and backupCount is 5, you will end up with six log files: 'log.txt', 'log.txt.1', 'log.txt.3', + up to 'log.txt.5' Therefore, the maximum amount of disk space the logs can use is + maxBytes*(backupCount+1). + + If either maxBytes or backupCount is not set, or set to zero, the log rotation is disabled. + This will result in a single log file with a name `filename` that will grow without bound. + + :param str filename: The filename of the log file + :param str mode: Whether to write ('w') or append ('a'); default is to append + :param int maxBytes: The max allowable size of the log file in bytes. + :param int backupCount: The number of old log files to keep. """ def __init__( self, filename: str, mode: str = "a", - LogFileSizeLimit: int = None, - OldFilePostfix: str = "_old", + maxBytes: int = 0, + backupCount: int = 0, ) -> None: - if mode == "r": - raise ValueError("Can't write to a read only file") + if maxBytes < 0: + raise ValueError("maxBytes must be a positive number") + if backupCount < 0: + raise ValueError("backupCount must be a positive number") self._LogFileName = filename self._WriteMode = mode - self._OldFilePostfix = OldFilePostfix - self._LogFileSizeLimit = LogFileSizeLimit - - # Here we are assuming that if there is a period in the filename, the stuff after the period - # is the extension of the file. It is possible that is not the case, but probably unlikely. - if "." in filename: - [basefilename, extension] = filename.rsplit(".", 1) - self._OldLogFileName = basefilename + self._OldFilePostfix + "." + extension - else: - basefilename = filename - self._OldLogFileName = basefilename + self._OldFilePostfix - - # pylint: disable=consider-using-with - super().__init__(open(self._LogFileName, mode=self._WriteMode)) - self.CheckLogSize() + self._maxBytes = maxBytes + self._backupCount = backupCount - def CheckLogSize(self) -> None: - """Check the size of the log file and rotate if needed.""" - if self._LogFileSizeLimit is None: - # No log limit set - return + # Open the file and save the handle to self.stream + super().__init__(self._LogFileName, mode=self._WriteMode) + # TODO: What do we do if the log file already exsists? + def doRollover(self) -> None: + """Roll over the log files. This should not need to be called directly""" + # At this point, we have already determined that we need to roll the log files. # Close the log file. Probably needed if we want to delete/rename files. self.close() - # Get the size of the log file. + for i in range(self._backupCount, 0, -1): + CurrentFileName = self._LogFileName + "." + str(i) + CurrentFileNamePlus = self._LogFileName + "." + str(i + 1) + try: + if i == self._backupCount: + # This is the oldest log file. Delete this one. + os.remove(CurrentFileName) + else: + # Rename the current file to the next number in the sequence. + os.rename(CurrentFileName, CurrentFileNamePlus) + except OSError as e: + if e.args[0] == 2: + # File does not exsist. This is okay. + pass + else: + raise e + + # Rename the current log to the first backup + os.rename(self._LogFileName, CurrentFileName) + + # Reopen the file. + # pylint: disable=consider-using-with + self.stream = open(self._LogFileName, mode=self._WriteMode) + + def GetLogSize(self) -> int: + """Check the size of the log file.""" + # TODO: Is this needed? I am catching the case where the file does not exsist, + # but I don't know if that is a realistic case anymore. try: + self.stream.flush() # We need to call this or the file size is always zero. LogFileSize = os.stat(self._LogFileName)[6] except OSError as e: if e.args[0] == 2: @@ -269,46 +327,19 @@ def CheckLogSize(self) -> None: LogFileSize = None else: raise e - - # Get the size of the old log file. - try: - OldLogFileSize = os.stat(self._OldLogFileName)[6] - except OSError as e: - if e.args[0] == 2: - # Log file does not exsist. This is okay. - OldLogFileSize = None - else: - raise e - - # This checks if the log file is too big. If so, it deletes the old log file and renames the - # log file to the old log filename. - if LogFileSize > self._LogFileSizeLimit: - if OldLogFileSize is not None: - os.remove(self._OldLogFileName) - os.rename(self._LogFileName, self._OldLogFileName) - - # Reopen the file. - # pylint: disable=consider-using-with - self.stream = open(self._LogFileName, mode=self._WriteMode) - - def close(self) -> None: - """Closes the file""" - self.stream.flush() - self.stream.close() - - def format(self, record: LogRecord) -> str: - """Generate a string to log - - :param record: The record (message object) to be logged - """ - return super().format(record) + "\r\n" + return LogFileSize def emit(self, record: LogRecord) -> None: - """Generate the message and write it to the UART. + """Generate the message and write it to the file. :param record: The record (message object) to be logged """ - self.CheckLogSize() + if ( + (self.GetLogSize() >= self._maxBytes) + and (self._maxBytes > 0) + and (self._backupCount > 0) + ): + self.doRollover() self.stream.write(self.format(record)) From c81d30e8a825f91642a7b57c076bce017dec36d3 Mon Sep 17 00:00:00 2001 From: Pat Date: Thu, 16 May 2024 23:22:36 -0500 Subject: [PATCH 9/9] Update adafruit_logging.py Remove some debug statements --- adafruit_logging.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/adafruit_logging.py b/adafruit_logging.py index 42d87d9..370f252 100644 --- a/adafruit_logging.py +++ b/adafruit_logging.py @@ -282,7 +282,6 @@ def __init__( # Open the file and save the handle to self.stream super().__init__(self._LogFileName, mode=self._WriteMode) - # TODO: What do we do if the log file already exsists? def doRollover(self) -> None: """Roll over the log files. This should not need to be called directly""" @@ -316,8 +315,6 @@ def doRollover(self) -> None: def GetLogSize(self) -> int: """Check the size of the log file.""" - # TODO: Is this needed? I am catching the case where the file does not exsist, - # but I don't know if that is a realistic case anymore. try: self.stream.flush() # We need to call this or the file size is always zero. LogFileSize = os.stat(self._LogFileName)[6]