From 5a65192fda312733f280e854181ba163d00b0170 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 02:14:42 +0000 Subject: [PATCH 01/14] adding run example --- log_example.py | 111 +++++++++++++++++++++++++++++++++++++++++++++++++ makefile | 8 +++- 2 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 log_example.py diff --git a/log_example.py b/log_example.py new file mode 100644 index 00000000..1d02fe8a --- /dev/null +++ b/log_example.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +""" +Author: Mike Ryan +Date: 2024/05/16 +License: MIT +""" +import logging +import multiprocessing +import secrets +import threading + +from loguru import logger +from tqdm import tqdm + +from dsg_lib.common_functions import logging_config + +# Configure logging as before +logging_config.config_log( + logging_directory='log', + log_name='log', + logging_level='DEBUG', + log_rotation='100 MB', + log_retention='10 days', + log_backtrace=True, + log_serializer=True, + log_diagnose=True, + # app_name='my_app', + # append_app_name=True, + intercept_standard_logging=True, + enqueue=True, +) + + +# @logger.catch +def div_zero(x, y): + try: + return x / y + except ZeroDivisionError as e: + logger.error(f'{e}') + logging.error(f'{e}') + + +# @logger.catch +def div_zero_two(x, y): + try: + return x / y + except ZeroDivisionError as e: + logger.error(f'{e}') + logging.error(f'{e}') + + +def log_big_string(lqty=100, size=256): + big_string = secrets.token_urlsafe(size) + for _ in range(lqty): + logging.debug(f'Lets make this a big message {big_string}') + div_zero(x=1, y=0) + div_zero_two(x=1, y=0) + # after configuring logging + # use loguru to log messages + logger.debug('This is a loguru debug message') + logger.info('This is an loguru info message') + logger.error('This is an loguru error message') + logger.warning('This is a loguru warning message') + logger.critical('This is a loguru critical message') + + # will intercept all standard logging messages also + logging.debug('This is a standard logging debug message') + logging.info('This is an standard logging info message') + logging.error('This is an standard logging error message') + logging.warning('This is a standard logging warning message') + logging.critical('This is a standard logging critical message') + + +def worker(wqty=1000, lqty=100, size=256): + for _ in tqdm(range(wqty), ascii=True, leave=True): # Adjusted for demonstration + log_big_string(lqty=lqty, size=size) + + +def main(wqty: int = 100, lqty: int = 10, size: int = 256, workers: int = 16, thread_test: bool = False, process_test: bool = False): + if process_test: + processes = [] + # Create worker processes + for _ in tqdm(range(workers), desc="Multi-Processing Start", leave=True): + p = multiprocessing.Process( + target=worker, args=(wqty, lqty, size,)) + processes.append(p) + p.start() + + for p in tqdm((processes), desc="Multi-Processing Start", leave=False): + p.join(timeout=60) # Timeout after 60 seconds + if p.is_alive(): + logger.error(f"Process {p.name} is hanging. Terminating.") + p.terminate() + p.join() + + if thread_test: + threads = [] + for _ in tqdm(range(workers), desc="Threading Start", leave=True): # Create worker threads + t = threading.Thread(target=worker, args=(wqty, lqty, size,)) + threads.append(t) + t.start() + + for t in tqdm(threads, desc="Threading Gather", leave=False): + t.join() + + +if __name__ == "__main__": + from time import time + start = time() + main(wqty=20, lqty=100, size=64, workers=8, thread_test=False, process_test=True) + print(f"Execution time: {time()-start:.2f} seconds") diff --git a/makefile b/makefile index f1d79810..09f92e97 100644 --- a/makefile +++ b/makefile @@ -1,5 +1,5 @@ # Variables -REPONAME = PyJSONSchemaForm +REPONAME = devsetgo_lib PYTHON = python3 PIP = $(PYTHON) -m pip @@ -96,3 +96,9 @@ ruff: ## Format Python code with Ruff ruff check --fix --exit-non-zero-on-fix --show-fixes $(SERVICE_PATH) ruff check --fix --exit-non-zero-on-fix --show-fixes $(TESTS_PATH) ruff check --fix --exit-non-zero-on-fix --show-fixes $(EXAMPLE_PATH) + + +example-log: ## Run the example logging script + cp /workspaces/devsetgo_lib/examples/log_example.py /workspaces/devsetgo_lib/ + python3 log_example.py + cp /workspaces/devsetgo_lib/log_example.py /workspaces/devsetgo_lib/examples From dffb49f313784885e3489c35d56f518efe361425 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 02:14:54 +0000 Subject: [PATCH 02/14] adding run example --- examples/log_example.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/log_example.py b/examples/log_example.py index bb51b12b..1d02fe8a 100644 --- a/examples/log_example.py +++ b/examples/log_example.py @@ -57,18 +57,18 @@ def log_big_string(lqty=100, size=256): div_zero_two(x=1, y=0) # after configuring logging # use loguru to log messages - logger.debug('This is a debug message') - logger.info('This is an info message') - logger.error('This is an error message') - logger.warning('This is a warning message') - logger.critical('This is a critical message') + logger.debug('This is a loguru debug message') + logger.info('This is an loguru info message') + logger.error('This is an loguru error message') + logger.warning('This is a loguru warning message') + logger.critical('This is a loguru critical message') # will intercept all standard logging messages also - logging.debug('This is a debug message') - logging.info('This is an info message') - logging.error('This is an error message') - logging.warning('This is a warning message') - logging.critical('This is a critical message') + logging.debug('This is a standard logging debug message') + logging.info('This is an standard logging info message') + logging.error('This is an standard logging error message') + logging.warning('This is a standard logging warning message') + logging.critical('This is a standard logging critical message') def worker(wqty=1000, lqty=100, size=256): @@ -107,5 +107,5 @@ def main(wqty: int = 100, lqty: int = 10, size: int = 256, workers: int = 16, th if __name__ == "__main__": from time import time start = time() - main(wqty=10, lqty=100, size=64, workers=8, thread_test=False, process_test=True) + main(wqty=20, lqty=100, size=64, workers=8, thread_test=False, process_test=True) print(f"Execution time: {time()-start:.2f} seconds") From d5fd6bca70cd6e49acf98e1fe6b8fc82f6dbac91 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 02:21:54 +0000 Subject: [PATCH 03/14] switching from loguru to standard logging --- coverage-badge.svg | 2 +- coverage.xml | 802 ++++++++++-------- .../__import_sqlalchemy.py | 3 +- .../async_database.py | 3 +- .../database_config.py | 3 +- .../database_operations.py | 3 +- .../common_functions/calendar_functions.py | 3 +- dsg_lib/common_functions/email_validation.py | 3 +- dsg_lib/common_functions/file_functions.py | 3 +- dsg_lib/common_functions/folder_functions.py | 3 +- dsg_lib/common_functions/patterns.py | 3 +- dsg_lib/fastapi_functions/http_codes.py | 3 +- .../system_health_endpoints.py | 3 +- examples/fastapi_example.py | 3 +- examples/log_example.py | 3 +- log_example.py | 3 +- tests/conftest.py | 3 +- unreleased/__tools_endpoint.py | 6 +- unreleased/_converters.py | 3 +- .../middleware/access_logging_middleware.py | 3 +- unreleased/pdf_processing.py | 6 +- unreleased/system_tools_endpoints.py | 6 +- unreleased/user_lib.py | 3 +- 23 files changed, 474 insertions(+), 402 deletions(-) diff --git a/coverage-badge.svg b/coverage-badge.svg index b639d978..9a3a87e7 100644 --- a/coverage-badge.svg +++ b/coverage-badge.svg @@ -1 +1 @@ -coverage: 0.00%coverage0.00% +coverage: 94.55%coverage94.55% diff --git a/coverage.xml b/coverage.xml index 0f8e29ed..71bbc13d 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,11 +1,59 @@ - + - /workspaces/devsetgo_lib + /github/workspace + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -22,8 +70,8 @@ - + @@ -33,23 +81,23 @@ - - - - - + + + + - - + + - - + + - - - + + + + @@ -99,28 +147,28 @@ - - - - - - + + + + + - + - + - - - - + + + + - + - - + + + @@ -128,185 +176,185 @@ - - - - - - + + + + + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + - - - - - - - + + + + + + + - + - + - - - + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - + - - + + - + - + - - + + - - - - - - - + + + + + + + - + - - + + - + - - + + - - - - - - - + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - + + + + @@ -320,29 +368,29 @@ - - - - + + + - + - + - - - - - + + + + + - - + + - + + @@ -351,48 +399,48 @@ - - - + + - - - - + + + + - + - + - - - + + + - + - - + + - + - + - + + @@ -405,165 +453,165 @@ - - - - - - + + + + + - - + + - + - + - + - + - - - - + + + + - + - + - - - - + + + + - - - + + + - - + + - + - + - - - + + + - - + + - + - + - + - + - + - - + + - + - - - + + + - + - + - + - + - + - + - - - - - + + + + + - + - + - - + + - - + + - + - - + + - + - - - - - - - + + + + + + + - + - + - - + + - + - + - + - + + @@ -573,52 +621,52 @@ - - + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - + - + - + - - - - - - - - - - - - + + + + + + + + + + + + + @@ -676,22 +724,22 @@ - - - + + - - + + - - - - - + + + + + - + + @@ -711,25 +759,25 @@ - - - - - - - - - - - - + + + + + + + + + + + - + - + - + + @@ -737,51 +785,51 @@ - - - - + + + - + - - - - - - - - - - - + + + + + + + + + + + - - + + - - - - - - - - - + + + + + + + + + - - - + + + - + - + + diff --git a/dsg_lib/async_database_functions/__import_sqlalchemy.py b/dsg_lib/async_database_functions/__import_sqlalchemy.py index ac424a5a..d05eb18c 100644 --- a/dsg_lib/async_database_functions/__import_sqlalchemy.py +++ b/dsg_lib/async_database_functions/__import_sqlalchemy.py @@ -26,7 +26,8 @@ """ from typing import Tuple -from loguru import logger +# from loguru import logger +import logging as logger from packaging import version as packaging_version # Importing AsyncDatabase class from local module async_database diff --git a/dsg_lib/async_database_functions/async_database.py b/dsg_lib/async_database_functions/async_database.py index 25e61a62..150f7ee9 100644 --- a/dsg_lib/async_database_functions/async_database.py +++ b/dsg_lib/async_database_functions/async_database.py @@ -50,7 +50,8 @@ """ -from loguru import logger +# from loguru import logger +import logging as logger from .database_config import BASE, DBConfig diff --git a/dsg_lib/async_database_functions/database_config.py b/dsg_lib/async_database_functions/database_config.py index 8f63ec08..875dae07 100644 --- a/dsg_lib/async_database_functions/database_config.py +++ b/dsg_lib/async_database_functions/database_config.py @@ -46,7 +46,8 @@ from contextlib import asynccontextmanager from typing import Dict -from loguru import logger +# from loguru import logger +import logging as logger from .__import_sqlalchemy import import_sqlalchemy diff --git a/dsg_lib/async_database_functions/database_operations.py b/dsg_lib/async_database_functions/database_operations.py index d5eb3118..a1b666e5 100644 --- a/dsg_lib/async_database_functions/database_operations.py +++ b/dsg_lib/async_database_functions/database_operations.py @@ -32,7 +32,8 @@ import time from typing import Dict, List, Type -from loguru import logger +# from loguru import logger +import logging as logger from sqlalchemy import delete from sqlalchemy.ext.declarative import DeclarativeMeta diff --git a/dsg_lib/common_functions/calendar_functions.py b/dsg_lib/common_functions/calendar_functions.py index efc4432e..7574cfb2 100644 --- a/dsg_lib/common_functions/calendar_functions.py +++ b/dsg_lib/common_functions/calendar_functions.py @@ -50,7 +50,8 @@ Date: 2024/05/16 License: MIT """ -from loguru import logger +# from loguru import logger +import logging as logger def get_month(month: int) -> str: diff --git a/dsg_lib/common_functions/email_validation.py b/dsg_lib/common_functions/email_validation.py index af35b50f..365a1978 100644 --- a/dsg_lib/common_functions/email_validation.py +++ b/dsg_lib/common_functions/email_validation.py @@ -45,7 +45,8 @@ caching_resolver, validate_email, ) -from loguru import logger +# from loguru import logger +import logging as logger class DNSType(Enum): diff --git a/dsg_lib/common_functions/file_functions.py b/dsg_lib/common_functions/file_functions.py index 6fd43ef7..6ce6d4a0 100644 --- a/dsg_lib/common_functions/file_functions.py +++ b/dsg_lib/common_functions/file_functions.py @@ -46,7 +46,8 @@ from pathlib import Path from typing import List -from loguru import logger +# from loguru import logger +import logging as logger # Set the path to the directory where the files are located directory_to_files: str = "data" diff --git a/dsg_lib/common_functions/folder_functions.py b/dsg_lib/common_functions/folder_functions.py index e8fb701b..c5d882b6 100644 --- a/dsg_lib/common_functions/folder_functions.py +++ b/dsg_lib/common_functions/folder_functions.py @@ -42,7 +42,8 @@ from pathlib import Path from typing import List, Tuple -from loguru import logger +# from loguru import logger +import logging as logger # Define the directory where the files are located directory_to__files: str = "data" diff --git a/dsg_lib/common_functions/patterns.py b/dsg_lib/common_functions/patterns.py index 394eb492..83575378 100644 --- a/dsg_lib/common_functions/patterns.py +++ b/dsg_lib/common_functions/patterns.py @@ -40,7 +40,8 @@ """ import re -from loguru import logger +# from loguru import logger +import logging as logger def pattern_between_two_char( diff --git a/dsg_lib/fastapi_functions/http_codes.py b/dsg_lib/fastapi_functions/http_codes.py index 55282088..095b8fe0 100644 --- a/dsg_lib/fastapi_functions/http_codes.py +++ b/dsg_lib/fastapi_functions/http_codes.py @@ -28,7 +28,8 @@ Date: 2024/05/16 License: MIT """ -from loguru import logger +# from loguru import logger +import logging as logger from ._all_codes import ALL_HTTP_CODES diff --git a/dsg_lib/fastapi_functions/system_health_endpoints.py b/dsg_lib/fastapi_functions/system_health_endpoints.py index b7a32cb4..1d3392df 100644 --- a/dsg_lib/fastapi_functions/system_health_endpoints.py +++ b/dsg_lib/fastapi_functions/system_health_endpoints.py @@ -73,7 +73,8 @@ import tracemalloc # Importing database connector module -from loguru import logger +# from loguru import logger +import logging as logger from packaging import version as packaging_version from dsg_lib.fastapi_functions.http_codes import generate_code_dict diff --git a/examples/fastapi_example.py b/examples/fastapi_example.py index dd34e927..15b13db4 100644 --- a/examples/fastapi_example.py +++ b/examples/fastapi_example.py @@ -11,7 +11,8 @@ from fastapi import Body, FastAPI, Query from fastapi.responses import RedirectResponse -from loguru import logger +# from loguru import logger +import logging as logger from pydantic import BaseModel, EmailStr from sqlalchemy import Column, ForeignKey, Select, String from sqlalchemy.orm import relationship diff --git a/examples/log_example.py b/examples/log_example.py index 1d02fe8a..7a62785c 100644 --- a/examples/log_example.py +++ b/examples/log_example.py @@ -9,7 +9,8 @@ import secrets import threading -from loguru import logger +# from loguru import logger +import logging as logger from tqdm import tqdm from dsg_lib.common_functions import logging_config diff --git a/log_example.py b/log_example.py index 1d02fe8a..7a62785c 100644 --- a/log_example.py +++ b/log_example.py @@ -9,7 +9,8 @@ import secrets import threading -from loguru import logger +# from loguru import logger +import logging as logger from tqdm import tqdm from dsg_lib.common_functions import logging_config diff --git a/tests/conftest.py b/tests/conftest.py index 8b1b3947..a73fc90f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,8 @@ import logging import pytest -from loguru import logger +# from loguru import logger +import logging as logger @pytest.fixture diff --git a/unreleased/__tools_endpoint.py b/unreleased/__tools_endpoint.py index 5047c651..7414ad2d 100644 --- a/unreleased/__tools_endpoint.py +++ b/unreleased/__tools_endpoint.py @@ -12,7 +12,8 @@ # from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile, status # from fastapi.responses import ORJSONResponse # from fastapi.templating import Jinja2Templates -# from loguru import logger # Import the Loguru logger +# # from loguru import logger +import logging as logger # Import the Loguru logger # from pydantic import BaseModel, Field # from xmltodict import parse as xml_parse # from xmltodict import unparse as xml_unparse @@ -478,7 +479,8 @@ # # -*- coding: utf-8 -*- # import psutil -# from loguru import logger +# # from loguru import logger +import logging as logger # async def get_processes() -> list: diff --git a/unreleased/_converters.py b/unreleased/_converters.py index a430b055..7a71d924 100644 --- a/unreleased/_converters.py +++ b/unreleased/_converters.py @@ -24,7 +24,8 @@ import toml # Used for reading and writing TOML data import xmltodict # Used for converting XML data to Python dictionary and vice versa import yaml # Used for reading and writing YAML data -from loguru import logger # Used for logging +# from loguru import logger +import logging as logger # Used for logging from tqdm import tqdm diff --git a/unreleased/middleware/access_logging_middleware.py b/unreleased/middleware/access_logging_middleware.py index 8d1e566f..a53112f9 100644 --- a/unreleased/middleware/access_logging_middleware.py +++ b/unreleased/middleware/access_logging_middleware.py @@ -2,7 +2,8 @@ # import logging -# from loguru import logger +# # from loguru import logger +import logging as logger # from starlette.middleware.base import BaseHTTPMiddleware diff --git a/unreleased/pdf_processing.py b/unreleased/pdf_processing.py index e3be9e49..69dbb19b 100644 --- a/unreleased/pdf_processing.py +++ b/unreleased/pdf_processing.py @@ -3,7 +3,8 @@ import time import io from pypdf import PdfReader -from loguru import logger +# from loguru import logger +import logging as logger app = FastAPI() @@ -120,7 +121,8 @@ async def check_pdf( import re from functools import lru_cache -from loguru import logger # Import the Loguru logger +# from loguru import logger +import logging as logger # Import the Loguru logger from pypdf import PdfReader, PaperSize from tqdm import tqdm from unsync import unsync diff --git a/unreleased/system_tools_endpoints.py b/unreleased/system_tools_endpoints.py index 4c7d5dec..e84e3316 100644 --- a/unreleased/system_tools_endpoints.py +++ b/unreleased/system_tools_endpoints.py @@ -19,7 +19,8 @@ from jinja2 import Template # Importing database connector module -from loguru import logger +# from loguru import logger +import logging as logger from dsg_lib._converters import ( FileFormatEnum, @@ -643,7 +644,8 @@ async def get_example_json(qty: int = 10) -> StreamingResponse: # # Importing database connector module # from loguru import logger -# def create_tool_router(config: dict): +# # from loguru import logger +import logging as loggernfig: dict): # router = APIRouter() diff --git a/unreleased/user_lib.py b/unreleased/user_lib.py index 9c2d21a1..e7e8682d 100644 --- a/unreleased/user_lib.py +++ b/unreleased/user_lib.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -# from loguru import logger +# # from loguru import logger +import logging as logger # from passlib.hash import bcrypt From 93948efc5f60ccebd2ac3b80bdb1a5ceafcbbbd6 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 02:41:31 +0000 Subject: [PATCH 04/14] adding run commands and fixes of examples --- cal_example.py | 41 +++++++++++++++ coverage-badge.svg | 2 +- coverage.xml | 46 ++++------------- examples/log_example.py | 2 +- log_example.py | 112 ---------------------------------------- makefile | 58 ++++++++++++++++++--- 6 files changed, 102 insertions(+), 159 deletions(-) create mode 100644 cal_example.py delete mode 100644 log_example.py diff --git a/cal_example.py b/cal_example.py new file mode 100644 index 00000000..ec84c3e5 --- /dev/null +++ b/cal_example.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Author: Mike Ryan +Date: 2024/05/16 +License: MIT +""" +from dsg_lib.common_functions import calendar_functions + +month_list: list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] +month_names: list = [ + "january", + "february", + "march", + "april", + "may", + "june", + "july", + "august", + "september", + "october", + "november", + "december", + "bob", +] + + +def calendar_check_number(): + for i in month_list: + month = calendar_functions.get_month(month=i) + print(month) + + +def calendar_check_name(): + for i in month_names: + month = calendar_functions.get_month_number(month_name=i) + print(month) + + +if __name__ == "__main__": + calendar_check_number() + calendar_check_name() diff --git a/coverage-badge.svg b/coverage-badge.svg index 9a3a87e7..b0cb7cf5 100644 --- a/coverage-badge.svg +++ b/coverage-badge.svg @@ -1 +1 @@ -coverage: 94.55%coverage94.55% +coverage: 98.40%coverage98.40% diff --git a/coverage.xml b/coverage.xml index 71bbc13d..baaa3c83 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -8,48 +8,20 @@ - + - - - - - + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/examples/log_example.py b/examples/log_example.py index 7a62785c..ba0316cc 100644 --- a/examples/log_example.py +++ b/examples/log_example.py @@ -108,5 +108,5 @@ def main(wqty: int = 100, lqty: int = 10, size: int = 256, workers: int = 16, th if __name__ == "__main__": from time import time start = time() - main(wqty=20, lqty=100, size=64, workers=8, thread_test=False, process_test=True) + main(wqty=5, lqty=50, size=64, workers=8, thread_test=False, process_test=True) print(f"Execution time: {time()-start:.2f} seconds") diff --git a/log_example.py b/log_example.py deleted file mode 100644 index 7a62785c..00000000 --- a/log_example.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Author: Mike Ryan -Date: 2024/05/16 -License: MIT -""" -import logging -import multiprocessing -import secrets -import threading - -# from loguru import logger -import logging as logger -from tqdm import tqdm - -from dsg_lib.common_functions import logging_config - -# Configure logging as before -logging_config.config_log( - logging_directory='log', - log_name='log', - logging_level='DEBUG', - log_rotation='100 MB', - log_retention='10 days', - log_backtrace=True, - log_serializer=True, - log_diagnose=True, - # app_name='my_app', - # append_app_name=True, - intercept_standard_logging=True, - enqueue=True, -) - - -# @logger.catch -def div_zero(x, y): - try: - return x / y - except ZeroDivisionError as e: - logger.error(f'{e}') - logging.error(f'{e}') - - -# @logger.catch -def div_zero_two(x, y): - try: - return x / y - except ZeroDivisionError as e: - logger.error(f'{e}') - logging.error(f'{e}') - - -def log_big_string(lqty=100, size=256): - big_string = secrets.token_urlsafe(size) - for _ in range(lqty): - logging.debug(f'Lets make this a big message {big_string}') - div_zero(x=1, y=0) - div_zero_two(x=1, y=0) - # after configuring logging - # use loguru to log messages - logger.debug('This is a loguru debug message') - logger.info('This is an loguru info message') - logger.error('This is an loguru error message') - logger.warning('This is a loguru warning message') - logger.critical('This is a loguru critical message') - - # will intercept all standard logging messages also - logging.debug('This is a standard logging debug message') - logging.info('This is an standard logging info message') - logging.error('This is an standard logging error message') - logging.warning('This is a standard logging warning message') - logging.critical('This is a standard logging critical message') - - -def worker(wqty=1000, lqty=100, size=256): - for _ in tqdm(range(wqty), ascii=True, leave=True): # Adjusted for demonstration - log_big_string(lqty=lqty, size=size) - - -def main(wqty: int = 100, lqty: int = 10, size: int = 256, workers: int = 16, thread_test: bool = False, process_test: bool = False): - if process_test: - processes = [] - # Create worker processes - for _ in tqdm(range(workers), desc="Multi-Processing Start", leave=True): - p = multiprocessing.Process( - target=worker, args=(wqty, lqty, size,)) - processes.append(p) - p.start() - - for p in tqdm((processes), desc="Multi-Processing Start", leave=False): - p.join(timeout=60) # Timeout after 60 seconds - if p.is_alive(): - logger.error(f"Process {p.name} is hanging. Terminating.") - p.terminate() - p.join() - - if thread_test: - threads = [] - for _ in tqdm(range(workers), desc="Threading Start", leave=True): # Create worker threads - t = threading.Thread(target=worker, args=(wqty, lqty, size,)) - threads.append(t) - t.start() - - for t in tqdm(threads, desc="Threading Gather", leave=False): - t.join() - - -if __name__ == "__main__": - from time import time - start = time() - main(wqty=20, lqty=100, size=64, workers=8, thread_test=False, process_test=True) - print(f"Execution time: {time()-start:.2f} seconds") diff --git a/makefile b/makefile index 09f92e97..76431e3f 100644 --- a/makefile +++ b/makefile @@ -73,9 +73,6 @@ isort: ## Sort imports in Python code isort $(TESTS_PATH) isort $(EXAMPLE_PATH) -run-fastapi: ## Run the example application - uvicorn examples.fastapi_example:app --port ${PORT} --reload --log-level $(LOG_LEVEL) - # uvicorn examples.fastapi_example:app --port ${PORT} --workers ${WORKER} --log-level $(LOG_LEVEL) speedtest: ## Run a speed test if [ ! -f speedtest/http_request.so ]; then gcc -shared -o speedtest/http_request.so speedtest/http_request.c -lcurl -fPIC; fi @@ -87,7 +84,7 @@ test: ## Run the project's tests sed -i 's|/workspaces/$(REPONAME)|/github/workspace|' /workspaces/$(REPONAME)/coverage.xml genbadge coverage -i /workspaces/$(REPONAME)/coverage.xml flake8 --tee . > htmlcov/_flake8Report.txt -#flake8 --max-doc-length=132 --tee . > htmlcov/_flake8Report.txt + make ex-all build: ## Build the project python -m build @@ -98,7 +95,52 @@ ruff: ## Format Python code with Ruff ruff check --fix --exit-non-zero-on-fix --show-fixes $(EXAMPLE_PATH) -example-log: ## Run the example logging script - cp /workspaces/devsetgo_lib/examples/log_example.py /workspaces/devsetgo_lib/ - python3 log_example.py - cp /workspaces/devsetgo_lib/log_example.py /workspaces/devsetgo_lib/examples +ex-fastapi: ## Run the example fast application + uvicorn examples.fastapi_example:app --port ${PORT} --reload --log-level $(LOG_LEVEL) + # uvicorn examples.fastapi_example:app --port ${PORT} --workers ${WORKER} --log-level $(LOG_LEVEL) + +ex-log: ## Run the example logging script + cp /workspaces/devsetgo_lib/examples/log_example.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + + +ex-cal: ## Run the example calendar script + cp /workspaces/devsetgo_lib/examples/cal_example.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + +ex-csv: ## Run the example calendar script + cp /workspaces/devsetgo_lib/examples/csv_example.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + +ex-json: ## Run the example calendar script + cp /workspaces/devsetgo_lib/examples/json_example.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + +ex-pattern: ## Run the example calendar script + cp /workspaces/devsetgo_lib/examples/pattern_example.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + +ex-text: ## Run the example calendar script + cp /workspaces/devsetgo_lib/examples/text_example.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + +ex-email: ## Run the example calendar script + cp /workspaces/devsetgo_lib/examples/validate_emails.py /workspaces/devsetgo_lib/ex.py + python3 ex.py + rm /workspaces/devsetgo_lib/ex.py + +ex-all: ## Run all the examples, but fastapi + + make ex-log + make ex-cal + make ex-csv + make ex-json + make ex-pattern + make ex-text + make ex-email From b06d1b1da38837dcf8e0b3e634f68414730229a3 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:21:54 +0000 Subject: [PATCH 05/14] Updates for documentation --- CHANGELOG.md | 146 +--- coverage.xml | 1001 ++++++++++++----------- docs/about.md | 12 + docs/recipes/emailValidation.md | 180 +++- docs/recipes/fastapi.md | 133 +-- docs/recipes/loggingExample.md | 137 ++-- docs/recipes/patterns.md | 5 + dsg_lib/fastapi_functions/http_codes.py | 6 +- mkdocs.yml | 3 +- 9 files changed, 873 insertions(+), 750 deletions(-) create mode 100644 docs/about.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e0d0472..690c75f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,42 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## Latest Changes +### Adding DB Disconnect ([v0.14.1](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.14.1)) + +#### What's Changed +* Adding Database Disconnect (#439) @devsetgo +* pip(deps): bump pre-commit from 3.7.1 to 3.8.0 (#434) @dependabot +* updates to deal with stashing pages (#437) @devsetgo +* working on issue for deployment (#436) @devsetgo +* Adding MKDocs Workflow (#435) @devsetgo +* Version 0.14.0 (#433) @devsetgo + + +Published Date: 2024 August 25, 18:47 + +### Fix of version for Pypi ([v0.14.0-a](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.14.0-a)) + +#### What's Changed +* Version 0.14.0 (#433) @devsetgo + + +Published Date: 2024 July 27, 22:40 + +### High Speed Multi-Processing Improvements ([v0.14.0](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.14.0)) + +#### What's Changed +* High Speed Logging for Loguru Multi-Processing (#432) @devsetgo +* Resilient Sink Fixes (#431) @devsetgo +* Fix of bug in resilient sink (#430) @devsetgo +* Adding Resiliency to Logging Config (#429) @devsetgo +* pip(deps): bump mkdocs-print-site-plugin from 2.4.1 to 2.5.0 (#422) @dependabot +* pip(deps): bump ruff from 0.4.5 to 0.4.7 (#420) @dependabot +* pip(deps): bump autopep8 from 2.1.1 to 2.2.0 (#421) @dependabot +* pip(deps): bump mkdocs-material from 9.5.24 to 9.5.25 (#423) @dependabot + + +Published Date: 2024 July 27, 22:28 + ### ([v0.13.0-republish](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.13.0-republish)) #### What's Changed @@ -660,113 +696,3 @@ Published Date: 2022 January 29, 01:42 Published Date: 2021 July 16, 23:44 - -### Fixing Publish ([v0.5.0-2](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.5.0-2)) - -# What's Changed -* adding update for publish (#117) @devsetgo - - -Published Date: 2021 March 18, 17:19 - -### Calendar and RegEx Function + Documentation ([v0.5.0](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.5.0)) - -# What's Changed -* Adding Calendar Functions (#116) @devsetgo -* Bump pre-commit from 2.10.1 to 2.11.1 (#113) @dependabot -* update to Saturday (#115) @devsetgo -* Bump tqdm from 4.58.0 to 4.59.0 (#112) @dependabot -* Bump mkdocs-material from 7.0.4 to 7.0.5 (#114) @dependabot -* fixes for mkdoc material update (#111) @devsetgo -* Bump tox from 3.22.0 to 3.23.0 (#109) @dependabot -* Bump mkdocs-material from 7.0.2 to 7.0.4 (#108) @dependabot -* Bump pylint from 2.7.1 to 2.7.2 (#107) @dependabot -* Bump coverage from 5.4 to 5.5 (#110) @dependabot -* Bump pylint from 2.6.2 to 2.7.1 (#103) @dependabot -* Bump mkdocs-material from 6.2.8 to 7.0.2 (#104) @dependabot -* Bump watchdog from 2.0.1 to 2.0.2 (#105) @dependabot -* Bump tqdm from 4.57.0 to 4.58.0 (#106) @dependabot -* Bump tox from 3.21.4 to 3.22.0 (#101) @dependabot -* Bump watchdog from 2.0.0 to 2.0.1 (#99) @dependabot -* Bump pylint from 2.6.0 to 2.6.2 (#102) @dependabot -* Bump tqdm from 4.56.2 to 4.57.0 (#100) @dependabot -* Bump pytest-runner from 5.2 to 5.3.0 (#98) @dependabot -* Bump tqdm from 4.56.0 to 4.56.2 (#97) @dependabot -* Bump watchdog from 1.0.2 to 2.0.0 (#96) @dependabot -* Bump pre-commit from 2.10.0 to 2.10.1 (#95) @dependabot -* Bump mkdocs-material from 6.2.6 to 6.2.8 (#94) @dependabot -* Bump tox from 3.21.3 to 3.21.4 (#93) @dependabot -* Bump autopep8 from 1.5.4 to 1.5.5 (#92) @dependabot -* Bump tox from 3.21.2 to 3.21.3 (#87) @dependabot -* Bump mkdocs-material from 6.2.5 to 6.2.6 (#88) @dependabot -* Bump pytest from 6.2.1 to 6.2.2 (#89) @dependabot -* Bump coverage from 5.3.1 to 5.4 (#91) @dependabot -* Bump pre-commit from 2.9.3 to 2.10.0 (#90) @dependabot -* Bump tox from 3.21.1 to 3.21.2 (#84) @dependabot -* Bump mkdocs-material from 6.2.4 to 6.2.5 (#85) @dependabot -* Bump pytest-cov from 2.10.1 to 2.11.1 (#86) @dependabot -* Bump tox from 3.20.1 to 3.21.1 (#81) @dependabot -* Bump mkdocs-material from 6.2.3 to 6.2.4 (#82) @dependabot -* Bump tqdm from 4.55.1 to 4.56.0 (#83) @dependabot -* Bump tqdm from 4.55.0 to 4.55.1 (#80) @dependabot -* Bump mkdocs-material from 6.2.2 to 6.2.3 (#79) @dependabot - - -Published Date: 2021 March 18, 17:06 - -### Minor updates and library updates. ([v0.4.1](https://github.com/devsetgo/devsetgo_lib/releases/tag/v0.4.1)) - -# What's Changed -* Updates and Minor updates (#78) @devsetgo -* Bump tqdm from 4.54.1 to 4.55.0 (#77) @dependabot -* Bump twine from 3.2.0 to 3.3.0 (#76) @dependabot -* Bump coverage from 5.3 to 5.3.1 (#74) @dependabot -* Bump mkdocs-material from 6.1.7 to 6.2.2 (#75) @dependabot -* Bump watchdog from 0.10.4 to 1.0.2 (#73) @dependabot -* Bump pytest from 6.1.2 to 6.2.1 (#71) @dependabot -* Bump wheel from 0.36.1 to 0.36.2 (#70) @dependabot -* Bump tqdm from 4.54.0 to 4.54.1 (#67) @dependabot -* Bump mkdocs-material from 6.1.6 to 6.1.7 (#68) @dependabot -* Bump pre-commit from 2.9.2 to 2.9.3 (#69) @dependabot -* Bump wheel from 0.36.0 to 0.36.1 (#66) @dependabot -* Bump wheel from 0.35.1 to 0.36.0 (#64) @dependabot -* Bump tqdm from 4.53.0 to 4.54.0 (#65) @dependabot -* Bump pre-commit from 2.8.2 to 2.9.2 (#61) @dependabot -* Bump mkdocs-material from 6.1.5 to 6.1.6 (#60) @dependabot -* Bump tqdm from 4.52.0 to 4.53.0 (#62) @dependabot -* Bump watchdog from 0.10.3 to 0.10.4 (#63) @dependabot -* Bump tqdm from 4.51.0 to 4.52.0 (#59) @dependabot -* Bump mkdocs-material from 6.1.4 to 6.1.5 (#58) @dependabot -* Bump mkdocs-material from 6.1.2 to 6.1.4 (#57) @dependabot -* Bump pre-commit from 2.8.0 to 2.8.2 (#55) @dependabot -* Bump mkdocs-material from 6.1.0 to 6.1.2 (#56) @dependabot -* Bump pytest from 6.1.1 to 6.1.2 (#52) @dependabot -* Bump pre-commit from 2.7.1 to 2.8.0 (#53) @dependabot -* Bump tqdm from 4.50.2 to 4.51.0 (#54) @dependabot -* Bump mkdocs-material from 6.0.2 to 6.1.0 (#51) @dependabot -* Bump tqdm from 4.50.1 to 4.50.2 (#49) @dependabot -* Bump tox from 3.20.0 to 3.20.1 (#50) @dependabot -* Bump pytest from 6.1.0 to 6.1.1 (#48) @dependabot -* Bump mkdocs-material from 6.0.1 to 6.0.2 (#47) @dependabot -* Bump flake8 from 3.8.3 to 3.8.4 (#45) @dependabot -* Bump tqdm from 4.50.0 to 4.50.1 (#44) @dependabot -* Bump bump2version from 1.0.0 to 1.0.1 (#46) @dependabot -* Bump tqdm from 4.49.0 to 4.50.0 (#42) @dependabot -* Bump black from 19.10b0 to 20.8b1 (#43) @dependabot -* Bump tqdm from 4.46.0 to 4.49.0 (#40) @dependabot -* Bump pytest from 5.4.2 to 6.1.0 (#39) @dependabot -* Bump coverage from 5.1 to 5.3 (#38) @dependabot -* Bump autoflake from 1.3.1 to 1.4 (#41) @dependabot -* Bump twine from 3.1.1 to 3.2.0 (#37) @dependabot -* Bump wheel from 0.34.2 to 0.35.1 (#34) @dependabot -* Bump pytest-cov from 2.9.0 to 2.10.1 (#36) @dependabot -* Bump watchdog from 0.10.2 to 0.10.3 (#35) @dependabot -* Bump mkdocs-material from 5.2.2 to 6.0.1 (#33) @dependabot -* Bump pylint from 2.5.2 to 2.6.0 (#32) @dependabot-preview -* Bump pre-commit from 2.4.0 to 2.7.1 (#31) @dependabot-preview -* Bump tox from 3.15.1 to 3.20.0 (#30) @dependabot-preview -* Bump flake8 from 3.8.2 to 3.8.3 (#29) @dependabot-preview -* Bump autopep8 from 1.5.2 to 1.5.4 (#28) @dependabot-preview - - -Published Date: 2020 December 26, 23:51 diff --git a/coverage.xml b/coverage.xml index baaa3c83..84d6c679 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,9 +1,9 @@ - + - /github/workspace + /workspaces/devsetgo_lib @@ -36,7 +36,7 @@ - + @@ -50,7 +50,7 @@ - + @@ -61,15 +61,15 @@ - - + + - - - - - - + + + + + + @@ -114,7 +114,7 @@ - + @@ -129,21 +129,21 @@ - - - + + + - - - - + + + + - + @@ -153,219 +153,219 @@ - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - + @@ -376,46 +376,46 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -429,164 +429,164 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - + @@ -598,50 +598,50 @@ - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - + @@ -655,68 +655,68 @@ - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + @@ -728,15 +728,16 @@ - + + - - + + @@ -752,7 +753,7 @@ - + @@ -774,33 +775,33 @@ - - + + - - - - - - + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/docs/about.md b/docs/about.md new file mode 100644 index 00000000..30d392ee --- /dev/null +++ b/docs/about.md @@ -0,0 +1,12 @@ +# About +DevSetGo Library is a collection of tools and utilities to help developers with their projects. It has been primarily developed to help with the development of the [DevSetGo.com](https://www.devsetgo.com) website and other projects in my personal and professional life. It is a work in progress and will be updated as needed. + +The driving force behind the library is to limit the amount of boilerplate code that I (Mike Ryan) have to write for each project. Copying code from one project to another causes issues to get fixed in one project, but not always get updated in others. This library is an attempt to fix that issue and make it easier to maintain code across multiple projects. + +The library is written in Python and is available on PyPi. It is open source and available on GitHub. Feel free to use it in your projects and contribute to the library. + +## About Me +I am a software engineering manager with an eclectic background in various industries (finance, manufacturing, and metrology). I am passionate about software development and love to learn new things. + +- [DevSetGo.com](https://www.devsetgo.com) +- [Github](https://github.com/devsetgo) diff --git a/docs/recipes/emailValidation.md b/docs/recipes/emailValidation.md index 12d0fb54..3a8cbde9 100644 --- a/docs/recipes/emailValidation.md +++ b/docs/recipes/emailValidation.md @@ -2,12 +2,49 @@ Example of how to use in a script ```python - -from dsg_lib.common_functions.email_validation import validate_email_address - +# -*- coding: utf-8 -*- +""" +This module is used to validate a list of email addresses using various configurations. + +The module imports the `validate_email_address` function from the `dsg_lib.common_functions.email_validation` +module and uses it to validate a list of email addresses. The email addresses and configurations are hard-coded +into the module. + +The module measures the time taken to validate all the email addresses with all the configurations and prints +the results in a sorted order. + +The module can be run as a standalone script. + +Example: + $ python validate_emails.py + +Attributes: + email_addresses (list of str): A list of email addresses to validate. + configurations (list of dict): A list of configurations to use for validation. Each configuration is a + dictionary with the following keys: + - check_deliverability (bool): Whether to check if the email address is deliverable. + - test_environment (bool): Whether the function is being run in a test environment. + - allow_smtputf8 (bool): Whether to allow non-ASCII characters in the email address. + - allow_empty_local (bool): Whether to allow email addresses with an empty local part. + - allow_quoted_local (bool): Whether to allow email addresses with a quoted local part. + - allow_display_name (bool): Whether to allow email addresses with a display name. + - allow_domain_literal (bool): Whether to allow email addresses with a domain literal. + - globally_deliverable (bool): Whether the email address should be globally deliverable. + - timeout (int): The timeout for the validation in seconds. + - dns_type (str): The type of DNS to use for the validation. Can be 'dns' or 'timeout'. + +Functions: + validate_email_address(email: str, **kwargs: dict) -> dict: Validates an email address using the provided + configuration and returns a dictionary with the results. + +Author: Mike Ryan +Date: 2024/05/16 +License: MIT +""" import pprint import time +from dsg_lib.common_functions.email_validation import validate_email_address if __name__ == "__main__": @@ -38,27 +75,20 @@ if __name__ == "__main__": 'this is"not\\allowed@example.com', # spaces, quotes, and backslashes may only exist when within quoted strings and preceded by a backslash 'this\\ still\\"not\\\\allowed@example.com', # even if escaped (preceded by a backslash), spaces, quotes, and backslashes must still be contained by quotes "1234567890123456789012345678901234567890123456789012345678901234+x@example.com", # local part is longer than 64 characters - # Emails with empty local part "@example.com", # only valid if allow_empty_local is True - # Emails with non-ASCII characters "üñîçøðé@example.com", # only valid if allow_smtputf8 is True "user@üñîçøðé.com", # only valid if allow_smtputf8 is True - # Emails with quoted local part '"john.doe"@example.com', # only valid if allow_quoted_local is True '"john..doe"@example.com', # only valid if allow_quoted_local is True - # Emails with display name - 'John Doe ', # only valid if allow_display_name is True - + "John Doe ", # only valid if allow_display_name is True # Emails with domain literal - 'user@[192.0.2.1]', # only valid if allow_domain_literal is True - + "user@[192.0.2.1]", # only valid if allow_domain_literal is True # Emails with long local part - "a"*65 + "@example.com", # local part is longer than 64 characters - + "a" * 65 + "@example.com", # local part is longer than 64 characters # Emails with invalid characters "john doe@example.com", # space is not allowed "john@doe@example.com", # only one @ is allowed @@ -69,20 +99,119 @@ if __name__ == "__main__": # create a list of configurations configurations = [ - {"check_deliverability": True, "test_environment": False, "allow_smtputf8": False, "allow_empty_local": False, "allow_quoted_local": False, "allow_display_name": False, "allow_domain_literal": False, "globally_deliverable": None, "timeout": 10, "dns_type": 'timeout'}, - {"check_deliverability": False, "test_environment": True, "allow_smtputf8": True, "allow_empty_local": True, "allow_quoted_local": True, "allow_display_name": True, "allow_domain_literal": True, "globally_deliverable": None, "timeout": 5, "dns_type": 'dns'}, + { + "check_deliverability": True, + "test_environment": False, + "allow_smtputf8": False, + "allow_empty_local": False, + "allow_quoted_local": False, + "allow_display_name": False, + "allow_domain_literal": False, + "globally_deliverable": None, + "timeout": 10, + "dns_type": "timeout", + }, + { + "check_deliverability": False, + "test_environment": True, + "allow_smtputf8": True, + "allow_empty_local": True, + "allow_quoted_local": True, + "allow_display_name": True, + "allow_domain_literal": True, + "globally_deliverable": None, + "timeout": 5, + "dns_type": "dns", + }, {"check_deliverability": True}, - {"check_deliverability": False, "test_environment": False, "allow_smtputf8": True, "allow_empty_local": False, "allow_quoted_local": True, "allow_display_name": False, "allow_domain_literal": True, "globally_deliverable": None, "timeout": 15, "dns_type": 'timeout'}, - {"check_deliverability": True, "test_environment": True, "allow_smtputf8": False, "allow_empty_local": True, "allow_quoted_local": False, "allow_display_name": True, "allow_domain_literal": False, "globally_deliverable": None, "timeout": 20, "dns_type": 'dns'}, - {"check_deliverability": False, "test_environment": False, "allow_smtputf8": True, "allow_empty_local": True, "allow_quoted_local": True, "allow_display_name": True, "allow_domain_literal": True, "globally_deliverable": None, "timeout": 25, "dns_type": 'timeout'}, - {"check_deliverability": True, "test_environment": True, "allow_smtputf8": False, "allow_empty_local": False, "allow_quoted_local": False, "allow_display_name": False, "allow_domain_literal": False, "globally_deliverable": None, "timeout": 30, "dns_type": 'dns'}, - {"check_deliverability": False, "test_environment": True, "allow_smtputf8": True, "allow_empty_local": False, "allow_quoted_local": True, "allow_display_name": True, "allow_domain_literal": False, "globally_deliverable": None, "timeout": 35, "dns_type": 'timeout'}, - {"check_deliverability": True, "test_environment": False, "allow_smtputf8": False, "allow_empty_local": True, "allow_quoted_local": True, "allow_display_name": False, "allow_domain_literal": True, "globally_deliverable": None, "timeout": 40, "dns_type": 'dns'}, - {"check_deliverability": False, "test_environment": True, "allow_smtputf8": True, "allow_empty_local": False, "allow_quoted_local": False, "allow_display_name": True, "allow_domain_literal": True, "globally_deliverable": None, "timeout": 45, "dns_type": 'timeout'}, + { + "check_deliverability": False, + "test_environment": False, + "allow_smtputf8": True, + "allow_empty_local": False, + "allow_quoted_local": True, + "allow_display_name": False, + "allow_domain_literal": True, + "globally_deliverable": None, + "timeout": 15, + "dns_type": "timeout", + }, + { + "check_deliverability": True, + "test_environment": True, + "allow_smtputf8": False, + "allow_empty_local": True, + "allow_quoted_local": False, + "allow_display_name": True, + "allow_domain_literal": False, + "globally_deliverable": None, + "timeout": 20, + "dns_type": "dns", + }, + { + "check_deliverability": False, + "test_environment": False, + "allow_smtputf8": True, + "allow_empty_local": True, + "allow_quoted_local": True, + "allow_display_name": True, + "allow_domain_literal": True, + "globally_deliverable": None, + "timeout": 25, + "dns_type": "timeout", + }, + { + "check_deliverability": True, + "test_environment": True, + "allow_smtputf8": False, + "allow_empty_local": False, + "allow_quoted_local": False, + "allow_display_name": False, + "allow_domain_literal": False, + "globally_deliverable": None, + "timeout": 30, + "dns_type": "dns", + }, + { + "check_deliverability": False, + "test_environment": True, + "allow_smtputf8": True, + "allow_empty_local": False, + "allow_quoted_local": True, + "allow_display_name": True, + "allow_domain_literal": False, + "globally_deliverable": None, + "timeout": 35, + "dns_type": "timeout", + }, + { + "check_deliverability": True, + "test_environment": False, + "allow_smtputf8": False, + "allow_empty_local": True, + "allow_quoted_local": True, + "allow_display_name": False, + "allow_domain_literal": True, + "globally_deliverable": None, + "timeout": 40, + "dns_type": "dns", + }, + { + "check_deliverability": False, + "test_environment": True, + "allow_smtputf8": True, + "allow_empty_local": False, + "allow_quoted_local": False, + "allow_display_name": True, + "allow_domain_literal": True, + "globally_deliverable": None, + "timeout": 45, + "dns_type": "timeout", + }, ] t0 = time.time() - validity=[] + validity = [] for email in email_addresses: for config in configurations: @@ -90,10 +219,11 @@ if __name__ == "__main__": res = validate_email_address(email, **config) validity.append(res) t1 = time.time() - validity = sorted(validity, key=lambda x: x['email']) + validity = sorted(validity, key=lambda x: x["email"]) for v in validity: - pprint.pprint(v, indent=4) + pprint.pprint(v, indent=4) print(f"Time taken: {t1 - t0:.2f}") + ``` diff --git a/docs/recipes/fastapi.md b/docs/recipes/fastapi.md index 6c1030b0..82c706e7 100644 --- a/docs/recipes/fastapi.md +++ b/docs/recipes/fastapi.md @@ -13,7 +13,11 @@ Copy the fastapi code below after installing. (assumption is main.py) ```python # -*- coding: utf-8 -*- - +""" +Author: Mike Ryan +Date: 2024/05/16 +License: MIT +""" import datetime import secrets import time @@ -21,7 +25,8 @@ from contextlib import asynccontextmanager from fastapi import Body, FastAPI, Query from fastapi.responses import RedirectResponse -from loguru import logger +# from loguru import logger +import logging as logger from pydantic import BaseModel, EmailStr from sqlalchemy import Column, ForeignKey, Select, String from sqlalchemy.orm import relationship @@ -34,68 +39,11 @@ from dsg_lib.async_database_functions import ( database_operations, ) from dsg_lib.common_functions import logging_config +from dsg_lib.fastapi_functions import system_health_endpoints # , system_tools_endpoints logging_config.config_log( logging_level="INFO", log_serializer=False, log_name="log.log" ) - - -@asynccontextmanager -async def lifespan(app: FastAPI): - logger.info("starting up") - # Create the tables in the database - await async_db.create_tables() - - create_users = True - if create_users: - await create_a_bunch_of_users(single_entry=24, many_entries=2000) - yield - logger.info("shutting down") - - -# Create an instance of the FastAPI class -app = FastAPI( - title="FastAPI Example", # The title of the API - description="This is an example of a FastAPI application using the DevSetGo Toolkit.", # A brief description of the API - version="0.1.0", # The version of the API - docs_url="/docs", # The URL where the API documentation will be served - redoc_url="/redoc", # The URL where the ReDoc documentation will be served - openapi_url="/openapi.json", # The URL where the OpenAPI schema will be served - debug=True, # Enable debug mode - middleware=[], # A list of middleware to include in the application - routes=[], # A list of routes to include in the application - lifespan=lifespan, # this is the replacement for the startup and shutdown events -) - - -@app.get("/") -async def root(): - """ - Root endpoint of API - Returns: - Redrects to openapi document - """ - # redirect to openapi docs - logger.info("Redirecting to OpenAPI docs") - response = RedirectResponse(url="/docs") - return response - - -from dsg_lib.fastapi_functions import ( # , system_tools_endpoints - system_health_endpoints, -) - -config_health = { - "enable_status_endpoint": True, - "enable_uptime_endpoint": True, - "enable_heapdump_endpoint": True, -} -app.include_router( - system_health_endpoints.create_health_router(config=config_health), - prefix="/api/health", - tags=["system-health"], -) - # Create a DBConfig instance config = { # "database_uri": "postgresql+asyncpg://postgres:postgres@postgresdb/postgres", @@ -108,6 +56,7 @@ config = { "pool_recycle": 3600, # "pool_timeout": 30, } + # create database configuration db_config = database_config.DBConfig(config) # Create an AsyncDatabase instance @@ -117,7 +66,7 @@ async_db = async_database.AsyncDatabase(db_config) db_ops = database_operations.DatabaseOperations(async_db) -class User(base_schema.SchemaBase, async_db.Base): +class User(base_schema.SchemaBaseSQLite, async_db.Base): """ User table storing user details like first name, last name, and email """ @@ -137,7 +86,7 @@ class User(base_schema.SchemaBase, async_db.Base): ) # Relationship to the Address class -class Address(base_schema.SchemaBase, async_db.Base): +class Address(base_schema.SchemaBaseSQLite, async_db.Base): """ Address table storing address details like street, city, and zip code """ @@ -157,6 +106,64 @@ class Address(base_schema.SchemaBase, async_db.Base): "User", back_populates="addresses" ) # Relationship to the User class +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("starting up") + # Create the tables in the database + await async_db.create_tables() + + create_users = True + if create_users: + await create_a_bunch_of_users(single_entry=2000, many_entries=20000) + yield + logger.info("shutting down") + await async_db.disconnect() + logger.info("database disconnected") + print("That's all folks!") + + + +# Create an instance of the FastAPI class +app = FastAPI( + title="FastAPI Example", # The title of the API + description="This is an example of a FastAPI application using the DevSetGo Toolkit.", # A brief description of the API + version="0.1.0", # The version of the API + docs_url="/docs", # The URL where the API documentation will be served + redoc_url="/redoc", # The URL where the ReDoc documentation will be served + openapi_url="/openapi.json", # The URL where the OpenAPI schema will be served + debug=True, # Enable debug mode + middleware=[], # A list of middleware to include in the application + routes=[], # A list of routes to include in the application + lifespan=lifespan, # this is the replacement for the startup and shutdown events +) + + +@app.get("/") +async def root(): + """ + Root endpoint of API + Returns: + Redrects to openapi document + """ + # redirect to openapi docs + logger.info("Redirecting to OpenAPI docs") + response = RedirectResponse(url="/docs") + return response + + +config_health = { + "enable_status_endpoint": True, + "enable_uptime_endpoint": True, + "enable_heapdump_endpoint": True, +} +app.include_router( + system_health_endpoints.create_health_router(config=config_health), + prefix="/api/health", + tags=["system-health"], +) + + + async def create_a_bunch_of_users(single_entry=0, many_entries=0): logger.info(f"single_entry: {single_entry}") @@ -165,7 +172,7 @@ async def create_a_bunch_of_users(single_entry=0, many_entries=0): # Create a loop to generate user data - for i in tqdm(range(single_entry), desc="executing one"): + for _ in tqdm(range(single_entry), desc="executing one"): value = secrets.token_hex(16) user = User( first_name=f"First{value}", diff --git a/docs/recipes/loggingExample.md b/docs/recipes/loggingExample.md index bd18ed7e..e5f39787 100644 --- a/docs/recipes/loggingExample.md +++ b/docs/recipes/loggingExample.md @@ -2,75 +2,116 @@ ```python # -*- coding: utf-8 -*- +""" +Author: Mike Ryan +Date: 2024/05/16 +License: MIT +""" import logging +import multiprocessing import secrets -from uuid import uuid4 +import threading -from loguru import logger +# from loguru import logger +import logging as logger from tqdm import tqdm from dsg_lib.common_functions import logging_config +# Configure logging as before logging_config.config_log( - logging_directory="log", - # or None and defaults to logging - # log_name="log.json", - # or None and defaults to "log.log" - logging_level="debug", - # or "info" or "debug" or "warning" or "error" or "critical" - # or None and defaults to "info" - log_rotation="10 MB", - # or None and default is 10 MB - log_retention="1 Day", - # or None and defaults to "14 Days" + logging_directory='log', + log_name='log', + logging_level='DEBUG', + log_rotation='100 MB', + log_retention='10 days', log_backtrace=True, - # or None and defaults to False - app_name="my_app", - # app name is used to identify the application - # this is an optional field - enable_trace_id=True, - # service id is used to identify the service - # this is an optional field - append_app_name=True, - # append app name to log file name defaults to false - append_trace_id=True, - # append app name and service name to log file name defaults to false + log_serializer=True, + log_diagnose=True, + # app_name='my_app', + # append_app_name=True, + intercept_standard_logging=True, + enqueue=True, ) -# after configuring logging -# user loguru to log messages -logger.debug("This is a debug message") -logger.info("This is an info message") -logger.error("This is an error message") -logger.warning("This is a warning message") -logger.critical("This is a critical message") - -# will intercept all standard logging messages also -logging.debug("This is a debug message") -logging.info("This is an info message") -logging.error("This is an error message") -logging.warning("This is a warning message") -logging.critical("This is a critical message") - +# @logger.catch def div_zero(x, y): try: return x / y except ZeroDivisionError as e: - logger.error(f"{e}") - logging.error(f"{e}") + logger.error(f'{e}') + logging.error(f'{e}') -@logger.catch +# @logger.catch def div_zero_two(x, y): - return x / y + try: + return x / y + except ZeroDivisionError as e: + logger.error(f'{e}') + logging.error(f'{e}') + + +def log_big_string(lqty=100, size=256): + big_string = secrets.token_urlsafe(size) + for _ in range(lqty): + logging.debug(f'Lets make this a big message {big_string}') + div_zero(x=1, y=0) + div_zero_two(x=1, y=0) + # after configuring logging + # use loguru to log messages + logger.debug('This is a loguru debug message') + logger.info('This is an loguru info message') + logger.error('This is an loguru error message') + logger.warning('This is a loguru warning message') + logger.critical('This is a loguru critical message') + + # will intercept all standard logging messages also + logging.debug('This is a standard logging debug message') + logging.info('This is an standard logging info message') + logging.error('This is an standard logging error message') + logging.warning('This is a standard logging warning message') + logging.critical('This is a standard logging critical message') + + +def worker(wqty=1000, lqty=100, size=256): + for _ in tqdm(range(wqty), ascii=True, leave=True): # Adjusted for demonstration + log_big_string(lqty=lqty, size=size) + + +def main(wqty: int = 100, lqty: int = 10, size: int = 256, workers: int = 16, thread_test: bool = False, process_test: bool = False): + if process_test: + processes = [] + # Create worker processes + for _ in tqdm(range(workers), desc="Multi-Processing Start", leave=True): + p = multiprocessing.Process( + target=worker, args=(wqty, lqty, size,)) + processes.append(p) + p.start() + + for p in tqdm((processes), desc="Multi-Processing Start", leave=False): + p.join(timeout=60) # Timeout after 60 seconds + if p.is_alive(): + logger.error(f"Process {p.name} is hanging. Terminating.") + p.terminate() + p.join() + + if thread_test: + threads = [] + for _ in tqdm(range(workers), desc="Threading Start", leave=True): # Create worker threads + t = threading.Thread(target=worker, args=(wqty, lqty, size,)) + threads.append(t) + t.start() + for t in tqdm(threads, desc="Threading Gather", leave=False): + t.join() -a = div_zero(x=1, y=0) -b = div_zero_two(x=1, y=0) -for _ in tqdm(range(5), ascii=True): - # log a lot of data - logging.debug(f"Lets make this a big message {secrets.token_urlsafe(32)}") +if __name__ == "__main__": + from time import time + start = time() + main(wqty=5, lqty=50, size=64, workers=8, thread_test=False, process_test=True) + print(f"Execution time: {time()-start:.2f} seconds") ``` diff --git a/docs/recipes/patterns.md b/docs/recipes/patterns.md index d126a0db..e591f679 100644 --- a/docs/recipes/patterns.md +++ b/docs/recipes/patterns.md @@ -2,6 +2,11 @@ ```python # -*- coding: utf-8 -*- +""" +Author: Mike Ryan +Date: 2024/05/16 +License: MIT +""" import pprint from random import randint diff --git a/dsg_lib/fastapi_functions/http_codes.py b/dsg_lib/fastapi_functions/http_codes.py index 095b8fe0..793a303e 100644 --- a/dsg_lib/fastapi_functions/http_codes.py +++ b/dsg_lib/fastapi_functions/http_codes.py @@ -30,7 +30,7 @@ """ # from loguru import logger import logging as logger - +from typing import List, Dict, Union from ._all_codes import ALL_HTTP_CODES # TODO: Create a way to extend the ALL_HTTP_CODES dictionary with custom codes. @@ -44,7 +44,7 @@ # # type: ignore""" -def generate_code_dict(codes, description_only=False): +def generate_code_dict(codes: List[int], description_only: bool = False) -> Dict[int, Union[str, Dict[str, str]]]: """ Generate a dictionary of specific HTTP error codes from the http_codes dictionary. @@ -109,7 +109,7 @@ def generate_code_dict(codes, description_only=False): # Usage: A list of common HTTP status codes used in various HTTP methods -common_codes = [200, 400, 401, 403, 404, 408, 429, 500, 503] +common_codes:list = [200, 400, 401, 403, 404, 408, 429, 500, 503] # A dictionary of common HTTP status codes and additional codes specific to GET # requests diff --git a/mkdocs.yml b/mkdocs.yml index 53b2f3d9..de5233ce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ -site_name: DSG_LIB +site_name: 'DevSetGo Library' # Project information site_description: 'Common functions for Python applications. This is to increase reusability and limit rewritting the same functions in multiple applications. It also allows for defects to be addressed quickly and then be propigated across applications.' site_author: 'Mike Ryan' @@ -36,6 +36,7 @@ nav: - Patterns: 'recipes/patterns.md' - EmailValidation: 'recipes/emailValidation.md' - About: + - About: 'about.md' - Contributing: 'contribute.md' - Release Notes: release-notes.md # Theme From 4c8bfbeebc13c3400856ccaf59912c6669f4e40f Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:47:57 +0000 Subject: [PATCH 06/14] adding configuration examples --- docs/recipes/asyncDatabase.md | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/docs/recipes/asyncDatabase.md b/docs/recipes/asyncDatabase.md index 5bf833a4..bcf895fc 100644 --- a/docs/recipes/asyncDatabase.md +++ b/docs/recipes/asyncDatabase.md @@ -227,3 +227,56 @@ if __name__ == "__main__": uvicorn.run(app, host="127.0.0.1", port=5000) ``` + + +### Configuration Examples + +```python +# SQLite in memory database +config = { + "database_uri": "sqlite+aiosqlite:///:memory:?cache=shared", + "echo": False, + "future": True, + "pool_recycle": 3600, +} +``` + +```python +# PostgreSQL database +config = { + "database_uri": "postgresql+asyncpg://postgres:postgres@postgresdb/postgres", + "echo": False, + "future": True, + "pool_recycle": 3600, +} +``` + +```python +# MySQL database +config = { + "database_uri": "mysql+aiomysql://root:root@localhost/test", + "echo": False, + "future": True, + "pool_recycle": 3600, +} +``` + +```python +# SQL Server database +config = { + "database_uri": "mssql+aiomssql://sa:yourStrong(!)Password@localhost:1433/master", + "echo": False, + "future": True, + "pool_recycle": 3600, +} +``` + +```python +# Oracle database +config = { + "database_uri": "oracle+oracledb_async://scott:tiger@localhost/?service_name=XEPDB1", + "echo": False, + "future": True, + "pool_recycle": 3600, +} +``` From 45f654ce4905ab28a889f22d95ca4adc12ca4f3a Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:48:05 +0000 Subject: [PATCH 07/14] moving --- cal_example.py | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 cal_example.py diff --git a/cal_example.py b/cal_example.py deleted file mode 100644 index ec84c3e5..00000000 --- a/cal_example.py +++ /dev/null @@ -1,41 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Author: Mike Ryan -Date: 2024/05/16 -License: MIT -""" -from dsg_lib.common_functions import calendar_functions - -month_list: list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] -month_names: list = [ - "january", - "february", - "march", - "april", - "may", - "june", - "july", - "august", - "september", - "october", - "november", - "december", - "bob", -] - - -def calendar_check_number(): - for i in month_list: - month = calendar_functions.get_month(month=i) - print(month) - - -def calendar_check_name(): - for i in month_names: - month = calendar_functions.get_month_number(month_name=i) - print(month) - - -if __name__ == "__main__": - calendar_check_number() - calendar_check_name() From 04a09b6d39431ce767687633cce4493bf4454c84 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:48:17 +0000 Subject: [PATCH 08/14] working on make commands --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 76431e3f..1e0730a2 100644 --- a/makefile +++ b/makefile @@ -84,7 +84,7 @@ test: ## Run the project's tests sed -i 's|/workspaces/$(REPONAME)|/github/workspace|' /workspaces/$(REPONAME)/coverage.xml genbadge coverage -i /workspaces/$(REPONAME)/coverage.xml flake8 --tee . > htmlcov/_flake8Report.txt - make ex-all + build: ## Build the project python -m build From bad7a39f19c78bf9536d48d23d7d70920b073f7c Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:48:52 +0000 Subject: [PATCH 09/14] changing to python-oracledb library, as it supports async --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c7146f72..006bbec5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ [project.optional-dependencies] postgres = ["asyncpg>=0.21.0", "sqlalchemy>=2.0.10,<2.0.99"] sqlite = ["aiosqlite>=0.17.0", "sqlalchemy>=2.0.10,<2.0.99"] -oracle = ["cx_Oracle>=8.0.0", "sqlalchemy>=2.0.10,<2.0.99"] +oracle = ["oracledb>=2.4.1,<2.5.0", "sqlalchemy>=2.0.10,<2.0.99"] mssql = ["aioodbc>=0.4.1", "sqlalchemy>=2.0.10,<2.0.99"] fastapi = ["fastapi>=0.100.0", "pydantic[email]>=2.0"] all = [ From 3df897673cf537a669027b61d66658e48a0a3921 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:49:49 +0000 Subject: [PATCH 10/14] change to not add .txt or add if not given --- dsg_lib/common_functions/folder_functions.py | 5 ++--- .../test_file_functions/test_save_text.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dsg_lib/common_functions/folder_functions.py b/dsg_lib/common_functions/folder_functions.py index c5d882b6..de69d1e2 100644 --- a/dsg_lib/common_functions/folder_functions.py +++ b/dsg_lib/common_functions/folder_functions.py @@ -37,14 +37,13 @@ License: MIT """ +# from loguru import logger +import logging as logger import re from datetime import datetime from pathlib import Path from typing import List, Tuple -# from loguru import logger -import logging as logger - # Define the directory where the files are located directory_to__files: str = "data" file_directory = f"{directory_to__files}/csv" diff --git a/tests/test_common_functions/test_file_functions/test_save_text.py b/tests/test_common_functions/test_file_functions/test_save_text.py index c4a9af47..f1d81b82 100644 --- a/tests/test_common_functions/test_file_functions/test_save_text.py +++ b/tests/test_common_functions/test_file_functions/test_save_text.py @@ -29,7 +29,7 @@ def test_save_text(self, mock_save_text): mock_save_text(file_name=file_name, data=text, root_folder=self.data_dir) # Check that file was created and contains the correct text - file_path = os.path.join(self.data_dir, "text", f"{file_name}.txt") + file_path = os.path.join(self.data_dir, "text", file_name) self.assertTrue(os.path.exists(file_path)) with open(file_path, "r") as file: self.assertEqual(file.read(), text) From 3c9f2c78c8443598f77035215e6e6eb7dc393ff8 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:50:06 +0000 Subject: [PATCH 11/14] formatting --- coverage-badge.svg | 2 +- coverage.xml | 1022 ++++++++--------- .../__import_sqlalchemy.py | 4 +- .../database_config.py | 5 +- .../database_operations.py | 4 +- dsg_lib/common_functions/email_validation.py | 4 +- dsg_lib/common_functions/file_functions.py | 10 +- dsg_lib/common_functions/patterns.py | 3 +- dsg_lib/fastapi_functions/http_codes.py | 3 +- .../system_health_endpoints.py | 6 +- examples/fastapi_example.py | 4 +- examples/log_example.py | 4 +- tests/conftest.py | 4 +- 13 files changed, 529 insertions(+), 546 deletions(-) diff --git a/coverage-badge.svg b/coverage-badge.svg index b0cb7cf5..05796758 100644 --- a/coverage-badge.svg +++ b/coverage-badge.svg @@ -1 +1 @@ -coverage: 98.40%coverage98.40% +coverage: 100.00%coverage100.00% diff --git a/coverage.xml b/coverage.xml index 84d6c679..65f551e3 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,31 +1,11 @@ - + - /workspaces/devsetgo_lib + /github/workspace - - - - - - - - - - - - - - - - - - - - @@ -36,7 +16,7 @@ - + @@ -50,7 +30,7 @@ - + @@ -61,15 +41,15 @@ - - + + - - - - - - + + + + + + @@ -114,7 +94,7 @@ - + @@ -129,21 +109,21 @@ - - - + + + - - - - + + + + - + @@ -153,219 +133,219 @@ - - - - - - - - - + + + + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - + - + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - + + + + + + + + + - + @@ -376,46 +356,46 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -429,164 +409,166 @@ - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + - + @@ -598,50 +580,50 @@ - - - - - - - - + + + + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - + @@ -655,68 +637,68 @@ - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + @@ -728,7 +710,7 @@ - + @@ -736,8 +718,8 @@ - - + + @@ -753,7 +735,7 @@ - + @@ -775,33 +757,33 @@ - - + + - - - - - - + + + + + + - - - - - - - - - - - - - + + + + + + + + + + + + + diff --git a/dsg_lib/async_database_functions/__import_sqlalchemy.py b/dsg_lib/async_database_functions/__import_sqlalchemy.py index d05eb18c..ef1aa06b 100644 --- a/dsg_lib/async_database_functions/__import_sqlalchemy.py +++ b/dsg_lib/async_database_functions/__import_sqlalchemy.py @@ -24,10 +24,10 @@ Date Updated: 2024/07/26 """ -from typing import Tuple - # from loguru import logger import logging as logger +from typing import Tuple + from packaging import version as packaging_version # Importing AsyncDatabase class from local module async_database diff --git a/dsg_lib/async_database_functions/database_config.py b/dsg_lib/async_database_functions/database_config.py index 875dae07..1a2d1b85 100644 --- a/dsg_lib/async_database_functions/database_config.py +++ b/dsg_lib/async_database_functions/database_config.py @@ -43,11 +43,10 @@ Date: 2024/05/16 License: MIT """ -from contextlib import asynccontextmanager -from typing import Dict - # from loguru import logger import logging as logger +from contextlib import asynccontextmanager +from typing import Dict from .__import_sqlalchemy import import_sqlalchemy diff --git a/dsg_lib/async_database_functions/database_operations.py b/dsg_lib/async_database_functions/database_operations.py index a1b666e5..0557695c 100644 --- a/dsg_lib/async_database_functions/database_operations.py +++ b/dsg_lib/async_database_functions/database_operations.py @@ -29,11 +29,11 @@ License: MIT """ +# from loguru import logger +import logging as logger import time from typing import Dict, List, Type -# from loguru import logger -import logging as logger from sqlalchemy import delete from sqlalchemy.ext.declarative import DeclarativeMeta diff --git a/dsg_lib/common_functions/email_validation.py b/dsg_lib/common_functions/email_validation.py index 365a1978..59ebe9fe 100644 --- a/dsg_lib/common_functions/email_validation.py +++ b/dsg_lib/common_functions/email_validation.py @@ -36,6 +36,8 @@ Date: 2024/05/16 License: MIT """ +# from loguru import logger +import logging as logger from enum import Enum from typing import Dict, List, Union @@ -45,8 +47,6 @@ caching_resolver, validate_email, ) -# from loguru import logger -import logging as logger class DNSType(Enum): diff --git a/dsg_lib/common_functions/file_functions.py b/dsg_lib/common_functions/file_functions.py index 6ce6d4a0..4140744a 100644 --- a/dsg_lib/common_functions/file_functions.py +++ b/dsg_lib/common_functions/file_functions.py @@ -40,15 +40,14 @@ # Import required modules import csv import json +# from loguru import logger +import logging as logger import os import random from datetime import datetime from pathlib import Path from typing import List -# from loguru import logger -import logging as logger - # Set the path to the directory where the files are located directory_to_files: str = "data" @@ -575,7 +574,7 @@ def save_text(file_name: str, data: str, root_folder: str = None) -> str: text_directory = Path(root_folder) / "text" # Construct the file path for text files - file_path = text_directory / f"{file_name}.txt" + file_path = text_directory / file_name # Create the text directory if it does not exist text_directory.mkdir(parents=True, exist_ok=True) @@ -589,6 +588,9 @@ def save_text(file_name: str, data: str, root_folder: str = None) -> str: logger.error(f"{file_name} cannot contain \\ or /") raise ValueError(f"{file_name} cannot contain \\ or /") + # Add extension to file_name if needed + if not file_name.endswith(".txt"): + file_name += ".txt" # Open or create the file and write the data with open(file_path, "w+", encoding="utf-8") as file: file.write(data) diff --git a/dsg_lib/common_functions/patterns.py b/dsg_lib/common_functions/patterns.py index 83575378..3bc97ba8 100644 --- a/dsg_lib/common_functions/patterns.py +++ b/dsg_lib/common_functions/patterns.py @@ -38,10 +38,9 @@ License: MIT """ -import re - # from loguru import logger import logging as logger +import re def pattern_between_two_char( diff --git a/dsg_lib/fastapi_functions/http_codes.py b/dsg_lib/fastapi_functions/http_codes.py index 793a303e..586163a8 100644 --- a/dsg_lib/fastapi_functions/http_codes.py +++ b/dsg_lib/fastapi_functions/http_codes.py @@ -30,7 +30,8 @@ """ # from loguru import logger import logging as logger -from typing import List, Dict, Union +from typing import Dict, List, Union + from ._all_codes import ALL_HTTP_CODES # TODO: Create a way to extend the ALL_HTTP_CODES dictionary with custom codes. diff --git a/dsg_lib/fastapi_functions/system_health_endpoints.py b/dsg_lib/fastapi_functions/system_health_endpoints.py index 1d3392df..1a2f9647 100644 --- a/dsg_lib/fastapi_functions/system_health_endpoints.py +++ b/dsg_lib/fastapi_functions/system_health_endpoints.py @@ -68,13 +68,13 @@ License: MIT """ +# Importing database connector module +# from loguru import logger +import logging as logger # Import necessary modules import time import tracemalloc -# Importing database connector module -# from loguru import logger -import logging as logger from packaging import version as packaging_version from dsg_lib.fastapi_functions.http_codes import generate_code_dict diff --git a/examples/fastapi_example.py b/examples/fastapi_example.py index 15b13db4..837e22fd 100644 --- a/examples/fastapi_example.py +++ b/examples/fastapi_example.py @@ -5,14 +5,14 @@ License: MIT """ import datetime +# from loguru import logger +import logging as logger import secrets import time from contextlib import asynccontextmanager from fastapi import Body, FastAPI, Query from fastapi.responses import RedirectResponse -# from loguru import logger -import logging as logger from pydantic import BaseModel, EmailStr from sqlalchemy import Column, ForeignKey, Select, String from sqlalchemy.orm import relationship diff --git a/examples/log_example.py b/examples/log_example.py index ba0316cc..5f879c0a 100644 --- a/examples/log_example.py +++ b/examples/log_example.py @@ -4,13 +4,13 @@ Date: 2024/05/16 License: MIT """ +# from loguru import logger import logging +import logging as logger import multiprocessing import secrets import threading -# from loguru import logger -import logging as logger from tqdm import tqdm from dsg_lib.common_functions import logging_config diff --git a/tests/conftest.py b/tests/conftest.py index a73fc90f..70a28cf4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- +# from loguru import logger import logging +import logging as logger import pytest -# from loguru import logger -import logging as logger @pytest.fixture From 06c3b78f736e5e66bb9c93a366ff2becabb6270f Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:51:56 +0000 Subject: [PATCH 12/14] bump to 0.14.2 --- .bumpversion.cfg | 2 +- dsg_lib/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index affbf924..ca7daa5a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.14.1 +current_version = 0.14.2 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(-(?P[a-z]+)(?P\d+))? diff --git a/dsg_lib/__init__.py b/dsg_lib/__init__.py index 27209659..fda496d8 100644 --- a/dsg_lib/__init__.py +++ b/dsg_lib/__init__.py @@ -9,4 +9,4 @@ License: MIT """ -__version__ = "0.14.1" +__version__ = "0.14.2" diff --git a/pyproject.toml b/pyproject.toml index 006bbec5..8b972c66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "hatchling.build" [project] name = "devsetgo_lib" -version = "0.14.1" +version = "0.14.2" requires-python = ">=3.9" description = """ DevSetGo Library is a Python library offering reusable functions for efficient coding. It includes file operations, calendar utilities, pattern matching, advanced logging with loguru, FastAPI endpoints, async database handling, and email validation. Designed for ease of use and versatility, it's a valuable tool for Python developers. From b64af586ba65aebde0d97ee41dfaefc0ea3dd44b Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:53:10 +0000 Subject: [PATCH 13/14] update of requirements --- requirements.txt | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/requirements.txt b/requirements.txt index d6516e06..75477ca7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,41 +3,41 @@ aiosqlite==0.20.0 # Vulnerabilities: None asyncpg==0.29.0 # Vulnerabilities: None autoflake==2.3.1 # Vulnerabilities: None autopep8==2.3.1 # Vulnerabilities: None -black==24.4.2 # Vulnerabilities: None +black==24.8.0 # From 24.4.2 | Vulnerabilities: None bump2version==1.0.1 # Vulnerabilities: None click==8.1.7 # Vulnerabilities: None cx-Oracle==8.3.0 # Vulnerabilities: None -fastapi[all]==0.111.1 # Vulnerabilities: None -flake8==7.1.0 # Vulnerabilities: None +fastapi[all]==0.112.2 # From 0.111.1 | Vulnerabilities: None +flake8==7.1.1 # From 7.1.0 | Vulnerabilities: None genbadge[all]==1.1.1 # Vulnerabilities: None hatchling==1.25.0 # Vulnerabilities: None loguru==0.7.2 # Vulnerabilities: None -mkdocs-material==9.5.30 # From 9.5.29 | Vulnerabilities: None +mike==2.1.3 # From none | Vulnerabilities: None +mkdocs-material==9.5.33 # From 9.5.30 | Vulnerabilities: None mkdocs-print-site-plugin==2.5.0 # Vulnerabilities: None -mkdocstrings[python,shell]==0.25.2 # From 0.25.1 | Vulnerabilities: None +mkdocstrings[python,shell]==0.25.2 # Vulnerabilities: None packaging==24.1 # Vulnerabilities: None -pre-commit==3.7.1 # Vulnerabilities: None +pre-commit==3.8.0 # From 3.7.1 | Vulnerabilities: None psycopg2==2.9.9 # Vulnerabilities: None Pygments==2.18.0 # Vulnerabilities: None -pylint==3.2.6 # From 3.2.5 | Vulnerabilities: None -pymdown-extensions==10.9 # From 10.8.1 | Vulnerabilities: None -pytest==8.3.2 # From 8.3.1 | Vulnerabilities: None -pytest-asyncio==0.23.8 # Vulnerabilities: None +pylint==3.2.6 # Vulnerabilities: None +pymdown-extensions==10.9 # Vulnerabilities: None +pytest==8.3.2 # Vulnerabilities: None +pytest-asyncio==0.24.0 # From 0.23.8 | Vulnerabilities: None pytest-cov==5.0.0 # Vulnerabilities: None pytest-mock==3.14.0 # Vulnerabilities: None pytest-runner==6.0.1 # Vulnerabilities: None pytest-xdist==3.6.1 # Vulnerabilities: None python-json-logger==2.0.7 # Vulnerabilities: None pytz==2024.1 # Vulnerabilities: None -PyYAML==6.0.1 # Vulnerabilities: None -ruff==0.5.5 # From 0.5.4 | Vulnerabilities: None -SQLAlchemy==2.0.31 # Vulnerabilities: None +PyYAML==6.0.2 # From 6.0.1 | Vulnerabilities: None +ruff==0.6.3 # From 0.5.5 | Vulnerabilities: None +SQLAlchemy==2.0.32 # From 2.0.31 | Vulnerabilities: None structlog==24.4.0 # Vulnerabilities: None toml==0.10.2 # Vulnerabilities: None -tox==4.16.0 # Vulnerabilities: None -tqdm==4.66.4 # Vulnerabilities: None +tox==4.18.0 # From 4.16.0 | Vulnerabilities: None +tqdm==4.66.5 # From 4.66.4 | Vulnerabilities: None twine==5.1.1 # Vulnerabilities: None -watchdog==4.0.1 # Vulnerabilities: None -wheel==0.43.0 # Vulnerabilities: None +watchdog==5.0.0 # From 4.0.1 | Vulnerabilities: None +wheel==0.44.0 # From 0.43.0 | Vulnerabilities: None xmltodict==0.13.0 # Vulnerabilities: None -mike From e63c01e398a90be42481eec61f762a571b979abc Mon Sep 17 00:00:00 2001 From: devsetgo Date: Sat, 31 Aug 2024 03:54:06 +0000 Subject: [PATCH 14/14] run of tests --- coverage.xml | 338 +++++++++++++++++++++++++-------------------------- 1 file changed, 169 insertions(+), 169 deletions(-) diff --git a/coverage.xml b/coverage.xml index 65f551e3..4bbc9403 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -21,8 +21,8 @@ - - + + @@ -97,38 +97,38 @@ - - - - - - + + + + + + + - + - + - - - - + + + + - + - - - + + - - + + @@ -348,10 +348,10 @@ - + - + @@ -400,227 +400,227 @@ - - - - - - + + + + + + + - - + + - + - + - + - + - - - - + + + + - + - + - - - - + + + + - - - + + + - - + + - + - + - - - + + + - - + + - + - + - + - + - + - - + + - + - - - + + + - + - + - + - + - + - + - - - - - + + + + + - + - + - - + + - - + + - + - - + + - + - - - - - - - + + + + + + + - + - + - + - - + + - + - + - + - + - - - + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + - + - + - + - - - - - - - - - - - - - + + + + + + + + + + + + @@ -677,23 +677,23 @@ - - - + + + + - - + + - - - - - + + + + + - + - @@ -715,32 +715,32 @@ - - - - - - - - - - - + + + + + + + + + + - + - + - + + - - + +