From bb0ddd19fca890f68e0e380b77ff948cbf7c4307 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Thu, 17 Apr 2025 19:11:23 +0000 Subject: [PATCH 1/5] Improving examples --- dsg_lib/common_functions/file_mover.py | 13 +- examples/cal_example.py | 47 +++- examples/csv_example.py | 121 +++++++-- examples/csv_example_with_timer.py | 3 +- examples/fastapi_example.py | 361 ++++++++++++++++++++++++- examples/file_monitor.py | 37 ++- examples/json_example.py | 99 ++++++- examples/text_example.py | 117 +++++++- examples/validate_emails.py | 63 +++-- 9 files changed, 773 insertions(+), 88 deletions(-) diff --git a/dsg_lib/common_functions/file_mover.py b/dsg_lib/common_functions/file_mover.py index ff3e90d4..f91666fb 100644 --- a/dsg_lib/common_functions/file_mover.py +++ b/dsg_lib/common_functions/file_mover.py @@ -37,13 +37,14 @@ """ import shutil +from datetime import datetime +from itertools import islice # Import islice to limit generator iterations from pathlib import Path from time import sleep +from typing import Optional, Generator, Set, Tuple + from loguru import logger from watchfiles import watch -from datetime import datetime -from typing import Optional -from itertools import islice # Import islice to limit generator iterations def process_files_flow( @@ -103,7 +104,7 @@ def process_files_flow( ) # Monitor the source directory for changes - changes_generator = watch(source_dir) + changes_generator: Generator[Set[Tuple[int, str]], None, None] = watch(source_dir) if max_iterations is not None: changes_generator = islice(changes_generator, max_iterations) @@ -111,6 +112,7 @@ def process_files_flow( logger.debug(f"Detected changes: {changes}") for _change_type, file_str in changes: file_path: Path = Path(file_str) + # Only process files matching the pattern and that are files if file_path.is_file() and file_path.match(file_pattern): try: logger.info(f"Detected file for processing: {file_path}") @@ -152,7 +154,8 @@ def _process_file( if compress: try: logger.debug(f"Starting compression for file: {temp_file_path}") - timestamp_suffix = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + # Add a timestamp to the zip file name to avoid collisions + timestamp_suffix: str = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") base_for_zip: Path = ( temp_file_path.parent / f"{temp_file_path.stem}_{timestamp_suffix}" ) diff --git a/examples/cal_example.py b/examples/cal_example.py index 0f08f68c..c6bef11f 100644 --- a/examples/cal_example.py +++ b/examples/cal_example.py @@ -76,9 +76,13 @@ This module is licensed under the MIT License. """ from dsg_lib.common_functions import calendar_functions +from typing import List, Any -month_list: list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] -month_names: list = [ +# List of month numbers to test, including invalid values (0, 13) +month_list: List[int] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] + +# List of month names to test, including an invalid value ("bob") +month_names: List[str] = [ "january", "february", "march", @@ -94,19 +98,50 @@ "bob", ] - -def calendar_check_number(): +def calendar_check_number() -> None: + """ + Example: Demonstrates converting month numbers to month names. + Iterates through `month_list` and prints the result of get_month for each. + """ for i in month_list: month = calendar_functions.get_month(month=i) print(month) - -def calendar_check_name(): +def calendar_check_name() -> None: + """ + Example: Demonstrates converting month names to month numbers. + Iterates through `month_names` and prints the result of get_month_number for each. + """ for i in month_names: month = calendar_functions.get_month_number(month_name=i) print(month) +def calendar_check_float_and_invalid_types() -> None: + """ + Example: Tests get_month with float values and various invalid types. + Shows how the function handles non-integer and unexpected input types. + """ + print("\nTesting get_month with float and invalid types:") + test_values: List[Any] = [1.0, 12.0, 5.5, "3", None, [1], {"month": 2}] + for val in test_values: + print(f"Input: {val!r} -> Output: {calendar_functions.get_month(month=val)}") + +def calendar_check_name_variants() -> None: + """ + Example: Tests get_month_number with name variants and invalid types. + Includes extra spaces, different cases, abbreviations, and non-string types. + """ + print("\nTesting get_month_number with name variants and invalid types:") + test_names: List[Any] = [ + " January ", "FEBRUARY", "mar", "Apr", "may", "JUNE", "July", "august", "Sept", "oct", "nov", "december", + 5, None, ["March"], {"month": "April"} + ] + for name in test_names: + print(f"Input: {name!r} -> Output: {calendar_functions.get_month_number(month_name=name)}") if __name__ == "__main__": + # Run all example checks to demonstrate library usage and edge case handling calendar_check_number() calendar_check_name() + calendar_check_float_and_invalid_types() + calendar_check_name_variants() diff --git a/examples/csv_example.py b/examples/csv_example.py index ec6cd8a8..5eeb7a3d 100644 --- a/examples/csv_example.py +++ b/examples/csv_example.py @@ -2,7 +2,7 @@ """ # CSV Example Module -This module provides examples of how to work with CSV files using the `dsg_lib` library. It includes functions for saving data to a CSV file, opening and reading data from a CSV file, and creating sample files for testing purposes. The module is designed to demonstrate the usage of the `file_functions` and `logging_config` utilities provided by `dsg_lib`. +This module provides examples of how to work with CSV files using the `dsg_lib` library. It includes functions for saving data to a CSV file, opening and reading data from a CSV file, appending data to an existing CSV file, deleting a CSV file, and creating sample files for testing purposes. The module is designed to demonstrate the usage of the `file_functions` and `logging_config` utilities provided by `dsg_lib`. ## Functions @@ -27,6 +27,18 @@ - Additional options such as delimiter, quote level, and space handling can be configured. - Refer to the Python CSV documentation for more details: [Python CSV Documentation](https://docs.python.org/3/library/csv.html). +### `append_some_data(rows: list)` +Appends rows to an existing CSV file. The function uses the `append_csv` utility from `dsg_lib`. + +- **Parameters**: + - `rows` (list): A list of lists containing the rows to append. The header must match the existing file. + +### `delete_example_file(file_name: str)` +Deletes a CSV file. The function uses the `delete_file` utility from `dsg_lib`. + +- **Parameters**: + - `file_name` (str): The name of the file to delete. + ### `sample_files()` Creates sample files for testing purposes. This function uses the `create_sample_files` utility from `dsg_lib`. @@ -42,7 +54,18 @@ # Open and read data from a CSV file opened_file = open_some_data("your-file-name.csv") - print(opened_file) + print("Opened CSV data:", opened_file) + + # Append data to an existing CSV file + rows_to_append = [ + ["thing_one", "thing_two"], # header row (must match) + ["i", "j"], + ["k", "l"], + ] + append_some_data(rows_to_append) + + # Delete the CSV file + delete_example_file("your-file-name.csv") # Create sample files for testing sample_files() @@ -55,11 +78,8 @@ ## License This module is licensed under the MIT License. """ -from dsg_lib.common_functions.file_functions import ( - create_sample_files, - open_csv, - save_csv, -) +from typing import List, Dict, Any +from dsg_lib.common_functions.file_functions import create_sample_files, open_csv, save_csv from dsg_lib.common_functions.logging_config import config_log config_log(logging_level="DEBUG") @@ -73,9 +93,14 @@ ] -def save_some_data(example_list: list): - # function requires file_name and data list to be sent. - # see documentation for additonal information +def save_some_data(example_list: List[List[str]]) -> None: + """ + Save a list of lists to a CSV file using dsg_lib's save_csv. + + Args: + example_list (List[List[str]]): Data to save, including header as first row. + """ + # Save data to CSV with custom delimiter and quote character save_csv( file_name="your-file-name.csv", data=example_list, @@ -85,31 +110,75 @@ def save_some_data(example_list: list): ) -def open_some_data(the_file_name: str) -> dict: - """ - function requires file_name and a dictionary will be returned - this function is designed with the idea that the CSV file has a header row. - see documentation for additonal information - options - file_name: str | "myfile.csv" - delimit: str | example - ":" single character only inside quotes - quote_level:str | ["none","non-numeric","minimal","all"] default is minimal - skip_initial_space:bool | default is True - See Python documentation as needed https://docs.python.org/3/library/csv.html +def open_some_data(the_file_name: str) -> List[Dict[str, Any]]: """ + Open a CSV file and return its contents as a list of dictionaries. + + Args: + the_file_name (str): Name of the CSV file to open. - result: dict = open_csv(file_name=the_file_name) + Returns: + List[Dict[str, Any]]: List of rows as dictionaries. + """ + result = open_csv(file_name=the_file_name) return result -def sample_files(): +def append_some_data(rows: List[List[str]]) -> None: + """ + Append rows to an existing CSV file. + + Args: + rows (List[List[str]]): Rows to append, header must match existing file. + """ + from dsg_lib.common_functions.file_functions import append_csv + append_csv( + file_name="your-file-name.csv", + data=rows, + root_folder="/data", + delimiter="|", + quotechar='"', + ) + + +def delete_example_file(file_name: str) -> None: + """ + Delete a CSV file using dsg_lib's delete_file. + + Args: + file_name (str): Name of the file to delete. + """ + from dsg_lib.common_functions.file_functions import delete_file + delete_file(file_name) + + +def sample_files() -> None: + """ + Create sample files for testing. + """ filename = "test_sample" samplesize = 1000 create_sample_files(filename, samplesize) if __name__ == "__main__": - # save_some_data(example_list) - # opened_file: dict = open_some_data("your-file-name.csv") - # print(opened_file) + # Example: Save data to CSV + save_some_data(example_list) + + # Example: Open and read data from CSV + opened_file = open_some_data("your-file-name.csv") + print("Opened CSV data:", opened_file) + + # Example: Append data to CSV (header must match) + rows_to_append = [ + ["thing_one", "thing_two"], # header row (must match) + ["i", "j"], + ["k", "l"], + ] + append_some_data(rows_to_append) + + # Example: Delete the CSV file + delete_example_file("your-file-name.csv") + + # Example: Create sample files sample_files() diff --git a/examples/csv_example_with_timer.py b/examples/csv_example_with_timer.py index 7a0e9ee2..74aa6874 100644 --- a/examples/csv_example_with_timer.py +++ b/examples/csv_example_with_timer.py @@ -50,11 +50,12 @@ ## License This module is licensed under the MIT License. """ +import random import time from datetime import datetime + from dsg_lib.common_functions.file_functions import save_csv from dsg_lib.common_functions.logging_config import config_log -import random config_log(logging_level="DEBUG") diff --git a/examples/fastapi_example.py b/examples/fastapi_example.py index cab7939c..a839b161 100644 --- a/examples/fastapi_example.py +++ b/examples/fastapi_example.py @@ -74,7 +74,7 @@ from fastapi.responses import RedirectResponse from loguru import logger from pydantic import BaseModel, EmailStr -from sqlalchemy import Column, ForeignKey, Select, String, insert +from sqlalchemy import Column, ForeignKey, Select, String, and_, delete, insert, or_, update from sqlalchemy.orm import relationship from tqdm import tqdm @@ -204,11 +204,14 @@ async def lifespan(app: FastAPI): @app.get("/") async def root(): """ - Root endpoint of API + Redirect to the OpenAPI documentation. + + Example: + GET / + Returns: - Redrects to openapi document + Redirects to /docs for interactive API documentation. """ - # redirect to openapi docs logger.info("Redirecting to OpenAPI docs") response = RedirectResponse(url="/docs") return response @@ -294,6 +297,15 @@ async def create_a_bunch_of_users(single_entry=0, many_entries=0): @app.get("/database/get-primary-key", tags=["Database Examples"]) async def table_primary_key(): + """ + Get the primary key(s) of the User table. + + Example: + GET /database/get-primary-key + + Returns: + The primary key column(s) for the User table. + """ logger.info("Getting primary key of User table") pk = await db_ops.get_primary_keys(User) logger.info(f"Primary key of User table: {pk}") @@ -302,6 +314,15 @@ async def table_primary_key(): @app.get("/database/get-column-details", tags=["Database Examples"]) async def table_column_details(): + """ + Get details about all columns in the User table. + + Example: + GET /database/get-column-details + + Returns: + Metadata for each column in the User table. + """ logger.info("Getting column details of User table") columns = await db_ops.get_columns_details(User) logger.info(f"Column details of User table: {columns}") @@ -310,6 +331,15 @@ async def table_column_details(): @app.get("/database/get-tables", tags=["Database Examples"]) async def table_table_details(): + """ + List all table names in the database. + + Example: + GET /database/get-tables + + Returns: + A list of all table names. + """ logger.info("Getting table names") tables = await db_ops.get_table_names() logger.info(f"Table names: {tables}") @@ -318,6 +348,15 @@ async def table_table_details(): @app.get("/database/get-count", tags=["Database Examples"]) async def get_count(): + """ + Get the total number of User records. + + Example: + GET /database/get-count + + Returns: + The count of User records. + """ logger.info("Getting count of users") count = await db_ops.count_query(Select(User)) logger.info(f"Count of users: {count}") @@ -326,6 +365,15 @@ async def get_count(): @app.get("/database/get-all", tags=["Database Examples"]) async def get_all(offset: int = 0, limit: int = Query(100, le=100000, ge=1)): + """ + Retrieve all User records with pagination. + + Example: + GET /database/get-all?offset=0&limit=10 + + Returns: + A list of User records. + """ logger.info(f"Getting all users with offset {offset} and limit {limit}") records = await db_ops.read_query(Select(User).offset(offset).limit(limit)) logger.info(f"Retrieved {len(records)} users") @@ -334,6 +382,15 @@ async def get_all(offset: int = 0, limit: int = Query(100, le=100000, ge=1)): @app.get("/database/get-one-record", tags=["Database Examples"]) async def read_one_record(record_id: str): + """ + Retrieve a single User record by primary key. + + Example: + GET /database/get-one-record?record_id=some-uuid + + Returns: + The User record with the given primary key. + """ logger.info(f"Reading one record with id {record_id}") record = await db_ops.read_one_record(Select(User).where(User.pkid == record_id)) logger.info(f"Record with id {record_id}: {record}") @@ -352,6 +409,20 @@ class UserCreate(UserBase): @app.post("/database/create-one-record", status_code=201, tags=["Database Examples"]) async def create_one_record(new_user: UserCreate): + """ + Create a new User record. + + Example: + POST /database/create-one-record + { + "first_name": "Alice", + "last_name": "Smith", + "email": "alice@example.com" + } + + Returns: + The created User record. + """ logger.info(f"Creating one record: {new_user}") user = User(**new_user.dict()) record = await db_ops.create_one(user) @@ -361,6 +432,15 @@ async def create_one_record(new_user: UserCreate): @app.post("/database/create-many-records", status_code=201, tags=["Database Examples"]) async def create_many_records(number_of_users: int = Query(100, le=1000, ge=1)): + """ + Create multiple User records in bulk. + + Example: + POST /database/create-many-records?number_of_users=10 + + Returns: + The number of users created and the process time. + """ logger.info(f"Creating {number_of_users} records") t0 = time.time() users = [] @@ -395,6 +475,21 @@ async def update_one_record( last_name: str = Body(..., examples=["Smith"]), email: str = Body(..., examples=["jim@something.com"]), ): + """ + Update a User record by primary key. + + Example: + PUT /database/update-one-record + { + "id": "some-uuid", + "first_name": "Agent", + "last_name": "Smith", + "email": "jim@something.com" + } + + Returns: + The updated User record. + """ logger.info(f"Updating one record with id {id}") # adding date_updated to new_values as it is not supported in sqlite \ # and other database may not either. @@ -411,6 +506,18 @@ async def update_one_record( @app.delete("/database/delete-one-record", status_code=200, tags=["Database Examples"]) async def delete_one_record(record_id: str = Body(...)): + """ + Delete a User record by primary key. + + Example: + DELETE /database/delete-one-record + { + "record_id": "some-uuid" + } + + Returns: + Success message or error. + """ logger.info(f"Deleting one record with id {record_id}") record = await db_ops.delete_one(table=User, record_id=record_id) logger.info(f"Deleted record with id {record_id}") @@ -425,6 +532,18 @@ async def delete_one_record(record_id: str = Body(...)): async def delete_many_records( id_values: list = Body(...), id_column_name: str = "pkid" ): + """ + Delete multiple User records by a list of primary keys. + + Example: + DELETE /database/delete-many-records-aka-this-is-a-bad-idea + { + "id_values": ["uuid1", "uuid2", "uuid3"] + } + + Returns: + The number of records deleted. + """ logger.info(f"Deleting many records with ids {id_values}") record = await db_ops.delete_many( table=User, id_column_name="pkid", id_values=id_values @@ -440,6 +559,15 @@ async def delete_many_records( async def read_list_of_records( offset: int = Query(0, le=1000, ge=0), limit: int = Query(100, le=10000, ge=1) ): + """ + Get a list of User primary keys for use in bulk delete. + + Example: + GET /database/get-list-of-records-to-paste-into-delete-many-records?offset=0&limit=10 + + Returns: + A list of User primary keys. + """ logger.info(f"Reading list of records with offset {offset} and limit {limit}") records = await db_ops.read_query(Select(User), offset=offset, limit=limit) records_list = [] @@ -451,7 +579,15 @@ async def read_list_of_records( @app.get("/database/get-list-of-distinct-records", tags=["Database Examples"]) async def read_list_of_distinct_records(): + """ + Insert many similar User records and return distinct last names. + Example: + GET /database/get-list-of-distinct-records + + Returns: + A list of distinct last names. + """ # create many similar records to test distinct queries = [] for i in tqdm(range(100), desc="executing many fake users"): @@ -474,13 +610,24 @@ async def read_list_of_distinct_records(): logger.info(f"Executing query: {distinct_last_name_query}") records = await db_ops.read_query(query=distinct_last_name_query) - logger.info(f"Read list of distinct records: {records}") return records @app.post("/database/execute-one", tags=["Database Examples"]) async def execute_query(query: str = Body(...)): + """ + Example of running a single SQL query (insert) using execute_one. + + Example: + POST /database/execute-one + { + "query": "insert example (not used, see code)" + } + + Returns: + The inserted User record(s) with first_name "John". + """ # add a user with execute_one logger.info(f"Executing query: {query}") @@ -495,6 +642,18 @@ async def execute_query(query: str = Body(...)): @app.post("/database/execute-many", tags=["Database Examples"]) async def execute_many(query: str = Body(...)): + """ + Example of running multiple SQL queries (bulk insert) using execute_many. + + Example: + POST /database/execute-many + { + "query": "bulk insert example (not used, see code)" + } + + Returns: + All User records after bulk insert. + """ # multiple users with execute_many logger.info(f"Executing query: {query}") queries = [] @@ -511,6 +670,198 @@ async def execute_many(query: str = Body(...)): return query_return +@app.get("/database/get-distinct-emails", tags=["Database Examples"]) +async def get_distinct_emails(): + """ + Get a list of distinct emails from the User table. + + Example: + GET /database/get-distinct-emails + + Returns: + A list of unique email addresses. + """ + from sqlalchemy import select + + query = select(User.email).distinct() + logger.info("Getting distinct emails") + records = await db_ops.read_query(query) + return {"distinct_emails": records} + + +@app.get("/database/get-users-by-email", tags=["Database Examples"]) +async def get_users_by_email(email: str): + """ + Get User records by email address. + + Example: + GET /database/get-users-by-email?email=alice@example.com + + Returns: + A list of User records matching the email. + """ + query = Select(User).where(User.email == email) + logger.info(f"Getting users with email: {email}") + records = await db_ops.read_query(query) + return {"users": records} + + +@app.get("/database/get-users-by-name", tags=["Database Examples"]) +async def get_users_by_name(first_name: str = "", last_name: str = ""): + """ + Get User records by first and/or last name. + + Example: + GET /database/get-users-by-name?first_name=Alice&last_name=Smith + + Returns: + A list of User records matching the name. + """ + filters = [] + if first_name: + filters.append(User.first_name == first_name) + if last_name: + filters.append(User.last_name == last_name) + query = Select(User).where(and_(*filters)) if filters else Select(User) + logger.info(f"Getting users by name: {first_name} {last_name}") + records = await db_ops.read_query(query) + return {"users": records} + + +@app.get("/database/get-users-or", tags=["Database Examples"]) +async def get_users_or(first_name: str = "", last_name: str = ""): + """ + Get User records where first name OR last name matches. + + Example: + GET /database/get-users-or?first_name=Alice + + Returns: + A list of User records matching either name. + """ + filters = [] + if first_name: + filters.append(User.first_name == first_name) + if last_name: + filters.append(User.last_name == last_name) + query = Select(User).where(or_(*filters)) if filters else Select(User) + logger.info(f"Getting users by OR: {first_name} {last_name}") + records = await db_ops.read_query(query) + return {"users": records} + + +@app.get("/database/get-multi-query", tags=["Database Examples"]) +async def get_multi_query(): + """ + Run multiple queries at once and return results as a dictionary. + + Example: + GET /database/get-multi-query + + Returns: + A dictionary with results for each query. + """ + queries = { + "all_users": Select(User), + "distinct_emails": Select(User.email).distinct(), + "first_10": Select(User).limit(10), + } + logger.info("Running multi-query example") + results = await db_ops.read_multi_query(queries) + return results + + +@app.put("/database/update-email", tags=["Database Examples"]) +async def update_email(record_id: str = Body(...), new_email: str = Body(...)): + """ + Update a User's email address by primary key. + + Example: + PUT /database/update-email + { + "record_id": "some-uuid", + "new_email": "new@email.com" + } + + Returns: + Result of the update operation. + """ + query = update(User).where(User.pkid == record_id).values(email=new_email) + logger.info(f"Updating email for user {record_id} to {new_email}") + result = await db_ops.execute_one(query) + return {"result": result} + + +@app.delete("/database/delete-by-email", tags=["Database Examples"]) +async def delete_by_email(email: str = Body(...)): + """ + Delete User records by email address. + + Example: + DELETE /database/delete-by-email + { + "email": "alice@example.com" + } + + Returns: + Result of the delete operation. + """ + query = delete(User).where(User.email == email) + logger.info(f"Deleting users with email {email}") + result = await db_ops.execute_one(query) + return {"result": result} + + +@app.post("/database/insert-bulk", tags=["Database Examples"]) +async def insert_bulk(count: int = Body(5)): + """ + Bulk insert User records using execute_many. + + Example: + POST /database/insert-bulk + { + "count": 10 + } + + Returns: + Result of the bulk insert operation. + """ + queries = [] + for i in range(count): + value = secrets.token_hex(4) + q = ( + insert(User), + { + "first_name": f"Bulk{value}{i}", + "last_name": f"User{value}{i}", + "email": f"bulk{value}{i}@example.com", + }, + ) + queries.append(q) + logger.info(f"Bulk inserting {count} users") + result = await db_ops.execute_many(queries) + return {"result": result} + + +@app.get("/database/error-example", tags=["Database Examples"]) +async def error_example(): + """ + Trigger an error to demonstrate error handling. + + Example: + GET /database/error-example + + Returns: + Error details from a failed query. + """ + # Try to select from a non-existent table + from sqlalchemy import text + + query = text("SELECT * FROM non_existent_table") + logger.info("Triggering error example") + result = await db_ops.read_query(query) + return {"result": result} + if __name__ == "__main__": import uvicorn diff --git a/examples/file_monitor.py b/examples/file_monitor.py index 1b5aefdf..375dedc2 100644 --- a/examples/file_monitor.py +++ b/examples/file_monitor.py @@ -60,19 +60,21 @@ This module is licensed under the MIT License. """ -import os import asyncio +import os from pathlib import Path + from loguru import logger + from dsg_lib.common_functions.file_mover import process_files_flow # Define source, temporary, and destination directories -SOURCE_DIRECTORY = "/workspaces/devsetgo_lib/data/move/source/csv" -TEMPORARY_DIRECTORY = "/workspaces/devsetgo_lib/data/move/temp" -DESTINATION_DIRECTORY = "/workspaces/devsetgo_lib/data/move/destination" -FILE_PATTERN = "*.csv" # File pattern to monitor (e.g., '*.txt') -COMPRESS_FILES = True # Set to True to compress files before moving -CLEAR_SOURCE = True # Set to True to clear the source directory before starting +SOURCE_DIRECTORY: str = "/workspaces/devsetgo_lib/data/move/source/csv" +TEMPORARY_DIRECTORY: str = "/workspaces/devsetgo_lib/data/move/temp" +DESTINATION_DIRECTORY: str = "/workspaces/devsetgo_lib/data/move/destination" +FILE_PATTERN: str = "*.csv" # File pattern to monitor (e.g., '*.txt') +COMPRESS_FILES: bool = True # Set to True to compress files before moving +CLEAR_SOURCE: bool = True # Set to True to clear the source directory before starting # Ensure directories exist os.makedirs(SOURCE_DIRECTORY, exist_ok=True) @@ -80,27 +82,34 @@ os.makedirs(DESTINATION_DIRECTORY, exist_ok=True) -async def create_sample_files(): +async def create_sample_files() -> None: """ Periodically create sample files in the source directory for demonstration purposes. + + This coroutine creates a new sample file every 10 seconds in the source directory. """ while True: - file_name = f"sample_{Path(SOURCE_DIRECTORY).glob('*').__len__() + 1}.txt" - file_path = Path(SOURCE_DIRECTORY) / file_name + # Count existing files to generate a unique file name + file_count: int = len(list(Path(SOURCE_DIRECTORY).glob('*'))) + file_name: str = f"sample_{file_count + 1}.txt" + file_path: Path = Path(SOURCE_DIRECTORY) / file_name file_path.write_text("This is a sample file for testing the file mover.") logger.info(f"Created sample file: {file_path}") await asyncio.sleep(10) # Create a new file every 10 seconds -async def main(): +async def main() -> None: """ Main function to demonstrate the file mover library. + + Starts the sample file creation task and runs the file processing flow in a separate thread. + Cancels the file creation task when processing is complete. """ # Start the sample file creation task - file_creator_task = asyncio.create_task(create_sample_files()) + file_creator_task: asyncio.Task = asyncio.create_task(create_sample_files()) - # Run the file processing flow in a separate thread - loop = asyncio.get_event_loop() + # Run the file processing flow in a separate thread (to avoid blocking the event loop) + loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() await loop.run_in_executor( None, process_files_flow, diff --git a/examples/json_example.py b/examples/json_example.py index 2cb9985c..3e3dbb09 100644 --- a/examples/json_example.py +++ b/examples/json_example.py @@ -22,18 +22,28 @@ ## Functions -### `save_some_data(example_json: str)` +### `save_some_data(example_json: Dict[str, Any])` Saves the provided JSON data to a file named `your-file-name.json`. -### `open_some_data(the_file_name: str) -> dict` +### `open_some_data(the_file_name: str) -> Dict[str, Any]` Loads JSON data from the specified file and returns it as a dictionary. +### `save_list_json(data: list, file_name: str)` +Saves a list of dictionaries as JSON to the specified file. + +### `open_list_json(file_name: str) -> list` +Loads a list of dictionaries from the specified JSON file. + +### `try_open_nonexistent_json(file_name: str)` +Attempts to open a non-existent JSON file and handles the error. + ## Usage Run the module directly to: 1. Save the `example_json` data to a file. 2. Load the data back from the file. -3. Print the loaded data to the console. +3. Save and load a list of dictionaries. +4. Attempt to open a non-existent file. ## Notes @@ -48,9 +58,11 @@ ## License This module is licensed under the MIT License. """ +from typing import Any, Dict + from dsg_lib.common_functions.file_functions import open_json, save_json -example_json = { +example_json: Dict[str, Any] = { "super_cool_people": [ { "name": "Blaise Pascal", @@ -74,21 +86,82 @@ "sources": "wikipedia via Google search.", } +def save_some_data(example_json: Dict[str, Any]) -> None: + """ + Save the provided JSON data to a file named 'your-file-name.json'. -def save_some_data(example_json: str): - # function requires file_name and data as a string to be sent. - # see documentation for additonal information + Args: + example_json (Dict[str, Any]): The JSON data to save. + """ save_json(file_name="your-file-name.json", data=example_json) +def open_some_data(the_file_name: str) -> Dict[str, Any]: + """ + Load JSON data from the specified file. -def open_some_data(the_file_name: str) -> dict: - # function requires file_name and a string will be returned - # see documentation for additonal information - result: dict = open_json(file_name=the_file_name) + Args: + the_file_name (str): The name of the JSON file to open. + + Returns: + Dict[str, Any]: The loaded JSON data. + """ + result: Dict[str, Any] = open_json(file_name=the_file_name) return result +# --- Additional Examples --- + +simple_list_json: list = [ + {"id": 1, "value": "foo"}, + {"id": 2, "value": "bar"}, +] + +def save_list_json(data: list, file_name: str) -> None: + """ + Save a list of dictionaries as JSON. + + Args: + data (list): The list of dictionaries to save. + file_name (str): The file name to save to. + """ + save_json(file_name=file_name, data=data) + +def open_list_json(file_name: str) -> list: + """ + Load a list of dictionaries from a JSON file. + + Args: + file_name (str): The file name to load from. + + Returns: + list: The loaded list of dictionaries. + """ + return open_json(file_name=file_name) + +def try_open_nonexistent_json(file_name: str) -> None: + """ + Attempt to open a non-existent JSON file and handle the error. + + Args: + file_name (str): The file name to attempt to open. + """ + try: + open_json(file_name=file_name) + except FileNotFoundError as e: + print(f"Handled error: {e}") if __name__ == "__main__": + # Example 1: Save and load a complex dictionary + print("Saving and loading example_json...") save_some_data(example_json) - opened_file: dict = open_some_data("your-file-name.json") - print(opened_file) + opened_file: Dict[str, Any] = open_some_data("your-file-name.json") + print("Loaded example_json:", opened_file) + + # Example 2: Save and load a list of dictionaries + print("\nSaving and loading a list of dictionaries...") + save_list_json(simple_list_json, "list-example.json") + loaded_list = open_list_json("list-example.json") + print("Loaded list-example.json:", loaded_list) + + # Example 3: Attempt to open a non-existent file + print("\nAttempting to open a non-existent file...") + try_open_nonexistent_json("does_not_exist.json") diff --git a/examples/text_example.py b/examples/text_example.py index f27a2689..e2a86644 100644 --- a/examples/text_example.py +++ b/examples/text_example.py @@ -23,6 +23,32 @@ - **Behavior**: Calls the `open_text` function from `dsg_lib.common_functions.file_functions` to read the content of the file. +### `save_csv_example(csv_data: list[list[str]], file_name: str = "example.csv")` +Saves example rows to a CSV file. +- **Parameters**: + - `csv_data` (list[list[str]]): Rows for CSV (first row is header). + - `file_name` (str): Target CSV file name. + +### `open_csv_example(file_name: str = "example.csv") -> list[dict]` +Opens a CSV file and returns its content as a list of dictionaries. +- **Parameters**: + - `file_name` (str): Name of the CSV file to read. +- **Returns**: + - `list[dict]`: Parsed CSV rows. + +### `save_json_example(data: dict | list, file_name: str = "example.json")` +Saves a dictionary or list as JSON. +- **Parameters**: + - `data` (dict|list): Data to serialize. + - `file_name` (str): Target JSON file name. + +### `open_json_example(file_name: str = "example.json") -> dict | list` +Opens a JSON file and returns its content. +- **Parameters**: + - `file_name` (str): Name of the JSON file to read. +- **Returns**: + - `dict|list`: Parsed JSON content. + ## Example Usage ```python @@ -30,6 +56,19 @@ save_some_data(example_text) opened_file: str = open_some_data("your-file-name.txt") print(opened_file) + + # CSV example + csv_rows = [ + ["header1", "header2"], + ["row1col1", "row1col2"] + ] + save_csv_example(csv_rows) + print(open_csv_example()) + + # JSON example + json_obj = {"foo": "bar", "count": 1} + save_json_example(json_obj) + print(open_json_example()) ``` ## Notes @@ -39,7 +78,11 @@ ## License This module is licensed under the MIT License. """ -from dsg_lib.common_functions.file_functions import open_text, save_text +from dsg_lib.common_functions.file_functions import ( + open_text, save_text, + save_csv, open_csv, + save_json, open_json +) example_text = """ @@ -70,7 +113,79 @@ def open_some_data(the_file_name: str) -> str: return result +def save_csv_example( + csv_data: list[list[str]], + file_name: str = "example.csv" +) -> None: + """ + Save example rows to a CSV file. + + Args: + csv_data (list[list[str]]): Rows for CSV (first row is header). + file_name (str): Target CSV file name. + """ + # write rows out + save_csv(file_name=file_name, data=csv_data) + + +def open_csv_example( + file_name: str = "example.csv" +) -> list[dict]: + """ + Open a CSV file and return its content as list of dicts. + + Args: + file_name (str): Name of CSV to read. + + Returns: + list[dict]: Parsed CSV rows. + """ + return open_csv(file_name=file_name) + + +def save_json_example( + data: dict | list, + file_name: str = "example.json" +) -> None: + """ + Save a dict or list as JSON. + + Args: + data (dict|list): Data to serialize. + file_name (str): Target JSON file name. + """ + save_json(file_name=file_name, data=data) + + +def open_json_example( + file_name: str = "example.json" +) -> dict | list: + """ + Open a JSON file and return its content. + + Args: + file_name (str): Name of JSON to read. + + Returns: + dict|list: Parsed JSON content. + """ + return open_json(file_name=file_name) + + if __name__ == "__main__": save_some_data(example_text) opened_file: str = open_some_data("your-file-name.txt") print(opened_file) + + # CSV example + csv_rows = [ + ["header1", "header2"], + ["row1col1", "row1col2"] + ] + save_csv_example(csv_rows) + print(open_csv_example()) + + # JSON example + json_obj = {"foo": "bar", "count": 1} + save_json_example(json_obj) + print(open_json_example()) diff --git a/examples/validate_emails.py b/examples/validate_emails.py index 4b41c712..da6b3266 100644 --- a/examples/validate_emails.py +++ b/examples/validate_emails.py @@ -64,12 +64,43 @@ import pprint import time +from typing import List, Dict, Any + from dsg_lib.common_functions.email_validation import validate_email_address -if __name__ == "__main__": +def run_validation( + email_addresses: List[str], + configurations: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """ + Validate each email against multiple configurations. + + Args: + email_addresses: List of email strings to validate. + configurations: List of parameter dicts for validation. + + Returns: + A sorted list of result dicts (sorted by "email" key). + """ + results: List[Dict[str, Any]] = [] + # iterate over every email and config combination + for email in email_addresses: + for config in configurations: + # call the core email validator and collect its output + res = validate_email_address(email, **config) + results.append(res) + # sort by email for consistent output + return sorted(results, key=lambda x: x["email"]) - # create a list of email addresses to check if valid - email_addresses = [ +def main() -> None: + """ + Entry point for the email validation example. + + Defines a list of emails and configurations, measures execution time, + runs validation, and pretty‑prints the results. + """ + # list of example email addresses + email_addresses: List[str] = [ "bob@devsetgo.com", "bob@devset.go", "foo@yahoo.com", @@ -117,8 +148,8 @@ "test@google.com", ] - # create a list of configurations - configurations = [ + # various validation parameter sets to exercise different rules + configurations: List[Dict[str, Any]] = [ { "check_deliverability": True, "test_environment": False, @@ -230,18 +261,16 @@ }, ] - t0 = time.time() - validity = [] - - for email in email_addresses: - for config in configurations: + # measure and run + start_time: float = time.time() + results = run_validation(email_addresses, configurations) + elapsed: float = time.time() - start_time - res = validate_email_address(email, **config) - validity.append(res) - t1 = time.time() - validity = sorted(validity, key=lambda x: x["email"]) + # output each result + for record in results: + pprint.pprint(record, indent=4) - for v in validity: - pprint.pprint(v, indent=4) + print(f"Time taken: {elapsed:.2f}s") - print(f"Time taken: {t1 - t0:.2f}") +if __name__ == "__main__": + main() From 7c6f7a3dd6c88af83e7c1c88676ec48e3a84a205 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Thu, 17 Apr 2025 19:11:32 +0000 Subject: [PATCH 2/5] formatting --- dsg_lib/async_database_functions/database_operations.py | 1 - .../test_file_functions/test_append_csv.py | 1 + .../test_common_functions/test_file_mover/test_file_mover.py | 3 ++- tests/test_database_functions/test_async_database.py | 3 ++- tests/test_database_functions/test_base_schema.py | 5 +---- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dsg_lib/async_database_functions/database_operations.py b/dsg_lib/async_database_functions/database_operations.py index 34dbe143..d34dd0aa 100644 --- a/dsg_lib/async_database_functions/database_operations.py +++ b/dsg_lib/async_database_functions/database_operations.py @@ -41,7 +41,6 @@ from .. import LOGGER as logger from .__import_sqlalchemy import import_sqlalchemy - # Importing AsyncDatabase class from local module async_database from .async_database import AsyncDatabase diff --git a/tests/test_common_functions/test_file_functions/test_append_csv.py b/tests/test_common_functions/test_file_functions/test_append_csv.py index 8eea6b12..c1cf07ea 100644 --- a/tests/test_common_functions/test_file_functions/test_append_csv.py +++ b/tests/test_common_functions/test_file_functions/test_append_csv.py @@ -1,6 +1,7 @@ import unittest from pathlib import Path from unittest.mock import patch + from dsg_lib.common_functions.file_functions import append_csv, save_csv diff --git a/tests/test_common_functions/test_file_mover/test_file_mover.py b/tests/test_common_functions/test_file_mover/test_file_mover.py index 48b96082..e23d74ca 100644 --- a/tests/test_common_functions/test_file_mover/test_file_mover.py +++ b/tests/test_common_functions/test_file_mover/test_file_mover.py @@ -1,9 +1,10 @@ +import logging import tempfile import unittest from pathlib import Path from unittest.mock import patch + from watchfiles import Change # Import Change from watchfiles -import logging from dsg_lib.common_functions.file_mover import process_files_flow diff --git a/tests/test_database_functions/test_async_database.py b/tests/test_database_functions/test_async_database.py index cf402132..9f3e2c3e 100644 --- a/tests/test_database_functions/test_async_database.py +++ b/tests/test_database_functions/test_async_database.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- import asyncio import secrets -import pytest_asyncio + import pytest +import pytest_asyncio from sqlalchemy import Column, Integer, String, delete, insert, select from sqlalchemy.exc import IntegrityError, SQLAlchemyError diff --git a/tests/test_database_functions/test_base_schema.py b/tests/test_database_functions/test_base_schema.py index d759fdfd..10395e75 100644 --- a/tests/test_database_functions/test_base_schema.py +++ b/tests/test_database_functions/test_base_schema.py @@ -7,10 +7,7 @@ from sqlalchemy import Column, String, create_engine from sqlalchemy.orm import declarative_base, sessionmaker -from dsg_lib.async_database_functions.base_schema import ( - SchemaBasePostgres, - SchemaBaseSQLite, -) +from dsg_lib.async_database_functions.base_schema import SchemaBasePostgres, SchemaBaseSQLite # Get the database URL from the environment variable database_url = os.getenv( From 918a9215486d49c437cb7b6b81d2c10bf2ffec07 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Thu, 17 Apr 2025 19:13:56 +0000 Subject: [PATCH 3/5] Add type hints, docstrings, and inline comments to log_example.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Annotate functions with precise parameter and return types Provide comprehensive docstrings for all public functions Insert inline comments explaining key logging and error‑handling steps --- examples/log_example.py | 79 +++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/examples/log_example.py b/examples/log_example.py index 18a1bfbc..9d69f50e 100644 --- a/examples/log_example.py +++ b/examples/log_example.py @@ -88,17 +88,27 @@ ) -# @logger.catch -def div_zero(x, y): +def div_zero(x: float, y: float) -> float | None: + """ + Safely divide x by y and log ZeroDivisionError if encountered. + + Args: + x (float): Numerator. + y (float): Denominator. + Returns: + float | None: Quotient or None on error. + """ try: return x / y except ZeroDivisionError as e: - logger.error(f"{e}") - logging.error(f"{e}") + logger.error(f"{e}") # log via loguru + logging.error(f"{e}") # log via standard logging -# @logger.catch -def div_zero_two(x, y): +def div_zero_two(x: float, y: float) -> float | None: + """ + Mirror of div_zero demonstrating identical error handling. + """ try: return x / y except ZeroDivisionError as e: @@ -106,30 +116,42 @@ def div_zero_two(x, y): logging.error(f"{e}") -def log_big_string(lqty=100, size=256): - big_string = secrets.token_urlsafe(size) +def log_big_string(lqty: int = 100, size: int = 256) -> None: + """ + Generate a large random string and log various messages repeatedly. + + Args: + lqty (int): Number of log iterations. + size (int): Length of each random string. + """ + big_string = secrets.token_urlsafe(size) # create URL-safe token for _ in range(lqty): - logging.debug(f"Lets make this a big message {big_string}") - div_zero(x=1, y=0) - div_zero_two(x=1, y=0) - # after configuring logging - # use loguru to log messages + logging.debug(f"Lets make this a big message {big_string}") # standard debug + div_zero(1, 0) # trigger/log ZeroDivisionError + div_zero_two(1, 0) + # loguru messages logger.debug("This is a loguru debug message") - logger.info("This is an loguru info message") - logger.error("This is an loguru error message") + logger.info("This is a loguru info message") logger.warning("This is a loguru warning message") + logger.error("This is a loguru error message") logger.critical("This is a loguru critical message") - - # will intercept all standard logging messages also - logging.debug("This is a standard logging debug message") - logging.info("This is an standard logging info message") - logging.error("This is an standard logging error message") + # continued standard logging + logging.info("This is a standard logging info message") logging.warning("This is a standard logging warning message") + logging.error("This is a standard logging error message") logging.critical("This is a standard logging critical message") -def worker(wqty=1000, lqty=100, size=256): - for _ in tqdm(range(wqty), ascii=True, leave=True): # Adjusted for demonstration +def worker(wqty: int = 1000, lqty: int = 100, size: int = 256) -> None: + """ + Worker routine performing log_big_string in a progress loop. + + Args: + wqty (int): Number of outer iterations. + lqty (int): Messages per iteration. + size (int): Random string length. + """ + for _ in tqdm(range(wqty), ascii=True, leave=True): log_big_string(lqty=lqty, size=size) @@ -140,7 +162,18 @@ def main( workers: int = 16, thread_test: bool = False, process_test: bool = False, -): +) -> None: + """ + Configure and launch concurrent logging workers. + + Args: + wqty (int): Iterations per worker. + lqty (int): Logs per iteration. + size (int): Random string size. + workers (int): Thread/process count. + thread_test (bool): Run threads if True. + process_test (bool): Run processes if True. + """ if process_test: processes = [] # Create worker processes From fc071e8865ef8486371f8ce7aa676ceeb040a200 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Thu, 17 Apr 2025 19:14:40 +0000 Subject: [PATCH 4/5] run of tests --- coverage.xml | 354 +++++++++++++++++++++++++-------------------------- report.xml | 2 +- 2 files changed, 178 insertions(+), 178 deletions(-) diff --git a/coverage.xml b/coverage.xml index 9937d12b..69b8f33d 100644 --- a/coverage.xml +++ b/coverage.xml @@ -1,5 +1,5 @@ - + @@ -143,108 +143,109 @@ - - - - + + + + + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - + + + + - - - - - - - + + + + + + + - - + + - - + + - + - + - + - - - + + + @@ -254,27 +255,27 @@ - + - - + + - + - + - + - - + + @@ -285,8 +286,8 @@ - - + + @@ -298,96 +299,95 @@ - + - - - - - + + + + + - - - - + + + + - + - - - - - - + + + + + + - + - + - - - + + + - + - - - - - - + + + + + + - + - - + + - + - - + + - + - - - - - - + + + + + + - - - - - - - - - + + + + + + + + + - + - - - - - - - - - - + + + + + + + + + + - - - + + + - @@ -676,15 +676,14 @@ - - - + + - + - + @@ -693,51 +692,52 @@ - - + + - + - - - + + + - + - - + + - - - + + + - + - - - - - + + + + + - + - + + diff --git a/report.xml b/report.xml index 5d3d954b..fdca90f4 100644 --- a/report.xml +++ b/report.xml @@ -1 +1 @@ - + From 854c2799594a76a4f6b2a1a2eb9c276f37c8d924 Mon Sep 17 00:00:00 2001 From: devsetgo Date: Thu, 17 Apr 2025 19:16:54 +0000 Subject: [PATCH 5/5] running local documents --- docs/examples/cal_example.md | 47 ++- docs/examples/csv_example.md | 121 ++++++-- docs/examples/csv_example_with_timer.md | 3 +- docs/examples/fastapi_example.md | 390 +++++++++++++++++++++++- docs/examples/file_monitor.md | 37 ++- docs/examples/json_example.md | 99 +++++- docs/examples/log_example.md | 79 +++-- docs/examples/text_example.md | 117 ++++++- docs/examples/validate_emails.md | 63 ++-- 9 files changed, 851 insertions(+), 105 deletions(-) diff --git a/docs/examples/cal_example.md b/docs/examples/cal_example.md index 5e827fbe..5bf0147a 100644 --- a/docs/examples/cal_example.md +++ b/docs/examples/cal_example.md @@ -77,9 +77,13 @@ This module is licensed under the MIT License. ```python from dsg_lib.common_functions import calendar_functions +from typing import List, Any -month_list: list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] -month_names: list = [ +# List of month numbers to test, including invalid values (0, 13) +month_list: List[int] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] + +# List of month names to test, including an invalid value ("bob") +month_names: List[str] = [ "january", "february", "march", @@ -95,20 +99,51 @@ month_names: list = [ "bob", ] - -def calendar_check_number(): +def calendar_check_number() -> None: + """ + Example: Demonstrates converting month numbers to month names. + Iterates through `month_list` and prints the result of get_month for each. + """ for i in month_list: month = calendar_functions.get_month(month=i) print(month) - -def calendar_check_name(): +def calendar_check_name() -> None: + """ + Example: Demonstrates converting month names to month numbers. + Iterates through `month_names` and prints the result of get_month_number for each. + """ for i in month_names: month = calendar_functions.get_month_number(month_name=i) print(month) +def calendar_check_float_and_invalid_types() -> None: + """ + Example: Tests get_month with float values and various invalid types. + Shows how the function handles non-integer and unexpected input types. + """ + print("\nTesting get_month with float and invalid types:") + test_values: List[Any] = [1.0, 12.0, 5.5, "3", None, [1], {"month": 2}] + for val in test_values: + print(f"Input: {val!r} -> Output: {calendar_functions.get_month(month=val)}") + +def calendar_check_name_variants() -> None: + """ + Example: Tests get_month_number with name variants and invalid types. + Includes extra spaces, different cases, abbreviations, and non-string types. + """ + print("\nTesting get_month_number with name variants and invalid types:") + test_names: List[Any] = [ + " January ", "FEBRUARY", "mar", "Apr", "may", "JUNE", "July", "august", "Sept", "oct", "nov", "december", + 5, None, ["March"], {"month": "April"} + ] + for name in test_names: + print(f"Input: {name!r} -> Output: {calendar_functions.get_month_number(month_name=name)}") if __name__ == "__main__": + # Run all example checks to demonstrate library usage and edge case handling calendar_check_number() calendar_check_name() + calendar_check_float_and_invalid_types() + calendar_check_name_variants() ``` diff --git a/docs/examples/csv_example.md b/docs/examples/csv_example.md index 3324bb85..d5c9fe2a 100644 --- a/docs/examples/csv_example.md +++ b/docs/examples/csv_example.md @@ -2,7 +2,7 @@ # CSV Example Module -This module provides examples of how to work with CSV files using the `dsg_lib` library. It includes functions for saving data to a CSV file, opening and reading data from a CSV file, and creating sample files for testing purposes. The module is designed to demonstrate the usage of the `file_functions` and `logging_config` utilities provided by `dsg_lib`. +This module provides examples of how to work with CSV files using the `dsg_lib` library. It includes functions for saving data to a CSV file, opening and reading data from a CSV file, appending data to an existing CSV file, deleting a CSV file, and creating sample files for testing purposes. The module is designed to demonstrate the usage of the `file_functions` and `logging_config` utilities provided by `dsg_lib`. ## Functions @@ -27,6 +27,18 @@ Opens a CSV file and returns its contents as a dictionary. This function assumes - Additional options such as delimiter, quote level, and space handling can be configured. - Refer to the Python CSV documentation for more details: [Python CSV Documentation](https://docs.python.org/3/library/csv.html). +### `append_some_data(rows: list)` +Appends rows to an existing CSV file. The function uses the `append_csv` utility from `dsg_lib`. + +- **Parameters**: + - `rows` (list): A list of lists containing the rows to append. The header must match the existing file. + +### `delete_example_file(file_name: str)` +Deletes a CSV file. The function uses the `delete_file` utility from `dsg_lib`. + +- **Parameters**: + - `file_name` (str): The name of the file to delete. + ### `sample_files()` Creates sample files for testing purposes. This function uses the `create_sample_files` utility from `dsg_lib`. @@ -42,7 +54,18 @@ if __name__ == "__main__": # Open and read data from a CSV file opened_file = open_some_data("your-file-name.csv") - print(opened_file) + print("Opened CSV data:", opened_file) + + # Append data to an existing CSV file + rows_to_append = [ + ["thing_one", "thing_two"], # header row (must match) + ["i", "j"], + ["k", "l"], + ] + append_some_data(rows_to_append) + + # Delete the CSV file + delete_example_file("your-file-name.csv") # Create sample files for testing sample_files() @@ -56,11 +79,8 @@ The module configures logging using the `config_log` utility from `dsg_lib`. The This module is licensed under the MIT License. ```python -from dsg_lib.common_functions.file_functions import ( - create_sample_files, - open_csv, - save_csv, -) +from typing import List, Dict, Any +from dsg_lib.common_functions.file_functions import create_sample_files, open_csv, save_csv from dsg_lib.common_functions.logging_config import config_log config_log(logging_level="DEBUG") @@ -74,9 +94,14 @@ example_list = [ ] -def save_some_data(example_list: list): - # function requires file_name and data list to be sent. - # see documentation for additonal information +def save_some_data(example_list: List[List[str]]) -> None: + """ + Save a list of lists to a CSV file using dsg_lib's save_csv. + + Args: + example_list (List[List[str]]): Data to save, including header as first row. + """ + # Save data to CSV with custom delimiter and quote character save_csv( file_name="your-file-name.csv", data=example_list, @@ -86,32 +111,76 @@ def save_some_data(example_list: list): ) -def open_some_data(the_file_name: str) -> dict: - """ - function requires file_name and a dictionary will be returned - this function is designed with the idea that the CSV file has a header row. - see documentation for additonal information - options - file_name: str | "myfile.csv" - delimit: str | example - ":" single character only inside quotes - quote_level:str | ["none","non-numeric","minimal","all"] default is minimal - skip_initial_space:bool | default is True - See Python documentation as needed https://docs.python.org/3/library/csv.html +def open_some_data(the_file_name: str) -> List[Dict[str, Any]]: """ + Open a CSV file and return its contents as a list of dictionaries. + + Args: + the_file_name (str): Name of the CSV file to open. - result: dict = open_csv(file_name=the_file_name) + Returns: + List[Dict[str, Any]]: List of rows as dictionaries. + """ + result = open_csv(file_name=the_file_name) return result -def sample_files(): +def append_some_data(rows: List[List[str]]) -> None: + """ + Append rows to an existing CSV file. + + Args: + rows (List[List[str]]): Rows to append, header must match existing file. + """ + from dsg_lib.common_functions.file_functions import append_csv + append_csv( + file_name="your-file-name.csv", + data=rows, + root_folder="/data", + delimiter="|", + quotechar='"', + ) + + +def delete_example_file(file_name: str) -> None: + """ + Delete a CSV file using dsg_lib's delete_file. + + Args: + file_name (str): Name of the file to delete. + """ + from dsg_lib.common_functions.file_functions import delete_file + delete_file(file_name) + + +def sample_files() -> None: + """ + Create sample files for testing. + """ filename = "test_sample" samplesize = 1000 create_sample_files(filename, samplesize) if __name__ == "__main__": - # save_some_data(example_list) - # opened_file: dict = open_some_data("your-file-name.csv") - # print(opened_file) + # Example: Save data to CSV + save_some_data(example_list) + + # Example: Open and read data from CSV + opened_file = open_some_data("your-file-name.csv") + print("Opened CSV data:", opened_file) + + # Example: Append data to CSV (header must match) + rows_to_append = [ + ["thing_one", "thing_two"], # header row (must match) + ["i", "j"], + ["k", "l"], + ] + append_some_data(rows_to_append) + + # Example: Delete the CSV file + delete_example_file("your-file-name.csv") + + # Example: Create sample files sample_files() ``` diff --git a/docs/examples/csv_example_with_timer.md b/docs/examples/csv_example_with_timer.md index 605514e9..39d4c07f 100644 --- a/docs/examples/csv_example_with_timer.md +++ b/docs/examples/csv_example_with_timer.md @@ -51,11 +51,12 @@ The script will continuously generate and save CSV files until manually stopped. This module is licensed under the MIT License. ```python +import random import time from datetime import datetime + from dsg_lib.common_functions.file_functions import save_csv from dsg_lib.common_functions.logging_config import config_log -import random config_log(logging_level="DEBUG") diff --git a/docs/examples/fastapi_example.md b/docs/examples/fastapi_example.md index cfa30d5b..fe7fbf95 100644 --- a/docs/examples/fastapi_example.md +++ b/docs/examples/fastapi_example.md @@ -75,7 +75,7 @@ from fastapi import Body, FastAPI, Query from fastapi.responses import RedirectResponse from loguru import logger from pydantic import BaseModel, EmailStr -from sqlalchemy import Column, ForeignKey, Select, String, insert +from sqlalchemy import Column, ForeignKey, Select, String, and_, delete, insert, or_, update from sqlalchemy.orm import relationship from tqdm import tqdm @@ -205,11 +205,14 @@ app = FastAPI( @app.get("/") async def root(): """ - Root endpoint of API + Redirect to the OpenAPI documentation. + + Example: + GET / + Returns: - Redrects to openapi document + Redirects to /docs for interactive API documentation. """ - # redirect to openapi docs logger.info("Redirecting to OpenAPI docs") response = RedirectResponse(url="/docs") return response @@ -295,6 +298,15 @@ async def create_a_bunch_of_users(single_entry=0, many_entries=0): @app.get("/database/get-primary-key", tags=["Database Examples"]) async def table_primary_key(): + """ + Get the primary key(s) of the User table. + + Example: + GET /database/get-primary-key + + Returns: + The primary key column(s) for the User table. + """ logger.info("Getting primary key of User table") pk = await db_ops.get_primary_keys(User) logger.info(f"Primary key of User table: {pk}") @@ -303,6 +315,15 @@ async def table_primary_key(): @app.get("/database/get-column-details", tags=["Database Examples"]) async def table_column_details(): + """ + Get details about all columns in the User table. + + Example: + GET /database/get-column-details + + Returns: + Metadata for each column in the User table. + """ logger.info("Getting column details of User table") columns = await db_ops.get_columns_details(User) logger.info(f"Column details of User table: {columns}") @@ -311,6 +332,15 @@ async def table_column_details(): @app.get("/database/get-tables", tags=["Database Examples"]) async def table_table_details(): + """ + List all table names in the database. + + Example: + GET /database/get-tables + + Returns: + A list of all table names. + """ logger.info("Getting table names") tables = await db_ops.get_table_names() logger.info(f"Table names: {tables}") @@ -319,6 +349,15 @@ async def table_table_details(): @app.get("/database/get-count", tags=["Database Examples"]) async def get_count(): + """ + Get the total number of User records. + + Example: + GET /database/get-count + + Returns: + The count of User records. + """ logger.info("Getting count of users") count = await db_ops.count_query(Select(User)) logger.info(f"Count of users: {count}") @@ -327,6 +366,15 @@ async def get_count(): @app.get("/database/get-all", tags=["Database Examples"]) async def get_all(offset: int = 0, limit: int = Query(100, le=100000, ge=1)): + """ + Retrieve all User records with pagination. + + Example: + GET /database/get-all?offset=0&limit=10 + + Returns: + A list of User records. + """ logger.info(f"Getting all users with offset {offset} and limit {limit}") records = await db_ops.read_query(Select(User).offset(offset).limit(limit)) logger.info(f"Retrieved {len(records)} users") @@ -335,6 +383,15 @@ async def get_all(offset: int = 0, limit: int = Query(100, le=100000, ge=1)): @app.get("/database/get-one-record", tags=["Database Examples"]) async def read_one_record(record_id: str): + """ + Retrieve a single User record by primary key. + + Example: + GET /database/get-one-record?record_id=some-uuid + + Returns: + The User record with the given primary key. + """ logger.info(f"Reading one record with id {record_id}") record = await db_ops.read_one_record(Select(User).where(User.pkid == record_id)) logger.info(f"Record with id {record_id}: {record}") @@ -353,6 +410,20 @@ class UserCreate(UserBase): @app.post("/database/create-one-record", status_code=201, tags=["Database Examples"]) async def create_one_record(new_user: UserCreate): + """ + Create a new User record. + + Example: + POST /database/create-one-record + { + "first_name": "Alice", + "last_name": "Smith", + "email": "alice@example.com" + } + + Returns: + The created User record. + """ logger.info(f"Creating one record: {new_user}") user = User(**new_user.dict()) record = await db_ops.create_one(user) @@ -362,6 +433,15 @@ async def create_one_record(new_user: UserCreate): @app.post("/database/create-many-records", status_code=201, tags=["Database Examples"]) async def create_many_records(number_of_users: int = Query(100, le=1000, ge=1)): + """ + Create multiple User records in bulk. + + Example: + POST /database/create-many-records?number_of_users=10 + + Returns: + The number of users created and the process time. + """ logger.info(f"Creating {number_of_users} records") t0 = time.time() users = [] @@ -396,6 +476,21 @@ async def update_one_record( last_name: str = Body(..., examples=["Smith"]), email: str = Body(..., examples=["jim@something.com"]), ): + """ + Update a User record by primary key. + + Example: + PUT /database/update-one-record + { + "id": "some-uuid", + "first_name": "Agent", + "last_name": "Smith", + "email": "jim@something.com" + } + + Returns: + The updated User record. + """ logger.info(f"Updating one record with id {id}") # adding date_updated to new_values as it is not supported in sqlite \ # and other database may not either. @@ -412,6 +507,18 @@ async def update_one_record( @app.delete("/database/delete-one-record", status_code=200, tags=["Database Examples"]) async def delete_one_record(record_id: str = Body(...)): + """ + Delete a User record by primary key. + + Example: + DELETE /database/delete-one-record + { + "record_id": "some-uuid" + } + + Returns: + Success message or error. + """ logger.info(f"Deleting one record with id {record_id}") record = await db_ops.delete_one(table=User, record_id=record_id) logger.info(f"Deleted record with id {record_id}") @@ -426,6 +533,18 @@ async def delete_one_record(record_id: str = Body(...)): async def delete_many_records( id_values: list = Body(...), id_column_name: str = "pkid" ): + """ + Delete multiple User records by a list of primary keys. + + Example: + DELETE /database/delete-many-records-aka-this-is-a-bad-idea + { + "id_values": ["uuid1", "uuid2", "uuid3"] + } + + Returns: + The number of records deleted. + """ logger.info(f"Deleting many records with ids {id_values}") record = await db_ops.delete_many( table=User, id_column_name="pkid", id_values=id_values @@ -441,6 +560,15 @@ async def delete_many_records( async def read_list_of_records( offset: int = Query(0, le=1000, ge=0), limit: int = Query(100, le=10000, ge=1) ): + """ + Get a list of User primary keys for use in bulk delete. + + Example: + GET /database/get-list-of-records-to-paste-into-delete-many-records?offset=0&limit=10 + + Returns: + A list of User primary keys. + """ logger.info(f"Reading list of records with offset {offset} and limit {limit}") records = await db_ops.read_query(Select(User), offset=offset, limit=limit) records_list = [] @@ -450,8 +578,57 @@ async def read_list_of_records( return records_list +@app.get("/database/get-list-of-distinct-records", tags=["Database Examples"]) +async def read_list_of_distinct_records(): + """ + Insert many similar User records and return distinct last names. + + Example: + GET /database/get-list-of-distinct-records + + Returns: + A list of distinct last names. + """ + # create many similar records to test distinct + queries = [] + for i in tqdm(range(100), desc="executing many fake users"): + value = f"Agent {i}" + queries.append( + ( + insert(User), + { + "first_name": value, + "last_name": "Smith", + "email": f"{value.lower()}@abc.com", + }, + ) + ) + + results = await db_ops.execute_many(queries) + print(results) + + distinct_last_name_query = Select(User.last_name).distinct() + logger.info(f"Executing query: {distinct_last_name_query}") + records = await db_ops.read_query(query=distinct_last_name_query) + + logger.info(f"Read list of distinct records: {records}") + return records + + @app.post("/database/execute-one", tags=["Database Examples"]) async def execute_query(query: str = Body(...)): + """ + Example of running a single SQL query (insert) using execute_one. + + Example: + POST /database/execute-one + { + "query": "insert example (not used, see code)" + } + + Returns: + The inserted User record(s) with first_name "John". + """ # add a user with execute_one logger.info(f"Executing query: {query}") @@ -466,6 +643,18 @@ async def execute_query(query: str = Body(...)): @app.post("/database/execute-many", tags=["Database Examples"]) async def execute_many(query: str = Body(...)): + """ + Example of running multiple SQL queries (bulk insert) using execute_many. + + Example: + POST /database/execute-many + { + "query": "bulk insert example (not used, see code)" + } + + Returns: + All User records after bulk insert. + """ # multiple users with execute_many logger.info(f"Executing query: {query}") queries = [] @@ -482,6 +671,199 @@ async def execute_many(query: str = Body(...)): return query_return +@app.get("/database/get-distinct-emails", tags=["Database Examples"]) +async def get_distinct_emails(): + """ + Get a list of distinct emails from the User table. + + Example: + GET /database/get-distinct-emails + + Returns: + A list of unique email addresses. + """ + from sqlalchemy import select + + query = select(User.email).distinct() + logger.info("Getting distinct emails") + records = await db_ops.read_query(query) + return {"distinct_emails": records} + + +@app.get("/database/get-users-by-email", tags=["Database Examples"]) +async def get_users_by_email(email: str): + """ + Get User records by email address. + + Example: + GET /database/get-users-by-email?email=alice@example.com + + Returns: + A list of User records matching the email. + """ + query = Select(User).where(User.email == email) + logger.info(f"Getting users with email: {email}") + records = await db_ops.read_query(query) + return {"users": records} + + +@app.get("/database/get-users-by-name", tags=["Database Examples"]) +async def get_users_by_name(first_name: str = "", last_name: str = ""): + """ + Get User records by first and/or last name. + + Example: + GET /database/get-users-by-name?first_name=Alice&last_name=Smith + + Returns: + A list of User records matching the name. + """ + filters = [] + if first_name: + filters.append(User.first_name == first_name) + if last_name: + filters.append(User.last_name == last_name) + query = Select(User).where(and_(*filters)) if filters else Select(User) + logger.info(f"Getting users by name: {first_name} {last_name}") + records = await db_ops.read_query(query) + return {"users": records} + + +@app.get("/database/get-users-or", tags=["Database Examples"]) +async def get_users_or(first_name: str = "", last_name: str = ""): + """ + Get User records where first name OR last name matches. + + Example: + GET /database/get-users-or?first_name=Alice + + Returns: + A list of User records matching either name. + """ + filters = [] + if first_name: + filters.append(User.first_name == first_name) + if last_name: + filters.append(User.last_name == last_name) + query = Select(User).where(or_(*filters)) if filters else Select(User) + logger.info(f"Getting users by OR: {first_name} {last_name}") + records = await db_ops.read_query(query) + return {"users": records} + + +@app.get("/database/get-multi-query", tags=["Database Examples"]) +async def get_multi_query(): + """ + Run multiple queries at once and return results as a dictionary. + + Example: + GET /database/get-multi-query + + Returns: + A dictionary with results for each query. + """ + queries = { + "all_users": Select(User), + "distinct_emails": Select(User.email).distinct(), + "first_10": Select(User).limit(10), + } + logger.info("Running multi-query example") + results = await db_ops.read_multi_query(queries) + return results + + +@app.put("/database/update-email", tags=["Database Examples"]) +async def update_email(record_id: str = Body(...), new_email: str = Body(...)): + """ + Update a User's email address by primary key. + + Example: + PUT /database/update-email + { + "record_id": "some-uuid", + "new_email": "new@email.com" + } + + Returns: + Result of the update operation. + """ + query = update(User).where(User.pkid == record_id).values(email=new_email) + logger.info(f"Updating email for user {record_id} to {new_email}") + result = await db_ops.execute_one(query) + return {"result": result} + + +@app.delete("/database/delete-by-email", tags=["Database Examples"]) +async def delete_by_email(email: str = Body(...)): + """ + Delete User records by email address. + + Example: + DELETE /database/delete-by-email + { + "email": "alice@example.com" + } + + Returns: + Result of the delete operation. + """ + query = delete(User).where(User.email == email) + logger.info(f"Deleting users with email {email}") + result = await db_ops.execute_one(query) + return {"result": result} + + +@app.post("/database/insert-bulk", tags=["Database Examples"]) +async def insert_bulk(count: int = Body(5)): + """ + Bulk insert User records using execute_many. + + Example: + POST /database/insert-bulk + { + "count": 10 + } + + Returns: + Result of the bulk insert operation. + """ + queries = [] + for i in range(count): + value = secrets.token_hex(4) + q = ( + insert(User), + { + "first_name": f"Bulk{value}{i}", + "last_name": f"User{value}{i}", + "email": f"bulk{value}{i}@example.com", + }, + ) + queries.append(q) + logger.info(f"Bulk inserting {count} users") + result = await db_ops.execute_many(queries) + return {"result": result} + + +@app.get("/database/error-example", tags=["Database Examples"]) +async def error_example(): + """ + Trigger an error to demonstrate error handling. + + Example: + GET /database/error-example + + Returns: + Error details from a failed query. + """ + # Try to select from a non-existent table + from sqlalchemy import text + + query = text("SELECT * FROM non_existent_table") + logger.info("Triggering error example") + result = await db_ops.read_query(query) + return {"result": result} + + if __name__ == "__main__": import uvicorn diff --git a/docs/examples/file_monitor.md b/docs/examples/file_monitor.md index a7c6d726..7addb962 100644 --- a/docs/examples/file_monitor.md +++ b/docs/examples/file_monitor.md @@ -61,19 +61,21 @@ Press `Ctrl+C` to stop the script. This module is licensed under the MIT License. ```python -import os import asyncio +import os from pathlib import Path + from loguru import logger + from dsg_lib.common_functions.file_mover import process_files_flow # Define source, temporary, and destination directories -SOURCE_DIRECTORY = "/workspaces/devsetgo_lib/data/move/source/csv" -TEMPORARY_DIRECTORY = "/workspaces/devsetgo_lib/data/move/temp" -DESTINATION_DIRECTORY = "/workspaces/devsetgo_lib/data/move/destination" -FILE_PATTERN = "*.csv" # File pattern to monitor (e.g., '*.txt') -COMPRESS_FILES = True # Set to True to compress files before moving -CLEAR_SOURCE = True # Set to True to clear the source directory before starting +SOURCE_DIRECTORY: str = "/workspaces/devsetgo_lib/data/move/source/csv" +TEMPORARY_DIRECTORY: str = "/workspaces/devsetgo_lib/data/move/temp" +DESTINATION_DIRECTORY: str = "/workspaces/devsetgo_lib/data/move/destination" +FILE_PATTERN: str = "*.csv" # File pattern to monitor (e.g., '*.txt') +COMPRESS_FILES: bool = True # Set to True to compress files before moving +CLEAR_SOURCE: bool = True # Set to True to clear the source directory before starting # Ensure directories exist os.makedirs(SOURCE_DIRECTORY, exist_ok=True) @@ -81,27 +83,34 @@ os.makedirs(TEMPORARY_DIRECTORY, exist_ok=True) os.makedirs(DESTINATION_DIRECTORY, exist_ok=True) -async def create_sample_files(): +async def create_sample_files() -> None: """ Periodically create sample files in the source directory for demonstration purposes. + + This coroutine creates a new sample file every 10 seconds in the source directory. """ while True: - file_name = f"sample_{Path(SOURCE_DIRECTORY).glob('*').__len__() + 1}.txt" - file_path = Path(SOURCE_DIRECTORY) / file_name + # Count existing files to generate a unique file name + file_count: int = len(list(Path(SOURCE_DIRECTORY).glob('*'))) + file_name: str = f"sample_{file_count + 1}.txt" + file_path: Path = Path(SOURCE_DIRECTORY) / file_name file_path.write_text("This is a sample file for testing the file mover.") logger.info(f"Created sample file: {file_path}") await asyncio.sleep(10) # Create a new file every 10 seconds -async def main(): +async def main() -> None: """ Main function to demonstrate the file mover library. + + Starts the sample file creation task and runs the file processing flow in a separate thread. + Cancels the file creation task when processing is complete. """ # Start the sample file creation task - file_creator_task = asyncio.create_task(create_sample_files()) + file_creator_task: asyncio.Task = asyncio.create_task(create_sample_files()) - # Run the file processing flow in a separate thread - loop = asyncio.get_event_loop() + # Run the file processing flow in a separate thread (to avoid blocking the event loop) + loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() await loop.run_in_executor( None, process_files_flow, diff --git a/docs/examples/json_example.md b/docs/examples/json_example.md index f5af145e..ca05b272 100644 --- a/docs/examples/json_example.md +++ b/docs/examples/json_example.md @@ -22,18 +22,28 @@ The `example_json` dictionary includes: ## Functions -### `save_some_data(example_json: str)` +### `save_some_data(example_json: Dict[str, Any])` Saves the provided JSON data to a file named `your-file-name.json`. -### `open_some_data(the_file_name: str) -> dict` +### `open_some_data(the_file_name: str) -> Dict[str, Any]` Loads JSON data from the specified file and returns it as a dictionary. +### `save_list_json(data: list, file_name: str)` +Saves a list of dictionaries as JSON to the specified file. + +### `open_list_json(file_name: str) -> list` +Loads a list of dictionaries from the specified JSON file. + +### `try_open_nonexistent_json(file_name: str)` +Attempts to open a non-existent JSON file and handles the error. + ## Usage Run the module directly to: 1. Save the `example_json` data to a file. 2. Load the data back from the file. -3. Print the loaded data to the console. +3. Save and load a list of dictionaries. +4. Attempt to open a non-existent file. ## Notes @@ -49,9 +59,11 @@ python json_example.py This module is licensed under the MIT License. ```python +from typing import Any, Dict + from dsg_lib.common_functions.file_functions import open_json, save_json -example_json = { +example_json: Dict[str, Any] = { "super_cool_people": [ { "name": "Blaise Pascal", @@ -75,22 +87,83 @@ example_json = { "sources": "wikipedia via Google search.", } +def save_some_data(example_json: Dict[str, Any]) -> None: + """ + Save the provided JSON data to a file named 'your-file-name.json'. -def save_some_data(example_json: str): - # function requires file_name and data as a string to be sent. - # see documentation for additonal information + Args: + example_json (Dict[str, Any]): The JSON data to save. + """ save_json(file_name="your-file-name.json", data=example_json) +def open_some_data(the_file_name: str) -> Dict[str, Any]: + """ + Load JSON data from the specified file. -def open_some_data(the_file_name: str) -> dict: - # function requires file_name and a string will be returned - # see documentation for additonal information - result: dict = open_json(file_name=the_file_name) + Args: + the_file_name (str): The name of the JSON file to open. + + Returns: + Dict[str, Any]: The loaded JSON data. + """ + result: Dict[str, Any] = open_json(file_name=the_file_name) return result +# --- Additional Examples --- + +simple_list_json: list = [ + {"id": 1, "value": "foo"}, + {"id": 2, "value": "bar"}, +] + +def save_list_json(data: list, file_name: str) -> None: + """ + Save a list of dictionaries as JSON. + + Args: + data (list): The list of dictionaries to save. + file_name (str): The file name to save to. + """ + save_json(file_name=file_name, data=data) + +def open_list_json(file_name: str) -> list: + """ + Load a list of dictionaries from a JSON file. + + Args: + file_name (str): The file name to load from. + + Returns: + list: The loaded list of dictionaries. + """ + return open_json(file_name=file_name) + +def try_open_nonexistent_json(file_name: str) -> None: + """ + Attempt to open a non-existent JSON file and handle the error. + + Args: + file_name (str): The file name to attempt to open. + """ + try: + open_json(file_name=file_name) + except FileNotFoundError as e: + print(f"Handled error: {e}") if __name__ == "__main__": + # Example 1: Save and load a complex dictionary + print("Saving and loading example_json...") save_some_data(example_json) - opened_file: dict = open_some_data("your-file-name.json") - print(opened_file) + opened_file: Dict[str, Any] = open_some_data("your-file-name.json") + print("Loaded example_json:", opened_file) + + # Example 2: Save and load a list of dictionaries + print("\nSaving and loading a list of dictionaries...") + save_list_json(simple_list_json, "list-example.json") + loaded_list = open_list_json("list-example.json") + print("Loaded list-example.json:", loaded_list) + + # Example 3: Attempt to open a non-existent file + print("\nAttempting to open a non-existent file...") + try_open_nonexistent_json("does_not_exist.json") ``` diff --git a/docs/examples/log_example.md b/docs/examples/log_example.md index 7c4e46e4..ebcb8a06 100644 --- a/docs/examples/log_example.md +++ b/docs/examples/log_example.md @@ -89,17 +89,27 @@ logging_config.config_log( ) -# @logger.catch -def div_zero(x, y): +def div_zero(x: float, y: float) -> float | None: + """ + Safely divide x by y and log ZeroDivisionError if encountered. + + Args: + x (float): Numerator. + y (float): Denominator. + Returns: + float | None: Quotient or None on error. + """ try: return x / y except ZeroDivisionError as e: - logger.error(f"{e}") - logging.error(f"{e}") + logger.error(f"{e}") # log via loguru + logging.error(f"{e}") # log via standard logging -# @logger.catch -def div_zero_two(x, y): +def div_zero_two(x: float, y: float) -> float | None: + """ + Mirror of div_zero demonstrating identical error handling. + """ try: return x / y except ZeroDivisionError as e: @@ -107,30 +117,42 @@ def div_zero_two(x, y): logging.error(f"{e}") -def log_big_string(lqty=100, size=256): - big_string = secrets.token_urlsafe(size) +def log_big_string(lqty: int = 100, size: int = 256) -> None: + """ + Generate a large random string and log various messages repeatedly. + + Args: + lqty (int): Number of log iterations. + size (int): Length of each random string. + """ + big_string = secrets.token_urlsafe(size) # create URL-safe token for _ in range(lqty): - logging.debug(f"Lets make this a big message {big_string}") - div_zero(x=1, y=0) - div_zero_two(x=1, y=0) - # after configuring logging - # use loguru to log messages + logging.debug(f"Lets make this a big message {big_string}") # standard debug + div_zero(1, 0) # trigger/log ZeroDivisionError + div_zero_two(1, 0) + # loguru messages logger.debug("This is a loguru debug message") - logger.info("This is an loguru info message") - logger.error("This is an loguru error message") + logger.info("This is a loguru info message") logger.warning("This is a loguru warning message") + logger.error("This is a loguru error message") logger.critical("This is a loguru critical message") - - # will intercept all standard logging messages also - logging.debug("This is a standard logging debug message") - logging.info("This is an standard logging info message") - logging.error("This is an standard logging error message") + # continued standard logging + logging.info("This is a standard logging info message") logging.warning("This is a standard logging warning message") + logging.error("This is a standard logging error message") logging.critical("This is a standard logging critical message") -def worker(wqty=1000, lqty=100, size=256): - for _ in tqdm(range(wqty), ascii=True, leave=True): # Adjusted for demonstration +def worker(wqty: int = 1000, lqty: int = 100, size: int = 256) -> None: + """ + Worker routine performing log_big_string in a progress loop. + + Args: + wqty (int): Number of outer iterations. + lqty (int): Messages per iteration. + size (int): Random string length. + """ + for _ in tqdm(range(wqty), ascii=True, leave=True): log_big_string(lqty=lqty, size=size) @@ -141,7 +163,18 @@ def main( workers: int = 16, thread_test: bool = False, process_test: bool = False, -): +) -> None: + """ + Configure and launch concurrent logging workers. + + Args: + wqty (int): Iterations per worker. + lqty (int): Logs per iteration. + size (int): Random string size. + workers (int): Thread/process count. + thread_test (bool): Run threads if True. + process_test (bool): Run processes if True. + """ if process_test: processes = [] # Create worker processes diff --git a/docs/examples/text_example.md b/docs/examples/text_example.md index b1677747..3ee20df3 100644 --- a/docs/examples/text_example.md +++ b/docs/examples/text_example.md @@ -23,6 +23,32 @@ Reads text data from a specified file. - **Behavior**: Calls the `open_text` function from `dsg_lib.common_functions.file_functions` to read the content of the file. +### `save_csv_example(csv_data: list[list[str]], file_name: str = "example.csv")` +Saves example rows to a CSV file. +- **Parameters**: + - `csv_data` (list[list[str]]): Rows for CSV (first row is header). + - `file_name` (str): Target CSV file name. + +### `open_csv_example(file_name: str = "example.csv") -> list[dict]` +Opens a CSV file and returns its content as a list of dictionaries. +- **Parameters**: + - `file_name` (str): Name of the CSV file to read. +- **Returns**: + - `list[dict]`: Parsed CSV rows. + +### `save_json_example(data: dict | list, file_name: str = "example.json")` +Saves a dictionary or list as JSON. +- **Parameters**: + - `data` (dict|list): Data to serialize. + - `file_name` (str): Target JSON file name. + +### `open_json_example(file_name: str = "example.json") -> dict | list` +Opens a JSON file and returns its content. +- **Parameters**: + - `file_name` (str): Name of the JSON file to read. +- **Returns**: + - `dict|list`: Parsed JSON content. + ## Example Usage ```python @@ -30,6 +56,19 @@ if __name__ == "__main__": save_some_data(example_text) opened_file: str = open_some_data("your-file-name.txt") print(opened_file) + + # CSV example + csv_rows = [ + ["header1", "header2"], + ["row1col1", "row1col2"] + ] + save_csv_example(csv_rows) + print(open_csv_example()) + + # JSON example + json_obj = {"foo": "bar", "count": 1} + save_json_example(json_obj) + print(open_json_example()) ``` ## Notes @@ -40,7 +79,11 @@ if __name__ == "__main__": This module is licensed under the MIT License. ```python -from dsg_lib.common_functions.file_functions import open_text, save_text +from dsg_lib.common_functions.file_functions import ( + open_text, save_text, + save_csv, open_csv, + save_json, open_json +) example_text = """ @@ -71,8 +114,80 @@ def open_some_data(the_file_name: str) -> str: return result +def save_csv_example( + csv_data: list[list[str]], + file_name: str = "example.csv" +) -> None: + """ + Save example rows to a CSV file. + + Args: + csv_data (list[list[str]]): Rows for CSV (first row is header). + file_name (str): Target CSV file name. + """ + # write rows out + save_csv(file_name=file_name, data=csv_data) + + +def open_csv_example( + file_name: str = "example.csv" +) -> list[dict]: + """ + Open a CSV file and return its content as list of dicts. + + Args: + file_name (str): Name of CSV to read. + + Returns: + list[dict]: Parsed CSV rows. + """ + return open_csv(file_name=file_name) + + +def save_json_example( + data: dict | list, + file_name: str = "example.json" +) -> None: + """ + Save a dict or list as JSON. + + Args: + data (dict|list): Data to serialize. + file_name (str): Target JSON file name. + """ + save_json(file_name=file_name, data=data) + + +def open_json_example( + file_name: str = "example.json" +) -> dict | list: + """ + Open a JSON file and return its content. + + Args: + file_name (str): Name of JSON to read. + + Returns: + dict|list: Parsed JSON content. + """ + return open_json(file_name=file_name) + + if __name__ == "__main__": save_some_data(example_text) opened_file: str = open_some_data("your-file-name.txt") print(opened_file) + + # CSV example + csv_rows = [ + ["header1", "header2"], + ["row1col1", "row1col2"] + ] + save_csv_example(csv_rows) + print(open_csv_example()) + + # JSON example + json_obj = {"foo": "bar", "count": 1} + save_json_example(json_obj) + print(open_json_example()) ``` diff --git a/docs/examples/validate_emails.md b/docs/examples/validate_emails.md index 6c1fb13f..36c49d39 100644 --- a/docs/examples/validate_emails.md +++ b/docs/examples/validate_emails.md @@ -65,12 +65,43 @@ This module is licensed under the MIT License. import pprint import time +from typing import List, Dict, Any + from dsg_lib.common_functions.email_validation import validate_email_address -if __name__ == "__main__": +def run_validation( + email_addresses: List[str], + configurations: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """ + Validate each email against multiple configurations. + + Args: + email_addresses: List of email strings to validate. + configurations: List of parameter dicts for validation. + + Returns: + A sorted list of result dicts (sorted by "email" key). + """ + results: List[Dict[str, Any]] = [] + # iterate over every email and config combination + for email in email_addresses: + for config in configurations: + # call the core email validator and collect its output + res = validate_email_address(email, **config) + results.append(res) + # sort by email for consistent output + return sorted(results, key=lambda x: x["email"]) - # create a list of email addresses to check if valid - email_addresses = [ +def main() -> None: + """ + Entry point for the email validation example. + + Defines a list of emails and configurations, measures execution time, + runs validation, and pretty‑prints the results. + """ + # list of example email addresses + email_addresses: List[str] = [ "bob@devsetgo.com", "bob@devset.go", "foo@yahoo.com", @@ -118,8 +149,8 @@ if __name__ == "__main__": "test@google.com", ] - # create a list of configurations - configurations = [ + # various validation parameter sets to exercise different rules + configurations: List[Dict[str, Any]] = [ { "check_deliverability": True, "test_environment": False, @@ -231,19 +262,17 @@ if __name__ == "__main__": }, ] - t0 = time.time() - validity = [] - - for email in email_addresses: - for config in configurations: + # measure and run + start_time: float = time.time() + results = run_validation(email_addresses, configurations) + elapsed: float = time.time() - start_time - res = validate_email_address(email, **config) - validity.append(res) - t1 = time.time() - validity = sorted(validity, key=lambda x: x["email"]) + # output each result + for record in results: + pprint.pprint(record, indent=4) - for v in validity: - pprint.pprint(v, indent=4) + print(f"Time taken: {elapsed:.2f}s") - print(f"Time taken: {t1 - t0:.2f}") +if __name__ == "__main__": + main() ```