Add NLP utilities and entity transformation scripts for Silver layer processing

Author: ByteMeDirk

README.md

@@ -34,6 +34,9 @@ semantic-medallion-data-platform/
 │   │   ├── brz_01_extract_newsapi.py        # Extract news articles from NewsAPI
 │   │   └── brz_01_extract_known_entities.py # Extract known entities from CSV files
 │   ├── silver/                   # Silver layer processing
+│   │   ├── slv_02_transform_nlp_known_entities.py  # Extract entities from known entities descriptions
+│   │   ├── slv_02_transform_nlp_newsapi.py         # Extract entities from news articles
+│   │   └── slv_03_transform_entity_to_entity_mapping.py  # Create entity mappings
 │   ├── gold/                     # Gold layer processing
 │   ├── common/                   # Shared utilities
 │   └── config/                   # Configuration
@@ -131,6 +134,50 @@ This will:
 2. Process and transform the data
 3. Store the entities in the bronze.known_entities table
 
+### Running Silver Layer Processes
+
+#### Processing Known Entities with NLP
+
+To extract entities from known entities descriptions:
+
+```bash
+cd semantic-medallion-data-platform
+python -m semantic_medallion_data_platform.silver.slv_02_transform_nlp_known_entities
+```
+
+This will:
+1. Copy known entities from bronze.known_entities to silver.known_entities
+2. Extract entities (locations, organizations, persons) from entity descriptions using NLP
+3. Store the extracted entities in the silver.known_entities_entities table
+
+#### Processing News Articles with NLP
+
+To extract entities from news articles:
+
+```bash
+cd semantic-medallion-data-platform
+python -m semantic_medallion_data_platform.silver.slv_02_transform_nlp_newsapi
+```
+
+This will:
+1. Copy news articles from bronze.newsapi to silver.newsapi
+2. Extract entities from article title, description, and content using NLP
+3. Store the extracted entities in the silver.newsapi_entities table
+
+#### Creating Entity Mappings
+
+To create entity-to-entity and entity-to-source mappings:
+
+```bash
+cd semantic-medallion-data-platform
+python -m semantic_medallion_data_platform.silver.slv_03_transform_entity_to_entity_mapping
+```
+
+This will:
+1. Create entity-to-source mappings between known_entities_entities and newsapi_entities
+2. Create entity-to-entity mappings within known_entities_entities
+3. Store the mappings in silver.entity_to_source_mapping and silver.entity_to_entity_mapping tables
+
 
 ## Contributing
 

local_run.sh

@@ -0,0 +1,10 @@
+# A local script to run the entire ETL pipeline
+
+# Bronze Layer
+#python semantic_medallion_data_platform/bronze/brz_01_extract_known_entities.py --raw_data_filepath "./data/known_entities/*.csv"
+#python semantic_medallion_data_platform/bronze/brz_01_extract_newsapi.py --days_back "1"
+
+# Silver Layer
+python semantic_medallion_data_platform/silver/slv_02_transform_nlp_newsapi.py
+python semantic_medallion_data_platform/silver/slv_02_transform_nlp_known_entities.py
+python semantic_medallion_data_platform/silver/slv_03_transform_entity_to_entity_mapping.py

poetry.lock

@@ -24,6 +24,7 @@ dotenv = "^0.9.9"
 spacy = "^3.8.7"
 textblob = "^0.19.0"
 newsapi-python = "^0.2.7"
+rapidfuzz = "^3.13.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.3.1"

pyproject.toml

@@ -0,0 +1,39 @@
+"""
+Common NLP utilities for the Semantic Medallion Data Platform.
+
+This module provides reusable NLP functions for entity extraction and other text processing tasks.
+"""
+import spacy
+from pyspark.sql.types import ArrayType, StringType, StructField, StructType
+
+# Load spaCy NLP model
+NLP = spacy.load("en_core_web_lg")
+
+# Define schema for entity extraction
+ENTITY_STRUCT = StructType(
+    [StructField("text", StringType(), True), StructField("type", StringType(), True)]
+)
+
+ENTITIES_SCHEMA = ArrayType(ENTITY_STRUCT)
+
+
+def extract_entities(text: str) -> list:
+    """
+    Extract location, organization, and person entities from text using spaCy.
+
+    Args:
+        text: The input text to process
+
+    Returns:
+        A list of dictionaries with entity text and type
+    """
+    if not text:
+        return []
+
+    doc = NLP(text)
+    entities = [
+        {"text": ent.text, "type": ent.label_}
+        for ent in doc.ents
+        if ent.label_ in ("LOC", "GPE", "ORG", "PERSON")
+    ]
+    return entities