build(clp-package): Adopt uv + hatchling as the build and packaging backend for Python components (resolves #1396); Upgrade dependencies for Python components.

components/clp-mcp-server/pyproject.toml

@@ -8,9 +8,9 @@ requires-python = ">=3.10"
 dependencies = [
     "click>=8.3.0",
     "fastmcp>=2.12.4",
-    "pymongo>=4.15.1",
+    "pymongo>=4.15.3",
     "aiomysql>=0.2.0",
-    "msgpack>=1.1.1",
+    "msgpack>=1.1.2",
 ]
 
 [project.scripts]

components/clp-package-utils/pyproject.toml

@@ -1,23 +1,19 @@
 [project]
-requires-python = ">= 3.9"
-
-[tool.poetry]
 name = "clp-package-utils"
 version = "0.5.2-dev"
 description = "Utilities for the CLP package."
-authors = ["YScope Inc. <dev@yscope.com>"]
+authors = [{name = "YScope Inc.", email = "dev@yscope.com"}]
 readme = "README.md"
-
-[tool.poetry.dependencies]
-python = ">= 3.9"
-Brotli = "^1.1.0"
-msgpack = "^1.0.7"
-psutil = "^7.1.0"
-pydantic = "^2.11.9"
-pymongo = "^4.7.2"
-PyYAML = "^6.0.1"
+requires-python = ">=3.9"
+dependencies = [
+    "Brotli>=1.1.0",
+    "msgpack>=1.1.2",
+    "psutil>=7.1.0",
+    "pydantic>=2.12.0",
+    "pymongo>=4.15.3",
+    "PyYAML>=6.0.3",
+]
 
 [build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
+requires = ["hatchling"]
+build-backend = "hatchling.build"

components/clp-py-utils/pyproject.toml

@@ -1,28 +1,25 @@
 [project]
-requires-python = ">= 3.9"
-
-[tool.poetry]
 name = "clp-py-utils"
 version = "0.5.2-dev"
 description = "Utilities for other Python packages in CLP."
-authors = ["YScope Inc. <dev@yscope.com>"]
+authors = [{name = "YScope Inc.", email = "dev@yscope.com"}]
 readme = "README.md"
-
-[tool.poetry.dependencies]
-python = ">= 3.9"
-boto3 = "^1.35.81"
-# mariadb version must be compatible with libmariadev installed in runtime env.
-# See https://mariadb.com/docs/server/connect/programming-languages/python/install/#Dependencies
-mariadb = "~1.0.11"
-mysql-connector-python = "^8.2.0"
-pydantic = "^2.11.9"
-python-dotenv = "^1.0.1"
-python-Levenshtein = "~0.22"
-sqlalchemy = "~2.0"
-PyYAML = "^6.0.1"
-result = "^0.17.0"
-StrEnum = "^0.4.15"
+requires-python = ">=3.9"
+dependencies = [
+    "boto3>=1.40.49",
+    # mariadb version must be compatible with libmariadev installed in runtime env.
+    # See https://mariadb.com/docs/server/connect/programming-languages/python/install/#Dependencies
+    "mariadb>=1.0.11,<1.1.dev0",
+    "mysql-connector-python>=9.4.0",
+    "pydantic>=2.12.0",
+    "python-dotenv>=1.1.1",
+    "python-Levenshtein>=0.27.1",
+    "sqlalchemy>=2.0.43",
+    "PyYAML>=6.0.3",
+    "result>=0.17.0",
+    "StrEnum>=0.4.15",
+]
 
 [build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
+requires = ["hatchling"]
+build-backend = "hatchling.build"

components/job-orchestration/job_orchestration/executor/compress/celery_compress.py

@@ -1,6 +1,7 @@
 from celery import signals
 from celery.app.task import Task
 from celery.utils.log import get_task_logger
+
 from job_orchestration.executor.compress.celery import app
 from job_orchestration.executor.compress.compression_task import compression_entry_point
 

components/job-orchestration/job_orchestration/executor/compress/compression_task.py

@@ -29,6 +29,7 @@
     s3_put,
 )
 from clp_py_utils.sql_adapter import SQL_Adapter
+
 from job_orchestration.scheduler.constants import CompressionTaskStatus
 from job_orchestration.scheduler.job_config import (
     ClpIoConfig,

components/job-orchestration/job_orchestration/executor/compress/spider_compress.py

@@ -1,9 +1,10 @@
 import json
 
 from clp_py_utils.clp_logging import get_logger
+from spider_py import Int8, Int64, TaskContext
+
 from job_orchestration.executor.compress.compression_task import compression_entry_point
 from job_orchestration.utils.spider_utils import int8_list_to_utf8_str, utf8_str_to_int8_list
-from spider_py import Int8, Int64, TaskContext
 
 # Setup logging
 logger = get_logger("spider_compression")

components/job-orchestration/job_orchestration/executor/query/extract_stream_task.py

@@ -20,6 +20,7 @@
     s3_put,
 )
 from clp_py_utils.sql_adapter import SQL_Adapter
+
 from job_orchestration.executor.query.celery import app
 from job_orchestration.executor.query.utils import (
     report_task_failure,

components/job-orchestration/job_orchestration/executor/query/fs_search_task.py

@@ -14,6 +14,7 @@
 from clp_py_utils.clp_logging import set_logging_level
 from clp_py_utils.s3_utils import generate_s3_virtual_hosted_style_url, get_credential_env_vars
 from clp_py_utils.sql_adapter import SQL_Adapter
+
 from job_orchestration.executor.query.celery import app
 from job_orchestration.executor.query.utils import (
     report_task_failure,

components/job-orchestration/job_orchestration/executor/query/utils.py

@@ -10,6 +10,7 @@
 
 from clp_py_utils.clp_config import QUERY_TASKS_TABLE_NAME
 from clp_py_utils.sql_adapter import SQL_Adapter
+
 from job_orchestration.scheduler.scheduler_data import QueryTaskResult, QueryTaskStatus
 
 

components/job-orchestration/job_orchestration/scheduler/compress/compression_scheduler.py

@@ -29,6 +29,8 @@
 from clp_py_utils.core import read_yaml_config_file
 from clp_py_utils.s3_utils import s3_get_object_metadata
 from clp_py_utils.sql_adapter import SQL_Adapter
+from pydantic import ValidationError
+
 from job_orchestration.scheduler.compress.partition import PathsToCompressBuffer
 from job_orchestration.scheduler.compress.task_manager.celery_task_manager import CeleryTaskManager
 from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager
@@ -47,7 +49,6 @@
     CompressionJob,
 )
 from job_orchestration.scheduler.utils import kill_hanging_jobs
-from pydantic import ValidationError
 
 # Setup logging
 logger = get_logger("compression_scheduler")

components/job-orchestration/job_orchestration/scheduler/compress/partition.py

@@ -9,6 +9,7 @@
     group_files_by_similar_filenames,
 )
 from clp_py_utils.core import FileMetadata
+
 from job_orchestration.scheduler.job_config import ClpIoConfig, PathsToCompress
 
 

components/job-orchestration/job_orchestration/scheduler/compress/task_manager/celery_task_manager.py

@@ -3,6 +3,7 @@
 from typing import Any
 
 import celery
+
 from job_orchestration.executor.compress.celery_compress import compress
 from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager
 from job_orchestration.scheduler.task_result import CompressionTaskResult

components/job-orchestration/job_orchestration/scheduler/compress/task_manager/spider_task_manager.py

@@ -4,11 +4,12 @@
 from typing import Any
 
 import spider_py
+from spider_py.client.job import Job
+
 from job_orchestration.executor.compress.spider_compress import compress
 from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager
 from job_orchestration.scheduler.task_result import CompressionTaskResult
 from job_orchestration.utils.spider_utils import int8_list_to_utf8_str, utf8_str_to_int8_list
-from spider_py.client.job import Job
 
 
 class SpiderTaskManager(TaskManager):

components/job-orchestration/job_orchestration/scheduler/query/query_scheduler.py

@@ -47,6 +47,8 @@
 from clp_py_utils.core import read_yaml_config_file
 from clp_py_utils.decorators import exception_default_value
 from clp_py_utils.sql_adapter import SQL_Adapter
+from pydantic import ValidationError
+
 from job_orchestration.executor.query.extract_stream_task import extract_stream
 from job_orchestration.executor.query.fs_search_task import search
 from job_orchestration.garbage_collector.constants import MIN_TO_SECONDS, SECOND_TO_MILLISECOND
@@ -77,7 +79,6 @@
     SearchJob,
 )
 from job_orchestration.scheduler.utils import kill_hanging_jobs
-from pydantic import ValidationError
 
 # Setup logging
 logger = get_logger("search-job-handler")

components/job-orchestration/job_orchestration/scheduler/query/reducer_handler.py

@@ -5,6 +5,7 @@
 
 import msgpack
 from clp_py_utils.clp_logging import get_logger
+
 from job_orchestration.scheduler.job_config import AggregationConfig
 
 # Setup logging

components/job-orchestration/job_orchestration/scheduler/scheduler_data.py

@@ -4,6 +4,8 @@
 from enum import auto, Enum
 from typing import Any, Dict, List, Optional
 
+from pydantic import BaseModel, ConfigDict
+
 from job_orchestration.scheduler.compress.task_manager.task_manager import TaskManager
 from job_orchestration.scheduler.constants import (
     QueryJobType,
@@ -16,7 +18,6 @@
     SearchJobConfig,
 )
 from job_orchestration.scheduler.query.reducer_handler import ReducerHandlerMessageQueues
-from pydantic import BaseModel, ConfigDict, field_validator
 
 
 class CompressionJob(BaseModel):

components/job-orchestration/job_orchestration/scheduler/task_result.py

@@ -1,8 +1,9 @@
 from typing import Optional
 
-from job_orchestration.scheduler.constants import CompressionTaskStatus
 from pydantic import BaseModel, field_validator
 
+from job_orchestration.scheduler.constants import CompressionTaskStatus
+
 
 class CompressionTaskResult(BaseModel):
     task_id: int

components/job-orchestration/job_orchestration/scheduler/utils.py

@@ -10,6 +10,7 @@
     QUERY_TASKS_TABLE_NAME,
 )
 from clp_py_utils.sql_adapter import SQL_Adapter
+
 from job_orchestration.scheduler.constants import (
     CompressionJobStatus,
     CompressionTaskStatus,