Connector secrets and pydantic migration

.gitignore

@@ -12,4 +12,5 @@ dist/
 build
 .env
 tests/reports
-.coverage
+.coverage
+.idea/

docs/reference.md

@@ -1,7 +1,8 @@
 ::: src.glassflow.client
 ::: src.glassflow.pipeline
 ::: src.glassflow.pipeline_data
+::: src.glassflow.secret
 ::: src.glassflow.space
 ::: src.glassflow.config
 ::: src.glassflow.models.errors
-::: src.glassflow.models.operations
+::: src.glassflow.models.responses

makefile

@@ -9,16 +9,7 @@ add-noqa: generate-api-data-models
 	echo "Add noqa comment ..."
 	sed -i '' -e '1s/^/# ruff: noqa\n/' $(API_DATA_MODELS)
 
-
-add-dataclass-json-decorators: add-noqa
-	echo "Import dataclass_json ..."
-	sed -i '' -e '/^from __future__ import annotations/a\'$$'\n''from dataclasses_json import dataclass_json' $(API_DATA_MODELS)
-
-
-	echo "Add dataclass_json decorators ..."
-	sed  -i '' -e '/@dataclass/ i\'$$'\n''@dataclass_json\''' $(API_DATA_MODELS)
-
-generate: add-dataclass-json-decorators
+generate: add-noqa
 
 include .env
 export
@@ -32,3 +23,14 @@ lint:
 
 formatter:
 	ruff format --check .
+
+fix-format:
+	ruff format .
+
+fix-lint:
+	ruff check --fix .
+
+fix: fix-format fix-lint
+
+serve-docs-locally:
+	mkdocs serve

mkdocs.yml

@@ -43,7 +43,11 @@ plugins:
       handlers:
         python:
           import:
-            - url: https://docs.python-requests.org/en/master/objects.inv
+            - url: https://docs.python.org/3/objects.inv  # Add Python's objects.inv
+              domains: [ std, py ]
+            - url: https://docs.python-requests.org/en/master/objects.inv  # Add requests objects.inv
+              domains: [ std, py ]
+            - url: https://docs.pydantic.dev/latest/objects.inv  # Add Pydantic's objects.inv
               domains: [ std, py ]
           options:
             members_order: source

pyproject.toml

@@ -40,11 +40,12 @@ convention = "google"
 field-constraints = true
 snake-case-field = true
 strip-default-none = false
-target-python-version = "3.7"
+target-python-version = "3.8"
 use-title-as-name = true
 disable-timestamp = true
 enable-version-header = true
 use-double-quotes = true
 use-subclass-enum=true
+use-standard-collections=true
 input-file-type = "openapi"
-output-model-type = "dataclasses.dataclass"
+output-model-type = "pydantic_v2.BaseModel"

setup.py

@@ -22,7 +22,7 @@
         "urllib3==1.26.15",
         "certifi>=2023.7.22",
         "charset-normalizer>=3.2.0",
-        "dataclasses-json>=0.6.4",
+        "pydantic>=2.10.6",
         "idna>=3.4",
         "jsonpath-python>=1.0.6 ",
         "marshmallow>=3.19.0",
@@ -34,16 +34,17 @@
         "typing-inspect>=0.9.0",
         "typing_extensions>=4.7.1",
         "python-dotenv==1.0.1",
+        "eval_type_backport>=0.2.0",
     ],
     extras_require={
         "dev": [
-            "pylint==2.16.2",
-            "pytest==8.3.2",
-            "pytest-cov==5.0.0",
-            "datamodel-code-generator[http]==0.26.0",
-            "requests-mock==1.12.1",
-            "isort==5.13.2",
-            "ruff==0.6.3",
+            "pylint>=2.16.2",
+            "pytest>=8.3.2",
+            "pytest-cov>=5.0.0",
+            "datamodel-code-generator[http]>=0.27.0",
+            "requests-mock>=1.12.1",
+            "isort>=5.13.2",
+            "ruff>=0.9.0",
         ]
     },
     package_dir={"": "src"},

src/glassflow/__init__.py

@@ -1,7 +1,10 @@
 from .client import GlassFlowClient as GlassFlowClient
 from .config import GlassFlowConfig as GlassFlowConfig
+from .models import api as internal  # noqa: F401
 from .models import errors as errors
+from .models import responses as responses
 from .pipeline import Pipeline as Pipeline
 from .pipeline_data import PipelineDataSink as PipelineDataSink
 from .pipeline_data import PipelineDataSource as PipelineDataSource
+from .secret import Secret as Secret
 from .space import Space as Space

src/glassflow/api_client.py

@@ -6,89 +6,56 @@
 
 from .config import GlassFlowConfig
 from .models import errors
-from .models.operations.base import BaseRequest, BaseResponse
-from .utils import utils as utils
 
 
 class APIClient:
     glassflow_config = GlassFlowConfig()
 
     def __init__(self):
+        """API client constructor"""
         super().__init__()
         self.client = requests_http.Session()
 
-    def _get_headers(
-        self, request: BaseRequest, req_content_type: str | None = None
-    ) -> dict:
-        headers = utils.get_req_specific_headers(request)
-        headers["Accept"] = "application/json"
-        headers["Gf-Client"] = self.glassflow_config.glassflow_client
-        headers["User-Agent"] = self.glassflow_config.user_agent
-        headers["Gf-Python-Version"] = (
-            f"{sys.version_info.major}."
-            f"{sys.version_info.minor}."
-            f"{sys.version_info.micro}"
-        )
-
-        if req_content_type and req_content_type not in (
-            "multipart/form-data",
-            "multipart/mixed",
-        ):
-            headers["content-type"] = req_content_type
-
+    def _get_core_headers(self) -> dict:
+        headers = {
+            "Accept": "application/json",
+            "Gf-Client": self.glassflow_config.glassflow_client,
+            "User-Agent": self.glassflow_config.user_agent,
+            "Gf-Python-Version": (
+                f"{sys.version_info.major}."
+                f"{sys.version_info.minor}."
+                f"{sys.version_info.micro}"
+            ),
+        }
         return headers
 
     def _request(
         self,
-        method: str,
-        endpoint: str,
-        request: BaseRequest,
-        serialization_method: str = "json",
-    ) -> BaseResponse:
-        request_type = type(request)
-
-        url = utils.generate_url(
-            request_type,
-            self.glassflow_config.server_url,
-            endpoint,
-            request,
-        )
-
-        req_content_type, data, form = utils.serialize_request_body(
-            request=request,
-            request_type=request_type,
-            request_field_name="request_body",
-            nullable=False,
-            optional=True,
-            serialization_method=serialization_method,
-        )
-        if method == "GET":
-            data = None
-
-        headers = self._get_headers(request, req_content_type)
-        query_params = utils.get_query_params(request_type, request)
-
-        # make the request
-        http_res = self.client.request(
-            method, url=url, params=query_params, headers=headers, data=data, files=form
-        )
-        content_type = http_res.headers.get("Content-Type")
-
-        res = BaseResponse(
-            status_code=http_res.status_code,
-            content_type=content_type,
-            raw_response=http_res,
-        )
-
-        if http_res.status_code in [400, 500]:
-            out = utils.unmarshal_json(http_res.text, errors.Error)
-            out.raw_response = http_res
-            raise out
-        elif http_res.status_code == 429:
-            pass
-        elif 400 < http_res.status_code < 600:
-            raise errors.ClientError(
-                "API error occurred", http_res.status_code, http_res.text, http_res
+        method,
+        endpoint,
+        request_headers=None,
+        json=None,
+        request_query_params=None,
+        files=None,
+        data=None,
+    ):
+        headers = self._get_core_headers()
+        if request_headers:
+            headers.update(request_headers)
+
+        url = self.glassflow_config.server_url + endpoint
+
+        try:
+            http_res = self.client.request(
+                method,
+                url=url,
+                params=request_query_params,
+                headers=headers,
+                json=json,
+                files=files,
+                data=data,
             )
-
-        return res
+            http_res.raise_for_status()
+        except requests_http.HTTPError as http_err:
+            raise errors.UnknownError(http_err.response) from http_err
+        return http_res