Release/v2.1.0

.gitignore

@@ -12,4 +12,5 @@ dist/
 build
 .env
 tests/reports
-.coverage
+.coverage
+.idea/

docs/reference.md

@@ -1,7 +1,8 @@
 ::: src.glassflow.client
 ::: src.glassflow.pipeline
 ::: src.glassflow.pipeline_data
+::: src.glassflow.secret
 ::: src.glassflow.space
 ::: src.glassflow.config
 ::: src.glassflow.models.errors
-::: src.glassflow.models.operations
+::: src.glassflow.models.responses

makefile

@@ -9,16 +9,7 @@ add-noqa: generate-api-data-models
 	echo "Add noqa comment ..."
 	sed -i '' -e '1s/^/# ruff: noqa\n/' $(API_DATA_MODELS)
 
-
-add-dataclass-json-decorators: add-noqa
-	echo "Import dataclass_json ..."
-	sed -i '' -e '/^from __future__ import annotations/a\'$$'\n''from dataclasses_json import dataclass_json' $(API_DATA_MODELS)
-
-
-	echo "Add dataclass_json decorators ..."
-	sed  -i '' -e '/@dataclass/ i\'$$'\n''@dataclass_json\''' $(API_DATA_MODELS)
-
-generate: add-dataclass-json-decorators
+generate: add-noqa
 
 include .env
 export
@@ -32,3 +23,14 @@ lint:
 
 formatter:
 	ruff format --check .
+
+fix-format:
+	ruff format .
+
+fix-lint:
+	ruff check --fix .
+
+fix: fix-format fix-lint
+
+serve-docs-locally:
+	mkdocs serve

mkdocs.yml

@@ -43,7 +43,11 @@ plugins:
       handlers:
         python:
           import:
-            - url: https://docs.python-requests.org/en/master/objects.inv
+            - url: https://docs.python.org/3/objects.inv  # Add Python's objects.inv
+              domains: [ std, py ]
+            - url: https://docs.python-requests.org/en/master/objects.inv  # Add requests objects.inv
+              domains: [ std, py ]
+            - url: https://docs.pydantic.dev/latest/objects.inv  # Add Pydantic's objects.inv
               domains: [ std, py ]
           options:
             members_order: source

pyproject.toml

@@ -40,11 +40,12 @@ convention = "google"
 field-constraints = true
 snake-case-field = true
 strip-default-none = false
-target-python-version = "3.7"
+target-python-version = "3.8"
 use-title-as-name = true
 disable-timestamp = true
 enable-version-header = true
 use-double-quotes = true
 use-subclass-enum=true
+use-standard-collections=true
 input-file-type = "openapi"
-output-model-type = "dataclasses.dataclass"
+output-model-type = "pydantic_v2.BaseModel"

setup.py

@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name="glassflow",
-    version="2.0.8",
+    version="2.1.0",
     author="glassflow",
     description="GlassFlow Python Client SDK",
     url="https://www.glassflow.dev/docs",
@@ -22,7 +22,7 @@
         "urllib3==1.26.15",
         "certifi>=2023.7.22",
         "charset-normalizer>=3.2.0",
-        "dataclasses-json>=0.6.4",
+        "pydantic>=2.10.6",
         "idna>=3.4",
         "jsonpath-python>=1.0.6 ",
         "marshmallow>=3.19.0",
@@ -34,19 +34,25 @@
         "typing-inspect>=0.9.0",
         "typing_extensions>=4.7.1",
         "python-dotenv==1.0.1",
+        "eval_type_backport>=0.2.0",
     ],
     extras_require={
         "dev": [
-            "pylint==2.16.2",
-            "pytest==8.3.2",
-            "pytest-cov==5.0.0",
-            "datamodel-code-generator[http]==0.26.0",
-            "requests-mock==1.12.1",
-            "isort==5.13.2",
-            "ruff==0.6.3",
+            "pylint>=2.16.2",
+            "pytest>=8.3.2",
+            "pytest-cov>=5.0.0",
+            "datamodel-code-generator[http]>=0.27.0",
+            "requests-mock>=1.12.1",
+            "isort>=5.13.2",
+            "ruff>=0.9.0",
         ]
     },
     package_dir={"": "src"},
     python_requires=">=3.8",
     package_data={"glassflow": ["py.typed"]},
+    entry_points={
+        "console_scripts": [
+            "glassflow = cli.cli:glassflow",
+        ],
+    },
 )

src/cli/cli.py

@@ -0,0 +1,43 @@
+import click
+
+from .commands import get_started
+
+
+@click.group()
+def glassflow():
+    """Glassflow CLI - Manage and control Glassflow SDK"""
+    pass
+
+
+@click.command()
+@click.argument("command", required=False)
+def help(command):
+    """Displays help information about Glassflow CLI and its commands."""
+
+    commands = {
+        "get-started": "Initialize Glassflow with an access token.\nUsage: "
+        "glassflow get-started --token YOUR_TOKEN",
+        "help": "Shows help information.\nUsage: glassflow help [command]",
+    }
+
+    if command:
+        if command in commands:
+            click.echo(f"ℹ️  Help for `{command}`:\n{commands[command]}")
+        else:
+            click.echo(
+                f"❌ Unknown command: `{command}`. Run `glassflow help` for a "
+                f"list of commands."
+            )
+    else:
+        click.echo("📖 Glassflow CLI Help:")
+        for cmd, desc in commands.items():
+            click.echo(f"  ➜ {cmd}: {desc.splitlines()[0]}")
+        click.echo("\nRun `glassflow help <command>` for more details.")
+
+
+# Add commands to CLI group
+glassflow.add_command(get_started)
+glassflow.add_command(help)
+
+if __name__ == "__main__":
+    glassflow()

src/cli/commands/__init__.py

@@ -0,0 +1 @@
+from .get_started import get_started as get_started

src/cli/commands/get_started.py

@@ -0,0 +1,107 @@
+import os
+
+import click
+from dotenv import load_dotenv
+
+
+@click.command()
+@click.option(
+    "--personal-access-token", "-pat", default=None, help="Personal access token."
+)
+@click.option(
+    "--env-file",
+    "-e",
+    default=".env",
+    help="Path to the .env file (default: .env in current directory).",
+)
+def get_started(personal_access_token, env_file):
+    """Displays a welcome message and setup instructions."""
+
+    # Load token from .env if not provided in CLI
+    if personal_access_token is None:
+        if os.path.exists(env_file):
+            load_dotenv(env_file)  # Load environment variables
+            personal_access_token = os.getenv("PERSONAL_ACCESS_TOKEN")
+        else:
+            click.echo("⚠️ No token provided and .env file not found!", err=True)
+            return
+
+    if not personal_access_token:
+        click.echo("❌ Error: Personal access token is required.", err=True)
+        return
+
+    click.echo("🚀 Welcome to Glassflow! \n")
+    click.echo(
+        f"🔑 Using Personal Access Token: {personal_access_token[:4]}... "
+        f"(hidden for security)"
+    )
+    click.echo("\n📝 In this getting started guide, we will do the following:")
+    click.echo("1. Define a data transformation function in Python.\n")
+    click.echo("2. Create a pipeline with the function.\n")
+    click.echo("3. Send events to the pipeline.\n")
+    click.echo("4. Consume transformed events in real-time from the pipeline\n")
+    click.echo("5. Monitor the pipeline and view logs.\n")
+
+    filename = create_transformation_function()
+    pipeline = create_space_pipeline(personal_access_token, filename)
+    send_consume_events(pipeline)
+
+    click.echo(
+        "\n🎉 Congratulations! You have successfully created a pipeline and sent"
+        " events to it.\n"
+    )
+    click.echo(
+        "💻 View the logs and monitor the Pipeline in the "
+        f"Glassflow Web App at https://app.glassflow.dev/pipelines/{pipeline.id}"
+    )
+
+
+def create_transformation_function(filename="transform_getting_started.py"):
+    file_content = """import json
+import logging
+
+def handler(data: dict, log: logging.Logger):
+    log.info("Echo: " + json.dumps(data))
+    data['transformed_by'] = "glassflow"
+
+    return data
+"""
+    with open(filename, "w") as f:
+        f.write(file_content)
+    click.echo(f"✅ Transformation function created in {filename}")
+    click.echo("The transformation function is:\n")
+    click.echo(file_content)
+    click.echo("📝 You can modify the transformation function in the file.")
+    return filename
+
+
+def create_space_pipeline(personal_access_token, transform_filename):
+    import glassflow
+
+    # create glassflow client to interact with GlassFlow
+    client = glassflow.GlassFlowClient(personal_access_token=personal_access_token)
+    example_space = client.create_space(name="getting-started")
+    pipeline = client.create_pipeline(
+        name="getting-started-pipeline",
+        transformation_file=transform_filename,
+        space_id=example_space.id,
+    )
+    click.echo(f"✅ Created a pipeline with pipeline_id {pipeline.id}")
+    return pipeline
+
+
+def send_consume_events(pipeline):
+    click.echo("🔄 Sending some generated events to pipeline .....")
+    data_source = pipeline.get_source()
+    for i in range(10):
+        event = {"data": f"hello GF {i}"}
+        res = data_source.publish(event)
+        if res.status_code == 200:
+            click.echo(f"Sent event: {event}")
+
+    click.echo("📡 Consuming transformed events from the pipeline")
+    data_sink = pipeline.get_sink()
+    for _ in range(10):
+        resp = data_sink.consume()
+        if resp.status_code == 200:
+            click.echo(f"Consumed event: {resp.event()} ")

src/glassflow/__init__.py

@@ -1,7 +1,10 @@
 from .client import GlassFlowClient as GlassFlowClient
 from .config import GlassFlowConfig as GlassFlowConfig
+from .models import api as internal  # noqa: F401
 from .models import errors as errors
+from .models import responses as responses
 from .pipeline import Pipeline as Pipeline
 from .pipeline_data import PipelineDataSink as PipelineDataSink
 from .pipeline_data import PipelineDataSource as PipelineDataSource
+from .secret import Secret as Secret
 from .space import Space as Space