Release/v2.0.0

.github/workflows/on_pr.yaml

@@ -4,6 +4,12 @@ on:
   pull_request:
     branches:
       - main
+      - release/*
+
+permissions:
+  contents: write
+  checks: write
+  pull-requests: write
 
 jobs:
   tests:
@@ -24,11 +30,16 @@ jobs:
       - name: Install dependencies
         run: pip install -e .[dev]
 
-      - name: Run Integration Tests
-        run: pytest tests/glassflow/integration_tests/
+      - name: Run Tests
+        run: pytest
         env:
-          PIPELINE_ID: ${{ secrets.PIPELINE_ID }}
-          PIPELINE_ACCESS_TOKEN: ${{ secrets.PIPELINE_ACCESS_TOKEN }}
+          PERSONAL_ACCESS_TOKEN: ${{ secrets.INTEGRATION_PERSONAL_ACCESS_TOKEN }}
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverageReport
+          path: tests/reports/
 
   checks:
     name: Run code checks
@@ -48,8 +59,46 @@ jobs:
       - name: Install dependencies
         run: pip install -e .[dev]
 
-      - name: Run isort
-        run: isort .
-
-      - name: Run linter and code formatter
+      - name: Run ruff linter checks
         run: ruff check .
+
+      - name: Run ruff formatting checks
+        run: ruff format --check .
+
+  coverage:
+    runs-on: ubuntu-latest
+    needs: [tests]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
+          fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
+
+      - name: Download coverage report
+        uses: actions/download-artifact@v4
+        with:
+          name: coverageReport
+
+      - name: Pytest coverage comment
+        id: coverageComment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-xml-coverage-path: ./coverage.xml
+          junitxml-path: ./pytest.xml
+
+      - name: Update Readme with Coverage Html
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          sed -i '/<!-- Pytest Coverage Comment:Begin -->/,/<!-- Pytest Coverage Comment:End -->/c\<!-- Pytest Coverage Comment:Begin -->\n\${{ steps.coverageComment.outputs.coverageHtml }}\n<!-- Pytest Coverage Comment:End -->' ./README.md
+
+      - name: Commit & Push changes to Readme
+        id: commit
+        if: ${{ github.ref == 'refs/heads/main' }}
+        run: |
+          git config --global user.name 'GitHub Actions Workflow glassflow-python-sdk/.github/workflows/on_pr.yaml'
+          git config --global user.email 'glassflow-actions-workflow@users.noreply.github.com'
+
+          git add README.md
+          git commit -m "Update coverage on Readme"
+
+          git push origin master

.gitignore

@@ -10,4 +10,6 @@ site**
 .pypirc
 dist/
 build
-.env
+.env
+tests/reports
+.coverage

README.md

@@ -8,6 +8,8 @@
         <img src="https://img.shields.io/badge/slack-join-community?logo=slack&amp;logoColor=white&amp;style=flat"
             alt="Chat on Slack"></a>
 <a href="https://github.com/astral-sh/ruff"><img src="https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json" alt="Ruff" style="max-width:100%;"></a>
+<!-- Pytest Coverage Comment:Begin -->
+<!-- Pytest Coverage Comment:End -->
 
 
 # GlassFlow Python SDK
@@ -22,11 +24,12 @@ You can install the GlassFlow Python SDK using pip:
 pip install glassflow
 ```
 
-## Available Operations
+## Data Operations
 
 * [publish](#publish) - Publish a new event into the pipeline
 * [consume](#consume) - Consume the transformed event from the pipeline
 * [consume failed](#consume-failed) - Consume the events that failed from the pipeline
+* [validate credentials](#validate-credentials) - Validate pipeline credentials
 
 
 ## publish
@@ -36,12 +39,11 @@ Publish a new event into the pipeline
 ### Example Usage
 
 ```python
-import glassflow
+from glassflow import PipelineDataSource
 
-client = glassflow.GlassFlowClient()
-pipeline_client = client.pipeline_client(pipeline_id="<str value", pipeline_access_token="<str value>")
+source = PipelineDataSource(pipeline_id="<str value", pipeline_access_token="<str token>")
 data = {} # your json event
-res = pipeline_client.publish(request_body=data)
+res = source.publish(request_body=data)
 
 if res.status_code == 200:
     print("Published sucessfully")
@@ -55,33 +57,163 @@ Consume the transformed event from the pipeline
 ### Example Usage
 
 ```python
-import glassflow
+from glassflow import PipelineDataSink
 
-client = glassflow.GlassFlowClient()
-pipeline_client = client.pipeline_client(pipeline_id="<str value", pipeline_access_token="<str value>")
-res = pipeline_client.consume()
+sink = PipelineDataSink(pipeline_id="<str value", pipeline_access_token="<str value>")
+res = sink.consume()
 
 if res.status_code == 200:
-    print(res.body.event)
+    print(res.json())
 ```
 
+
 ## consume failed
 
 If the transformation failed for any event, they are available in a failed queue. You can consume those events from the pipeline
 
 ### Example Usage
 
 ```python
-import glassflow
+from glassflow import PipelineDataSink
 
-client = glassflow.GlassFlowClient()
-pipeline_client = client.pipeline_client(pipeline_id="<str value", pipeline_access_token="<str value>")
-res = pipeline_client.consume_failed()
+sink = PipelineDataSink(pipeline_id="<str value", pipeline_access_token="<str value>")
+res = sink.consume_failed()
 
 if res.status_code == 200:
-    print(res.body.event)
+    print(res.json())
+```
+
+
+## validate credentials
+
+Validate pipeline credentials (`pipeline_id` and `pipeline_access_token`) from source or sink
+
+### Example Usage
+
+```python
+from glassflow import PipelineDataSource, errors
+
+try:
+    source = PipelineDataSource(pipeline_id="<str value", pipeline_access_token="<str value>")
+    source.validate_credentials()
+except errors.PipelineNotFoundError as e:
+    print("Pipeline ID does not exist!")
+    raise e
+except errors.PipelineAccessTokenInvalidError as e:
+    print("Pipeline Access Token is invalid!")
+    raise e
+```
+
+
+## Pipeline Management
+
+In order to manage your pipelines with this SDK, one needs to provide the `PERSONAL_ACCESS_TOKEN` 
+to the GlassFlow client.
+
+```python
+from glassflow import GlassFlowClient
+
+client = GlassFlowClient(personal_access_token="<your personal access token>")
+```
+
+Now you can perform CRUD operations on your pipelines:
+
+* [list_pipelines](#list_pipelines) - Returns the list of pipelines available
+* [get_pipeline](#get_pipeline) - Returns a pipeline object from a given pipeline ID
+* [create](#create) - Create a new pipeline
+* [delete](#delete) - Delete an existing pipeline
+
+## list_pipelines
+
+Returns information about the available pipelines. It can be restricted to a
+specific space by passing the `space_id`.
+
+### Example Usage
+
+```python
+from glassflow import GlassFlowClient
+
+client = GlassFlowClient(personal_access_token="<your access token>")
+res = client.list_pipelines()
+```
+
+## get_pipeline
+
+Gets information about a pipeline from a given pipeline ID. It returns a Pipeline object
+which can be used manage the Pipeline. 
+
+### Example Usage
+
+```python
+from glassflow import GlassFlowClient
+
+client = GlassFlowClient(personal_access_token="<your access token>")
+pipeline = client.get_pipeline(pipeline_id="<your pipeline id>")
+
+print("Name:", pipeline.name)
 ```
 
+## create
+
+The Pipeline object has a create method that creates a new GlassFlow pipeline.
+
+### Example Usage
+
+```python
+from glassflow import Pipeline
+
+pipeline = Pipeline(
+    name="<your pipeline name>",
+    transformation_file="path/to/transformation.py",
+    space_id="<your space id>",
+    personal_access_token="<your personal access token>"
+).create()
+```
+
+In the next example we create a pipeline with Google PubSub source 
+and a webhook sink:
+
+```python
+from glassflow import Pipeline
+
+pipeline = Pipeline(
+    name="<your pipeline name>",
+    transformation_file="path/to/transformation.py",
+    space_id="<your space id>",
+    personal_access_token="<your personal access token>",
+    source_kind="google_pubsub",
+    source_config={
+        "project_id": "<your gcp project id>",
+        "subscription_id": "<your subscription id>",
+        "credentials_json": "<your credentials json>"
+    },
+    sink_kind="webhook",
+    sink_config={
+        "url": "<webhook url>",
+        "method": "<GET | POST | PUT | PATCH | DELETE>",
+        "headers": [{"header1": "header1_value"}]
+    }
+).create()
+```
+
+## delete
+
+The Pipeline object has a delete method to delete a pipeline
+
+### Example Usage
+
+```python
+from glassflow import Pipeline
+
+pipeline = Pipeline(
+    name="<your pipeline name>",
+    transformation_file="path/to/transformation.py",
+    space_id="<your space id>",
+    personal_access_token="<your personal access token>"
+).create()
+
+pipeline.delete()
+```
 
 ## Quickstart
 

USAGE.md

@@ -1,11 +1,13 @@
 ```python
 import glassflow
 
-client = glassflow.GlassFlowClient()
-pipeline = client.pipeline_client(pipeline_id="<pipeline_id>", pipeline_access_token="<pipeline_token>")
+source = glassflow.PipelineDataSource(pipeline_id="<pipeline_id>", pipeline_access_token="<pipeline_token>")
 data = {
     "name": "Hello World",
     "id": 1
 }
-res = pipeline.publish(request_body=data)
+source_res = source.publish(request_body=data)
+
+sink = glassflow.PipelineDataSink(pipeline_id="<pipeline_id>", pipeline_access_token="<pipeline_token>")
+sink_res = sink.consume()
 ```