fix workflow

.github/workflows/ci_workflow.yml

@@ -15,45 +15,50 @@ jobs:
       with:
         python-version: '${{ matrix.python-version }}'
 
-    - name: Install Poetry
-      run: |
-        python -m pip install --upgrade pip
-        pip install poetry==1.6.*
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v6
+
     - name: Install dependencies
-      run: |
-        poetry install
+      run: uv sync
+
     - name: Run pre-commit
       run: |
-        poetry run pre-commit install
-        poetry run pre-commit run --all-files
+        uv run pre-commit install
+        uv run pre-commit run --all-files
   pytest:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12"]
+        python-version: [ "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python '${{ matrix.python-version }}'
       uses: actions/setup-python@v2
       with:
         python-version: '${{ matrix.python-version }}'
-    - name: Install Poetry
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v6
+    - name: Set up Docker Compose
       run: |
-        pip install typing-extensions
-        python -m pip install --upgrade pip
-        pip install poetry==1.6.*
+        sudo curl -L "https://github.com/docker/compose/releases/download/v2.24.5/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+        sudo chmod +x /usr/local/bin/docker-compose
+        docker-compose --version # Verify installation
+      shell: bash
     - name: Install dependencies
-      run: |
-        poetry install
+      run: uv sync
+    - uses: actions/cache@v4
+      with:
+        path: ~/elasticsearch
+        key: ${{ runner.os }}-elasticsearch-9
+    - uses: ankane/setup-elasticsearch@v1
     - name: Test with pytest
-      run: |
-        poetry run pytest --capture=no
+      run: uv run pytest --capture=no
 
   integration:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.9", "3.12"]
+        python-version: [ "3.10", "3.11", "3.12"]
     steps:
     - uses: actions/checkout@v2
 
@@ -63,18 +68,21 @@ jobs:
         sudo chmod +x /usr/local/bin/docker-compose
         docker-compose --version # Verify installation
       shell: bash
-
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v6
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install meltano
-        meltano install
+      run: uv tool install meltano
+    - name: Install Meltano plugins
+      run: meltano install --clean
+    - uses: actions/cache@v4
+      with:
+        path: ~/elasticsearch
+        key: ${{ runner.os }}-elasticsearch-9
+    - uses: ankane/setup-elasticsearch@v1
     - name: smoke-test-tap
-      run: |
-        docker-compose up -d elasticsearch
-        sleep 30
-        meltano run tap-smoke-test target-elasticsearch
+
+      run: meltano run tap-smoke-test target-elasticsearch

meltano.yml

@@ -21,8 +21,6 @@ plugins:
     namespace: target_elasticsearch
     pip_url: -e .
     config:
-      username: $TARGET_ELASTICSEARCH_USERNAME
-      password: $TARGET_ELASTICSEARCH_PASSWORD
       index_format: "ecs-{{ stream_name }}-an-{{ foo }}-{{ current_timestamp_daily }}"
       stream_maps: # use stream maps to do more complex base record manipulation
         animals:

target_elasticsearch/sinks.py

@@ -32,7 +32,9 @@ def __init__(
     ):
         super().__init__(target, stream_name, schema, key_properties)
         self.client = self._authenticated_client()
-        self.index_schema_fields = self.config.get("index_schema_fields", {}).get(self.stream_name, {})
+        self.index_schema_fields = self.config.get("index_schema_fields", {}).get(
+            self.stream_name, {}
+        )
         self.metadata_fields = self.config.get("metadata_fields", {}).get(self.stream_name, {})
         self.index_mappings = self.config.get("index_mappings", {}).get(self.stream_name, {})
         self.index_name = None
@@ -97,7 +99,9 @@ def _build_fields(
         for k, v in mapping.items():
             match = jsonpath_ng.parse(v).find(record)
             if len(match) == 0:
-                self.logger.warning(f"schema key {k} with json path {v} could not be found in record: {record}")
+                self.logger.warning(
+                    f"schema key {k} with json path {v} could not be found in record: {record}"
+                )
                 schemas[k] = v
             else:
                 if len(match) > 1:
@@ -208,7 +212,9 @@ def process_batch(self, context: dict[str, Any]) -> None:
         Args:
             context: Dictionary containing batch processing context including records.
         """
-        updated_records, distinct_indices = self.build_request_body_and_distinct_indices(context["records"])
+        updated_records, distinct_indices = self.build_request_body_and_distinct_indices(
+            context["records"]
+        )
         for index in distinct_indices:
             self.create_index(index)
         try:

target_elasticsearch/target.py

@@ -149,10 +149,17 @@ def __init__(
         setup_mapper: bool = True,
     ) -> None:
         super().__init__(
-            config=config, parse_env_config=parse_env_config, validate_config=validate_config, setup_mapper=setup_mapper
+            config=config,
+            parse_env_config=parse_env_config,
+            validate_config=validate_config,
+            setup_mapper=setup_mapper,
+        )
+        assert bool(self.config.get("username") is None) == bool(
+            self.config.get("password") is None
+        )
+        assert bool(self.config.get("api_key_id") is None) == bool(
+            self.config.get("api_key") is None
         )
-        assert bool(self.config.get("username") is None) == bool(self.config.get("password") is None)
-        assert bool(self.config.get("api_key_id") is None) == bool(self.config.get("api_key") is None)
 
     @property
     def state(self) -> Dict:

tests/test_core.py

@@ -1,4 +1,3 @@
-import pytest
 import os
 from typing import Any
 
@@ -7,7 +6,7 @@
 
 SAMPLE_CONFIG: dict[str, Any] = {
     "username": "elastic",
-    "password": os.environ["TARGET_ELASTICSEARCH_PASSWORD"],
+    "password": os.environ.get("TARGET_ELASTICSEARCH_PASSWORD", "changeme"),
 }
 
 # Run standard built-in target tests from the SDK: