Skip to content

Commit 892535d

Browse files
authored
Chore/bump version 2.7.0 (#395)
* Bump version to 2.7.0 * Re-generate API docs
1 parent c02e759 commit 892535d

File tree

9 files changed

+4203
-4083
lines changed

9 files changed

+4203
-4083
lines changed

docs/api-reference/application.md

Lines changed: 48 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
class Application()
1111
```
1212

13-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L55)
13+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L56)
1414

1515
The main Application class.
1616

@@ -40,7 +40,7 @@ Most functionality is explained the various methods, except for
4040
```python
4141
from quixstreams import Application
4242

43-
# Set up an `app = Application` and `sdf = StreamingDataFrame`;
43+
# Set up an `app = Application` and `sdf = StreamingDataFrame`;
4444
# add some operations to `sdf` and then run everything.
4545

4646
app = Application(broker_address='localhost:9092', consumer_group='group')
@@ -79,10 +79,11 @@ def __init__(broker_address: Optional[Union[str, ConnectionConfig]] = None,
7979
quix_config_builder: Optional[QuixKafkaConfigsBuilder] = None,
8080
topic_manager: Optional[TopicManager] = None,
8181
request_timeout: float = 30,
82-
topic_create_timeout: float = 60)
82+
topic_create_timeout: float = 60,
83+
processing_guarantee: ProcessingGuarantee = "at-least-once")
8384
```
8485

85-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L93)
86+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L94)
8687

8788

8889
<br>
@@ -134,6 +135,7 @@ Default - `True`
134135
- `topic_manager`: A `TopicManager` instance
135136
- `request_timeout`: timeout (seconds) for REST-based requests
136137
- `topic_create_timeout`: timeout (seconds) for topic create finalization
138+
- `processing_guarantee`: Use "exactly-once" or "at-least-once" processing.
137139
<br><br>***Error Handlers***<br>
138140
To handle errors, `Application` accepts callbacks triggered when
139141
exceptions occur on different stages of stream processing. If the callback
@@ -158,29 +160,33 @@ instead of the default one.
158160

159161
```python
160162
@classmethod
161-
def Quix(cls,
162-
consumer_group: Optional[str] = None,
163-
auto_offset_reset: AutoOffsetReset = "latest",
164-
consumer_extra_config: Optional[dict] = None,
165-
producer_extra_config: Optional[dict] = None,
166-
state_dir: str = "state",
167-
rocksdb_options: Optional[RocksDBOptionsType] = None,
168-
on_consumer_error: Optional[ConsumerErrorCallback] = None,
169-
on_processing_error: Optional[ProcessingErrorCallback] = None,
170-
on_producer_error: Optional[ProducerErrorCallback] = None,
171-
on_message_processed: Optional[MessageProcessedCallback] = None,
172-
consumer_poll_timeout: float = 1.0,
173-
producer_poll_timeout: float = 0.0,
174-
loglevel: Optional[LogLevel] = "INFO",
175-
quix_config_builder: Optional[QuixKafkaConfigsBuilder] = None,
176-
auto_create_topics: bool = True,
177-
use_changelog_topics: bool = True,
178-
topic_manager: Optional[QuixTopicManager] = None,
179-
request_timeout: float = 30,
180-
topic_create_timeout: float = 60) -> Self
163+
def Quix(
164+
cls,
165+
consumer_group: Optional[str] = None,
166+
auto_offset_reset: AutoOffsetReset = "latest",
167+
consumer_extra_config: Optional[dict] = None,
168+
producer_extra_config: Optional[dict] = None,
169+
state_dir: str = "state",
170+
rocksdb_options: Optional[RocksDBOptionsType] = None,
171+
on_consumer_error: Optional[ConsumerErrorCallback] = None,
172+
on_processing_error: Optional[ProcessingErrorCallback] = None,
173+
on_producer_error: Optional[ProducerErrorCallback] = None,
174+
on_message_processed: Optional[MessageProcessedCallback] = None,
175+
consumer_poll_timeout: float = 1.0,
176+
producer_poll_timeout: float = 0.0,
177+
loglevel: Optional[LogLevel] = "INFO",
178+
quix_config_builder: Optional[QuixKafkaConfigsBuilder] = None,
179+
auto_create_topics: bool = True,
180+
use_changelog_topics: bool = True,
181+
topic_manager: Optional[QuixTopicManager] = None,
182+
request_timeout: float = 30,
183+
topic_create_timeout: float = 60,
184+
processing_guarantee: Literal["at-least-once",
185+
"exactly-once"] = "exactly-once"
186+
) -> Self
181187
```
182188

183-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L313)
189+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L330)
184190

185191
>***NOTE:*** DEPRECATED: use Application with `quix_sdk_token` argument instead.
186192

@@ -253,6 +259,7 @@ Default - `True`
253259
- `topic_manager`: A `QuixTopicManager` instance
254260
- `request_timeout`: timeout (seconds) for REST-based requests
255261
- `topic_create_timeout`: timeout (seconds) for topic create finalization
262+
- `processing_guarantee`: Use "exactly-once" or "at-least-once" processing.
256263
<br><br>***Error Handlers***<br>
257264
To handle errors, `Application` accepts callbacks triggered when
258265
exceptions occur on different stages of stream processing. If the callback
@@ -290,7 +297,7 @@ def topic(name: str,
290297
timestamp_extractor: Optional[TimestampExtractor] = None) -> Topic
291298
```
292299

293-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L451)
300+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L471)
294301

295302
Create a topic definition.
296303

@@ -371,7 +378,7 @@ topic = app.topic("input-topic", timestamp_extractor=custom_ts_extractor)
371378
def dataframe(topic: Topic) -> StreamingDataFrame
372379
```
373380

374-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L531)
381+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L551)
375382

376383
A simple helper method that generates a `StreamingDataFrame`, which is used
377384

@@ -421,7 +428,7 @@ to be used as an input topic.
421428
def stop(fail: bool = False)
422429
```
423430

424-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L570)
431+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L590)
425432

426433
Stop the internal poll loop and the message processing.
427434

@@ -448,7 +455,7 @@ to unhandled exception, and it shouldn't commit the current checkpoint.
448455
def get_producer() -> Producer
449456
```
450457

451-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L593)
458+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L613)
452459

453460
Create and return a pre-configured Producer instance.
454461
The Producer is initialized with params passed to Application.
@@ -483,19 +490,23 @@ with app.get_producer() as producer:
483490
def get_consumer(auto_commit_enable: bool = True) -> Consumer
484491
```
485492

486-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L623)
493+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L643)
487494

488495
Create and return a pre-configured Consumer instance.
489496
The Consumer is initialized with params passed to Application.
490497

491-
It's useful for consuming data from Kafka outside the standard Application processing flow.
492-
(e.g. to consume test data from a topic).
493-
Using it within the StreamingDataFrame functions is not recommended, as it creates a new Consumer instance
498+
It's useful for consuming data from Kafka outside the standard
499+
Application processing flow.
500+
(e.g., to consume test data from a topic).
501+
Using it within the StreamingDataFrame functions is not recommended, as it
502+
creates a new Consumer instance
494503
each time, which is not optimized for repeated use in a streaming pipeline.
495504

496-
Note: By default this consumer does not autocommit consumed offsets to allow exactly-once processing.
505+
Note: By default, this consumer does not autocommit the consumed offsets to allow
506+
at-least-once processing.
497507
To store the offset call store_offsets() after processing a message.
498-
If autocommit is necessary set `enable.auto.offset.store` to True in the consumer config when creating the app.
508+
If autocommit is necessary set `enable.auto.offset.store` to True in
509+
the consumer config when creating the app.
499510

500511

501512
<br>
@@ -528,7 +539,7 @@ with app.get_consumer() as consumer:
528539
def clear_state()
529540
```
530541

531-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L666)
542+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L690)
532543

533544
Clear the state of the application.
534545

@@ -542,7 +553,7 @@ Clear the state of the application.
542553
def run(dataframe: StreamingDataFrame)
543554
```
544555

545-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/app.py#L672)
556+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/app.py#L696)
546557

547558
Start processing data from Kafka using provided `StreamingDataFrame`
548559

docs/api-reference/context.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
def set_message_context(context: Optional[MessageContext])
1313
```
1414

15-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/context.py#L20)
15+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/context.py#L20)
1616

1717
Set a MessageContext for the current message in the given `contextvars.Context`
1818

@@ -55,7 +55,7 @@ sdf = sdf.update(lambda value: alter_context(value))
5555
def message_context() -> MessageContext
5656
```
5757

58-
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/ea3d07177df3f11deb3c51e8337534408f5f68c1/quixstreams/context.py#L51)
58+
[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/209d74b1262aa6d90dddd1a01473c83d9d10fdce/quixstreams/context.py#L51)
5959

6060
Get a MessageContext for the current message, which houses most of the message
6161

0 commit comments

Comments
 (0)