diff --git a/docs/api-reference/application.md b/docs/api-reference/application.md index a63a4c15c..de8d8f2a8 100644 --- a/docs/api-reference/application.md +++ b/docs/api-reference/application.md @@ -10,7 +10,7 @@ class Application() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L53) The main Application class. @@ -82,7 +82,7 @@ def __init__(broker_address: Optional[Union[str, ConnectionConfig]] = None, topic_create_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L91) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L91)
@@ -179,7 +179,7 @@ def Quix(cls, topic_create_timeout: float = 60) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L304) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L304) >***NOTE:*** DEPRECATED: use Application with `quix_sdk_token` argument instead. @@ -289,7 +289,7 @@ def topic(name: str, timestamp_extractor: Optional[TimestampExtractor] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L442) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L442) Create a topic definition. @@ -370,7 +370,7 @@ topic = app.topic("input-topic", timestamp_extractor=custom_ts_extractor) def dataframe(topic: Topic) -> StreamingDataFrame ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L522) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L522) A simple helper method that generates a `StreamingDataFrame`, which is used @@ -420,7 +420,7 @@ to be used as an input topic. def stop(fail: bool = False) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L561) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L561) Stop the internal poll loop and the message processing. @@ -447,7 +447,7 @@ to unhandled exception, and it shouldn't commit the current checkpoint. def get_producer() -> Producer ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L584) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L584) Create and return a pre-configured Producer instance. The Producer is initialized with params passed to Application. @@ -482,7 +482,7 @@ with app.get_producer() as producer: def get_consumer(auto_commit_enable: bool = True) -> Consumer ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L614) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L614) Create and return a pre-configured Consumer instance. The Consumer is initialized with params passed to Application. @@ -527,7 +527,7 @@ with app.get_consumer() as consumer: def clear_state() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L657) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L657) Clear the state of the application. @@ -541,7 +541,7 @@ Clear the state of the application. def run(dataframe: StreamingDataFrame) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L663) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L663) Start processing data from Kafka using provided `StreamingDataFrame` diff --git a/docs/api-reference/context.md b/docs/api-reference/context.md index cf542b5ec..4c07058fd 100644 --- a/docs/api-reference/context.md +++ b/docs/api-reference/context.md @@ -12,7 +12,7 @@ def set_message_context(context: Optional[MessageContext]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/context.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/context.py#L20) Set a MessageContext for the current message in the given `contextvars.Context` @@ -55,7 +55,7 @@ sdf = sdf.update(lambda value: alter_context(value)) def message_context() -> MessageContext ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/context.py#L51) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/context.py#L51) Get a MessageContext for the current message, which houses most of the message diff --git a/docs/api-reference/dataframe.md b/docs/api-reference/dataframe.md index 3dd21d1ae..ca22b7160 100644 --- a/docs/api-reference/dataframe.md +++ b/docs/api-reference/dataframe.md @@ -10,7 +10,7 @@ class StreamingDataFrame(BaseStreaming) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L60) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L62) `StreamingDataFrame` is the main object you will use for ETL work. @@ -81,7 +81,7 @@ def apply(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L175) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L177) Apply a function to transform the value and return a new value. @@ -139,7 +139,7 @@ def update(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L264) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L266) Apply a function to mutate value in-place or to perform a side effect @@ -197,7 +197,7 @@ def filter(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L352) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L354) Filter value using provided function. @@ -249,7 +249,7 @@ def group_by(key: Union[str, Callable[[Any], Any]], key_serializer: Optional[SerializerType] = "json") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L438) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L440) "Groups" messages by re-keying them via the provided group_by operation @@ -314,7 +314,7 @@ a clone with this operation added (assign to keep its effect). def contains(key: str) -> StreamingSeries ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L516) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L518) Check if the key is present in the Row value. @@ -353,7 +353,7 @@ or False otherwise. def to_topic(topic: Topic, key: Optional[Callable[[Any], Any]] = None) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L541) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L543) Produce current value to a topic. You can optionally specify a new key. @@ -396,7 +396,7 @@ By default, the current message key will be used. def set_timestamp(func: Callable[[Any, Any, int, Any], int]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L582) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L584) Set a new timestamp based on the current message value and its metadata. @@ -425,10 +425,64 @@ sdf = sdf.set_timestamp(lambda value, key, timestamp, headers: value['new_timest
***Arguments:*** -- `func`: callable accepting the current value and the current timestamp. +- `func`: callable accepting the current value, key, timestamp, and headers. It's expected to return a new timestamp as integer in milliseconds. +
+***Returns:*** + +a new StreamingDataFrame instance + + + +

+ +#### StreamingDataFrame.set\_headers + +```python +def set_headers( + func: Callable[ + [Any, Any, int, List[Tuple[str, HeaderValue]]], + Collection[Tuple[str, HeaderValue]], + ] +) -> Self +``` + +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L625) + +Set new message headers based on the current message value and metadata. + +The new headers will be used when producing messages to the output topics. + +The provided callback must accept value, key, timestamp, and headers, +and return a new collection of (header, value) tuples. + + +
+***Example Snippet:*** + +```python +from quixstreams import Application + + +app = Application() +input_topic = app.topic("data") + +sdf = app.dataframe(input_topic) +# Updating the record's headers based on the value and metadata +sdf = sdf.set_headers(lambda value, key, timestamp, headers: [('id', value['id'])]) +``` + + +
+***Arguments:*** + +- `func`: callable accepting the current value, key, timestamp, and headers. +It's expected to return a new set of headers +as a collection of (header, value) tuples. + +
***Returns:*** @@ -446,7 +500,7 @@ def compose( ) -> Dict[str, VoidExecutor] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L623) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L676) Compose all functions of this StreamingDataFrame into one big closure. @@ -500,7 +554,7 @@ def test(value: Any, topic: Optional[Topic] = None) -> List[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L660) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L713) A shorthand to test `StreamingDataFrame` with provided value @@ -537,7 +591,7 @@ def tumbling_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> TumblingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L697) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L750) Create a tumbling window transformation on this StreamingDataFrame. @@ -623,7 +677,7 @@ def hopping_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> HoppingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L773) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L826) Create a hopping window transformation on this StreamingDataFrame. @@ -717,7 +771,7 @@ sdf = ( class StreamingSeries(BaseStreaming) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L47) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L47) `StreamingSeries` are typically generated by `StreamingDataframes` when getting elements from, or performing certain operations on, a `StreamingDataframe`, @@ -783,7 +837,7 @@ sdf = sdf[["column_a"] & (sdf["new_sum_field"] >= 10)] def from_apply_callback(cls, func: ApplyWithMetadataCallback) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L107) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L107) Create a StreamingSeries from a function. @@ -811,7 +865,7 @@ instance of `StreamingSeries` def apply(func: ApplyCallback) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L121) Add a callable to the execution list for this series. @@ -863,7 +917,7 @@ a new `StreamingSeries` with the new callable added def compose_returning() -> ReturningExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L155) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L155) Compose a list of functions from this StreamingSeries and its parents into one @@ -894,7 +948,7 @@ def compose( None]] = None) -> VoidExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L170) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L170) Compose all functions of this StreamingSeries into one big closure. @@ -952,7 +1006,7 @@ def test(value: Any, ctx: Optional[MessageContext] = None) -> Any ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L214) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L214) A shorthand to test `StreamingSeries` with provided value @@ -984,7 +1038,7 @@ result of `StreamingSeries` def isin(other: Container) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L269) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L269) Check if series value is in "other". @@ -1029,7 +1083,7 @@ new StreamingSeries def contains(other: Union[Self, object]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L296) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L296) Check if series value contains "other" @@ -1074,7 +1128,7 @@ new StreamingSeries def is_(other: Union[Self, object]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L321) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L321) Check if series value refers to the same object as `other` @@ -1116,7 +1170,7 @@ new StreamingSeries def isnot(other: Union[Self, object]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L344) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L344) Check if series value does not refer to the same object as `other` @@ -1159,7 +1213,7 @@ new StreamingSeries def isnull() -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L368) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L368) Check if series value is None. @@ -1196,7 +1250,7 @@ new StreamingSeries def notnull() -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L391) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L391) Check if series value is not None. @@ -1233,7 +1287,7 @@ new StreamingSeries def abs() -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L414) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L414) Get absolute value of the series value. diff --git a/docs/api-reference/kafka.md b/docs/api-reference/kafka.md index cb42d1f21..7a2e105d3 100644 --- a/docs/api-reference/kafka.md +++ b/docs/api-reference/kafka.md @@ -10,7 +10,7 @@ class Producer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L44) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L44) @@ -25,7 +25,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], extra_config: Optional[dict] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L45) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L45) A wrapper around `confluent_kafka.Producer`. @@ -64,7 +64,7 @@ def produce(topic: str, on_delivery: Optional[DeliveryCallback] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L80) Produce a message to a topic. @@ -99,7 +99,7 @@ for the produced message. def poll(timeout: float = 0) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L141) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L141) Polls the producer for events and calls `on_delivery` callbacks. @@ -120,7 +120,7 @@ Polls the producer for events and calls `on_delivery` callbacks. def flush(timeout: Optional[float] = None) -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L149) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L149) Wait for all messages in the Producer queue to be delivered. @@ -149,7 +149,7 @@ number of messages remaining to flush class Consumer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L64) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L64) @@ -169,7 +169,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], extra_config: Optional[dict] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L65) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L65) A wrapper around `confluent_kafka.Consumer`. @@ -212,7 +212,7 @@ Note: values passed as arguments override values in `extra_config`. def poll(timeout: Optional[float] = None) -> Optional[Message] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L128) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L128) Consumes a single message, calls callbacks and returns events. @@ -253,7 +253,7 @@ def subscribe(topics: List[str], on_lost: Optional[RebalancingCallback] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L146) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L146) Set subscription to supplied list of topics @@ -296,7 +296,7 @@ for example, may fail. def unsubscribe() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L240) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L240) Remove current subscription. @@ -316,7 +316,7 @@ def store_offsets(message: Optional[Message] = None, offsets: Optional[List[TopicPartition]] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L248) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L248) .. py:function:: store_offsets([message=None], [offsets=None]) @@ -351,7 +351,7 @@ def commit(message: Optional[Message] = None, asynchronous: bool = True) -> Optional[List[TopicPartition]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L282) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L282) Commit a message or a list of offsets. @@ -389,7 +389,7 @@ def committed(partitions: List[TopicPartition], timeout: Optional[float] = None) -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L322) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L322) .. py:function:: committed(partitions, [timeout=None]) @@ -426,7 +426,7 @@ def get_watermark_offsets(partition: TopicPartition, cached: bool = False) -> Tuple[int, int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L342) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L342) Retrieve low and high offsets for the specified partition. @@ -465,7 +465,7 @@ def list_topics(topic: Optional[str] = None, timeout: Optional[float] = None) -> ClusterMetadata ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L368) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L368) .. py:function:: list_topics([topic=None], [timeout=-1]) @@ -498,7 +498,7 @@ None or -1 is infinite. Default: None def memberid() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L391) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L391) Return this client's broker-assigned group member id. @@ -521,7 +521,7 @@ def offsets_for_times(partitions: List[TopicPartition], timeout: Optional[float] = None) -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L404) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L404) Look up offsets by timestamp for the specified partitions. @@ -550,7 +550,7 @@ last message in the partition, a value of -1 will be returned. def pause(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L430) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L430) Pause consumption for the provided list of partitions. @@ -578,7 +578,7 @@ Does NOT affect the result of Consumer.assignment(). def resume(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L444) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L444) .. py:function:: resume(partitions) @@ -604,7 +604,7 @@ Resume consumption for the provided list of partitions. def position(partitions: List[TopicPartition]) -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L456) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L456) Retrieve current positions (offsets) for the specified partitions. @@ -637,7 +637,7 @@ the last consumed message + 1. def seek(partition: TopicPartition) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L470) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L470) Set consume position for partition to offset. @@ -669,7 +669,7 @@ pass the offset in an `assign()` call. def assignment() -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L487) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L487) Returns the current partition assignment. @@ -694,7 +694,7 @@ Returns the current partition assignment. def set_sasl_credentials(username: str, password: str) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L500) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L500) Sets the SASL credentials used for this client. These credentials will overwrite the old ones, and will be used the next @@ -713,7 +713,7 @@ This method is applicable only to SASL PLAIN and SCRAM mechanisms. def incremental_assign(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L512) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L512) Assign new partitions. @@ -733,7 +733,7 @@ Any additional partitions besides the ones passed during the `Consumer` def incremental_unassign(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L524) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L524) Revoke partitions. @@ -749,7 +749,7 @@ Can be called outside an on_revoke callback. def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L532) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L532) Close down and terminate the Kafka Consumer. diff --git a/docs/api-reference/quixstreams.md b/docs/api-reference/quixstreams.md index 8643006e1..3ef115715 100644 --- a/docs/api-reference/quixstreams.md +++ b/docs/api-reference/quixstreams.md @@ -14,7 +14,7 @@ def configure_logging(loglevel: Optional[LogLevel]) -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/logging.py#L26) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/logging.py#L26) Configure "quixstreams" logger. @@ -51,7 +51,7 @@ True if logging config has been updated, otherwise False. def strip_workspace_id_prefix(workspace_id: str, s: str) -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L31) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L31) Remove the workspace ID from a given string if it starts with it, @@ -74,7 +74,7 @@ the string with workspace_id prefix removed def prepend_workspace_id(workspace_id: str, s: str) -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L43) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L43) Add the workspace ID as a prefix to a given string if it does not have it, @@ -98,7 +98,7 @@ the string with workspace_id prepended class QuixApplicationConfig() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L56) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L56) A convenience container class for Quix Application configs. @@ -110,7 +110,7 @@ A convenience container class for Quix Application configs. class QuixKafkaConfigsBuilder() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L66) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L66) Retrieves all the necessary information from the Quix API and builds all the objects required to connect a confluent-kafka client to the Quix Platform. @@ -136,7 +136,7 @@ def __init__(quix_sdk_token: Optional[str] = None, topic_create_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L82) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L82) **Arguments**: @@ -151,7 +151,7 @@ def __init__(quix_sdk_token: Optional[str] = None, def strip_workspace_id_prefix(s: str) -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L160) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L160) Remove the workspace ID from a given string if it starts with it, @@ -173,7 +173,7 @@ the string with workspace_id prefix removed def prepend_workspace_id(s: str) -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L170) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L170) Add the workspace ID as a prefix to a given string if it does not have it, @@ -196,7 +196,7 @@ def search_for_workspace(workspace_name_or_id: Optional[str] = None, timeout: Optional[float] = None) -> Optional[dict] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L180) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L180) Search for a workspace given an expected workspace name or id. @@ -218,7 +218,7 @@ def get_workspace_info(known_workspace_topic: Optional[str] = None, timeout: Optional[float] = None) -> dict ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L222) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L222) Queries for workspace data from the Quix API, regardless of instance cache, @@ -240,7 +240,7 @@ def search_workspace_for_topic( timeout: Optional[float] = None) -> Optional[str] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L250) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L250) Search through all the topics in the given workspace id to see if there is a @@ -266,7 +266,7 @@ def search_for_topic_workspace(topic: str, ) -> Optional[dict] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L271) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L271) Find what workspace a topic belongs to. @@ -292,7 +292,7 @@ def create_topics(topics: List[Topic], finalize_timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L369) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L369) Create topics in a Quix cluster. @@ -312,7 +312,7 @@ def get_topic(topic_name: str, timeout: Optional[float] = None) -> Optional[dict] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L419) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L419) return the topic ID (the actual cluster topic name) if it exists, else None @@ -338,7 +338,7 @@ def confirm_topics_exist(topics: Union[List[Topic], List[str]], timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L451) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L451) Confirm whether the desired set of topics exists in the Quix workspace. @@ -355,7 +355,7 @@ Confirm whether the desired set of topics exists in the Quix workspace. def get_application_config(consumer_group_id: str) -> QuixApplicationConfig ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/config.py#L483) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/config.py#L483) Get all the necessary attributes for an Application to run on Quix Cloud. @@ -379,7 +379,7 @@ a QuixApplicationConfig instance class QuixEnvironment() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/env.py#L7) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/env.py#L7) Class to access various Quix platform environment settings @@ -392,7 +392,7 @@ Class to access various Quix platform environment settings def state_management_enabled() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/env.py#L19) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/env.py#L19) Check whether "State management" is enabled for the current deployment @@ -409,7 +409,7 @@ True if state management is enabled, otherwise False def deployment_id() -> Optional[str] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/env.py#L27) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/env.py#L27) Return current Quix deployment id. @@ -429,7 +429,7 @@ deployment id or None def workspace_id() -> Optional[str] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/env.py#L39) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/env.py#L39) Return Quix workspace id if set @@ -446,7 +446,7 @@ workspace id or None def portal_api() -> Optional[str] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/env.py#L47) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/env.py#L47) Return Quix Portal API url if set @@ -463,7 +463,7 @@ portal API URL or None def state_dir() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/env.py#L56) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/env.py#L56) Return application state directory on Quix. @@ -483,7 +483,7 @@ path to state dir def check_state_management_enabled() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/checks.py#L11) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/checks.py#L11) Check if State Management feature is enabled for the current deployment on Quix platform. @@ -497,7 +497,7 @@ If it's disabled, the exception will be raised. def check_state_dir(state_dir: str) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/checks.py#L28) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/checks.py#L28) Check if Application "state_dir" matches the state dir on Quix platform. @@ -523,7 +523,7 @@ If it doesn't match, the warning will be logged. class QuixPortalApiService() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/api.py#L19) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/api.py#L19) A light wrapper around the Quix Portal Api. If used in the Quix Platform, it will use that workspaces auth token and portal endpoint, else you must provide it. @@ -545,7 +545,7 @@ def get_workspace_certificate(workspace_id: Optional[str] = None, timeout: float = 30) -> Optional[bytes] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/api.py#L119) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/api.py#L119) Get a workspace TLS certificate if available. @@ -576,7 +576,7 @@ certificate as bytes if present, or None class QuixTopicManager(TopicManager) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/topic_manager.py#L9) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/topic_manager.py#L9) The source of all topic management with quixstreams. @@ -600,7 +600,7 @@ def __init__(topic_admin: TopicAdmin, create_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/platforms/quix/topic_manager.py#L30) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/platforms/quix/topic_manager.py#L30) **Arguments**: @@ -622,7 +622,7 @@ generated for you. class StreamingDataFrame(BaseStreaming) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L60) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L62) `StreamingDataFrame` is the main object you will use for ETL work. @@ -685,7 +685,7 @@ def apply(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L175) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L177) Apply a function to transform the value and return a new value. @@ -737,7 +737,7 @@ def update(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L264) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L266) Apply a function to mutate value in-place or to perform a side effect @@ -789,7 +789,7 @@ def filter(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L352) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L354) Filter value using provided function. @@ -835,7 +835,7 @@ def group_by(key: Union[str, Callable[[Any], Any]], key_serializer: Optional[SerializerType] = "json") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L438) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L440) "Groups" messages by re-keying them via the provided group_by operation @@ -892,7 +892,7 @@ a clone with this operation added (assign to keep its effect). def contains(key: str) -> StreamingSeries ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L516) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L518) Check if the key is present in the Row value. @@ -923,7 +923,7 @@ or False otherwise. def to_topic(topic: Topic, key: Optional[Callable[[Any], Any]] = None) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L541) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L543) Produce current value to a topic. You can optionally specify a new key. @@ -960,7 +960,7 @@ By default, the current message key will be used. def set_timestamp(func: Callable[[Any, Any, int, Any], int]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L582) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L584) Set a new timestamp based on the current message value and its metadata. @@ -985,13 +985,59 @@ sdf = sdf.set_timestamp(lambda value, key, timestamp, headers: value['new_timest **Arguments**: -- `func`: callable accepting the current value and the current timestamp. +- `func`: callable accepting the current value, key, timestamp, and headers. It's expected to return a new timestamp as integer in milliseconds. **Returns**: a new StreamingDataFrame instance + + +#### StreamingDataFrame.set\_headers + +```python +def set_headers( + func: Callable[ + [Any, Any, int, List[Tuple[str, HeaderValue]]], + Collection[Tuple[str, HeaderValue]], + ] +) -> Self +``` + +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L625) + +Set new message headers based on the current message value and metadata. + +The new headers will be used when producing messages to the output topics. + +The provided callback must accept value, key, timestamp, and headers, +and return a new collection of (header, value) tuples. + +Example Snippet: + +```python +from quixstreams import Application + + +app = Application() +input_topic = app.topic("data") + +sdf = app.dataframe(input_topic) +# Updating the record's headers based on the value and metadata +sdf = sdf.set_headers(lambda value, key, timestamp, headers: [('id', value['id'])]) +``` + +**Arguments**: + +- `func`: callable accepting the current value, key, timestamp, and headers. +It's expected to return a new set of headers +as a collection of (header, value) tuples. + +**Returns**: + +a new StreamingDataFrame instance + #### StreamingDataFrame.compose @@ -1002,7 +1048,7 @@ def compose( ) -> Dict[str, VoidExecutor] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L623) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L676) Compose all functions of this StreamingDataFrame into one big closure. @@ -1048,7 +1094,7 @@ def test(value: Any, topic: Optional[Topic] = None) -> List[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L660) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L713) A shorthand to test `StreamingDataFrame` with provided value @@ -1079,7 +1125,7 @@ def tumbling_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> TumblingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L697) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L750) Create a tumbling window transformation on this StreamingDataFrame. @@ -1156,7 +1202,7 @@ def hopping_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> HoppingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/dataframe.py#L773) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/dataframe.py#L826) Create a hopping window transformation on this StreamingDataFrame. @@ -1243,7 +1289,7 @@ like `sum`, `count`, etc. and applied to the StreamingDataFrame. class StreamingSeries(BaseStreaming) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L47) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L47) `StreamingSeries` are typically generated by `StreamingDataframes` when getting elements from, or performing certain operations on, a `StreamingDataframe`, @@ -1301,7 +1347,7 @@ sdf = sdf[["column_a"] & (sdf["new_sum_field"] >= 10)] def from_apply_callback(cls, func: ApplyWithMetadataCallback) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L107) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L107) Create a StreamingSeries from a function. @@ -1323,7 +1369,7 @@ instance of `StreamingSeries` def apply(func: ApplyCallback) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L121) Add a callable to the execution list for this series. @@ -1367,7 +1413,7 @@ a new `StreamingSeries` with the new callable added def compose_returning() -> ReturningExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L155) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L155) Compose a list of functions from this StreamingSeries and its parents into one @@ -1394,7 +1440,7 @@ def compose( None]] = None) -> VoidExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L170) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L170) Compose all functions of this StreamingSeries into one big closure. @@ -1444,7 +1490,7 @@ def test(value: Any, ctx: Optional[MessageContext] = None) -> Any ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L214) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L214) A shorthand to test `StreamingSeries` with provided value @@ -1470,7 +1516,7 @@ result of `StreamingSeries` def isin(other: Container) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L269) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L269) Check if series value is in "other". @@ -1507,7 +1553,7 @@ new StreamingSeries def contains(other: Union[Self, object]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L296) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L296) Check if series value contains "other" @@ -1544,7 +1590,7 @@ new StreamingSeries def is_(other: Union[Self, object]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L321) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L321) Check if series value refers to the same object as `other` @@ -1578,7 +1624,7 @@ new StreamingSeries def isnot(other: Union[Self, object]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L344) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L344) Check if series value does not refer to the same object as `other` @@ -1613,7 +1659,7 @@ new StreamingSeries def isnull() -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L368) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L368) Check if series value is None. @@ -1644,7 +1690,7 @@ new StreamingSeries def notnull() -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L391) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L391) Check if series value is not None. @@ -1675,7 +1721,7 @@ new StreamingSeries def abs() -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/series.py#L414) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/series.py#L414) Get absolute value of the series value. @@ -1711,7 +1757,7 @@ new StreamingSeries def ensure_milliseconds(delta: Union[int, timedelta]) -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/utils.py#L5) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/utils.py#L5) Convert timedelta to milliseconds. @@ -1743,7 +1789,7 @@ timedelta value in milliseconds as `int` class FixedTimeWindowDefinition(abc.ABC) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L20) @@ -1753,7 +1799,7 @@ class FixedTimeWindowDefinition(abc.ABC) def sum() -> "FixedTimeWindow" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L67) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L67) Configure the window to aggregate data by summing up values within @@ -1771,7 +1817,7 @@ an instance of `FixedTimeWindow` configured to perform sum aggregation. def count() -> "FixedTimeWindow" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L94) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L94) Configure the window to aggregate data by counting the number of values @@ -1789,7 +1835,7 @@ an instance of `FixedTimeWindow` configured to perform record count. def mean() -> "FixedTimeWindow" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L121) Configure the window to aggregate data by calculating the mean of the values @@ -1809,7 +1855,7 @@ def reduce(reducer: Callable[[Any, Any], Any], initializer: Callable[[Any], Any]) -> "FixedTimeWindow" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L152) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L152) Configure the window to perform a custom aggregation using `reducer` @@ -1858,7 +1904,7 @@ A window configured to perform custom reduce aggregation on the data. def max() -> "FixedTimeWindow" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L212) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L212) Configure a window to aggregate the maximum value within each window period. @@ -1875,7 +1921,7 @@ value within each window period. def min() -> "FixedTimeWindow" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/definitions.py#L241) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/definitions.py#L241) Configure a window to aggregate the minimum value within each window period. @@ -1900,7 +1946,7 @@ value within each window period. class FixedTimeWindow() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/time_based.py#L32) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/time_based.py#L32) @@ -1910,7 +1956,7 @@ class FixedTimeWindow() def final() -> "StreamingDataFrame" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/time_based.py#L107) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/time_based.py#L107) Apply the window aggregation and return results only when the windows are closed. @@ -1941,7 +1987,7 @@ can remain unprocessed until the message the same key is received. def current() -> "StreamingDataFrame" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/time_based.py#L145) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/time_based.py#L145) Apply the window transformation to the StreamingDataFrame to return results for each updated window. @@ -1972,7 +2018,7 @@ def get_window_ranges(timestamp_ms: int, step_ms: Optional[int] = None) -> List[Tuple[int, int]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/dataframe/windows/base.py#L18) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/dataframe/windows/base.py#L18) Get a list of window ranges for the given timestamp. @@ -2002,7 +2048,7 @@ a list of (, ) tuples class RowProducer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowproducer.py#L18) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowproducer.py#L18) A producer class that is capable of serializing Rows to bytes and send them to Kafka. @@ -2034,7 +2080,7 @@ def produce_row(row: Row, timestamp: Optional[int] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowproducer.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowproducer.py#L53) Serialize Row to bytes according to the Topic serialization settings @@ -2058,7 +2104,7 @@ If this method fails, it will trigger the provided "on_error" callback. def poll(timeout: float = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowproducer.py#L93) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowproducer.py#L93) Polls the producer for events and calls `on_delivery` callbacks. @@ -2080,7 +2126,7 @@ If `poll()` fails, it will trigger the provided "on_error" callback class StreamFunction(abc.ABC) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L65) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L65) A base class for all the streaming operations in Quix Streams. @@ -2096,7 +2142,7 @@ values. def get_executor(child_executor: VoidExecutor) -> VoidExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L79) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L79) Returns a wrapper to be called on a value, key and timestamp. @@ -2108,7 +2154,7 @@ Returns a wrapper to be called on a value, key and timestamp. class ApplyFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L85) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L85) Wrap a function into "Apply" function. @@ -2123,7 +2169,7 @@ and its result will always be passed downstream. class ApplyWithMetadataFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L125) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L125) Wrap a function into "Apply" function. @@ -2139,7 +2185,7 @@ and its result will always be passed downstream. class FilterFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L166) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L166) Wraps a function into a "Filter" function. The result of a Filter function is interpreted as boolean. @@ -2155,7 +2201,7 @@ value is filtered out. class FilterWithMetadataFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L187) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L187) Wraps a function into a "Filter" function. @@ -2173,7 +2219,7 @@ Otherwise, the value will be filtered out. class UpdateFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L210) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L210) Wrap a function into an "Update" function. @@ -2191,7 +2237,7 @@ downstream. class UpdateWithMetadataFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L233) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L233) Wrap a function into an "Update" function. @@ -2209,7 +2255,7 @@ downstream. class TransformFunction(StreamFunction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/functions.py#L256) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/functions.py#L256) Wrap a function into a "Transform" function. @@ -2239,7 +2285,7 @@ The result of the callback will always be passed downstream. class Stream() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L34) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L34) @@ -2250,7 +2296,7 @@ def __init__(func: Optional[StreamFunction] = None, parent: Optional[Self] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L35) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L35) A base class for all streaming operations. @@ -2301,7 +2347,7 @@ def add_filter(func: Union[FilterCallback, FilterWithMetadataCallback], metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L97) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L97) Add a function to filter values from the Stream. @@ -2336,7 +2382,7 @@ def add_apply(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L122) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L122) Add an "apply" function to the Stream. @@ -2367,7 +2413,7 @@ def add_update(func: Union[UpdateCallback, UpdateWithMetadataCallback], metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L155) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L155) Add an "update" function to the Stream, that will mutate the input value. @@ -2395,7 +2441,7 @@ def add_transform(func: Union[TransformCallback, TransformExpandedCallback], expand: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L179) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L179) Add a "transform" function to the Stream, that will mutate the input value. @@ -2424,7 +2470,7 @@ a new Stream derived from the current one def diff(other: "Stream") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L204) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L204) Takes the difference between Streams `self` and `other` based on their last @@ -2456,7 +2502,7 @@ new `Stream` instance including all the Streams from the diff def tree() -> List[Self] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L233) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L233) Return a list of all parent Streams including the node itself. @@ -2474,7 +2520,7 @@ a list of `Stream` objects def compose_returning() -> ReturningExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L247) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L247) Compose a list of functions from this `Stream` and its parents into one big closure that always returns the transformed record. @@ -2499,7 +2545,7 @@ def compose( None]] = None) -> VoidExecutor ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/core/stream/stream.py#L284) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/core/stream/stream.py#L284) Compose a list of functions from this `Stream` and its parents into one @@ -2544,7 +2590,7 @@ the stream has transform functions in the tree. Default - True. class ProcessingContext() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/processing_context.py#L21) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/processing_context.py#L21) A class to share processing-related objects between `Application` and `StreamingDataFrame` instances. @@ -2557,7 +2603,7 @@ between `Application` and `StreamingDataFrame` instances. def store_offset(topic: str, partition: int, offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/processing_context.py#L41) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/processing_context.py#L41) Store the offset of the processed message to the checkpoint. @@ -2575,7 +2621,7 @@ Store the offset of the processed message to the checkpoint. def init_checkpoint() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/processing_context.py#L51) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/processing_context.py#L51) Initialize a new checkpoint @@ -2587,7 +2633,7 @@ Initialize a new checkpoint def commit_checkpoint(force: bool = False) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/processing_context.py#L62) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/processing_context.py#L62) Commit the current checkpoint. @@ -2616,7 +2662,7 @@ The actual commit will happen only when: def dict_values(d: object) -> List ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/utils/dicts.py#L4) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/utils/dicts.py#L4) Recursively unpacks a set of nested dicts to get a flattened list of leaves, @@ -2644,7 +2690,7 @@ a list with all the leaves of the various contained dicts def dumps(value: Any) -> bytes ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/utils/json.py#L8) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/utils/json.py#L8) Serialize to JSON using `orjson` package. @@ -2664,7 +2710,7 @@ bytes def loads(value: bytes) -> Any ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/utils/json.py#L18) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/utils/json.py#L18) Deserialize from JSON using `orjson` package. @@ -2696,7 +2742,7 @@ object class TimestampType(enum.IntEnum) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/timestamps.py#L8) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/timestamps.py#L8) @@ -2724,7 +2770,7 @@ broker receive time class MessageTimestamp() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/timestamps.py#L14) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/timestamps.py#L14) Represents a timestamp of incoming Kafka message. @@ -2740,7 +2786,7 @@ it should not be mutated during message processing. def create(cls, timestamp_type: int, milliseconds: int) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/timestamps.py#L41) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/timestamps.py#L41) Create a Timestamp object based on data @@ -2777,7 +2823,7 @@ Timestamp object class MessageContext() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/messagecontext.py#L4) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/messagecontext.py#L4) An object with Kafka message properties. @@ -2796,7 +2842,7 @@ it should not be mutated during message processing. class ConfluentKafkaMessageProto(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/types.py#L13) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/types.py#L13) An interface of `confluent_kafka.Message`. @@ -2821,7 +2867,7 @@ see https://github.com/confluentinc/confluent-kafka-python/issues/1535. class IgnoreMessage(exceptions.QuixException) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/exceptions.py#L46) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/exceptions.py#L46) Raise this exception from Deserializer.__call__ in order to ignore the processing of the particular message. @@ -2838,7 +2884,7 @@ of the particular message. class QuixDeserializer(JSONDeserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L73) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L73) Handles Deserialization for any Quix-formatted topic. @@ -2853,7 +2899,7 @@ def __init__(column_name: Optional[str] = None, loads: Callable[[Union[bytes, bytearray]], Any] = default_loads) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L80) **Arguments**: @@ -2871,7 +2917,7 @@ Default - :py:func:`quixstreams.utils.json.loads`. def split_values() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L100) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L100) Each Quix message might contain data for multiple Rows. This property informs the downstream processors about that, so they can @@ -2886,7 +2932,7 @@ def deserialize(model_key: str, value: Union[List[Mapping], Mapping]) -> Iterable[Mapping] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L153) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L153) Deserialization function for particular data types (Timeseries or EventData). @@ -2907,7 +2953,7 @@ Iterable of dicts class QuixSerializer(JSONSerializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L274) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L274) @@ -2918,7 +2964,7 @@ def __init__(as_legacy: bool = True, dumps: Callable[[Any], Union[str, bytes]] = default_dumps) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L278) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L278) Serializer that returns data in json format. @@ -2936,7 +2982,7 @@ Default - :py:func:`quixstreams.utils.json.dumps` class QuixTimeseriesSerializer(QuixSerializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L321) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L321) Serialize data to JSON formatted according to Quix Timeseries format. @@ -2968,7 +3014,7 @@ Output: class QuixEventsSerializer(QuixSerializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L409) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L409) Serialize data to JSON formatted according to Quix EventData format. The input value is expected to be a dictionary with the following keys: @@ -3009,7 +3055,7 @@ Output: class BytesDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L44) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L44) A deserializer to bypass bytes without any changes @@ -3021,7 +3067,7 @@ A deserializer to bypass bytes without any changes class BytesSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L55) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L55) A serializer to bypass bytes without any changes @@ -3033,7 +3079,7 @@ A serializer to bypass bytes without any changes class StringDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L64) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L64) @@ -3043,7 +3089,7 @@ class StringDeserializer(Deserializer) def __init__(column_name: Optional[str] = None, codec: str = "utf_8") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L65) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L65) Deserializes bytes to strings using the specified encoding. @@ -3060,7 +3106,7 @@ A wrapper around `confluent_kafka.serialization.StringDeserializer`. class IntegerDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L84) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L84) Deserializes bytes to integers. @@ -3074,7 +3120,7 @@ A wrapper around `confluent_kafka.serialization.IntegerDeserializer`. class DoubleDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L103) Deserializes float to IEEE 764 binary64. @@ -3088,7 +3134,7 @@ A wrapper around `confluent_kafka.serialization.DoubleDeserializer`. class StringSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L122) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L122) @@ -3098,7 +3144,7 @@ class StringSerializer(Serializer) def __init__(codec: str = "utf_8") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L123) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L123) Serializes strings to bytes using the specified encoding. @@ -3114,7 +3160,7 @@ Serializes strings to bytes using the specified encoding. class IntegerSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L135) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L135) Serializes integers to bytes @@ -3126,7 +3172,7 @@ Serializes integers to bytes class DoubleSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L148) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L148) Serializes floats to bytes @@ -3142,7 +3188,7 @@ Serializes floats to bytes class JSONSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/json.py#L13) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/json.py#L13) @@ -3152,7 +3198,7 @@ class JSONSerializer(Serializer) def __init__(dumps: Callable[[Any], Union[str, bytes]] = default_dumps) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/json.py#L14) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/json.py#L14) Serializer that returns data in json format. @@ -3169,7 +3215,7 @@ Default - :py:func:`quixstreams.utils.json.dumps` class JSONDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/json.py#L35) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/json.py#L35) @@ -3180,7 +3226,7 @@ def __init__(column_name: Optional[str] = None, loads: Callable[[Union[bytes, bytearray]], Any] = default_loads) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/json.py#L36) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/json.py#L36) Deserializer that parses data from JSON @@ -3203,7 +3249,7 @@ Default - :py:func:`quixstreams.utils.json.loads`. class SerializationContext() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L22) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L22) Provides additional context for message serialization/deserialization. @@ -3217,7 +3263,7 @@ Every `Serializer` and `Deserializer` receives an instance of `SerializationCont def to_confluent_ctx(field: MessageField) -> _SerializationContext ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L35) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L35) Convert `SerializationContext` to `confluent_kafka.SerializationContext` @@ -3239,7 +3285,7 @@ instance of `confluent_kafka.serialization.SerializationContext` class Deserializer(abc.ABC) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L47) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L47) @@ -3249,7 +3295,7 @@ class Deserializer(abc.ABC) def __init__(column_name: Optional[str] = None, *args, **kwargs) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L48) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L48) A base class for all Deserializers @@ -3267,7 +3313,7 @@ dictionary with `column_name` as a key. def split_values() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L58) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L58) Return True if the deserialized message should be considered as Iterable and each item in it should be processed as a separate message. @@ -3280,7 +3326,7 @@ and each item in it should be processed as a separate message. class Serializer(abc.ABC) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L74) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L74) A base class for all Serializers @@ -3293,7 +3339,7 @@ A base class for all Serializers def extra_headers() -> MessageHeadersMapping ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/base.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/base.py#L80) Informs producer to set additional headers @@ -3330,7 +3376,7 @@ dict with headers def convert_topic_list(topics: List[Topic]) -> List[ConfluentTopic] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L24) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L24) Converts `Topic`s to `ConfluentTopic`s as required for Confluent's @@ -3352,7 +3398,7 @@ list of confluent_kafka `ConfluentTopic`s class TopicAdmin() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L47) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L47) For performing "admin"-level operations on a Kafka cluster, mostly around topics. @@ -3368,7 +3414,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], extra_config: Optional[Mapping] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L54) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L54) **Arguments**: @@ -3386,7 +3432,7 @@ or a ConnectionConfig object if authentication is required. def list_topics(timeout: float = -1) -> Dict[str, ConfluentTopicMetadata] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L83) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L83) Get a list of topics and their metadata from a Kafka cluster @@ -3407,7 +3453,7 @@ def inspect_topics(topic_names: List[str], timeout: float = 30) -> Dict[str, Optional[TopicConfig]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L94) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L94) A simplified way of getting the topic configurations of the provided topics @@ -3433,7 +3479,7 @@ def create_topics(topics: List[Topic], finalize_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L176) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L176) Create the given list of topics and confirm they are ready. @@ -3460,7 +3506,7 @@ def merge_headers(original: Optional[MessageHeadersTuples], other: MessageHeadersMapping) -> MessageHeadersTuples ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/utils.py#L8) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/utils.py#L8) Merge two sets of Kafka message headers, overwriting headers in "origin" @@ -3488,7 +3534,7 @@ a list of (key, value) tuples. class TopicConfig() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L42) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L42) Represents all kafka-level configuration for a kafka topic. @@ -3502,7 +3548,7 @@ Generally used by Topic and any topic creation procedures. class Topic() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L83) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L83) A definition of a Kafka topic. @@ -3525,7 +3571,7 @@ def __init__( timestamp_extractor: Optional[TimestampExtractor] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L92) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L92) **Arguments**: @@ -3547,7 +3593,7 @@ milliseconds from a deserialized message. def name() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L121) Topic name @@ -3559,7 +3605,7 @@ Topic name def row_serialize(row: Row, key: Any) -> KafkaMessage ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L131) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L131) Serialize Row to a Kafka message structure @@ -3581,7 +3627,7 @@ def row_deserialize( message: ConfluentKafkaMessageProto) -> Union[Row, List[Row], None] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L162) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L162) Deserialize incoming Kafka message to a Row. @@ -3609,7 +3655,7 @@ Row, list of Rows or None if the message is ignored. def affirm_ready_for_create(topics: List[Topic]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L20) Validate a list of topics is ready for creation attempt @@ -3625,7 +3671,7 @@ Validate a list of topics is ready for creation attempt class TopicManager() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L30) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L30) The source of all topic management with quixstreams. @@ -3646,7 +3692,7 @@ def __init__(topic_admin: TopicAdmin, create_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L53) **Arguments**: @@ -3664,7 +3710,7 @@ def __init__(topic_admin: TopicAdmin, def changelog_topics() -> Dict[str, Dict[str, Topic]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L103) Note: `Topic`s are the changelogs. @@ -3679,7 +3725,7 @@ returns: the changelog topic dict, {topic_name: {suffix: Topic}} def all_topics() -> Dict[str, Topic] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L112) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L112) Every registered topic name mapped to its respective `Topic`. @@ -3695,7 +3741,7 @@ def topic_config(num_partitions: Optional[int] = None, extra_config: Optional[dict] = None) -> TopicConfig ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L220) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L220) Convenience method for generating a `TopicConfig` with default settings @@ -3723,7 +3769,7 @@ def topic(name: str, timestamp_extractor: Optional[TimestampExtractor] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L241) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L241) A convenience method for generating a `Topic`. Will use default config options @@ -3758,7 +3804,7 @@ def repartition_topic(operation: str, timeout: Optional[float] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L286) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L286) Create an internal repartition topic. @@ -3786,7 +3832,7 @@ def changelog_topic(topic_name: str, timeout: Optional[float] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L326) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L326) Performs all the logic necessary to generate a changelog topic based on a @@ -3827,7 +3873,7 @@ def create_topics(topics: List[Topic], create_timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L383) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L383) Creates topics via an explicit list of provided `Topics`. @@ -3849,7 +3895,7 @@ def create_all_topics(timeout: Optional[float] = None, create_timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L411) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L411) A convenience method to create all Topic objects stored on this TopicManager. @@ -3866,7 +3912,7 @@ A convenience method to create all Topic objects stored on this TopicManager. def validate_all_topics(timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L424) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L424) Validates all topics exist and changelogs have correct topic and rep factor. @@ -3884,7 +3930,7 @@ Issues are pooled and raised as an Exception once inspections are complete. class WindowedRocksDBStore(RocksDBStore) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/store.py#L10) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/store.py#L10) RocksDB-based windowed state store. @@ -3904,7 +3950,7 @@ def __init__( options: Optional[RocksDBOptionsType] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/store.py#L18) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/store.py#L18) **Arguments**: @@ -3927,7 +3973,7 @@ if using changelogs class WindowedRocksDBStorePartition(RocksDBStorePartition) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/partition.py#L24) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/partition.py#L24) A base class to access windowed state in RocksDB. @@ -3957,7 +4003,7 @@ stores the expiration index to delete expired windows. class WindowedRocksDBPartitionTransaction(RocksDBPartitionTransaction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/transaction.py#L22) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/transaction.py#L22) @@ -3969,7 +4015,7 @@ def expire_windows(duration_ms: int, grace_ms: int = 0) -> List[Tuple[Tuple[int, int], Any]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/transaction.py#L105) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/transaction.py#L105) Get a list of expired windows from RocksDB considering latest timestamp, @@ -4007,7 +4053,7 @@ sorted list of tuples in format `((start, end), value)` def parse_window_key(key: bytes) -> Tuple[bytes, int, int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/serialization.py#L12) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/serialization.py#L12) Parse the window key from Rocksdb into (message_key, start, end) structure. @@ -4030,7 +4076,7 @@ a tuple with message key, start timestamp, end timestamp def encode_window_key(start_ms: int, end_ms: int) -> bytes ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/serialization.py#L39) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/serialization.py#L39) Encode window start and end timestamps into bytes of the following format: @@ -4055,7 +4101,7 @@ window timestamps as bytes def encode_window_prefix(prefix: bytes, start_ms: int) -> bytes ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/serialization.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/serialization.py#L53) Encode window prefix and start time to iterate over keys in RocksDB @@ -4083,7 +4129,7 @@ bytes class WindowedTransactionState(WindowedState) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/state.py#L9) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/state.py#L9) @@ -4094,7 +4140,7 @@ def __init__(transaction: "WindowedRocksDBPartitionTransaction", prefix: bytes) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/state.py#L12) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/state.py#L12) A windowed state to be provided into `StreamingDataFrame` window functions. @@ -4112,7 +4158,7 @@ def get_window(start_ms: int, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/state.py#L23) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/state.py#L23) Get the value of the window defined by `start` and `end` timestamps @@ -4136,7 +4182,7 @@ value or None if the key is not found and `default` is not provided def update_window(start_ms: int, end_ms: int, value: Any, timestamp_ms: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/state.py#L39) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/state.py#L39) Set a value for the window. @@ -4158,7 +4204,7 @@ using the provided `timestamp`. def get_latest_timestamp() -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/state.py#L60) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/state.py#L60) Get the latest observed timestamp for the current state partition. @@ -4178,7 +4224,7 @@ def expire_windows(duration_ms: int, grace_ms: int = 0) -> List[Tuple[Tuple[int, int], Any]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/windowed/state.py#L72) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/windowed/state.py#L72) Get a list of expired windows from RocksDB considering the current latest timestamp, window duration and grace period. @@ -4200,7 +4246,7 @@ calling this method multiple times will yield different results for the same class RocksDBOptions(RocksDBOptionsType) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/options.py#L25) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/options.py#L25) RocksDB database options. @@ -4221,7 +4267,7 @@ Please see `rocksdict.Options` for a complete description of other options. def to_options() -> rocksdict.Options ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/options.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/options.py#L53) Convert parameters to `rocksdict.Options` @@ -4241,7 +4287,7 @@ instance of `rocksdict.Options` class RocksDBStore(Store) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L19) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L19) RocksDB-based state store. @@ -4261,7 +4307,7 @@ def __init__( options: Optional[options_type] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L29) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L29) **Arguments**: @@ -4281,7 +4327,7 @@ if using changelogs def topic() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L53) Store topic name @@ -4294,7 +4340,7 @@ Store topic name def name() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L60) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L60) Store name @@ -4307,7 +4353,7 @@ Store name def partitions() -> Dict[int, RocksDBStorePartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L67) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L67) Mapping of assigned store partitions @@ -4319,7 +4365,7 @@ Mapping of assigned store partitions def assign_partition(partition: int) -> RocksDBStorePartition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L80) Open and assign store partition. @@ -4342,7 +4388,7 @@ instance of`RocksDBStorePartition` def revoke_partition(partition: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L117) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L117) Revoke and close the assigned store partition. @@ -4360,7 +4406,7 @@ If the partition is not assigned, it will log the message and return. def start_partition_transaction(partition: int) -> RocksDBPartitionTransaction ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L138) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L138) Start a new partition transaction. @@ -4383,7 +4429,7 @@ instance of `RocksDBPartitionTransaction` def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/store.py#L160) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/store.py#L160) Close the store and revoke all assigned partitions @@ -4399,7 +4445,7 @@ Close the store and revoke all assigned partitions class RocksDBStorePartition(StorePartition) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L40) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L40) A base class to access state in RocksDB. @@ -4426,7 +4472,7 @@ it will retry according to `open_max_retries` and `open_retry_backoff` options. def begin() -> RocksDBPartitionTransaction ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L75) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L75) Create a new `RocksDBTransaction` object. @@ -4445,7 +4491,7 @@ def recover_from_changelog_message( changelog_message: ConfluentKafkaMessageProto, committed_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L128) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L128) Updates state from a given changelog message. @@ -4471,7 +4517,7 @@ messages and improves the state consistency guarantees. def set_changelog_offset(changelog_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L169) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L169) Set the changelog offset based on a message (usually an "offset-only" message). @@ -4489,7 +4535,7 @@ Used during recovery. def write(batch: WriteBatch) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L179) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L179) Write `WriteBatch` to RocksDB @@ -4507,7 +4553,7 @@ def get(key: bytes, cf_name: str = "default") -> Union[None, bytes, Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L186) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L186) Get a key from RocksDB. @@ -4529,7 +4575,7 @@ a value if the key is present in the DB. Otherwise, `default` def exists(key: bytes, cf_name: str = "default") -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L200) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L200) Check if a key is present in the DB. @@ -4550,7 +4596,7 @@ Check if a key is present in the DB. def get_processed_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L211) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L211) Get last processed offset for the given partition @@ -4566,7 +4612,7 @@ offset or `None` if there's no processed offset yet def get_changelog_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L223) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L223) Get offset that the changelog is up-to-date with. @@ -4582,7 +4628,7 @@ offset or `None` if there's no processed offset yet def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L233) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L233) Close the underlying RocksDB @@ -4595,7 +4641,7 @@ Close the underlying RocksDB def path() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L246) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L246) Absolute path to RocksDB database folder @@ -4612,7 +4658,7 @@ file path def destroy(cls, path: str) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L254) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L254) Delete underlying RocksDB database @@ -4630,7 +4676,7 @@ The database must be closed first. def get_column_family_handle(cf_name: str) -> ColumnFamily ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L264) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L264) Get a column family handle to pass to it WriteBatch. @@ -4653,7 +4699,7 @@ instance of `rocksdict.ColumnFamily` def get_column_family(cf_name: str) -> Rdict ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/partition.py#L285) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/partition.py#L285) Get a column family instance. @@ -4683,7 +4729,7 @@ instance of `rocksdict.Rdict` for the given column family class RocksDBPartitionTransaction(PartitionTransaction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L61) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L61) A transaction class to perform simple key-value operations like "get", "set", "delete" and "exists" on a single RocksDB partition. @@ -4723,7 +4769,7 @@ def __init__(partition: "RocksDBStorePartition", changelog_producer: Optional[ChangelogProducer] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L100) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L100) **Arguments**: @@ -4744,7 +4790,7 @@ def get(key: Any, cf_name: str = "default") -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L124) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L124) Get a key from the store. @@ -4774,7 +4820,7 @@ value or `default` def set(key: Any, value: Any, prefix: bytes, cf_name: str = "default") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L164) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L164) Set a key to the store. @@ -4796,7 +4842,7 @@ It first updates the key in the update cache. def delete(key: Any, prefix: bytes, cf_name: str = "default") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L187) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L187) Delete a key from the store. @@ -4817,7 +4863,7 @@ It first deletes the key from the update cache. def exists(key: Any, prefix: bytes, cf_name: str = "default") -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L208) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L208) Check if a key exists in the store. @@ -4842,7 +4888,7 @@ It first looks up the key in the update cache. def prepare(processed_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L235) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L235) Produce changelog messages to the changelog topic for all changes accumulated @@ -4870,7 +4916,7 @@ def flush(processed_offset: Optional[int] = None, changelog_offset: Optional[int] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L259) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L259) Flush the recent updates to the database. @@ -4899,7 +4945,7 @@ optional. def completed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L294) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L294) Check if the transaction is completed. @@ -4921,7 +4967,7 @@ The completed transaction should not be re-used. def prepared() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L308) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L308) Check if the transaction is in PREPARED status. @@ -4941,7 +4987,7 @@ updates anymore, but its state is not yet flushed to the disk def failed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L320) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L320) Check if the transaction has failed. @@ -4961,7 +5007,7 @@ and def changelog_topic_partition() -> Optional[Tuple[str, int]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L332) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L332) Return the changelog topic-partition for the StorePartition of this transaction. @@ -4979,7 +5025,7 @@ Returns `None` if changelog_producer is not provided. def as_state(prefix: Any = DEFAULT_PREFIX) -> TransactionState ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/transaction.py#L346) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/transaction.py#L346) Create a one-time use `TransactionState` object with a limited CRUD interface @@ -5024,7 +5070,7 @@ an instance of `TransactionState` class RecoveryPartition() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L24) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L24) A changelog topic partition mapped to a respective `StorePartition` with helper methods to determine its current recovery status. @@ -5040,7 +5086,7 @@ Since `StorePartition`s do recovery directly, it also handles recovery transacti def offset() -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L63) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L63) Get the changelog offset from the underlying `StorePartition`. @@ -5057,7 +5103,7 @@ changelog offset (int) def needs_recovery() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L72) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L72) Determine whether recovery is necessary for underlying `StorePartition`. @@ -5070,7 +5116,7 @@ Determine whether recovery is necessary for underlying `StorePartition`. def needs_offset_update() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L81) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L81) Determine if an offset update is required. @@ -5084,7 +5130,7 @@ Usually checked during assign if recovery was not required. def update_offset() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L89) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L89) Update only the changelog offset of a StorePartition. @@ -5097,7 +5143,7 @@ def recover_from_changelog_message( changelog_message: ConfluentKafkaMessageProto) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L109) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L109) Recover the StorePartition using a message read from its respective changelog. @@ -5113,7 +5159,7 @@ Recover the StorePartition using a message read from its respective changelog. def set_watermarks(lowwater: int, highwater: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L121) Set the changelog watermarks as gathered from Consumer.get_watermark_offsets() @@ -5130,7 +5176,7 @@ Set the changelog watermarks as gathered from Consumer.get_watermark_offsets() class ChangelogProducerFactory() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L132) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L132) Generates ChangelogProducers, which produce changelog messages to a StorePartition. @@ -5142,7 +5188,7 @@ Generates ChangelogProducers, which produce changelog messages to a StorePartiti def __init__(changelog_name: str, producer: RowProducer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L137) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L137) **Arguments**: @@ -5161,7 +5207,7 @@ a ChangelogWriter instance def get_partition_producer(partition_num) -> "ChangelogProducer" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L147) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L147) Generate a ChangelogProducer for producing to a specific partition number @@ -5179,7 +5225,7 @@ Generate a ChangelogProducer for producing to a specific partition number class ChangelogProducer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L161) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L161) Generated for a `StorePartition` to produce state changes to its respective kafka changelog partition. @@ -5192,7 +5238,7 @@ kafka changelog partition. def __init__(changelog_name: str, partition: int, producer: RowProducer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L167) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L167) **Arguments**: @@ -5210,7 +5256,7 @@ def produce(key: bytes, headers: Optional[MessageHeadersMapping] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L190) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L190) Produce a message to a changelog topic partition. @@ -5228,7 +5274,7 @@ Produce a message to a changelog topic partition. class RecoveryManager() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L215) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L215) Manages all consumer-related aspects of recovery, including: - assigning/revoking, pausing/resuming topic partitions (especially changelogs) @@ -5248,7 +5294,7 @@ Recovery is attempted from the `Application` after any new partition assignment. def partitions() -> Dict[int, Dict[str, RecoveryPartition]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L234) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L234) Returns a mapping of assigned RecoveryPartitions in the following format: {: {: }} @@ -5262,7 +5308,7 @@ Returns a mapping of assigned RecoveryPartitions in the following format: def has_assignments() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L242) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L242) Whether the Application has assigned RecoveryPartitions @@ -5279,7 +5325,7 @@ has assignments, as bool def recovering() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L251) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L251) Whether the Application is currently recovering @@ -5295,7 +5341,7 @@ is recovering, as bool def register_changelog(topic_name: str, store_name: str) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L259) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L259) Register a changelog Topic with the TopicManager. @@ -5312,7 +5358,7 @@ Register a changelog Topic with the TopicManager. def do_recovery() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L271) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L271) If there are any active RecoveryPartitions, do a recovery procedure. @@ -5327,7 +5373,7 @@ def assign_partition(topic: str, partition: int, committed_offset: int, store_partitions: Dict[str, StorePartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L324) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L324) Assigns `StorePartition`s (as `RecoveryPartition`s) ONLY IF recovery required. @@ -5341,7 +5387,7 @@ Pauses active consumer partitions as needed. def revoke_partition(partition_num: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/recovery.py#L391) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/recovery.py#L391) revoke ALL StorePartitions (across all Stores) for a given partition number @@ -5365,7 +5411,7 @@ revoke ALL StorePartitions (across all Stores) for a given partition number class Store(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L11) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L11) Abstract state store. @@ -5381,7 +5427,7 @@ partitions' transactions. def topic() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L22) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L22) Topic name @@ -5394,7 +5440,7 @@ Topic name def name() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L29) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L29) Store name @@ -5407,7 +5453,7 @@ Store name def partitions() -> Dict[int, "StorePartition"] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L36) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L36) Mapping of assigned store partitions @@ -5423,7 +5469,7 @@ dict of "{partition: }" def assign_partition(partition: int) -> "StorePartition" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L43) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L43) Assign new store partition @@ -5443,7 +5489,7 @@ instance of `StorePartition` def revoke_partition(partition: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L52) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L52) Revoke assigned store partition @@ -5459,7 +5505,7 @@ Revoke assigned store partition def start_partition_transaction(partition: int) -> "PartitionTransaction" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L60) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L60) Start a new partition transaction. @@ -5481,7 +5527,7 @@ instance of `PartitionTransaction` def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L69) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L69) Close store and revoke all store partitions @@ -5493,7 +5539,7 @@ Close store and revoke all store partitions class StorePartition(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L80) A base class to access state in the underlying storage. It represents a single instance of some storage (e.g. a single database for @@ -5508,7 +5554,7 @@ the persistent storage). def path() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L89) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L89) Absolute path to RocksDB database folder @@ -5520,7 +5566,7 @@ Absolute path to RocksDB database folder def begin() -> "PartitionTransaction" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L95) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L95) State new `PartitionTransaction` @@ -5533,7 +5579,7 @@ def recover_from_changelog_message( changelog_message: ConfluentKafkaMessageProto, committed_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L100) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L100) Updates state from a given changelog message. @@ -5550,7 +5596,7 @@ Updates state from a given changelog message. def get_processed_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L111) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L111) Get last processed offset for the given partition @@ -5566,7 +5612,7 @@ offset or `None` if there's no processed offset yet def get_changelog_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L118) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L118) Get offset that the changelog is up-to-date with. @@ -5582,7 +5628,7 @@ offset or `None` if there's no processed offset yet def set_changelog_offset(changelog_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L125) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L125) Set the changelog offset based on a message (usually an "offset-only" message). @@ -5600,7 +5646,7 @@ Used during recovery. class State(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L136) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L136) Primary interface for working with key-value state data from `StreamingDataFrame` @@ -5612,7 +5658,7 @@ Primary interface for working with key-value state data from `StreamingDataFrame def get(key: Any, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L141) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L141) Get the value for key if key is present in the state, else default @@ -5633,7 +5679,7 @@ value or None if the key is not found and `default` is not provided def set(key: Any, value: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L151) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L151) Set value for the key. @@ -5650,7 +5696,7 @@ Set value for the key. def delete(key: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L159) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L159) Delete value for the key. @@ -5668,7 +5714,7 @@ This function always returns `None`, even if value is not found. def exists(key: Any) -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L168) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L168) Check if the key exists in state. @@ -5688,7 +5734,7 @@ True if key exists, False otherwise class PartitionTransaction(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L177) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L177) A transaction class to perform simple key-value operations like "get", "set", "delete" and "exists" on a single storage partition. @@ -5701,7 +5747,7 @@ A transaction class to perform simple key-value operations like def as_state(prefix: Any) -> State ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L183) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L183) Create an instance implementing the `State` protocol to be provided @@ -5721,7 +5767,7 @@ an instance implementing the `State` protocol def get(key: Any, prefix: bytes, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L194) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L194) Get the value for key if key is present in the state, else default @@ -5743,7 +5789,7 @@ value or None if the key is not found and `default` is not provided def set(key: Any, prefix: bytes, value: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L205) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L205) Set value for the key. @@ -5761,7 +5807,7 @@ Set value for the key. def delete(key: Any, prefix: bytes) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L214) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L214) Delete value for the key. @@ -5780,7 +5826,7 @@ This function always returns `None`, even if value is not found. def exists(key: Any, prefix: bytes) -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L224) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L224) Check if the key exists in state. @@ -5802,7 +5848,7 @@ True if key exists, False otherwise def failed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L234) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L234) Return `True` if transaction failed to update data at some point. @@ -5821,7 +5867,7 @@ bool def completed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L244) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L244) Return `True` if transaction is successfully completed. @@ -5840,7 +5886,7 @@ bool def prepared() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L254) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L254) Return `True` if transaction is prepared completed. @@ -5858,7 +5904,7 @@ bool def prepare(processed_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L263) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L263) Produce changelog messages to the changelog topic for all changes accumulated @@ -5884,7 +5930,7 @@ to the changelog topic. def changelog_topic_partition() -> Optional[Tuple[str, int]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L279) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L279) Return the changelog topic-partition for the StorePartition of this transaction. @@ -5903,7 +5949,7 @@ def flush(processed_offset: Optional[int] = None, changelog_offset: Optional[int] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L288) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L288) Flush the recent updates to the storage. @@ -5921,7 +5967,7 @@ optional. class WindowedState(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L306) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L306) A windowed state to be provided into `StreamingDataFrame` window functions. @@ -5935,7 +5981,7 @@ def get_window(start_ms: int, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L311) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L311) Get the value of the window defined by `start` and `end` timestamps @@ -5959,7 +6005,7 @@ value or None if the key is not found and `default` is not provided def update_window(start_ms: int, end_ms: int, value: Any, timestamp_ms: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L325) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L325) Set a value for the window. @@ -5981,7 +6027,7 @@ using the provided `timestamp`. def get_latest_timestamp() -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L339) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L339) Get the latest observed timestamp for the current state partition. @@ -6001,7 +6047,7 @@ def expire_windows(duration_ms: int, grace_ms: int = 0) -> List[Tuple[Tuple[int, int], Any]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L350) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L350) Get a list of expired windows from RocksDB considering the current @@ -6024,7 +6070,7 @@ calling this method multiple times will yield different results for the same class WindowedPartitionTransaction(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L367) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L367) @@ -6035,7 +6081,7 @@ class WindowedPartitionTransaction(Protocol) def failed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L370) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L370) Return `True` if transaction failed to update data at some point. @@ -6054,7 +6100,7 @@ bool def completed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L380) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L380) Return `True` if transaction is successfully completed. @@ -6073,7 +6119,7 @@ bool def prepared() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L390) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L390) Return `True` if transaction is prepared completed. @@ -6091,7 +6137,7 @@ bool def prepare(processed_offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L399) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L399) Produce changelog messages to the changelog topic for all changes accumulated @@ -6119,7 +6165,7 @@ def get_window(start_ms: int, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L416) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L416) Get the value of the window defined by `start` and `end` timestamps @@ -6145,7 +6191,7 @@ def update_window(start_ms: int, end_ms: int, value: Any, timestamp_ms: int, prefix: bytes) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L435) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L435) Set a value for the window. @@ -6168,7 +6214,7 @@ using the provided `timestamp`. def get_latest_timestamp() -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L452) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L452) Get the latest observed timestamp for the current state partition. @@ -6187,7 +6233,7 @@ latest observed event timestamp in milliseconds def expire_windows(duration_ms: int, prefix: bytes, grace_ms: int = 0) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L463) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L463) Get a list of expired windows from RocksDB considering the current @@ -6212,7 +6258,7 @@ def flush(processed_offset: Optional[int] = None, changelog_offset: Optional[int] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L478) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L478) Flush the recent updates to the storage. @@ -6231,7 +6277,7 @@ optional. def changelog_topic_partition() -> Optional[Tuple[str, int]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L492) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L492) Return the changelog topic-partition for the StorePartition of this transaction. @@ -6249,7 +6295,7 @@ Returns `None` if changelog_producer is not provided. class PartitionRecoveryTransaction(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L506) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L506) A class for managing recovery for a StorePartition from a changelog message @@ -6261,7 +6307,7 @@ A class for managing recovery for a StorePartition from a changelog message def flush() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L513) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L513) Flush the recovery update to the storage. @@ -6273,7 +6319,7 @@ Flush the recovery update to the storage. class PartitionTransactionStatus(enum.Enum) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L520) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L520) @@ -6315,7 +6361,7 @@ Transaction is failed, it cannot be used anymore class StateStoreManager() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L24) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L24) Class for managing state stores and partitions. @@ -6333,7 +6379,7 @@ StateStoreManager is responsible for: def stores() -> Dict[str, Dict[str, Store]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L62) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L62) Map of registered state stores @@ -6350,7 +6396,7 @@ dict in format {topic: {store_name: store}} def recovery_required() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L70) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L70) Whether recovery needs to be done. @@ -6363,7 +6409,7 @@ Whether recovery needs to be done. def using_changelogs() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L79) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L79) Whether the StateStoreManager is using changelog topics @@ -6379,7 +6425,7 @@ using changelogs, as bool def do_recovery() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L87) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L87) Perform a state recovery, if necessary. @@ -6391,7 +6437,7 @@ Perform a state recovery, if necessary. def stop_recovery() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L93) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L93) Stop recovery (called during app shutdown). @@ -6403,7 +6449,7 @@ Stop recovery (called during app shutdown). def get_store(topic: str, store_name: str = DEFAULT_STATE_STORE_NAME) -> Store ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L99) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L99) Get a store for given name and topic @@ -6425,7 +6471,7 @@ def register_store(topic_name: str, store_name: str = DEFAULT_STATE_STORE_NAME) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L132) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L132) Register a state store to be managed by StateStoreManager. @@ -6447,7 +6493,7 @@ Each store can be registered only once for each topic. def register_windowed_store(topic_name: str, store_name: str) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L157) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L157) Register a windowed state store to be managed by StateStoreManager. @@ -6469,7 +6515,7 @@ Each window store can be registered only once for each topic. def clear_stores() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L180) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L180) Delete all state stores managed by StateStoreManager. @@ -6482,7 +6528,7 @@ def on_partition_assign(topic: str, partition: int, committed_offset: int) -> List[StorePartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L195) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L195) Assign store partitions for each registered store for the given `TopicPartition` @@ -6506,7 +6552,7 @@ list of assigned `StorePartition` def on_partition_revoke(topic: str, partition: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L221) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L221) Revoke store partitions for each registered store for the given `TopicPartition` @@ -6523,7 +6569,7 @@ Revoke store partitions for each registered store for the given `TopicPartition` def init() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L234) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L234) Initialize `StateStoreManager` and create a store directory @@ -6536,7 +6582,7 @@ Initialize `StateStoreManager` and create a store directory def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/manager.py#L241) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/manager.py#L241) Close all registered stores @@ -6552,7 +6598,7 @@ Close all registered stores class TransactionState(State) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/state.py#L6) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/state.py#L6) @@ -6562,7 +6608,7 @@ class TransactionState(State) def __init__(prefix: bytes, transaction: PartitionTransaction) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/state.py#L12) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/state.py#L12) Simple key-value state to be provided into `StreamingDataFrame` functions @@ -6578,7 +6624,7 @@ Simple key-value state to be provided into `StreamingDataFrame` functions def get(key: Any, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/state.py#L21) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/state.py#L21) Get the value for key if key is present in the state, else default @@ -6599,7 +6645,7 @@ value or None if the key is not found and `default` is not provided def set(key: Any, value: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/state.py#L31) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/state.py#L31) Set value for the key. @@ -6616,7 +6662,7 @@ Set value for the key. def delete(key: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/state.py#L39) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/state.py#L39) Delete value for the key. @@ -6634,7 +6680,7 @@ This function always returns `None`, even if value is not found. def exists(key: Any) -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/state.py#L48) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/state.py#L48) Check if the key exists in state. @@ -6662,7 +6708,7 @@ True if key exists, False otherwise class PartitionAssignmentError(QuixException) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/exceptions/assignment.py#L6) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/exceptions/assignment.py#L6) Error happened during partition rebalancing. Raised from `on_assign`, `on_revoke` and `on_lost` callbacks @@ -6683,7 +6729,7 @@ Raised from `on_assign`, `on_revoke` and `on_lost` callbacks def set_message_context(context: Optional[MessageContext]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/context.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/context.py#L20) Set a MessageContext for the current message in the given `contextvars.Context` @@ -6720,7 +6766,7 @@ sdf = sdf.update(lambda value: alter_context(value)) def message_context() -> MessageContext ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/context.py#L51) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/context.py#L51) Get a MessageContext for the current message, which houses most of the message @@ -6759,7 +6805,7 @@ instance of `MessageContext` class ConnectionConfig(BaseSettings) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/configuration.py#L17) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/configuration.py#L17) Provides an interface for all librdkafka connection-based configs. @@ -6782,7 +6828,7 @@ def settings_customise_sources( ) -> Tuple[PydanticBaseSettingsSource, ...] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/configuration.py#L96) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/configuration.py#L96) Included to ignore reading/setting values from the environment @@ -6797,7 +6843,7 @@ def from_librdkafka_dict(cls, ignore_extras: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/configuration.py#L110) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/configuration.py#L110) Create a `ConnectionConfig` from a librdkafka config dictionary. @@ -6818,7 +6864,7 @@ a ConnectionConfig def as_librdkafka_dict(plaintext_secrets=True) -> dict ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/configuration.py#L125) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/configuration.py#L125) Dump any non-empty config values as a librdkafka dictionary. @@ -6848,7 +6894,7 @@ a librdkafka-compatible dictionary class Producer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L44) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L44) @@ -6861,7 +6907,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], extra_config: Optional[dict] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L45) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L45) A wrapper around `confluent_kafka.Producer`. @@ -6896,7 +6942,7 @@ def produce(topic: str, on_delivery: Optional[DeliveryCallback] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L80) Produce a message to a topic. @@ -6927,7 +6973,7 @@ for the produced message. def poll(timeout: float = 0) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L141) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L141) Polls the producer for events and calls `on_delivery` callbacks. @@ -6944,7 +6990,7 @@ Polls the producer for events and calls `on_delivery` callbacks. def flush(timeout: Optional[float] = None) -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/producer.py#L149) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/producer.py#L149) Wait for all messages in the Producer queue to be delivered. @@ -6969,7 +7015,7 @@ number of messages remaining to flush class Consumer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L64) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L64) @@ -6987,7 +7033,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], extra_config: Optional[dict] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L65) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L65) A wrapper around `confluent_kafka.Consumer`. @@ -7026,7 +7072,7 @@ Note: values passed as arguments override values in `extra_config`. def poll(timeout: Optional[float] = None) -> Optional[Message] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L128) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L128) Consumes a single message, calls callbacks and returns events. @@ -7061,7 +7107,7 @@ def subscribe(topics: List[str], on_lost: Optional[RebalancingCallback] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L146) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L146) Set subscription to supplied list of topics @@ -7099,7 +7145,7 @@ for example, may fail. def unsubscribe() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L240) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L240) Remove current subscription. @@ -7117,7 +7163,7 @@ def store_offsets(message: Optional[Message] = None, offsets: Optional[List[TopicPartition]] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L248) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L248) .. py:function:: store_offsets([message=None], [offsets=None]) @@ -7148,7 +7194,7 @@ def commit(message: Optional[Message] = None, asynchronous: bool = True) -> Optional[List[TopicPartition]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L282) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L282) Commit a message or a list of offsets. @@ -7182,7 +7228,7 @@ def committed(partitions: List[TopicPartition], timeout: Optional[float] = None) -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L322) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L322) .. py:function:: committed(partitions, [timeout=None]) @@ -7213,7 +7259,7 @@ def get_watermark_offsets(partition: TopicPartition, cached: bool = False) -> Tuple[int, int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L342) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L342) Retrieve low and high offsets for the specified partition. @@ -7246,7 +7292,7 @@ def list_topics(topic: Optional[str] = None, timeout: Optional[float] = None) -> ClusterMetadata ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L368) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L368) .. py:function:: list_topics([topic=None], [timeout=-1]) @@ -7275,7 +7321,7 @@ None or -1 is infinite. Default: None def memberid() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L391) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L391) Return this client's broker-assigned group member id. @@ -7296,7 +7342,7 @@ def offsets_for_times(partitions: List[TopicPartition], timeout: Optional[float] = None) -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L404) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L404) Look up offsets by timestamp for the specified partitions. @@ -7323,7 +7369,7 @@ last message in the partition, a value of -1 will be returned. def pause(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L430) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L430) Pause consumption for the provided list of partitions. @@ -7347,7 +7393,7 @@ Does NOT affect the result of Consumer.assignment(). def resume(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L444) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L444) .. py:function:: resume(partitions) @@ -7369,7 +7415,7 @@ Resume consumption for the provided list of partitions. def position(partitions: List[TopicPartition]) -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L456) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L456) Retrieve current positions (offsets) for the specified partitions. @@ -7396,7 +7442,7 @@ the last consumed message + 1. def seek(partition: TopicPartition) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L470) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L470) Set consume position for partition to offset. @@ -7424,7 +7470,7 @@ pass the offset in an `assign()` call. def assignment() -> List[TopicPartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L487) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L487) Returns the current partition assignment. @@ -7445,7 +7491,7 @@ Returns the current partition assignment. def set_sasl_credentials(username: str, password: str) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L500) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L500) Sets the SASL credentials used for this client. These credentials will overwrite the old ones, and will be used the next @@ -7462,7 +7508,7 @@ This method is applicable only to SASL PLAIN and SCRAM mechanisms. def incremental_assign(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L512) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L512) Assign new partitions. @@ -7480,7 +7526,7 @@ Any additional partitions besides the ones passed during the `Consumer` def incremental_unassign(partitions: List[TopicPartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L524) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L524) Revoke partitions. @@ -7494,7 +7540,7 @@ Can be called outside an on_revoke callback. def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/kafka/consumer.py#L532) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/kafka/consumer.py#L532) Close down and terminate the Kafka Consumer. @@ -7524,7 +7570,7 @@ see `poll()` for more info. class Application() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L53) The main Application class. @@ -7590,7 +7636,7 @@ def __init__(broker_address: Optional[Union[str, ConnectionConfig]] = None, topic_create_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L91) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L91) **Arguments**: @@ -7683,7 +7729,7 @@ def Quix(cls, topic_create_timeout: float = 60) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L304) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L304) >***NOTE:*** DEPRECATED: use Application with `quix_sdk_token` argument instead. @@ -7785,7 +7831,7 @@ def topic(name: str, timestamp_extractor: Optional[TimestampExtractor] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L442) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L442) Create a topic definition. @@ -7856,7 +7902,7 @@ topic = app.topic("input-topic", timestamp_extractor=custom_ts_extractor) def dataframe(topic: Topic) -> StreamingDataFrame ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L522) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L522) A simple helper method that generates a `StreamingDataFrame`, which is used @@ -7898,7 +7944,7 @@ to be used as an input topic. def stop(fail: bool = False) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L561) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L561) Stop the internal poll loop and the message processing. @@ -7921,7 +7967,7 @@ to unhandled exception, and it shouldn't commit the current checkpoint. def get_producer() -> Producer ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L584) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L584) Create and return a pre-configured Producer instance. The Producer is initialized with params passed to Application. @@ -7952,7 +7998,7 @@ with app.get_producer() as producer: def get_consumer(auto_commit_enable: bool = True) -> Consumer ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L614) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L614) Create and return a pre-configured Consumer instance. The Consumer is initialized with params passed to Application. @@ -7993,7 +8039,7 @@ with app.get_consumer() as consumer: def clear_state() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L657) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L657) Clear the state of the application. @@ -8005,7 +8051,7 @@ Clear the state of the application. def run(dataframe: StreamingDataFrame) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/app.py#L663) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/app.py#L663) Start processing data from Kafka using provided `StreamingDataFrame` @@ -8045,7 +8091,7 @@ app.run(dataframe=df) class RowConsumer(Consumer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowconsumer.py#L19) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowconsumer.py#L19) @@ -8062,7 +8108,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], on_error: Optional[ConsumerErrorCallback] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowconsumer.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowconsumer.py#L20) A consumer class that is capable of deserializing Kafka messages to Rows @@ -8105,7 +8151,7 @@ def subscribe(topics: List[Topic], on_lost: Optional[RebalancingCallback] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowconsumer.py#L72) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowconsumer.py#L72) Set subscription to supplied list of topics. @@ -8134,7 +8180,7 @@ for example, may fail. def poll_row(timeout: float = None) -> Union[Row, List[Row], None] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/rowconsumer.py#L106) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/rowconsumer.py#L106) Consumes a single message and deserialize it to Row or a list of Rows. @@ -8162,7 +8208,7 @@ single Row, list of Rows or None class Checkpoint() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/checkpointing/checkpoint.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/checkpointing/checkpoint.py#L20) Class to keep track of state updates and consumer offsets and to checkpoint these updates on schedule. @@ -8175,7 +8221,7 @@ updates on schedule. def expired() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/checkpointing/checkpoint.py#L45) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/checkpointing/checkpoint.py#L45) Returns `True` if checkpoint deadline has expired. @@ -8187,7 +8233,7 @@ Returns `True` if checkpoint deadline has expired. def empty() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/checkpointing/checkpoint.py#L51) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/checkpointing/checkpoint.py#L51) Returns `True` if checkpoint doesn't have any offsets stored yet. @@ -8200,7 +8246,7 @@ Returns `True` if checkpoint doesn't have any offsets stored yet. def store_offset(topic: str, partition: int, offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/checkpointing/checkpoint.py#L58) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/checkpointing/checkpoint.py#L58) Store the offset of the processed message to the checkpoint. @@ -8221,7 +8267,7 @@ def get_store_transaction( store_name: str = DEFAULT_STATE_STORE_NAME) -> PartitionTransaction ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/checkpointing/checkpoint.py#L78) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/checkpointing/checkpoint.py#L78) Get a PartitionTransaction for the given store, topic and partition. @@ -8245,7 +8291,7 @@ instance of `PartitionTransaction` def commit() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/checkpointing/checkpoint.py#L101) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/checkpointing/checkpoint.py#L101) Commit the checkpoint. diff --git a/docs/api-reference/serialization.md b/docs/api-reference/serialization.md index d0c8d4bb0..b757fa8c5 100644 --- a/docs/api-reference/serialization.md +++ b/docs/api-reference/serialization.md @@ -10,7 +10,7 @@ class QuixDeserializer(JSONDeserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L73) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L73) Handles Deserialization for any Quix-formatted topic. @@ -27,7 +27,7 @@ def __init__(column_name: Optional[str] = None, loads: Callable[[Union[bytes, bytearray]], Any] = default_loads) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L80) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L80)
@@ -49,7 +49,7 @@ Default - :py:func:`quixstreams.utils.json.loads`. def split_values() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L100) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L100) Each Quix message might contain data for multiple Rows. This property informs the downstream processors about that, so they can @@ -66,7 +66,7 @@ def deserialize(model_key: str, value: Union[List[Mapping], Mapping]) -> Iterable[Mapping] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L153) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L153) Deserialization function for particular data types (Timeseries or EventData). @@ -91,7 +91,7 @@ Iterable of dicts class QuixTimeseriesSerializer(QuixSerializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L321) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L321) Serialize data to JSON formatted according to Quix Timeseries format. @@ -123,7 +123,7 @@ Output: class QuixEventsSerializer(QuixSerializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/quix.py#L409) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/quix.py#L409) Serialize data to JSON formatted according to Quix EventData format. The input value is expected to be a dictionary with the following keys: @@ -164,7 +164,7 @@ Output: class BytesDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L44) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L44) A deserializer to bypass bytes without any changes @@ -176,7 +176,7 @@ A deserializer to bypass bytes without any changes class BytesSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L55) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L55) A serializer to bypass bytes without any changes @@ -188,7 +188,7 @@ A serializer to bypass bytes without any changes class StringDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L64) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L64) @@ -200,7 +200,7 @@ class StringDeserializer(Deserializer) def __init__(column_name: Optional[str] = None, codec: str = "utf_8") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L65) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L65) Deserializes bytes to strings using the specified encoding. @@ -219,7 +219,7 @@ A wrapper around `confluent_kafka.serialization.StringDeserializer`. class IntegerDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L84) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L84) Deserializes bytes to integers. @@ -233,7 +233,7 @@ A wrapper around `confluent_kafka.serialization.IntegerDeserializer`. class DoubleDeserializer(Deserializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L103) Deserializes float to IEEE 764 binary64. @@ -247,7 +247,7 @@ A wrapper around `confluent_kafka.serialization.DoubleDeserializer`. class StringSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L122) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L122) @@ -259,7 +259,7 @@ class StringSerializer(Serializer) def __init__(codec: str = "utf_8") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L123) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L123) Serializes strings to bytes using the specified encoding. @@ -277,7 +277,7 @@ Serializes strings to bytes using the specified encoding. class IntegerSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L135) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L135) Serializes integers to bytes @@ -289,7 +289,7 @@ Serializes integers to bytes class DoubleSerializer(Serializer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/serializers/simple_types.py#L148) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/serializers/simple_types.py#L148) Serializes floats to bytes diff --git a/docs/api-reference/state.md b/docs/api-reference/state.md index b37bd89bd..b8e6f4bb9 100644 --- a/docs/api-reference/state.md +++ b/docs/api-reference/state.md @@ -10,7 +10,7 @@ class State(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L136) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L136) Primary interface for working with key-value state data from `StreamingDataFrame` @@ -24,7 +24,7 @@ Primary interface for working with key-value state data from `StreamingDataFrame def get(key: Any, default: Any = None) -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L141) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L141) Get the value for key if key is present in the state, else default @@ -51,7 +51,7 @@ value or None if the key is not found and `default` is not provided def set(key: Any, value: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L151) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L151) Set value for the key. @@ -72,7 +72,7 @@ Set value for the key. def delete(key: Any) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L159) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L159) Delete value for the key. @@ -94,7 +94,7 @@ This function always returns `None`, even if value is not found. def exists(key: Any) -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/types.py#L168) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/types.py#L168) Check if the key exists in state. @@ -123,7 +123,7 @@ True if key exists, False otherwise class RocksDBOptions(RocksDBOptionsType) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/options.py#L25) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/options.py#L25) RocksDB database options. @@ -148,7 +148,7 @@ Please see `rocksdict.Options` for a complete description of other options. def to_options() -> rocksdict.Options ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/state/rocksdb/options.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/state/rocksdb/options.py#L53) Convert parameters to `rocksdict.Options` diff --git a/docs/api-reference/topics.md b/docs/api-reference/topics.md index fe6adee5f..0ae568d36 100644 --- a/docs/api-reference/topics.md +++ b/docs/api-reference/topics.md @@ -16,7 +16,7 @@ def convert_topic_list(topics: List[Topic]) -> List[ConfluentTopic] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L24) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L24) Converts `Topic`s to `ConfluentTopic`s as required for Confluent's @@ -42,7 +42,7 @@ list of confluent_kafka `ConfluentTopic`s class TopicAdmin() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L47) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L47) For performing "admin"-level operations on a Kafka cluster, mostly around topics. @@ -60,7 +60,7 @@ def __init__(broker_address: Union[str, ConnectionConfig], extra_config: Optional[Mapping] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L54) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L54)
@@ -82,7 +82,7 @@ or a ConnectionConfig object if authentication is required. def list_topics(timeout: float = -1) -> Dict[str, ConfluentTopicMetadata] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L83) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L83) Get a list of topics and their metadata from a Kafka cluster @@ -109,7 +109,7 @@ def inspect_topics(topic_names: List[str], timeout: float = 30) -> Dict[str, Optional[TopicConfig]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L94) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L94) A simplified way of getting the topic configurations of the provided topics @@ -141,7 +141,7 @@ def create_topics(topics: List[Topic], finalize_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/admin.py#L176) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/admin.py#L176) Create the given list of topics and confirm they are ready. @@ -170,7 +170,7 @@ fail (it ignores issues for a topic already existing). class TopicConfig() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L42) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L42) Represents all kafka-level configuration for a kafka topic. @@ -184,7 +184,7 @@ Generally used by Topic and any topic creation procedures. class Topic() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L83) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L83) A definition of a Kafka topic. @@ -209,7 +209,7 @@ def __init__( timestamp_extractor: Optional[TimestampExtractor] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L92) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L92)
@@ -235,7 +235,7 @@ milliseconds from a deserialized message. def name() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L121) Topic name @@ -249,7 +249,7 @@ Topic name def row_serialize(row: Row, key: Any) -> KafkaMessage ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L131) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L131) Serialize Row to a Kafka message structure @@ -277,7 +277,7 @@ def row_deserialize( message: ConfluentKafkaMessageProto) -> Union[Row, List[Row], None] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/topic.py#L162) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/topic.py#L162) Deserialize incoming Kafka message to a Row. @@ -307,7 +307,7 @@ Row, list of Rows or None if the message is ignored. def affirm_ready_for_create(topics: List[Topic]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L20) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L20) Validate a list of topics is ready for creation attempt @@ -325,7 +325,7 @@ Validate a list of topics is ready for creation attempt class TopicManager() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L30) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L30) The source of all topic management with quixstreams. @@ -348,7 +348,7 @@ def __init__(topic_admin: TopicAdmin, create_timeout: float = 60) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L53) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L53)
@@ -370,7 +370,7 @@ def __init__(topic_admin: TopicAdmin, def changelog_topics() -> Dict[str, Dict[str, Topic]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L103) Note: `Topic`s are the changelogs. @@ -387,7 +387,7 @@ returns: the changelog topic dict, {topic_name: {suffix: Topic}} def all_topics() -> Dict[str, Topic] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L112) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L112) Every registered topic name mapped to its respective `Topic`. @@ -405,7 +405,7 @@ def topic_config(num_partitions: Optional[int] = None, extra_config: Optional[dict] = None) -> TopicConfig ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L220) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L220) Convenience method for generating a `TopicConfig` with default settings @@ -439,7 +439,7 @@ def topic(name: str, timestamp_extractor: Optional[TimestampExtractor] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L241) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L241) A convenience method for generating a `Topic`. Will use default config options @@ -480,7 +480,7 @@ def repartition_topic(operation: str, timeout: Optional[float] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L286) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L286) Create an internal repartition topic. @@ -514,7 +514,7 @@ def changelog_topic(topic_name: str, timeout: Optional[float] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L326) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L326) Performs all the logic necessary to generate a changelog topic based on a @@ -561,7 +561,7 @@ def create_topics(topics: List[Topic], create_timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L383) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L383) Creates topics via an explicit list of provided `Topics`. @@ -587,7 +587,7 @@ def create_all_topics(timeout: Optional[float] = None, create_timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L411) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L411) A convenience method to create all Topic objects stored on this TopicManager. @@ -608,7 +608,7 @@ A convenience method to create all Topic objects stored on this TopicManager. def validate_all_topics(timeout: Optional[float] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/1c8ae6251382eb13c734c048ab13f1ccc4c7352b/quixstreams/models/topics/manager.py#L424) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/0cb9393ae30a0cf9d4eec5bdb102d4081a963b9c/quixstreams/models/topics/manager.py#L424) Validates all topics exist and changelogs have correct topic and rep factor. diff --git a/docs/processing.md b/docs/processing.md index 45041a7ec..ed72466f6 100644 --- a/docs/processing.md +++ b/docs/processing.md @@ -585,6 +585,37 @@ sdf = sdf.set_timestamp(lambda value, key, timestamp, headers: int(time.time() * ``` +## Updating Kafka Headers + +Kafka headers are key-value pairs that can provide additional information about a message without modifying its payload. + +The message headers are represented as lists of ("header", "value") tuples, where "header" is a string, and "value" is bytes. +The same header can have multiple values. + +Example of the headers: +```[('clientID', b'client-123'), ('source', b'source-123')]``` + + +To set or update the Kafka message headers, use the `StreamingDataFrame.set_headers()` API. + +The updated headers will be attached when producing messages to the output topics. + +**Example:** + +```python +sdf = app.dataframe(...) + +APP_VERSION = "v0.1.1" + +# Add the value of APP_VERSION to the message headers for debugging purposes. +# The callback receives four positional arguments: value, key, current timestamp, and headers. +# It must return a new set of headers as a list of (header, value) tuples. + +sdf = sdf.set_headers( + lambda value, key, timestamp, headers: [('APP_VERSION', APP_VERSION.encode())] +) +``` + ## Accessing Kafka Keys, Timestamps and Headers By leveraging the power of custom functions in `apply()`, `update()`, and `filter()` methods of `StreamingDataFrame`, you can conveniently access message keys, timestamps and headers of the records. diff --git a/docs/windowing.md b/docs/windowing.md index aca327616..9887b91f9 100644 --- a/docs/windowing.md +++ b/docs/windowing.md @@ -115,7 +115,10 @@ sdf = ( ### Message headers of the aggregation results Currently, windowed aggregations do not store the original headers of the messages. -The results of the windowed aggregations will have headers set to `None`. +The results of the windowed aggregations will have headers set to `None`. + +You may set messages headers by using the `StreamingDataFrame.set_headers()` API, as +described in [the "Updating Kafka Headers" section](./processing.md#updating-kafka-headers). ## Tumbling Windows Tumbling windows slice time into non-overlapping intervals of a fixed size. diff --git a/quixstreams/dataframe/dataframe.py b/quixstreams/dataframe/dataframe.py index ba0bcc897..25e0b876f 100644 --- a/quixstreams/dataframe/dataframe.py +++ b/quixstreams/dataframe/dataframe.py @@ -16,6 +16,7 @@ Dict, Tuple, Literal, + Collection, ) from typing_extensions import Self @@ -39,6 +40,7 @@ TopicManager, Row, MessageContext, + HeaderValue, ) from quixstreams.models.serializers import SerializerType, DeserializerType from quixstreams.processing_context import ProcessingContext @@ -602,7 +604,7 @@ def set_timestamp(self, func: Callable[[Any, Any, int, Any], int]) -> Self: sdf = sdf.set_timestamp(lambda value, key, timestamp, headers: value['new_timestamp']) ``` - :param func: callable accepting the current value and the current timestamp. + :param func: callable accepting the current value, key, timestamp, and headers. It's expected to return a new timestamp as integer in milliseconds. :return: a new StreamingDataFrame instance """ @@ -620,6 +622,57 @@ def _set_timestamp_callback( stream = self.stream.add_transform(func=_set_timestamp_callback) return self.__dataframe_clone__(stream=stream) + def set_headers( + self, + func: Callable[ + [Any, Any, int, List[Tuple[str, HeaderValue]]], + Collection[Tuple[str, HeaderValue]], + ], + ) -> Self: + """ + Set new message headers based on the current message value and metadata. + + The new headers will be used when producing messages to the output topics. + + The provided callback must accept value, key, timestamp, and headers, + and return a new collection of (header, value) tuples. + + Example Snippet: + + ```python + from quixstreams import Application + + + app = Application() + input_topic = app.topic("data") + + sdf = app.dataframe(input_topic) + # Updating the record's headers based on the value and metadata + sdf = sdf.set_headers(lambda value, key, timestamp, headers: [('id', value['id'])]) + ``` + + :param func: callable accepting the current value, key, timestamp, and headers. + It's expected to return a new set of headers + as a collection of (header, value) tuples. + :return: a new StreamingDataFrame instance + """ + + @functools.wraps(func) + def _set_headers_callback( + value: Any, + key: Any, + timestamp: int, + headers: Collection[Tuple[str, HeaderValue]], + ) -> Tuple[Any, Any, int, Collection[Tuple[str, HeaderValue]]]: + # Create a shallow copy of original headers to prevent potential mutations + # of the same collection + headers = list(headers) if headers else [] + new_headers = func(value, key, timestamp, headers) + return value, key, timestamp, new_headers + + stream = self.stream.add_transform(func=_set_headers_callback) + return self.__dataframe_clone__(stream=stream) + def compose( self, sink: Optional[Callable[[Any, Any, int, Any], None]] = None, diff --git a/tests/test_quixstreams/test_app.py b/tests/test_quixstreams/test_app.py index 15d9680f6..b61b40953 100644 --- a/tests/test_quixstreams/test_app.py +++ b/tests/test_quixstreams/test_app.py @@ -138,7 +138,15 @@ def on_message_processed(topic_, partition, offset): processed_count = 0 total_messages = 3 # Produce messages to the topic and flush - data = {"key": b"key", "value": b'"value"', "partition": partition_num} + timestamp_ms = int(time.time() / 1000) + headers = [("header", b"value")] + data = { + "key": b"key", + "value": b'"value"', + "partition": partition_num, + "timestamp": timestamp_ms, + "headers": headers, + } with app.get_producer() as producer: for _ in range(total_messages): producer.produce(topic_in.name, **data) @@ -171,6 +179,8 @@ def on_message_processed(topic_, partition, offset): assert row.topic == topic_out.name assert row.key == data["key"] assert row.value == {column_name: loads(data["value"].decode())} + assert row.timestamp == timestamp_ms + assert row.headers == headers def test_run_fails_no_commit( self, diff --git a/tests/test_quixstreams/test_dataframe/test_dataframe.py b/tests/test_quixstreams/test_dataframe/test_dataframe.py index f6065a394..87f5a2624 100644 --- a/tests/test_quixstreams/test_dataframe/test_dataframe.py +++ b/tests/test_quixstreams/test_dataframe/test_dataframe.py @@ -348,6 +348,27 @@ def test_set_timestamp(self, dataframe_factory): result = sdf.test(value=value, key=key, timestamp=timestamp, headers=headers)[0] assert result == expected + @pytest.mark.parametrize( + "original_headers, new_headers", + [ + (None, None), + ([], []), + ([], [("key", b"value")]), + ([[("key", b"value")]], [("key2", b"value2")]), + ], + ) + def test_set_headers(self, original_headers, new_headers, dataframe_factory): + value, key, timestamp = 1, "key", 0 + expected = (1, "key", 0, new_headers) + sdf = dataframe_factory() + + sdf = sdf.set_headers(lambda value_, key_, timestamp_, headers_: new_headers) + + result = sdf.test( + value=value, key=key, timestamp=timestamp, headers=original_headers + )[0] + assert result == expected + class TestStreamingDataFrameApplyExpand: def test_apply_expand(self, dataframe_factory):