From bfab5d1c55f19690514298e324c5668ea1a61037 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:48:39 +0100 Subject: [PATCH] Update documentation (#668) Co-authored-by: quentin-quix <171557156+quentin-quix@users.noreply.github.com> --- docs/api-reference/dataframe.md | 43 ++--- docs/api-reference/quixstreams.md | 308 ++++++++++++++---------------- docs/api-reference/sinks.md | 20 +- 3 files changed, 171 insertions(+), 200 deletions(-) diff --git a/docs/api-reference/dataframe.md b/docs/api-reference/dataframe.md index e59c21d4b..3655dc0b4 100644 --- a/docs/api-reference/dataframe.md +++ b/docs/api-reference/dataframe.md @@ -10,7 +10,7 @@ class StreamingDataFrame(BaseStreaming) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L69) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L68) `StreamingDataFrame` is the main object you will use for ETL work. @@ -81,7 +81,7 @@ def apply(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L175) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L174) Apply a function to transform the value and return a new value. @@ -139,7 +139,7 @@ def update(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L264) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L263) Apply a function to mutate value in-place or to perform a side effect @@ -207,7 +207,7 @@ def filter(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L356) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L355) Filter value using provided function. @@ -259,7 +259,7 @@ def group_by(key: Union[str, Callable[[Any], Any]], key_serializer: Optional[SerializerType] = "json") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L442) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L441) "Groups" messages by re-keying them via the provided group_by operation @@ -323,7 +323,7 @@ a clone with this operation added (assign to keep its effect). def contains(key: str) -> StreamingSeries ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L515) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L514) Check if the key is present in the Row value. @@ -362,7 +362,7 @@ or False otherwise. def to_topic(topic: Topic, key: Optional[Callable[[Any], Any]] = None) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L540) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L539) Produce current value to a topic. You can optionally specify a new key. @@ -415,7 +415,7 @@ the updated StreamingDataFrame instance (reassignment NOT required). def set_timestamp(func: Callable[[Any, Any, int, Any], int]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L585) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L584) Set a new timestamp based on the current message value and its metadata. @@ -461,14 +461,13 @@ a new StreamingDataFrame instance ```python def set_headers( - func: Callable[ - [Any, Any, int, List[Tuple[str, HeaderValue]]], - Collection[Tuple[str, HeaderValue]], - ] -) -> Self + func: Callable[ + [Any, Any, int, HeadersTuples], + HeadersTuples, + ]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L626) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L625) Set new message headers based on the current message value and metadata. @@ -517,7 +516,7 @@ a new StreamingDataFrame instance def print(pretty: bool = True, metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L677) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L676) Print out the current message value (and optionally, the message metadata) to @@ -573,7 +572,7 @@ def compose( ) -> Dict[str, VoidExecutor] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L719) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L718) Compose all functions of this StreamingDataFrame into one big closure. @@ -627,7 +626,7 @@ def test(value: Any, topic: Optional[Topic] = None) -> List[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L753) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L752) A shorthand to test `StreamingDataFrame` with provided value @@ -664,7 +663,7 @@ def tumbling_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> TumblingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L790) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L789) Create a tumbling window transformation on this StreamingDataFrame. @@ -750,7 +749,7 @@ def hopping_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> HoppingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L866) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L865) Create a hopping window transformation on this StreamingDataFrame. @@ -844,7 +843,7 @@ def sliding_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> SlidingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L958) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L957) Create a sliding window transformation on this StreamingDataFrame. @@ -933,7 +932,7 @@ def drop(columns: Union[str, List[str]], errors: Literal["ignore", "raise"] = "raise") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1039) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1038) Drop column(s) from the message value (value must support `del`, like a dict). @@ -977,7 +976,7 @@ a new StreamingDataFrame instance def sink(sink: BaseSink) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1083) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1082) Sink the processed data to the specified destination. diff --git a/docs/api-reference/quixstreams.md b/docs/api-reference/quixstreams.md index 1b69f0436..6826ada4b 100644 --- a/docs/api-reference/quixstreams.md +++ b/docs/api-reference/quixstreams.md @@ -766,7 +766,7 @@ a {topic_name: composed} dict, where composed is a callable class StreamingDataFrame(BaseStreaming) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L69) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L68) `StreamingDataFrame` is the main object you will use for ETL work. @@ -829,7 +829,7 @@ def apply(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L175) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L174) Apply a function to transform the value and return a new value. @@ -881,7 +881,7 @@ def update(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L264) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L263) Apply a function to mutate value in-place or to perform a side effect @@ -941,7 +941,7 @@ def filter(func: Union[ metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L356) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L355) Filter value using provided function. @@ -987,7 +987,7 @@ def group_by(key: Union[str, Callable[[Any], Any]], key_serializer: Optional[SerializerType] = "json") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L442) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L441) "Groups" messages by re-keying them via the provided group_by operation @@ -1043,7 +1043,7 @@ a clone with this operation added (assign to keep its effect). def contains(key: str) -> StreamingSeries ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L515) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L514) Check if the key is present in the Row value. @@ -1074,7 +1074,7 @@ or False otherwise. def to_topic(topic: Topic, key: Optional[Callable[[Any], Any]] = None) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L540) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L539) Produce current value to a topic. You can optionally specify a new key. @@ -1119,7 +1119,7 @@ the updated StreamingDataFrame instance (reassignment NOT required). def set_timestamp(func: Callable[[Any, Any, int, Any], int]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L585) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L584) Set a new timestamp based on the current message value and its metadata. @@ -1157,14 +1157,13 @@ a new StreamingDataFrame instance ```python def set_headers( - func: Callable[ - [Any, Any, int, List[Tuple[str, HeaderValue]]], - Collection[Tuple[str, HeaderValue]], - ] -) -> Self + func: Callable[ + [Any, Any, int, HeadersTuples], + HeadersTuples, + ]) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L626) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L625) Set new message headers based on the current message value and metadata. @@ -1205,7 +1204,7 @@ a new StreamingDataFrame instance def print(pretty: bool = True, metadata: bool = False) -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L677) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L676) Print out the current message value (and optionally, the message metadata) to @@ -1253,7 +1252,7 @@ def compose( ) -> Dict[str, VoidExecutor] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L719) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L718) Compose all functions of this StreamingDataFrame into one big closure. @@ -1299,7 +1298,7 @@ def test(value: Any, topic: Optional[Topic] = None) -> List[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L753) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L752) A shorthand to test `StreamingDataFrame` with provided value @@ -1330,7 +1329,7 @@ def tumbling_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> TumblingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L790) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L789) Create a tumbling window transformation on this StreamingDataFrame. @@ -1407,7 +1406,7 @@ def hopping_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> HoppingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L866) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L865) Create a hopping window transformation on this StreamingDataFrame. @@ -1492,7 +1491,7 @@ def sliding_window(duration_ms: Union[int, timedelta], name: Optional[str] = None) -> SlidingWindowDefinition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L958) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L957) Create a sliding window transformation on this StreamingDataFrame. @@ -1572,7 +1571,7 @@ def drop(columns: Union[str, List[str]], errors: Literal["ignore", "raise"] = "raise") -> Self ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1039) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1038) Drop column(s) from the message value (value must support `del`, like a dict). @@ -1608,7 +1607,7 @@ a new StreamingDataFrame instance def sink(sink: BaseSink) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1083) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/dataframe/dataframe.py#L1082) Sink the processed data to the specified destination. @@ -4163,9 +4162,8 @@ Initialize the KinesisSink. #### KinesisSink.add ```python -def add(value: Any, key: Any, timestamp: int, - headers: list[tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) -> None +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) -> None ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/community/kinesis.py#L80) @@ -4445,9 +4443,8 @@ Default - `None`. #### PubSubSink.add ```python -def add(value: Any, key: Any, timestamp: int, - headers: list[tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) -> None +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) -> None ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/community/pubsub.py#L81) @@ -4511,9 +4508,8 @@ If flush() fails, the checkpoint will be aborted. ```python @abc.abstractmethod -def add(value: Any, key: Any, timestamp: int, - headers: List[Tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/base/sink.py#L33) @@ -4583,9 +4579,8 @@ corresponding topic partition. #### BatchingSink.add ```python -def add(value: Any, key: Any, timestamp: int, - headers: List[Tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/base/sink.py#L93) @@ -4924,7 +4919,7 @@ it should not be mutated during message processing. class ConfluentKafkaMessageProto(Protocol) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/models/types.py#L13) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/models/types.py#L16) An interface of `confluent_kafka.Message`. @@ -5677,7 +5672,7 @@ A base class for all Serializers ```python @property -def extra_headers() -> MessageHeadersMapping +def extra_headers() -> HeadersMapping ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/models/serializers/base.py#L68) @@ -5843,11 +5838,11 @@ fail (it ignores issues for a topic already existing). #### merge\_headers ```python -def merge_headers(original: Optional[MessageHeadersTuples], - other: MessageHeadersMapping) -> MessageHeadersTuples +def merge_headers(original: KafkaHeaders, + other: HeadersMapping) -> HeadersTuples ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/models/topics/utils.py#L8) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/models/topics/utils.py#L10) Merge two sets of Kafka message headers, overwriting headers in "origin" @@ -6364,7 +6359,7 @@ def expire_windows(max_start_time: int, delete: bool = True) -> list[tuple[tuple[int, int], Any]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/transaction.py#L120) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/transaction.py#L128) Get all expired windows from RocksDB up to the specified `max_start_time` timestamp. @@ -6398,7 +6393,7 @@ A sorted list of tuples in the format `((start, end), value)`. def delete_windows(max_start_time: int, prefix: bytes) -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/transaction.py#L177) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/transaction.py#L185) Delete windows from RocksDB up to the specified `max_start_time` timestamp. @@ -6430,7 +6425,7 @@ def get_windows(start_from_ms: int, backwards: bool = False) -> list[tuple[tuple[int, int], Any]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/transaction.py#L224) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/transaction.py#L232) Get all windows that start between "start_from_ms" and "start_to_ms" @@ -6619,7 +6614,7 @@ using the provided `timestamp`. #### WindowedTransactionState.get\_latest\_timestamp ```python -def get_latest_timestamp() -> int +def get_latest_timestamp() -> Optional[int] ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/windowed/state.py#L69) @@ -6770,13 +6765,13 @@ partitions' transactions. ```python def __init__( name: str, - topic: str, + topic: Optional[str], base_dir: str, changelog_producer_factory: Optional[ChangelogProducerFactory] = None, - options: Optional[options_type] = None) + options: Optional[RocksDBOptionsType] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/store.py#L28) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/store.py#L26) **Arguments**: @@ -6829,7 +6824,7 @@ def write(cache: PartitionTransactionCache, batch: Optional[WriteBatch] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L112) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L115) Write data to RocksDB @@ -6846,11 +6841,10 @@ Write data to RocksDB ```python def get(key: bytes, - default: Any = None, - cf_name: str = "default") -> Union[None, bytes, Any] + cf_name: str = "default") -> Union[bytes, Literal[Marker.UNDEFINED]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L177) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L180) Get a key from RocksDB. @@ -6872,7 +6866,7 @@ a value if the key is present in the DB. Otherwise, `default` def exists(key: bytes, cf_name: str = "default") -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L191) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L196) Check if a key is present in the DB. @@ -6893,7 +6887,7 @@ Check if a key is present in the DB. def get_processed_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L202) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L207) Get last processed offset for the given partition @@ -6909,7 +6903,7 @@ offset or `None` if there's no processed offset yet def get_changelog_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L214) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L222) Get offset that the changelog is up-to-date with. @@ -6925,7 +6919,7 @@ offset or `None` if there's no processed offset yet def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L224) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L234) Close the underlying RocksDB @@ -6938,7 +6932,7 @@ Close the underlying RocksDB def path() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L237) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L247) Absolute path to RocksDB database folder @@ -6955,7 +6949,7 @@ file path def destroy(cls, path: str) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L245) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L255) Delete underlying RocksDB database @@ -6973,7 +6967,7 @@ The database must be closed first. def get_column_family_handle(cf_name: str) -> ColumnFamily ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L255) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L265) Get a column family handle to pass to it WriteBatch. @@ -6996,7 +6990,7 @@ instance of `rocksdict.ColumnFamily` def get_column_family(cf_name: str) -> Rdict ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L276) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/rocksdb/partition.py#L286) Get a column family instance. @@ -7058,7 +7052,7 @@ Requires a full state recovery for each partition on assignment. ```python def __init__( name: str, - topic: str, + topic: Optional[str], changelog_producer_factory: Optional[ChangelogProducerFactory] = None ) -> None ``` @@ -7121,7 +7115,7 @@ Write data to the state def get_processed_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L122) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L125) Get last processed offset for the given partition @@ -7137,7 +7131,7 @@ offset or `None` if there's no processed offset yet def get_changelog_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L129) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L132) Get offset that the changelog is up-to-date with. @@ -7152,11 +7146,10 @@ offset or `None` if there's no processed offset yet ```python @_validate_partition_state() def get(key: bytes, - default: Any = None, - cf_name: str = "default") -> Union[None, bytes, Any] + cf_name: str = "default") -> Union[bytes, Literal[Marker.UNDEFINED]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L137) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L140) Get a key from the store @@ -7179,7 +7172,7 @@ a value if the key is present in the store. Otherwise, `default` def exists(key: bytes, cf_name: str = "default") -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L151) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/memory/partition.py#L154) Check if a key is present in the store. @@ -7224,7 +7217,7 @@ Since `StorePartition`s do recovery directly, it also handles recovery transacti def offset() -> int ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L62) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L64) Get the changelog offset from the underlying `StorePartition`. @@ -7241,7 +7234,7 @@ changelog offset (int) def needs_recovery_check() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L78) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L80) Determine whether to attempt recovery for underlying `StorePartition`. @@ -7256,7 +7249,7 @@ This does NOT mean that anything actually requires recovering. def has_invalid_offset() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L89) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L91) Determine if the current changelog offset stored in state is invalid. @@ -7269,7 +7262,7 @@ def recover_from_changelog_message( changelog_message: ConfluentKafkaMessageProto) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L108) Recover the StorePartition using a message read from its respective changelog. @@ -7277,23 +7270,6 @@ Recover the StorePartition using a message read from its respective changelog. - `changelog_message`: A confluent kafka message (everything as bytes) - - -#### RecoveryPartition.set\_watermarks - -```python -def set_watermarks(lowwater: int, highwater: int) -``` - -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L115) - -Set the changelog watermarks as gathered from Consumer.get_watermark_offsets() - -**Arguments**: - -- `lowwater`: topic partition lowwater -- `highwater`: topic partition highwater - #### RecoveryPartition.set\_recovery\_consume\_position @@ -7302,7 +7278,7 @@ Set the changelog watermarks as gathered from Consumer.get_watermark_offsets() def set_recovery_consume_position(offset: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L125) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L120) Update the recovery partition with the consumer's position (whenever @@ -7322,7 +7298,7 @@ It is possible that it may be set more than once. class ChangelogProducerFactory() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L137) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L132) Generates ChangelogProducers, which produce changelog messages to a StorePartition. @@ -7334,7 +7310,7 @@ Generates ChangelogProducers, which produce changelog messages to a StorePartiti def __init__(changelog_name: str, producer: RowProducer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L142) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L137) **Arguments**: @@ -7353,7 +7329,7 @@ a ChangelogWriter instance def get_partition_producer(partition_num) -> "ChangelogProducer" ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L152) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L147) Generate a ChangelogProducer for producing to a specific partition number @@ -7371,7 +7347,7 @@ Generate a ChangelogProducer for producing to a specific partition number class ChangelogProducer() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L166) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L161) Generated for a `StorePartition` to produce state changes to its respective kafka changelog partition. @@ -7384,7 +7360,7 @@ kafka changelog partition. def __init__(changelog_name: str, partition: int, producer: RowProducer) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L172) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L167) **Arguments**: @@ -7399,10 +7375,10 @@ def __init__(changelog_name: str, partition: int, producer: RowProducer) ```python def produce(key: bytes, value: Optional[bytes] = None, - headers: Optional[MessageHeadersMapping] = None) + headers: Optional[Headers] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L195) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L190) Produce a message to a changelog topic partition. @@ -7420,7 +7396,7 @@ Produce a message to a changelog topic partition. class RecoveryManager() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L220) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L215) Manages all consumer-related aspects of recovery, including: - assigning/revoking, pausing/resuming topic partitions (especially changelogs) @@ -7440,7 +7416,7 @@ Recovery is attempted from the `Application` after any new partition assignment. def partitions() -> Dict[int, Dict[str, RecoveryPartition]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L239) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L234) Returns a mapping of assigned RecoveryPartitions in the following format: {: {: }} @@ -7454,7 +7430,7 @@ Returns a mapping of assigned RecoveryPartitions in the following format: def has_assignments() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L247) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L242) Whether the Application has assigned RecoveryPartitions @@ -7471,7 +7447,7 @@ has assignments, as bool def recovering() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L256) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L251) Whether the Application is currently recovering @@ -7489,7 +7465,7 @@ def register_changelog(topic_name: Optional[str], topic_config: Optional[TopicConfig] = None) -> Topic ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L264) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L259) Register a changelog Topic with the TopicManager. @@ -7506,7 +7482,7 @@ Register a changelog Topic with the TopicManager. def do_recovery() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L282) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L277) If there are any active RecoveryPartitions, do a recovery procedure. @@ -7522,7 +7498,7 @@ def assign_partition(topic: Optional[str], partition: int, store_partitions: Dict[str, StorePartition]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L335) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L332) Assigns `StorePartition`s (as `RecoveryPartition`s) ONLY IF recovery required. @@ -7536,7 +7512,7 @@ Pauses active consumer partitions as needed. def revoke_partition(partition_num: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L412) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/recovery.py#L409) revoke ALL StorePartitions (across all Stores) for a given partition number @@ -7622,7 +7598,7 @@ using the provided `timestamp`. #### WindowedState.get\_latest\_timestamp ```python -def get_latest_timestamp() -> int +def get_latest_timestamp() -> Optional[int] ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/types.py#L48) @@ -8042,7 +8018,7 @@ StateStoreManager is responsible for: def stores() -> Dict[Optional[str], Dict[str, Store]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L73) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L79) Map of registered state stores @@ -8059,7 +8035,7 @@ dict in format {topic: {store_name: store}} def recovery_required() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L81) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L87) Whether recovery needs to be done. @@ -8072,7 +8048,7 @@ Whether recovery needs to be done. def using_changelogs() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L90) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L96) Whether the StateStoreManager is using changelog topics @@ -8085,10 +8061,10 @@ using changelogs, as bool #### StateStoreManager.do\_recovery ```python -def do_recovery() +def do_recovery() -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L102) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L108) Perform a state recovery, if necessary. @@ -8097,10 +8073,10 @@ Perform a state recovery, if necessary. #### StateStoreManager.stop\_recovery ```python -def stop_recovery() +def stop_recovery() -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L108) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L117) Stop recovery (called during app shutdown). @@ -8112,7 +8088,7 @@ Stop recovery (called during app shutdown). def get_store(topic: str, store_name: str = DEFAULT_STATE_STORE_NAME) -> Store ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L114) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L126) Get a store for given name and topic @@ -8133,10 +8109,10 @@ instance of `Store` def register_store(topic_name: Optional[str], store_name: str = DEFAULT_STATE_STORE_NAME, store_type: Optional[StoreTypes] = None, - topic_config: Optional[TopicConfig] = None) + topic_config: Optional[TopicConfig] = None) -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L149) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L163) Register a state store to be managed by StateStoreManager. @@ -8157,10 +8133,10 @@ Default to StateStoreManager `default_store_type` #### StateStoreManager.register\_windowed\_store ```python -def register_windowed_store(topic_name: str, store_name: str) +def register_windowed_store(topic_name: str, store_name: str) -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L194) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L208) Register a windowed state store to be managed by StateStoreManager. @@ -8179,10 +8155,10 @@ Each window store can be registered only once for each topic. #### StateStoreManager.clear\_stores ```python -def clear_stores() +def clear_stores() -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L220) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L234) Delete all state stores managed by StateStoreManager. @@ -8195,7 +8171,7 @@ def on_partition_assign(topic: Optional[str], partition: int, committed_offset: int) -> Dict[str, StorePartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L235) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L250) Assign store partitions for each registered store for the given `TopicPartition` @@ -8216,10 +8192,10 @@ list of assigned `StorePartition` #### StateStoreManager.on\_partition\_revoke ```python -def on_partition_revoke(topic: str, partition: int) +def on_partition_revoke(topic: str, partition: int) -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L261) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L276) Revoke store partitions for each registered store for the given `TopicPartition` @@ -8233,10 +8209,10 @@ Revoke store partitions for each registered store for the given `TopicPartition` #### StateStoreManager.init ```python -def init() +def init() -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L274) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L289) Initialize `StateStoreManager` and create a store directory @@ -8246,10 +8222,10 @@ Initialize `StateStoreManager` and create a store directory #### StateStoreManager.close ```python -def close() +def close() -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L281) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/manager.py#L296) Close all registered stores @@ -8282,10 +8258,10 @@ partitions' transactions. ```python @property -def topic() -> str +def topic() -> Optional[str] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L37) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L35) Topic name @@ -8298,7 +8274,7 @@ Topic name def name() -> str ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L44) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L42) Store name @@ -8311,7 +8287,7 @@ Store name def partitions() -> Dict[int, StorePartition] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L51) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L49) Mapping of assigned store partitions @@ -8327,7 +8303,7 @@ dict of "{partition: }" def assign_partition(partition: int) -> StorePartition ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L58) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L56) Assign new store partition @@ -8347,7 +8323,7 @@ instance of `StorePartition` def revoke_partition(partition: int) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L85) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L83) Revoke assigned store partition @@ -8363,7 +8339,7 @@ Revoke assigned store partition def start_partition_transaction(partition: int) -> PartitionTransaction ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L101) Start a new partition transaction. @@ -8385,7 +8361,7 @@ instance of `PartitionTransaction` def close() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L121) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/store.py#L119) Close store and revoke all store partitions @@ -8401,7 +8377,7 @@ Close store and revoke all store partitions class StorePartition(ABC) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L25) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L26) A base class to access state in the underlying storage. It represents a single instance of some storage (e.g. a single database for @@ -8416,7 +8392,7 @@ the persistent storage). def get_processed_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L56) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L57) Get last processed offset for the given partition @@ -8433,7 +8409,7 @@ offset or `None` if there's no processed offset yet def get_changelog_offset() -> Optional[int] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L64) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L65) Get offset that the changelog is up-to-date with. @@ -8451,7 +8427,7 @@ def write(cache: PartitionTransactionCache, processed_offset: Optional[int], changelog_offset: Optional[int]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L72) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L73) Update the state with data from the update cache @@ -8468,11 +8444,10 @@ Update the state with data from the update cache ```python @abstractmethod def get(key: bytes, - default: Any = None, - cf_name: str = "default") -> Union[None, bytes, Any] + cf_name: str = "default") -> Union[bytes, Literal[Marker.UNDEFINED]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L87) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L88) Get a key from the store @@ -8495,7 +8470,7 @@ a value if the key is present in the store. Otherwise, `default` def exists(key: bytes, cf_name: str = "default") -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L100) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L101) Check if a key is present in the store. @@ -8516,7 +8491,7 @@ Check if a key is present in the store. def begin() -> PartitionTransaction ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L109) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L110) Start a new `PartitionTransaction` @@ -8528,10 +8503,11 @@ Using `PartitionTransaction` is a recommended way for accessing the data. ```python def recover_from_changelog_message( - changelog_message: ConfluentKafkaMessageProto, committed_offset: int) + changelog_message: ConfluentKafkaMessageProto, + committed_offset: int) -> None ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L122) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/partition.py#L123) Updates state from a given changelog message. @@ -8552,7 +8528,7 @@ Updates state from a given changelog message. class PartitionTransactionCache() ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L37) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L36) A cache with the data updated in the current PartitionTransaction. It is used to read-your-own-writes before the transaction is committed to the Store. @@ -8567,10 +8543,10 @@ to simplify the querying over them. ```python def get(key: bytes, prefix: bytes, - cf_name: str = "default") -> Union[bytes, Undefined] + cf_name: str = "default") -> Union[bytes, Marker] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L60) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L59) Get a value for the key. @@ -8594,7 +8570,7 @@ If the key is not present in the cache, returns "UNDEFINED sentinel def set(key: bytes, value: bytes, prefix: bytes, cf_name: str = "default") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L90) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L89) Set a value for the key. @@ -8612,7 +8588,7 @@ Set a value for the key. def delete(key: Any, prefix: bytes, cf_name: str = "default") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L103) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L102) Delete a key. @@ -8630,7 +8606,7 @@ Delete a key. def is_empty() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L116) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L115) Return True if any changes have been made (updates or deletes), otherwise return False. @@ -8643,7 +8619,7 @@ return False. def get_column_families() -> Set[str] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L123) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L122) Get all update column families. @@ -8655,7 +8631,7 @@ Get all update column families. def get_updates(cf_name: str = "default") -> Dict[bytes, Dict[bytes, bytes]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L129) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L128) Get all updated keys (excluding deleted) @@ -8672,7 +8648,7 @@ in the format "{: {: }}". def get_deletes(cf_name: str = "default") -> Set[bytes] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L138) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L137) Get all deleted keys (excluding updated) as a set. @@ -8684,7 +8660,7 @@ Get all deleted keys (excluding updated) as a set. class PartitionTransactionStatus(enum.Enum) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L145) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L144) @@ -8718,7 +8694,7 @@ Transaction is failed, it cannot be used anymore def validate_transaction_status(*allowed: PartitionTransactionStatus) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L156) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L155) Check that the status of `RocksDBTransaction` is valid before calling a method @@ -8730,7 +8706,7 @@ Check that the status of `RocksDBTransaction` is valid before calling a method class PartitionTransaction(ABC) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L176) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L175) A transaction class to perform simple key-value operations like "get", "set", "delete" and "exists" on a single storage partition. @@ -8744,7 +8720,7 @@ A transaction class to perform simple key-value operations like def failed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L208) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L207) Return `True` if transaction failed to update data at some point. @@ -8763,7 +8739,7 @@ bool def completed() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L218) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L217) Return `True` if transaction is successfully completed. @@ -8782,7 +8758,7 @@ bool def prepared() -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L228) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L227) Return `True` if transaction is prepared completed. @@ -8801,7 +8777,7 @@ bool def changelog_topic_partition() -> Optional[Tuple[str, int]] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L238) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L237) Return the changelog topic-partition for the StorePartition of this transaction. @@ -8819,7 +8795,7 @@ Returns `None` if changelog_producer is not provided. def as_state(prefix: Any = DEFAULT_PREFIX) -> State ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L263) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L264) Create an instance implementing the `State` protocol to be provided @@ -8843,7 +8819,7 @@ def get(key: Any, cf_name: str = "default") -> Optional[Any] ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L282) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L283) Get a key from the store. @@ -8869,7 +8845,7 @@ value or None if the key is not found and `default` is not provided def set(key: Any, value: Any, prefix: bytes, cf_name: str = "default") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L318) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L319) Set value for the key. @@ -8889,7 +8865,7 @@ Set value for the key. def delete(key: Any, prefix: bytes, cf_name: str = "default") ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L341) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L342) Delete value for the key. @@ -8910,7 +8886,7 @@ This function always returns `None`, even if value is not found. def exists(key: Any, prefix: bytes, cf_name: str = "default") -> bool ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L360) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L361) Check if the key exists in state. @@ -8933,7 +8909,7 @@ True if key exists, False otherwise def prepare(processed_offset: Optional[int]) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L380) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L381) Produce changelog messages to the changelog topic for all changes accumulated @@ -8961,7 +8937,7 @@ def flush(processed_offset: Optional[int] = None, changelog_offset: Optional[int] = None) ``` -[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L441) +[[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/state/base/transaction.py#L442) Flush the recent updates to the database. diff --git a/docs/api-reference/sinks.md b/docs/api-reference/sinks.md index af8618b86..dfffbeaaa 100644 --- a/docs/api-reference/sinks.md +++ b/docs/api-reference/sinks.md @@ -47,9 +47,8 @@ If flush() fails, the checkpoint will be aborted. ```python @abc.abstractmethod -def add(value: Any, key: Any, timestamp: int, - headers: List[Tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/base/sink.py#L33) @@ -125,9 +124,8 @@ corresponding topic partition. #### BatchingSink.add ```python -def add(value: Any, key: Any, timestamp: int, - headers: List[Tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/base/sink.py#L93) @@ -1320,9 +1318,8 @@ Default - `None`. #### PubSubSink.add ```python -def add(value: Any, key: Any, timestamp: int, - headers: list[tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) -> None +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) -> None ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/community/pubsub.py#L81) @@ -1461,9 +1458,8 @@ Initialize the KinesisSink. #### KinesisSink.add ```python -def add(value: Any, key: Any, timestamp: int, - headers: list[tuple[str, HeaderValue]], topic: str, partition: int, - offset: int) -> None +def add(value: Any, key: Any, timestamp: int, headers: HeadersTuples, + topic: str, partition: int, offset: int) -> None ``` [[VIEW SOURCE]](https://github.com/quixio/quix-streams/blob/main/quixstreams/sinks/community/kinesis.py#L80)