From 79f0cd17cc6424a2d78e718ec5a255211855d935 Mon Sep 17 00:00:00 2001 From: Hammad Bashir Date: Wed, 13 Mar 2024 17:11:05 -0700 Subject: [PATCH] [ENH] Add Segment Interfaces (#1853) ## Description of changes *Summarize the changes made by this PR.* - Improvements & Bug fixes - Adds an interface type for segments. We can add flush etc here in the future. I will move hnsw to use this in a stacked PR. - New functionality - None. ## Test plan *How are these changes tested?* No logical changes. These are just interfaces. - [x] Tests pass locally with `pytest` for python, `yarn test` for js, `cargo test` for rust ## Documentation Changes None --- rust/worker/src/segment/mod.rs | 1 + rust/worker/src/segment/types.rs | 12 ++++++++++++ 2 files changed, 13 insertions(+) create mode 100644 rust/worker/src/segment/types.rs diff --git a/rust/worker/src/segment/mod.rs b/rust/worker/src/segment/mod.rs index 5b56f40422f..14237847e29 100644 --- a/rust/worker/src/segment/mod.rs +++ b/rust/worker/src/segment/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod config; mod distributed_hnsw_segment; mod segment_ingestor; mod segment_manager; +mod types; pub(crate) use segment_ingestor::*; pub(crate) use segment_manager::*; diff --git a/rust/worker/src/segment/types.rs b/rust/worker/src/segment/types.rs new file mode 100644 index 00000000000..27fae95d6c7 --- /dev/null +++ b/rust/worker/src/segment/types.rs @@ -0,0 +1,12 @@ +use crate::types::EmbeddingRecord; + +trait SegmentWriter { + fn begin_transaction(&self); + fn write_records(&self, records: Vec>, offset_ids: Vec); + fn commit_transaction(&self); + fn rollback_transaction(&self); +} + +trait OffsetIdAssigner: SegmentWriter { + fn assign_offset_ids(&self, records: Vec>) -> Vec; +}