Skip to content

Commit

Permalink
Introduce SnapshotsRepository backed by object_store
Browse files Browse the repository at this point in the history
  • Loading branch information
pcholakov committed Nov 18, 2024
1 parent 693c174 commit 768bddf
Show file tree
Hide file tree
Showing 13 changed files with 406 additions and 40 deletions.
120 changes: 96 additions & 24 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ arrow = { version = "53.1.0", default-features = false }
assert2 = "0.3.11"
async-channel = "2.1.1"
async-trait = "0.1.73"
aws-config = { version = "1.5.4", default-features = false, features = ["rt-tokio", "sso", "rustls"] }
aws-credential-types = { version = "1.2.0", default-features = false }
axum = { version = "0.7.5", default-features = false }
base64 = "0.22"
bitflags = { version = "2.6.0" }
Expand Down Expand Up @@ -134,6 +136,7 @@ metrics-exporter-prometheus = { version = "0.15", default-features = false, feat
"async-runtime",
] }
moka = "0.12.5"
object_store = { version = "0.11.1", features = ["aws"] }
once_cell = "1.18"
opentelemetry = { version = "0.24.0" }
opentelemetry-http = { version = "0.13.0" }
Expand Down
3 changes: 3 additions & 0 deletions crates/core/src/worker_api/partition_processor_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ pub enum SnapshotError {
SnapshotExportError(PartitionId, #[source] anyhow::Error),
#[error("Snapshot failed for partition {0}: {1}")]
SnapshotMetadataHeaderError(PartitionId, #[source] io::Error),
#[error("Error putting partition id {0} snapshot into repository: {1}")]
RepositoryPutError(PartitionId, #[source] anyhow::Error),
#[error("Internal error creating snapshot for partition {0}: {1}")]
Internal(PartitionId, String),
}
Expand All @@ -94,6 +96,7 @@ impl SnapshotError {
SnapshotError::InvalidState(partition_id) => *partition_id,
SnapshotError::SnapshotExportError(partition_id, _) => *partition_id,
SnapshotError::SnapshotMetadataHeaderError(partition_id, _) => *partition_id,
SnapshotError::RepositoryPutError(partition_id, _) => *partition_id,
SnapshotError::Internal(partition_id, _) => *partition_id,
}
}
Expand Down
4 changes: 2 additions & 2 deletions crates/service-client/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ options_schema = ["dep:schemars", "restate-types/schemars"]

[dependencies]
arc-swap = { workspace = true }
aws-config = { version = "1.5.4", default-features = false, features = ["rt-tokio", "sso"] }
aws-credential-types = {version = "1.2.0", default-features = false}
aws-config = { workspace = true }
aws-credential-types = { workspace = true }
aws-sdk-lambda = {version = "1.36.0", default-features = false, features = ["rt-tokio"]}
aws-sdk-sts = {version = "1.35.0", default-features = false, features = ["rt-tokio"]}
base64 = { workspace = true }
Expand Down
1 change: 1 addition & 0 deletions crates/types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ tracing = { workspace = true }
tracing-opentelemetry = { workspace = true }
ulid = { workspace = true }
xxhash-rust = { workspace = true, features = ["xxh3"] }
url = "2.5.3"

[dev-dependencies]
restate-test-util = { workspace = true }
Expand Down
Loading

0 comments on commit 768bddf

Please sign in to comment.