Skip to content

Commit

Permalink
0.5.0 (#24)
Browse files Browse the repository at this point in the history
- Refactor the project to make all WALs based on the generic implementation.
- Support different memtables based on [`crossbeam-skiplist`](https://github.com/crossbeam-rs/crossbeam) or [`skl`](https://github.com/al8n/skl)
- More user-friendly APIs
  • Loading branch information
al8n committed Oct 31, 2024
1 parent 7b33846 commit a0e9784
Show file tree
Hide file tree
Showing 81 changed files with 12,494 additions and 9,909 deletions.
10 changes: 2 additions & 8 deletions .codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,9 @@ ignore:
- "**/integration/"
- "**/examples/"
- "**/benches/"
- "src/tests.rs"
- "src/error.rs"
- "src/swmr/generic/tests.rs"
- "src/swmr/generic/tests/"
- "src/swmr/wal/tests.rs"
- "src/swmr/wal/tests/"
- "src/wal/type/"
- "src/unsync/tests.rs"
- "src/unsync/tests/"
- "src/swmr/tests.rs"
- "src/swmr/tests/"

coverage:
status:
Expand Down
26 changes: 11 additions & 15 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,12 +183,12 @@ jobs:
path: ~/.cargo
key: ${{ runner.os }}-coverage-dotcargo
- name: Run test (Unix)
run: RUSTFLAGS="--cfg all_tests" cargo test --all-features
run: RUSTFLAGS="--cfg all_orderwal_tests" cargo test --all-features
if: matrix.os != 'windows-latest'
- name: Run test (Windows)
shell: pwsh
run: |
$env:RUSTFLAGS="--cfg all_tests"
$env:RUSTFLAGS="--cfg all_orderwal_tests"
cargo test --all-features
if: matrix.os == 'windows-latest'

Expand Down Expand Up @@ -238,18 +238,14 @@ jobs:
- x86_64-apple-darwin
- aarch64-apple-darwin
cfg:
- unsync_insert
- unsync_iters
- unsync_get
- unsync_constructor
- swmr_insert
- swmr_iters
- swmr_get
- swmr_constructor
- swmr_generic_insert
- swmr_generic_iters
- swmr_generic_get
- swmr_generic_constructor
- swmr_multiple_version_insert
- swmr_multiple_version_iters
- swmr_multiple_version_get
- swmr_multiple_version_constructor
# Exclude invalid combinations
exclude:
- os: ubuntu-latest
Expand Down Expand Up @@ -303,10 +299,10 @@ jobs:
# - swmr_iters
# - swmr_get
# - swmr_constructor
# - swmr_generic_insert
# - swmr_generic_iters
# - swmr_generic_get
# - swmr_generic_constructor
# - swmr_insert
# - swmr_iters
# - swmr_get
# - swmr_constructor
# # Exclude invalid combinations
# exclude:
# - os: ubuntu-latest
Expand Down Expand Up @@ -433,7 +429,7 @@ jobs:
- name: Run tarpaulin
uses: actions-rs/cargo@v1
env:
RUSTFLAGS: "--cfg all_tests"
RUSTFLAGS: "--cfg all_orderwal_tests"
with:
command: tarpaulin
args: --all-features --run-types tests --run-types doctests --workspace --out xml
Expand Down
8 changes: 7 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
# Rleases

## 0.5.0 (Oct 27th, 2024)

- Refactor the project to make all of the WALs based on the generic implementation.
- Support different memtables based on [`crossbeam-skiplist`](https://github.com/crossbeam-rs/crossbeam) or [`skl`](https://github.com/al8n/skl)
- More user-friendly APIs

## 0.4.0 (Sep 30th, 2024)

FEATURES

- Support `K: ?Sized` and `V: ?Sized` for `GenericOrderWal`.
- Support `K: ?Sized` and `V: ?Sized` for `OrderWal`.
- Use `flush_header_and_range` instead of `flush_range` when insertion.

## 0.1.0 (Sep 14th, 2024)
Expand Down
26 changes: 12 additions & 14 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
[package]
name = "orderwal"
version = "0.4.1"
version = "0.5.0"
edition = "2021"
repository = "https://github.com/al8n/orderwal"
homepage = "https://github.com/al8n/orderwal"
documentation = "https://docs.rs/orderwal"
description = "A generic-purpose, atomic, ordered, zero-copy, Write-Ahead Log implementation for Rust."
license = "MIT OR Apache-2.0"
rust-version = "1.80"
rust-version = "1.81"
categories = ["filesystem", "database-implementations", "development-tools", "data-structures"]
keywords = ["wal", "write-ahead-log", "append-only", "append-only-log", "bitcask"]

Expand All @@ -18,7 +18,7 @@ harness = false

[features]
default = ["std"]
std = ["rarena-allocator/default", "crossbeam-skiplist/default", "bitflags/std", "dbutils/default", "among/default", "faststr?/default", "bytes?/default", "smol_str?/default"]
std = ["rarena-allocator/default", "crossbeam-skiplist/default", "bitflags/std", "dbutils/default", "among/default", "faststr?/default", "bytes?/default", "smol_str?/default", "skl/memmap"]

xxhash3 = ["dbutils/xxhash3", "std"]
xxhash64 = ["dbutils/xxhash64", "std"]
Expand All @@ -28,12 +28,14 @@ tracing = ["dep:tracing", "dbutils/tracing"]
[dependencies]
among = { version = "0.1", default-features = false, features = ["either"] }
bitflags = { version = "2", default-features = false }
dbutils = { version = "0.6", default-features = false, features = ["crc32fast"] }
dbutils = { version = "0.8", default-features = false, features = ["crc32fast"] }
derive-where = "1"
ref-cast = "1"
rarena-allocator = { version = "0.4", default-features = false, features = ["memmap"] }
crossbeam-skiplist = { version = "0.1", default-features = false, package = "crossbeam-skiplist-pr1132" }
crossbeam-skiplist-mvcc = "0.1"
skl = { version = "0.18", default-features = false, features = ["alloc"] }
paste = "1"
thiserror = "1"

bytes = { version = "1", default-features = false, optional = true }
smallvec = { version = "1", default-features = false, optional = true, features = ["const_generics"] }
Expand Down Expand Up @@ -68,17 +70,13 @@ rustdoc-args = ["--cfg", "docsrs"]
rust_2018_idioms = "warn"
single_use_lifetimes = "warn"
unexpected_cfgs = { level = "warn", check-cfg = [
'cfg(all_tests)',
'cfg(test_unsync_constructor)',
'cfg(test_unsync_insert)',
'cfg(test_unsync_iters)',
'cfg(test_unsync_get)',
'cfg(all_orderwal_tests)',
'cfg(test_swmr_constructor)',
'cfg(test_swmr_insert)',
'cfg(test_swmr_iters)',
'cfg(test_swmr_get)',
'cfg(test_swmr_generic_constructor)',
'cfg(test_swmr_generic_insert)',
'cfg(test_swmr_generic_iters)',
'cfg(test_swmr_generic_get)',
'cfg(test_swmr_multiple_version_constructor)',
'cfg(test_swmr_multiple_version_insert)',
'cfg(test_swmr_multiple_version_iters)',
'cfg(test_swmr_multiple_version_get)',
] }
31 changes: 22 additions & 9 deletions README-zh_CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
</div>
<div align="center">

An ordered, zero-copy, Write-Ahead Log implementation for Rust.
A generic-purpose, atomic, ordered, zero-copy, Write-Ahead Log implementation for Rust.

[<img alt="github" src="https://img.shields.io/badge/github-al8n/orderwal-8da0cb?style=for-the-badge&logo=Github" height="22">][Github-url]
<img alt="LoC" src="https://img.shields.io/endpoint?url=https%3A%2F%2Fgist.githubusercontent.com%2Fal8n%2F327b2a8aef9003246e45c6e47fe63937%2Fraw%2Forderwal" height="22">
Expand All @@ -15,17 +15,34 @@ An ordered, zero-copy, Write-Ahead Log implementation for Rust.
[<img alt="crates.io" src="https://img.shields.io/crates/d/orderwal?color=critical&logo=data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBzdGFuZGFsb25lPSJubyI/PjwhRE9DVFlQRSBzdmcgUFVCTElDICItLy9XM0MvL0RURCBTVkcgMS4xLy9FTiIgImh0dHA6Ly93d3cudzMub3JnL0dyYXBoaWNzL1NWRy8xLjEvRFREL3N2ZzExLmR0ZCI+PHN2ZyB0PSIxNjQ1MTE3MzMyOTU5IiBjbGFzcz0iaWNvbiIgdmlld0JveD0iMCAwIDEwMjQgMTAyNCIgdmVyc2lvbj0iMS4xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHAtaWQ9IjM0MjEiIGRhdGEtc3BtLWFuY2hvci1pZD0iYTMxM3guNzc4MTA2OS4wLmkzIiB3aWR0aD0iNDgiIGhlaWdodD0iNDgiIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIj48ZGVmcz48c3R5bGUgdHlwZT0idGV4dC9jc3MiPjwvc3R5bGU+PC9kZWZzPjxwYXRoIGQ9Ik00NjkuMzEyIDU3MC4yNHYtMjU2aDg1LjM3NnYyNTZoMTI4TDUxMiA3NTYuMjg4IDM0MS4zMTIgNTcwLjI0aDEyOHpNMTAyNCA2NDAuMTI4QzEwMjQgNzgyLjkxMiA5MTkuODcyIDg5NiA3ODcuNjQ4IDg5NmgtNTEyQzEyMy45MDQgODk2IDAgNzYxLjYgMCA1OTcuNTA0IDAgNDUxLjk2OCA5NC42NTYgMzMxLjUyIDIyNi40MzIgMzAyLjk3NiAyODQuMTYgMTk1LjQ1NiAzOTEuODA4IDEyOCA1MTIgMTI4YzE1Mi4zMiAwIDI4Mi4xMTIgMTA4LjQxNiAzMjMuMzkyIDI2MS4xMkM5NDEuODg4IDQxMy40NCAxMDI0IDUxOS4wNCAxMDI0IDY0MC4xOTJ6IG0tMjU5LjItMjA1LjMxMmMtMjQuNDQ4LTEyOS4wMjQtMTI4Ljg5Ni0yMjIuNzItMjUyLjgtMjIyLjcyLTk3LjI4IDAtMTgzLjA0IDU3LjM0NC0yMjQuNjQgMTQ3LjQ1NmwtOS4yOCAyMC4yMjQtMjAuOTI4IDIuOTQ0Yy0xMDMuMzYgMTQuNC0xNzguMzY4IDEwNC4zMi0xNzguMzY4IDIxNC43MiAwIDExNy45NTIgODguODMyIDIxNC40IDE5Ni45MjggMjE0LjRoNTEyYzg4LjMyIDAgMTU3LjUwNC03NS4xMzYgMTU3LjUwNC0xNzEuNzEyIDAtODguMDY0LTY1LjkyLTE2NC45MjgtMTQ0Ljk2LTE3MS43NzZsLTI5LjUwNC0yLjU2LTUuODg4LTMwLjk3NnoiIGZpbGw9IiNmZmZmZmYiIHAtaWQ9IjM0MjIiIGRhdGEtc3BtLWFuY2hvci1pZD0iYTMxM3guNzc4MTA2OS4wLmkwIiBjbGFzcz0iIj48L3BhdGg+PC9zdmc+&style=for-the-badge" height="22">][crates-url]
<img alt="license" src="https://img.shields.io/badge/License-Apache%202.0/MIT-blue.svg?style=for-the-badge&fontColor=white&logoColor=f5c076&logo=data:image/svg+xml;base64,PCFET0NUWVBFIHN2ZyBQVUJMSUMgIi0vL1czQy8vRFREIFNWRyAxLjEvL0VOIiAiaHR0cDovL3d3dy53My5vcmcvR3JhcGhpY3MvU1ZHLzEuMS9EVEQvc3ZnMTEuZHRkIj4KDTwhLS0gVXBsb2FkZWQgdG86IFNWRyBSZXBvLCB3d3cuc3ZncmVwby5jb20sIFRyYW5zZm9ybWVkIGJ5OiBTVkcgUmVwbyBNaXhlciBUb29scyAtLT4KPHN2ZyBmaWxsPSIjZmZmZmZmIiBoZWlnaHQ9IjgwMHB4IiB3aWR0aD0iODAwcHgiIHZlcnNpb249IjEuMSIgaWQ9IkNhcGFfMSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB4bWxuczp4bGluaz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94bGluayIgdmlld0JveD0iMCAwIDI3Ni43MTUgMjc2LjcxNSIgeG1sOnNwYWNlPSJwcmVzZXJ2ZSIgc3Ryb2tlPSIjZmZmZmZmIj4KDTxnIGlkPSJTVkdSZXBvX2JnQ2FycmllciIgc3Ryb2tlLXdpZHRoPSIwIi8+Cg08ZyBpZD0iU1ZHUmVwb190cmFjZXJDYXJyaWVyIiBzdHJva2UtbGluZWNhcD0icm91bmQiIHN0cm9rZS1saW5lam9pbj0icm91bmQiLz4KDTxnIGlkPSJTVkdSZXBvX2ljb25DYXJyaWVyIj4gPGc+IDxwYXRoIGQ9Ik0xMzguMzU3LDBDNjIuMDY2LDAsMCw2Mi4wNjYsMCwxMzguMzU3czYyLjA2NiwxMzguMzU3LDEzOC4zNTcsMTM4LjM1N3MxMzguMzU3LTYyLjA2NiwxMzguMzU3LTEzOC4zNTcgUzIxNC42NDgsMCwxMzguMzU3LDB6IE0xMzguMzU3LDI1OC43MTVDNzEuOTkyLDI1OC43MTUsMTgsMjA0LjcyMywxOCwxMzguMzU3UzcxLjk5MiwxOCwxMzguMzU3LDE4IHMxMjAuMzU3LDUzLjk5MiwxMjAuMzU3LDEyMC4zNTdTMjA0LjcyMywyNTguNzE1LDEzOC4zNTcsMjU4LjcxNXoiLz4gPHBhdGggZD0iTTE5NC43OTgsMTYwLjkwM2MtNC4xODgtMi42NzctOS43NTMtMS40NTQtMTIuNDMyLDIuNzMyYy04LjY5NCwxMy41OTMtMjMuNTAzLDIxLjcwOC0zOS42MTQsMjEuNzA4IGMtMjUuOTA4LDAtNDYuOTg1LTIxLjA3OC00Ni45ODUtNDYuOTg2czIxLjA3Ny00Ni45ODYsNDYuOTg1LTQ2Ljk4NmMxNS42MzMsMCwzMC4yLDcuNzQ3LDM4Ljk2OCwyMC43MjMgYzIuNzgyLDQuMTE3LDguMzc1LDUuMjAxLDEyLjQ5NiwyLjQxOGM0LjExOC0yLjc4Miw1LjIwMS04LjM3NywyLjQxOC0xMi40OTZjLTEyLjExOC0xNy45MzctMzIuMjYyLTI4LjY0NS01My44ODItMjguNjQ1IGMtMzUuODMzLDAtNjQuOTg1LDI5LjE1Mi02NC45ODUsNjQuOTg2czI5LjE1Miw2NC45ODYsNjQuOTg1LDY0Ljk4NmMyMi4yODEsMCw0Mi43NTktMTEuMjE4LDU0Ljc3OC0zMC4wMDkgQzIwMC4yMDgsMTY5LjE0NywxOTguOTg1LDE2My41ODIsMTk0Ljc5OCwxNjAuOTAzeiIvPiA8L2c+IDwvZz4KDTwvc3ZnPg==" height="22">

[English][en-url] | 简体中文
English | [简体中文][zh-cn-url]

</div>

## Introduction

`orderwal` is generic-purpose, atomic, ordered, zero-copy, concurrent-safe, pre-allocate style (memory map) write-ahead-log for developing databases.

`orderwal` also supports generic structured key and value types, which is not limited to just bytes like other implementations.

## Installation

```toml
[dependencies]
orderwal = "0.3"
orderwal = "0.5"
```

## Example

See [examples](./examples/) for more information.

## Related projects

- [`aol`](https://github.com/al8n/aol): Yet another generic purpose, append-only write-ahead log implementation based on `std::fs::File`.
- [`skl`](https://github.com/al8n/skl): A lock-free, ARNEA based skiplist implementation, which supports in-memory and on-disk, suitable for frozen durable data file or memtable for LSM database.
- [`valog`](https://github.com/al8n/valog): A lock-free, generic, lightweight value log for WiscKey or Bitcask architecture databases.
- [`dtlog`](https://github.com/al8n/dtlog): A log for tracking discard stats of multi-files databases.

#### License

`orderwal` is under the terms of both the MIT license and the
Expand All @@ -36,12 +53,8 @@ See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for details.
Copyright (c) 2024 Al Liu.

[Github-url]: https://github.com/al8n/orderwal/
[CI-url]: https://github.com/al8n/template/actions/workflows/template.yml
[CI-url]: https://github.com/al8n/orderwal/actions/workflows/ci.yml
[doc-url]: https://docs.rs/orderwal
[crates-url]: https://crates.io/crates/orderwal
[codecov-url]: https://app.codecov.io/gh/al8n/orderwal/
[license-url]: https://opensource.org/licenses/Apache-2.0
[rustc-url]: https://github.com/rust-lang/rust/blob/master/RELEASES.md
[license-apache-url]: https://opensource.org/licenses/Apache-2.0
[license-mit-url]: https://opensource.org/licenses/MIT
[en-url]: https://github.com/al8n/orderwal/tree/main/README.md
[zh-cn-url]: https://github.com/al8n/orderwal/tree/main/README-zh_CN.md
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ English | [简体中文][zh-cn-url]

```toml
[dependencies]
orderwal = "0.4"
orderwal = "0.5"
```

## Example
Expand All @@ -40,6 +40,8 @@ See [examples](./examples/) for more information.

- [`aol`](https://github.com/al8n/aol): Yet another generic purpose, append-only write-ahead log implementation based on `std::fs::File`.
- [`skl`](https://github.com/al8n/skl): A lock-free, ARNEA based skiplist implementation, which supports in-memory and on-disk, suitable for frozen durable data file or memtable for LSM database.
- [`valog`](https://github.com/al8n/valog): A lock-free, generic, lightweight value log for WiscKey or Bitcask architecture databases.
- [`dtlog`](https://github.com/al8n/dtlog): A log for tracking discard stats of multi-files databases.

#### License

Expand Down
11 changes: 8 additions & 3 deletions ci/miri_tb.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
#!/bin/bash
set -e
set -euxo pipefail
IFS=$'\n\t'

# We need 'ts' for the per-line timing
sudo apt-get -y install moreutils
echo

# Check if TARGET and CONFIG_FLAGS are provided, otherwise panic
if [ -z "$1" ]; then
Expand All @@ -19,9 +24,9 @@ rustup toolchain install nightly --component miri
rustup override set nightly
cargo miri setup

# Zmiri-ignore-leaks needed because of https://github.com/crossbeam-rs/crossbeam/issues/579
export MIRIFLAGS="-Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-tree-borrows -Zmiri-ignore-leaks"

export RUSTFLAGS="--cfg test_$CONFIG_FLAGS"

cargo miri test --tests --target $TARGET --lib
cargo miri test --tests --target $TARGET --lib 2>&1 | ts -i '%.s '

6 changes: 3 additions & 3 deletions ci/sanitizer.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ set -ex
export ASAN_OPTIONS="detect_odr_violation=0 detect_leaks=0"

# Run address sanitizer
RUSTFLAGS="-Z sanitizer=address --cfg all_tests" \
RUSTFLAGS="-Z sanitizer=address --cfg all_orderwal_tests" \
cargo test -Z build-std --all --release --tests --target x86_64-unknown-linux-gnu --all-features --exclude benchmarks -- --test-threads=1

# Run memory sanitizer
RUSTFLAGS="-Z sanitizer=memory --cfg all_tests" \
RUSTFLAGS="-Z sanitizer=memory --cfg all_orderwal_tests" \
cargo test -Z build-std --all --release --tests --target x86_64-unknown-linux-gnu --all-features --exclude benchmarks -- --test-threads=1

# Run thread sanitizer
cargo clean
TSAN_OPTIONS="suppressions=$(pwd)/ci/tsan" \
RUSTFLAGS="${RUSTFLAGS:-} -Z sanitizer=thread --cfg all_tests" \
RUSTFLAGS="${RUSTFLAGS:-} -Z sanitizer=thread --cfg all_orderwal_tests" \
cargo test -Z build-std --all --release --target x86_64-unknown-linux-gnu --all-features --tests --exclude benchmarks -- --test-threads=1
29 changes: 28 additions & 1 deletion examples/generic_not_sized.rs
Original file line number Diff line number Diff line change
@@ -1 +1,28 @@
fn main() {}
use orderwal::{
base::{OrderWal, Reader, Writer},
Builder,
};

fn main() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("not_sized.wal");

let mut wal = unsafe {
Builder::new()
.with_capacity(1024 * 1024)
.with_create_new(true)
.with_read(true)
.with_write(true)
.map_mut::<OrderWal<str, [u8]>, _>(&path)
.unwrap()
};

wal.insert("a", b"a1".as_slice()).unwrap();
wal.insert("c", b"c1".as_slice()).unwrap();

let a = wal.get("a").unwrap();
let c = wal.get("c").unwrap();

assert_eq!(a.value(), b"a1");
assert_eq!(c.value(), b"c1");
}
36 changes: 36 additions & 0 deletions examples/multiple_version.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
use orderwal::{
multiple_version::{OrderWal, Reader, Writer},
Builder,
};

fn main() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("not_sized.wal");

let mut wal = unsafe {
Builder::new()
.with_capacity(1024 * 1024)
.with_create_new(true)
.with_read(true)
.with_write(true)
.map_mut::<OrderWal<str, [u8]>, _>(&path)
.unwrap()
};

wal.insert(1, "a", b"a1".as_slice()).unwrap();
wal.insert(3, "a", b"a3".as_slice()).unwrap();
wal.insert(1, "c", b"c1".as_slice()).unwrap();
wal.insert(3, "c", b"c3".as_slice()).unwrap();

let a = wal.get(2, "a").unwrap();
let c = wal.get(2, "c").unwrap();

assert_eq!(a.value(), b"a1");
assert_eq!(c.value(), b"c1");

let a = wal.get(3, "a").unwrap();
let c = wal.get(3, "c").unwrap();

assert_eq!(a.value(), b"a3");
assert_eq!(c.value(), b"c3");
}
Loading

0 comments on commit a0e9784

Please sign in to comment.