9 releases
new 0.4.0-alpha.8 | May 8, 2025 |
---|---|
0.4.0-alpha.6 | Mar 20, 2025 |
0.4.0-alpha.4 | Mar 19, 2025 |
0.4.0-alpha.2 | Mar 18, 2025 |
0.3.0-alpha | Mar 18, 2025 |
#142 in Database implementations
161 downloads per month
150KB
1.5K
SLoC
SIMD R Drive Extensions
Work in progress.
simd-r-drive-extensions
provides optional utilities for working with Option<T>
and TTL-based caching in SIMD R Drive.
Install
cargo add simd-r-drive-extensions
Usage
Working with Option<T>
use simd_r_drive::DataStore;
use simd_r_drive_extensions::StorageOptionExt;
use std::path::PathBuf;
let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
// Write Some value
storage.write_option(b"key_with_some_value", Some(&42)).unwrap();
assert_eq!(
storage.read_option::<i32>(b"key_with_some_value").expect("Failed to read key1"),
Some(42)
);
// Write None
storage.write_option::<i32>(b"key_with_none_value", None).unwrap();
assert_eq!(
storage.read_option::<i32>(b"key_with_none_value").expect("Failed to read key2"),
None
);
// Errors on non-existent keys
assert!(storage.read_option::<i32>(b"non_existent_key").is_err());
Notes
- Uses a predefined tombstone marker (
[0xFF, 0xFE]
) to representNone
. - Values are serialized using bincode.
- ⚠️ Unlike SIMD R Drive, values are non-zero-copy, as they require deserialization.
Working with TTL-based Caching
use simd_r_drive::DataStore;
use simd_r_drive_extensions::StorageCacheExt;
use std::path::PathBuf;
use std::thread::sleep;
use std::time::Duration;
let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
// Write value with a TTL of 5 seconds
storage.write_with_ttl(b"key_with_ttl", &42, 5).unwrap();
assert_eq!(
storage.read_with_ttl::<i32>(b"key_with_ttl").expect("Failed to read key"),
Some(42)
);
// Wait for TTL to expire
sleep(Duration::from_secs(6));
assert_eq!(
storage.read_with_ttl::<i32>(b"key_with_ttl").expect("Failed to read key"),
None // Key should be expired and removed
);
Notes
- TTL values are stored as a binary prefix before the actual value.
- Values are serialized using bincode.
- ⚠️ Unlike SIMD R Drive, values are non-zero-copy, as they require deserialization.
- TTL-based storage will automatically evict expired values upon read to prevent stale data.
Importing Files from a Directory (Recursive + Streaming)
use simd_r_drive::DataStore;
use simd_r_drive_extensions::StorageFileImportExt;
use std::path::PathBuf;
let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
// Recursively stream and import all files under `./assets`
// Keys will use Unix-style paths like "subdir/file.txt"
let imported = storage
.import_dir_recursively("../assets", None)
.expect("Failed to import directory");
for (key, offset) in &imported {
println!(
"Imported file at key: {} (offset {})",
String::from_utf8_lossy(key),
offset
);
}
// Optional: use a namespace to avoid key collisions
let namespace: Option<&[u8]> = Some(b"assets");
let namespace_imported = storage
.import_dir_recursively("../assets", namespace)
.expect("Failed to import with namespace");
for (key, offset) in &namespace_imported {
println!(
"Imported (namespaced) file at key: {:02X?} (offset {})",
key,
offset
);
}
Note
- File import uses streaming I/O, avoiding full file loads into memory.
Reading a File Stream from Storage
Streaming Example
use simd_r_drive::DataStore;
use simd_r_drive_extensions::StorageFileImportExt;
use std::fs;
use std::io::{Read, BufReader};
use std::path::PathBuf;
let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
let import_dir = "../.github";
let relative_file = "workflows/rust-release.yml";
// Import directory first
storage
.import_dir_recursively(import_dir, Some(b"some-namespace"))
.expect("Failed to import directory");
// Read from storage stream
let mut stored_stream = storage
.open_file_stream(relative_file, Some(b"some-namespace"))
.expect("File not found in storage");
let mut stored_contents = String::new();
stored_stream
.read_to_string(&mut stored_contents)
.expect("Failed to read from stream");
// Read original for comparison
let full_path = PathBuf::from(import_dir).join(relative_file);
let mut file = BufReader::new(fs::File::open(&full_path).expect("Original file missing"));
let mut original_contents = String::new();
file.read_to_string(&mut original_contents)
.expect("Failed to read original file");
// Compare
assert_eq!(
stored_contents, original_contents,
"Streamed contents do not match original"
);
Non-Streaming Example
use simd_r_drive::DataStore;
use simd_r_drive_extensions::StorageFileImportExt;
use std::fs;
use std::io::{Read, BufReader};
use std::path::PathBuf;
let storage = DataStore::open(&PathBuf::from("test_store.bin")).unwrap();
let import_dir = "../.github";
let relative_file = "workflows/rust-release.yml";
// Import directory first
storage
.import_dir_recursively(import_dir, Some(b"some-namespace"))
.expect("Failed to import directory");
// Read from storage
let mut stored = storage
.open_file_stream(relative_file, Some(b"some-namespace"))
.expect("File not found in storage");
let mut stored_contents = String::new();
stored
.read_to_string(&mut stored_contents)
.expect("Failed to read from store");
// Read from original file
let full_path = PathBuf::from(import_dir).join(relative_file);
let mut file = BufReader::new(fs::File::open(&full_path).expect("Original file missing"));
let mut original_contents = String::new();
file.read_to_string(&mut original_contents)
.expect("Failed to read original file");
// Validate contents match
assert_eq!(
stored_contents, original_contents,
"Stored content does not match original file"
);
Notes
- These methods do not directly access the filesystem — they operate entirely within the DataStore.
- If a namespace was used during import, it must be provided again when reading.
- Relative paths must match those used during import, and must use the same namespace if provided.
- Internally uses zero-copy range handles (EntryStream) backed by memory-mapped file reads.
License
Licensed under the Apache-2.0 License.
Dependencies
~2–10MB
~85K SLoC