6 releases
0.18.1 | Nov 13, 2023 |
---|---|
0.17.2 | Sep 20, 2023 |
0.17.0-dev1 | Aug 31, 2023 |
0.16.5 | Aug 29, 2023 |
#17 in #distributed-computing
47 downloads per month
265KB
6K
SLoC
Vineyard Rust SDK
Note
Rust nightly is required. The vineyard Rust SDK is still under development. The API may change in the future.
Connecting to Vineyard
-
Resolve the UNIX-domain socket from the environment variable
VINEYARD_IPC_SOCKET
:use vineyard::client::*; let mut client = vineyard::default().unwrap();
-
Or, using explicit parameter:
use vineyard::client::*; let mut client = vineyard::connect("/var/run/vineyard.sock").unwrap();
Interact with Vineyard
-
Creating blob:
let mut blob_writer = client.create_blob(N)?;
-
Get object:
let mut meta_writer = client.get::<DataFrame>(object_id)?;
Inter-op with Python: numpy.ndarray
-
Python:
import numpy as np import vineyard client = vineyard.connect() np_array = np.random.rand(10, 20).astype(np.int32) object_id = int(client.put(np_array))
-
Rust:
let mut client = IPCClient::default()?; let tensor = client.get::<Int32Tensor>(object_id)?; assert_that!(tensor.shape().to_vec()).is_equal_to(vec![10, 20]);
Inter-op with Python: pandas.DataFrame
-
Python
import pandas as pd import vineyard client = vineyard.connect() df = pd.DataFrame({'a': ["1", "2", "3", "4"], 'b': ["5", "6", "7", "8"]}) object_id = int(client.put(df))
-
Rust
let mut client = IPCClient::default()?; let dataframe = client.get::<DataFrame>(object_id)?; assert_that!(dataframe.num_columns()).is_equal_to(2); assert_that!(dataframe.names().to_vec()).is_equal_to(vec!["a".into(), "b".into()]); for index in 0..dataframe.num_columns() { let column = dataframe.column(index); assert_that!(column.len()).is_equal_to(4); }
Inter-op with Python: pyarrow.RecordBatch
-
Python
import pandas as pd import pyarrow as pa import vineyard client = vineyard.connect() arrays = [ pa.array([1, 2, 3, 4]), pa.array(["foo", "bar", "baz", "qux"]), pa.array([3.0, 5.0, 7.0, 9.0]), ] batch = pa.RecordBatch.from_arrays(arrays, ["f0", "f1", "f2"]) object_id = int(client.put(batch))
-
Rust
let batch = client.get::<RecordBatch>(object_id)?; assert_that!(batch.num_columns()).is_equal_to(3); assert_that!(batch.num_rows()).is_equal_to(4); let schema = batch.schema(); let names = ["f0", "f1", "f2"]; let recordbatch = batch.as_ref().as_ref();
Inter-op with Python: pyarrow.Table
-
Python
batches = [batch] * 5 table = pa.Table.from_batches(batches) object_id = int(client.put(table))
-
Rust
let mut client = IPCClient::default()?; let table = client.get::<Table>(object_id)?; assert_that!(table.num_batches()).is_equal_to(5); for batch in table.batches().iter() { // ... }
Inter-op with Python: polars.DataFrame
-
Python
import polars dataframe = polars.DataFrame(table) object_id = int(client.put(dataframe))
-
Rust
use vineyard_polars::ds::dataframe::DataFrame; let mut client = IPCClient::default()?; let dataframe = client.get::<DataFrame>(object_id)?; let dataframe = dataframe.as_ref().as_ref(); assert_that!(dataframe.width()).is_equal_to(3); for column in dataframe.get_columns() { // ... }
Inter-op with Python: polars.DataFrame
-
Python
batches = [batch] * 5 table = pa.Table.from_batches(batches) object_id = int(client.put(table))
-
Rust
use vineyard_datafusion::ds::dataframe::DataFrame; let mut client = IPCClient::default()?; let dataframe = client.get::<DataFrame>(object_id)?; let ctx = SessionContext::new(); let table = ctx.read_table(dataframe.table_provider()).unwrap(); assert_that!(block_on(table.count()).unwrap()).is_equal_to(1000);
Dependencies
~23–32MB
~542K SLoC