2 stable releases
1.1.0 | Feb 17, 2020 |
---|---|
1.0.0 | May 13, 2019 |
#2200 in Parser implementations
22 downloads per month
13KB
223 lines
Rust-Warc
A high performance and easy to use Web Archive (WARC) file reader
use rust_warc::WarcReader;
use std::io;
fn main() {
// we're taking input from stdin here, but any BufRead will do
let stdin = io::stdin();
let handle = stdin.lock();
let warc = WarcReader::new(handle);
let mut response_counter = 0;
let mut response_size = 0;
for item in warc {
let record = item.unwrap(); // could be IO/malformed error
// header names are case insensitive
if record.header.get(&"WARC-Type".into()) == Some(&"response".into()) {
response_counter += 1;
response_size += record.content.len();
}
}
println!("response records: {}", response_counter);
println!("response size: {} MiB", response_size >> 20);
}
lib.rs
:
A high performance Web Archive (WARC) file parser
The WarcReader iterates over WarcRecords from a [BufRead] input.
Perfomance should be quite good, about ~500MiB/s on a single CPU core.
Usage
use rust_warc::WarcReader;
fn main() {
// we're taking input from stdin here, but any BufRead will do
let stdin = std::io::stdin();
let handle = stdin.lock();
let mut warc = WarcReader::new(handle);
let mut response_counter = 0;
for item in warc {
let record = item.expect("IO/malformed error");
// header names are case insensitive
if record.header.get(&"WARC-Type".into()) == Some(&"response".into()) {
response_counter += 1;
}
}
println!("# response records: {}", response_counter);
}