4 releases
0.2.1 | Jan 10, 2024 |
---|---|
0.2.0 | Jan 9, 2024 |
0.1.2 | Jan 5, 2024 |
0.1.1 | Jan 5, 2024 |
0.1.0 |
|
#12 in #spider
27KB
184 lines
Crawl
Rust crawl
demo
fn main() -> anyhow::Result<()> {
let download = Arc::new(Downloader::new(
String::from(r"data/book1"),
String::from("https://doc.rust-lang.org/book/")
));
let url = String::from("https://doc.rust-lang.org/book/index.html");
let manager = Arc::new(Mutex::new(Manager{datas:Vec::new(), added_urls:HashSet::from([url.clone()])}));
download.start_url(url, false, Arc::new(Flag::Page))?;
for _ in 0..16{
let res_arg = get_res_thread_arg(&download);
let r = Arc::clone(&manager);
thread::spawn(move || res_run(res_arg, r));
}
start_crawl(&download, 16);
download.wait_finish();
{
let m = manager.lock().unwrap();
println!("finish {}", m.datas.len());
for item in m.datas.iter(){
println!("{} {}", item.url, item.title);
}
}
Ok(())
}
Dependencies
~3–15MB
~219K SLoC