Add basic CLI for stork_http
Diff
Cargo.toml | 2 ++-
storkcli/Cargo.toml | 16 +++++++++++++++-
storkcli/src/main.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
3 files changed, 75 insertions(+)
@@ -4,6 +4,8 @@ members = [
"stork",
"stork_http",
"storkcli",
"examples",
]
\ No newline at end of file
@@ -0,0 +1,16 @@
[package]
name = "storkcli"
version = "0.1.0"
authors = ["Jordan Doyle <jordan@doyle.la>"]
edition = "2018"
[dependencies]
stork = { path = "../stork", version = "0.0.3" }
stork_http = { path = "../stork_http", version = "0.0.3" }
tokio = { version = "0.2", features = ["full"] }
futures = "0.3"
failure = ""
\ No newline at end of file
@@ -0,0 +1,57 @@
use futures::{pin_mut, StreamExt};
use std::collections::VecDeque;
use stork_http::HttpStorkable;
#[tokio::main]
async fn main() -> failure::Fallible<()> {
let args: Vec<String> = std::env::args().collect();
let url = args
.get(1)
.expect("Expecting URL parameter")
.parse()
.unwrap();
traverse(HttpStorkable::new(url)).await?;
Ok(())
}
async fn traverse(storkable: HttpStorkable) -> failure::Fallible<()> {
let stream = storkable.exec();
pin_mut!(stream);
let mut queue: VecDeque<_> = stream.map(|v| (v, 0)).collect::<VecDeque<_>>().await;
if queue.is_empty() {
panic!("Failed to find any links on the page!");
}
loop {
if queue.is_empty() {
break;
}
let (link, depth) = queue.pop_front().unwrap();
let link: HttpStorkable = link?;
println!("{}↳ {}", " ".repeat(depth), link.val().url());
let children = link.exec();
pin_mut!(children);
while let Some(v) = children.next().await {
queue.push_front((v, depth + 1));
}
}
Ok(())
}