🏡 index : ~doyle/stork.git

d4491476413a6c70017383d6f1cc66585fe88ce5 [patch]

f9b5a2b3a789bcd083504e4c2f70671a7f31c471

7761e3d56c0977b4bf6ce9984f6c7ea6f97bbf76

d4491476413a6c70017383d6f1cc66585fe88ce5.tar.gz

Add basic CLI for stork_http

Diff

 Cargo.toml           |  2 ++-
 storkcli/Cargo.toml  | 16 +++++++++++++++-
 storkcli/src/main.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 75 insertions(+)

diff --git a/Cargo.toml b/Cargo.toml
index c50d400..9248e51 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -4,6 +4,8 @@ members = [
  "stork",
  "stork_http",

  "storkcli",

  # Internal
  "examples",
]
\ No newline at end of file
diff --git a/storkcli/Cargo.toml b/storkcli/Cargo.toml
new file mode 100644
index 0000000..3758c2b
--- /dev/null
+++ b/storkcli/Cargo.toml
@@ -0,0 +1,16 @@
[package]
name = "storkcli"
version = "0.1.0"
authors = ["Jordan Doyle <jordan@doyle.la>"]
edition = "2018"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
stork = { path = "../stork", version = "0.0.3" }
stork_http = { path = "../stork_http", version = "0.0.3" }

tokio = { version = "0.2", features = ["full"] }
futures = "0.3"

failure = ""
\ No newline at end of file
diff --git a/storkcli/src/main.rs b/storkcli/src/main.rs
new file mode 100644
index 0000000..ed229e7
--- /dev/null
+++ b/storkcli/src/main.rs
@@ -0,0 +1,57 @@
use futures::{pin_mut, StreamExt};
use std::collections::VecDeque;
use stork_http::HttpStorkable;

#[tokio::main]
async fn main() -> failure::Fallible<()> {
    let args: Vec<String> = std::env::args().collect();
    let url = args
        .get(1)
        .expect("Expecting URL parameter")
        .parse()
        .unwrap();

    traverse(HttpStorkable::new(url)).await?;

    Ok(())
}

async fn traverse(storkable: HttpStorkable) -> failure::Fallible<()> {
    let stream = storkable.exec();
    pin_mut!(stream); // needed for iteration

    let mut queue: VecDeque<_> = stream.map(|v| (v, 0)).collect::<VecDeque<_>>().await;

    if queue.is_empty() {
        panic!("Failed to find any links on the page!");
    }

    // TODO: this is very synchronous at the moment
    loop {
        if queue.is_empty() {
            break;
        }

        let (link, depth) = queue.pop_front().unwrap();
        let link: HttpStorkable = link?;

        println!("{}↳ {}", " ".repeat(depth), link.val().url());

        // add children of this storkable to the front of the queue with
        // 1 depth added on
        let children = link.exec();
        pin_mut!(children);

        while let Some(v) = children.next().await {
            queue.push_front((v, depth + 1));
        }

        // TODO: get the library returning futures 0.3 asap
        //        link.exec()
        //            .map(|v| (v, depth + 1))
        //            .for_each(|v| queue.push_front(v))
        //            .await;
    }

    Ok(())
}