🏡 index : ~doyle/stork.git

c8713fa1f7e5f78075308a5568b6bafaecf2563b [patch]

57bcffa94bf6b0e7626bc0f13eb1ba5fd0301f25

18315fcbe1dbe54fb9082f33f98711596a44e81b

c8713fa1f7e5f78075308a5568b6bafaecf2563b.tar.gz

Fix some reqwest-related issues.

Some websites were throwing errors because we didn't
accept gzipped content and/or had a user-agent they
didn't like.

Diff

 crawler/Cargo.toml |  2 +-
 crawler/src/lib.rs | 11 +++++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/crawler/Cargo.toml b/crawler/Cargo.toml
index 450b578..b14032b 100644
--- a/crawler/Cargo.toml
+++ a/crawler/Cargo.toml
@@ -6,7 +6,7 @@

[dependencies]
select = ""
reqwest = "0.10.1"
reqwest = { version = "0.10.1", features = ["gzip"] }
url = ""

failure = ""
diff --git a/crawler/src/lib.rs b/crawler/src/lib.rs
index 4017480..d09869b 100644
--- a/crawler/src/lib.rs
+++ a/crawler/src/lib.rs
@@ -33,7 +33,16 @@
        Self {
            url,
            filters: Arc::new(Filters::default()),
            client: Arc::new(reqwest::Client::builder().build().unwrap()),
            client: Arc::new(
                reqwest::Client::builder()
                    .user_agent(concat!(
                        env!("CARGO_PKG_NAME"),
                        "/",
                        env!("CARGO_PKG_VERSION")
                    ))
                    .build()
                    .unwrap(),
            ),
            parent: None,
        }
    }