From c8713fa1f7e5f78075308a5568b6bafaecf2563b Mon Sep 17 00:00:00 2001
From: Jordan Doyle <jordan@doyle.la>
Date: Wed, 12 Feb 2020 20:56:37 +0000
Subject: [PATCH] Fix some reqwest-related issues.

Some websites were throwing errors because we didn't
accept gzipped content and/or had a user-agent they
didn't like.
---
 crawler/Cargo.toml |  2 +-
 crawler/src/lib.rs | 11 ++++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/crawler/Cargo.toml b/crawler/Cargo.toml
index 450b578..b14032b 100644
--- a/crawler/Cargo.toml
+++ b/crawler/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2018"
 
 [dependencies]
 select = ""
-reqwest = "0.10.1"
+reqwest = { version = "0.10.1", features = ["gzip"] }
 url = ""
 
 failure = ""
diff --git a/crawler/src/lib.rs b/crawler/src/lib.rs
index 4017480..d09869b 100644
--- a/crawler/src/lib.rs
+++ b/crawler/src/lib.rs
@@ -33,7 +33,16 @@ impl Storkable {
         Self {
             url,
             filters: Arc::new(Filters::default()),
-            client: Arc::new(reqwest::Client::builder().build().unwrap()),
+            client: Arc::new(
+                reqwest::Client::builder()
+                    .user_agent(concat!(
+                        env!("CARGO_PKG_NAME"),
+                        "/",
+                        env!("CARGO_PKG_VERSION")
+                    ))
+                    .build()
+                    .unwrap(),
+            ),
             parent: None,
         }
     }
--
libgit2 1.7.2