dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-06-24 17:21:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #11 from spk/use-request-http-client

Browse source 
Use request higher level Client crate for openssl updates
This commit is contained in:
messense 2016-11-14 16:10:18 +08:00 committed by GitHub
commit e3b9ca6c6b
2 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Cargo.toml
View file

@ -13,8 +13,8 @@ version = "0.7.0"
[dependencies] [dependencies]
url = "1.2" url = "1.2"
[dependencies.hyper] [dependencies.reqwest]
version = "0.9" version = "0.1.0"
optional = true optional = true
[dependencies.clippy] [dependencies.clippy]
@ -23,5 +23,5 @@ version = "^0.*"
[features] [features]
default = ["http"] default = ["http"]
http = ["hyper"] http = ["reqwest"]
unstable = [] unstable = []

15
src/lib.rs
View file

@ -34,7 +34,7 @@
extern crate url; extern crate url;
#[cfg(feature = "http")] #[cfg(feature = "http")]
extern crate hyper; extern crate reqwest;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use std::io::Read; use std::io::Read;
@ -45,13 +45,13 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use url::Url; use url::Url;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use hyper::Client; use reqwest::Client;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use hyper::header::UserAgent; use reqwest::header::UserAgent;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use hyper::status::StatusCode; use reqwest::StatusCode;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use hyper::client::Response; use reqwest::Response;
#[cfg(feature = "http")] #[cfg(feature = "http")]
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)"; const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
@ -254,7 +254,7 @@ impl<'a> RobotFileParser<'a> {
#[cfg(feature = "http")] #[cfg(feature = "http")]
/// Reads the robots.txt URL and feeds it to the parser. /// Reads the robots.txt URL and feeds it to the parser.
pub fn read(&self) { pub fn read(&self) {
let client = Client::new(); let client = Client::new().expect("client failed to construct");
let request = client.get(self.url.clone()) let request = client.get(self.url.clone())
.header(UserAgent(USER_AGENT.to_owned())); .header(UserAgent(USER_AGENT.to_owned()));
let mut res = match request.send() { let mut res = match request.send() {
@ -263,7 +263,8 @@ impl<'a> RobotFileParser<'a> {
return; return;
} }
}; };
match res.status { let status = res.status().clone();
match status {
StatusCode::Unauthorized | StatusCode::Forbidden => { StatusCode::Unauthorized | StatusCode::Forbidden => {
self.disallow_all.set(true); self.disallow_all.set(true);
} }