dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-06-18 06:01:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #11 from spk/use-request-http-client

Browse source 
Use request higher level Client crate for openssl updates
This commit is contained in:
messense 2016-11-14 16:10:18 +08:00 committed by GitHub
commit e3b9ca6c6b
2 changed files with 11 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Cargo.toml
View file

@ -13,8 +13,8 @@ version = "0.7.0"
[dependencies]
url = "1.2"
[dependencies.hyper]
version = "0.9"
[dependencies.reqwest]
version = "0.1.0"
optional = true
[dependencies.clippy]
@ -23,5 +23,5 @@ version = "^0.*"
[features]
default = ["http"]
http = ["hyper"]
http = ["reqwest"]
unstable = []

15
src/lib.rs
View file

@ -34,7 +34,7 @@
extern crate url;
#[cfg(feature = "http")]
extern crate hyper;
extern crate reqwest;
#[cfg(feature = "http")]
use std::io::Read;
@ -45,13 +45,13 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
use url::Url;
#[cfg(feature = "http")]
use hyper::Client;
use reqwest::Client;
#[cfg(feature = "http")]
use hyper::header::UserAgent;
use reqwest::header::UserAgent;
#[cfg(feature = "http")]
use hyper::status::StatusCode;
use reqwest::StatusCode;
#[cfg(feature = "http")]
use hyper::client::Response;
use reqwest::Response;
#[cfg(feature = "http")]
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
@ -254,7 +254,7 @@ impl<'a> RobotFileParser<'a> {
#[cfg(feature = "http")]
/// Reads the robots.txt URL and feeds it to the parser.
pub fn read(&self) {
let client = Client::new();
let client = Client::new().expect("client failed to construct");
let request = client.get(self.url.clone())
.header(UserAgent(USER_AGENT.to_owned()));
let mut res = match request.send() {
@ -263,7 +263,8 @@ impl<'a> RobotFileParser<'a> {
return;
}
};
match res.status {
let status = res.status().clone();
match status {
StatusCode::Unauthorized | StatusCode::Forbidden => {
self.disallow_all.set(true);
}