dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-07-05 19:20:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update reqwest to 0.9 (#16)

Browse source
This commit is contained in:
Laurent Arnoud 2018-09-22 05:30:41 +02:00 committed by messense
parent ed0f892875
commit 13e60c226a
2 changed files with 8 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Cargo.toml
View file

@ -14,7 +14,7 @@ version = "0.10.1"
url = "1" url = "1"
[dependencies.reqwest] [dependencies.reqwest]
version = "0.8" version = "0.9"
optional = true optional = true
[dependencies.clippy] [dependencies.clippy]

14
src/lib.rs
View file

@ -47,14 +47,14 @@ use url::Url;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use reqwest::Client; use reqwest::Client;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use reqwest::header::UserAgent; use reqwest::header::USER_AGENT;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use reqwest::StatusCode; use reqwest::StatusCode;
#[cfg(feature = "http")] #[cfg(feature = "http")]
use reqwest::Response; use reqwest::Response;
#[cfg(feature = "http")] #[cfg(feature = "http")]
const USER_AGENT: &str = "robotparser-rs (https://crates.io/crates/robotparser)"; const RP_USER_AGENT: &str = "robotparser-rs (https://crates.io/crates/robotparser)";
/// A rule line is a single "Allow:" (allowance==True) or "Disallow:" /// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
/// (allowance==False) followed by a path.""" /// (allowance==False) followed by a path."""
@ -255,8 +255,8 @@ impl<'a> RobotFileParser<'a> {
/// Reads the robots.txt URL and feeds it to the parser. /// Reads the robots.txt URL and feeds it to the parser.
pub fn read(&self) { pub fn read(&self) {
let client = Client::new(); let client = Client::new();
let mut request = client.get(self.url.clone()); let request = client.get(self.url.clone());
let request = request.header(UserAgent::new(USER_AGENT.to_owned())); let request = request.header(USER_AGENT, RP_USER_AGENT.to_owned());
let mut res = match request.send() { let mut res = match request.send() {
Ok(res) => res, Ok(res) => res,
Err(_) => { Err(_) => {
@ -265,13 +265,13 @@ impl<'a> RobotFileParser<'a> {
}; };
let status = res.status(); let status = res.status();
match status { match status {
StatusCode::Unauthorized | StatusCode::Forbidden => { StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {
self.disallow_all.set(true); self.disallow_all.set(true);
} }
status if status >= StatusCode::BadRequest && status < StatusCode::InternalServerError => { status if status >= StatusCode::BAD_REQUEST && status < StatusCode::INTERNAL_SERVER_ERROR => {
self.allow_all.set(true); self.allow_all.set(true);
} }
StatusCode::Ok => self.from_response(&mut res), StatusCode::OK => self.from_response(&mut res),
_ => {} _ => {}
} }
} }