dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-06-14 04:01:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add method read_from_response

Browse source
This commit is contained in:
Ernestas Poskus 2016-08-24 23:37:47 +03:00
parent d0351ecd2a
commit 36c90ec169
1 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/lib.rs
View file

@ -44,6 +44,7 @@ use url::Url;
use hyper::Client;
use hyper::header::UserAgent;
use hyper::status::StatusCode;
use hyper::client::Response;
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
@ -247,7 +248,7 @@ impl<'a> RobotFileParser<'a> {
let client = Client::new();
let request = client.get(self.url.clone())
.header(UserAgent(USER_AGENT.to_owned()));
let mut res = match request.send() {
let res = match request.send() {
Ok(res) => res,
Err(_) => {
return;
@ -260,16 +261,19 @@ impl<'a> RobotFileParser<'a> {
status if status >= StatusCode::BadRequest && status < StatusCode::InternalServerError => {
self.allow_all.set(true);
}
StatusCode::Ok => {
let mut buf = String::new();
res.read_to_string(&mut buf).unwrap();
let lines: Vec<&str> = buf.split('\n').collect();
self.parse(&lines);
}
StatusCode::Ok => self.from_response(res),
_ => {}
}
}
/// Reads the HTTP response and feeds it to the parser.
pub fn from_response(&self, mut response: Response) {
let mut buf = String::new();
response.read_to_string(&mut buf).unwrap();
let lines: Vec<&str> = buf.split('\n').collect();
self.parse(&lines);
}
fn _add_entry(&self, entry: Entry<'a>) {
if entry.has_useragent("*") {
// the default entry is considered last