dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-05-19 04:18:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3 from spk/set-user-agent

Browse source 
Set UserAgent as robotparser-rs
This commit is contained in:
messense 2016-05-05 09:26:15 +08:00
commit 5b3d3ea305
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/lib.rs
View file

@ -42,9 +42,12 @@ use std::cell::{Cell, RefCell};
use std::borrow::Cow;
use url::Url;
use hyper::{Client};
use hyper::header::{UserAgent};
use hyper::status::StatusCode;
use std::time::Duration;
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
/// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
/// (allowance==False) followed by a path."""
#[derive(Debug, Eq, PartialEq, Clone)]
@ -228,7 +231,9 @@ impl<'a> RobotFileParser<'a> {
/// Reads the robots.txt URL and feeds it to the parser.
pub fn read(&self) {
let client = Client::new();
let mut res = match client.get(self.url.clone()).send() {
let request = client.get(self.url.clone())
.header(UserAgent(USER_AGENT.to_owned()));
let mut res = match request.send() {
Ok(res) => res,
Err(_) => {
return;