diff --git a/Cargo.toml b/Cargo.toml index db96002..05dfafe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "robotparser" -version = "0.2.0" +version = "0.3.0" authors = ["messense "] description = "robots.txt parser for Rust" repository = "https://github.com/messense/robotparser-rs" diff --git a/src/lib.rs b/src/lib.rs index fe24fa9..d669754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ use std::cell::{Cell, RefCell}; use url::Url; use hyper::{Client}; use hyper::status::StatusCode; +use std::time::Duration; /// A rule line is a single "Allow:" (allowance==True) or "Disallow:" /// (allowance==False) followed by a path.""" @@ -27,6 +28,7 @@ struct RuleLine { struct Entry { useragents: RefCell>, rulelines: RefCell>, + crawl_delay: Option, } #[derive(Debug, Eq, PartialEq, Clone)] @@ -66,6 +68,7 @@ impl Entry { Entry { useragents: RefCell::new(vec![]), rulelines: RefCell::new(vec![]), + crawl_delay: None, } } @@ -118,6 +121,14 @@ impl Entry { let rulelines = self.rulelines.borrow(); useragents.is_empty() && rulelines.is_empty() } + + fn set_crawl_delay(&mut self,delay: Duration) { + self.crawl_delay = Some(delay); + } + + fn get_crawl_delay(&self) -> Option { + return self.crawl_delay.clone(); + } } @@ -270,6 +281,21 @@ impl RobotFileParser { state = 2; } }, + ref x if x == "crawl-delay" => { + if state != 0 { + let delay = part1.parse::(); + match delay { + Ok(delay) => { + let delay_seconds = delay.trunc(); + let delay_nanoseconds = delay.fract()* 10f64.powi(9); + let delay = Duration::new(delay_seconds as u64,delay_nanoseconds as u32); + entry.set_crawl_delay(delay); + }, + Err(_) => {} + } + state = 2; + } + } _ => {}, } } @@ -320,4 +346,19 @@ impl RobotFileParser { // agent not found ==> access granted true } + + /// Returns the crawl delay for this user agent as a `Duration`, or None if no crawl delay is defined. + pub fn get_crawl_delay>(&self,useragent: T) -> Option { + let useragent = useragent.as_ref(); + if self.last_checked.get() == 0 { + return None; + } + let entries = self.entries.borrow(); + for entry in &*entries { + if entry.applies_to(useragent) { + return entry.get_crawl_delay(); + } + } + return None; + } } diff --git a/tests/lib.rs b/tests/lib.rs index bc5f9d4..ada37cb 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -1,10 +1,10 @@ extern crate robotparser; use robotparser::RobotFileParser; +use std::time::Duration; const AGENT: &'static str = "test_robotparser"; - fn robot_test(doc: &str, good_urls: Vec<&str>, bad_urls: Vec<&str>, agent: &str) { let parser = RobotFileParser::new("http://www.baidu.com/robots.txt"); let lines: Vec<&str> = doc.split("\n").collect(); @@ -217,3 +217,14 @@ fn test_robots_txt_read() { parser.read(); assert!(parser.can_fetch("*", "http://www.python.org/robots.txt")); } + +#[test] +fn test_robots_text_crawl_delay() { + let parser = RobotFileParser::new("http://www.python.org/robots.txt"); + let doc = "User-agent: Yandex\n\ + Crawl-delay: 2.35\n\ + Disallow: /search/\n"; + let lines: Vec<&str> = doc.split("\n").collect(); + parser.parse(&lines); + assert_eq!(Duration::new(2,350 * 1000 * 1000), parser.get_crawl_delay("Yandex").unwrap()); +}