diff --git a/src/lib.rs b/src/lib.rs index fe24fa9..526e4b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,6 +27,7 @@ struct RuleLine { struct Entry { useragents: RefCell>, rulelines: RefCell>, + crawl_delay: Option, } #[derive(Debug, Eq, PartialEq, Clone)] @@ -66,6 +67,7 @@ impl Entry { Entry { useragents: RefCell::new(vec![]), rulelines: RefCell::new(vec![]), + crawl_delay: None, } } @@ -118,6 +120,14 @@ impl Entry { let rulelines = self.rulelines.borrow(); useragents.is_empty() && rulelines.is_empty() } + + fn set_crawl_delay(&mut self,delay: usize) { + self.crawl_delay = Some(delay); + } + + fn get_crawl_delay(&self) -> Option { + return self.crawl_delay.clone(); + } } @@ -270,6 +280,19 @@ impl RobotFileParser { state = 2; } }, + ref x if x == "crawl-delay" => { + if state != 0 { + let delay = part1.parse::(); + match delay { + Ok(delay) => { + let delay = delay * 1000.0; + entry.set_crawl_delay(delay.round() as usize); + }, + Err(_) => {} + } + state = 2; + } + } _ => {}, } } @@ -320,4 +343,19 @@ impl RobotFileParser { // agent not found ==> access granted true } + + /// Returns the crawl delay for this user agent as a `usize` in milliseconds, or None if no crawl delay is defined. + pub fn get_crawl_delay>(&self,useragent: T) -> Option { + let useragent = useragent.as_ref(); + if self.last_checked.get() == 0 { + return None; + } + let entries = self.entries.borrow(); + for entry in &*entries { + if entry.applies_to(useragent) { + return entry.get_crawl_delay(); + } + } + return None; + } } diff --git a/tests/lib.rs b/tests/lib.rs index bc5f9d4..6b4434b 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -217,3 +217,14 @@ fn test_robots_txt_read() { parser.read(); assert!(parser.can_fetch("*", "http://www.python.org/robots.txt")); } + +#[test] +fn test_robots_text_crawl_delay() { + let parser = RobotFileParser::new("http://www.python.org/robots.txt"); + let doc = "User-agent: Yandex\n\ + Crawl-delay: 2.35\n\ + Disallow: /search/\n"; + let lines: Vec<&str> = doc.split("\n").collect(); + parser.parse(&lines); + assert_eq!(2350, parser.get_crawl_delay("Yandex").unwrap()); +}