dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-05-22 22:09:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use host_str instead domain for HostInternal

Browse source
This commit is contained in:
Laurent Arnoud 2018-03-11 22:05:14 +01:00
parent 953d1f0c0b
commit 19bd3b4cf6
No known key found for this signature in database
GPG key ID: A79DAB2374F95DD5
2 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/lib.rs
View file

@ -221,7 +221,7 @@ impl<'a> RobotFileParser<'a> {
disallow_all: Cell::new(false),
allow_all: Cell::new(false),
url: parsed_url.clone(),
host: parsed_url.domain().unwrap().to_owned(),
host: parsed_url.host_str().unwrap().to_owned(),
path: parsed_url.path().to_owned(),
last_checked: Cell::new(0i64),
}
@ -246,7 +246,7 @@ impl<'a> RobotFileParser<'a> {
pub fn set_url<T: AsRef<str>>(&mut self, url: T) {
let parsed_url = Url::parse(url.as_ref()).unwrap();
self.url = parsed_url.clone();
self.host = parsed_url.domain().unwrap().to_owned();
self.host = parsed_url.host_str().unwrap().to_owned();
self.path = parsed_url.path().to_owned();
self.last_checked.set(0i64);
}

6
tests/lib.rs
View file

@ -268,3 +268,9 @@ fn test_robots_text_request_rate() {
let req_rate = parser.get_req_rate("Google");
assert!(req_rate.is_none());
}
#[test]
fn test_robots_127_0_0_1() {
// Ensure it does not panic
RobotFileParser::new("http://127.0.0.1:4000/robots.txt");
}