mirror of
https://github.com/danbulant/robotparser-rs
synced 2026-05-19 04:18:42 +00:00
6ba403aab9
* Inital github-actions most taken from starship project ref #21 * rustfmt config remove unknown configuration options * Run rustfmt * clippy: use any instead of find..is_some * clippy: Remove the `clone` call: `self.crawl_delay` * Clippy fixes * Rustfmt fixes * clippy: fix dont need to add `&` to all patterns * clippy: fix needless `fn main` in doctest * clippy: fix if-then-else expression returns a bool literal * clippy: fix very complex type BoxFuture response * clippy: fix variable `line_no` is used as a loop counter * clippy: dereference the expression on tests * clippy: fix assert(true) will be optimized out by the compiler * github: name workflow
26 lines
926 B
Rust
26 lines
926 B
Rust
use reqwest::blocking::Client;
|
|
use robotparser::http::RobotsTxtClient;
|
|
use robotparser::service::RobotsTxtService;
|
|
use url::Url;
|
|
use url::{Host, Origin};
|
|
|
|
#[test]
|
|
fn test_reqwest_blocking() {
|
|
let client = Client::new();
|
|
let robots_txt_url = Url::parse("https://www.python.org/robots.txt").unwrap();
|
|
let robots_txt = client.fetch_robots_txt(robots_txt_url.origin()).unwrap().get_result();
|
|
let fetch_url = Url::parse("https://www.python.org/robots.txt").unwrap();
|
|
assert!(robots_txt.can_fetch("*", &fetch_url));
|
|
let fetch_url = Url::parse("https://www.python.org/webstats/").unwrap();
|
|
assert!(!robots_txt.can_fetch("*", &fetch_url));
|
|
}
|
|
|
|
#[test]
|
|
fn test_reqwest_blocking_panic_url() {
|
|
let client = Client::new();
|
|
let host = Host::Domain("python.org::".into());
|
|
let origin = Origin::Tuple("https".into(), host, 80);
|
|
if client.fetch_robots_txt(origin).is_ok() {
|
|
panic!()
|
|
}
|
|
}
|