mirror of
https://github.com/danbulant/robotparser-rs
synced 2026-06-18 14:11:30 +00:00
2d19755779
* Migrated sites into robotsparser file. * Robots.txt refactoring. * Migrated to new version of url and reqwest.
16 lines
No EOL
636 B
Rust
16 lines
No EOL
636 B
Rust
use robotparser::http::RobotsTxtClient;
|
|
use robotparser::service::RobotsTxtService;
|
|
use reqwest::Client;
|
|
use url::Url;
|
|
use tokio::runtime::Runtime;
|
|
|
|
#[test]
|
|
fn test_reqwest_async() {
|
|
let mut runtime = Runtime::new().unwrap();
|
|
let client = Client::new();
|
|
let robots_txt_url = Url::parse("http://www.python.org/robots.txt").unwrap();
|
|
let robots_txt_response = runtime.block_on(client.fetch_robots_txt(robots_txt_url.origin()));
|
|
let robots_txt = robots_txt_response.unwrap().get_result();
|
|
let fetch_url = Url::parse("http://www.python.org/robots.txt").unwrap();
|
|
assert!(robots_txt.can_fetch("*", &fetch_url));
|
|
} |