mirror of
https://github.com/danbulant/robotparser-rs
synced 2026-05-19 04:18:42 +00:00
Feature gate hyper
This commit is contained in:
parent
7e3a47b7f2
commit
8ede3b4218
3 changed files with 19 additions and 3 deletions
|
|
@ -11,12 +11,17 @@ repository = "https://github.com/messense/robotparser-rs"
|
|||
version = "0.6.0"
|
||||
|
||||
[dependencies]
|
||||
hyper = "0.9"
|
||||
url = "1.1"
|
||||
url = "1.2"
|
||||
|
||||
[dependencies.hyper]
|
||||
version = "0.9"
|
||||
optional = true
|
||||
|
||||
[dependencies.clippy]
|
||||
optional = true
|
||||
version = "^0.*"
|
||||
|
||||
[features]
|
||||
default = ["http"]
|
||||
http = ["hyper"]
|
||||
unstable = []
|
||||
|
|
|
|||
12
src/lib.rs
12
src/lib.rs
|
|
@ -16,7 +16,7 @@
|
|||
//!
|
||||
//! # Examples
|
||||
//!
|
||||
//! ```
|
||||
//! ```ignore
|
||||
//! extern crate robotparser;
|
||||
//!
|
||||
//! use robotparser::RobotFileParser;
|
||||
|
|
@ -33,19 +33,27 @@
|
|||
#![cfg_attr(feature="clippy", warn(cyclomatic_complexity))]
|
||||
|
||||
extern crate url;
|
||||
#[cfg(feature = "http")]
|
||||
extern crate hyper;
|
||||
|
||||
#[cfg(feature = "http")]
|
||||
use std::io::Read;
|
||||
use std::cell::{Cell, RefCell};
|
||||
use std::borrow::Cow;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use url::Url;
|
||||
|
||||
#[cfg(feature = "http")]
|
||||
use hyper::Client;
|
||||
#[cfg(feature = "http")]
|
||||
use hyper::header::UserAgent;
|
||||
#[cfg(feature = "http")]
|
||||
use hyper::status::StatusCode;
|
||||
#[cfg(feature = "http")]
|
||||
use hyper::client::Response;
|
||||
|
||||
#[cfg(feature = "http")]
|
||||
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
|
||||
|
||||
/// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
|
||||
|
|
@ -243,6 +251,7 @@ impl<'a> RobotFileParser<'a> {
|
|||
self.last_checked.set(0i64);
|
||||
}
|
||||
|
||||
#[cfg(feature = "http")]
|
||||
/// Reads the robots.txt URL and feeds it to the parser.
|
||||
pub fn read(&self) {
|
||||
let client = Client::new();
|
||||
|
|
@ -266,6 +275,7 @@ impl<'a> RobotFileParser<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "http")]
|
||||
/// Reads the HTTP response and feeds it to the parser.
|
||||
pub fn from_response(&self, response: &mut Response) {
|
||||
let mut buf = String::new();
|
||||
|
|
|
|||
|
|
@ -213,6 +213,7 @@ fn test_robots_txt_13() {
|
|||
robot_test_simple(doc, good, bad);
|
||||
}
|
||||
|
||||
#[cfg(feature = "http")]
|
||||
#[test]
|
||||
fn test_robots_txt_read() {
|
||||
let parser = RobotFileParser::new("http://www.python.org/robots.txt");
|
||||
|
|
|
|||
Loading…
Reference in a new issue