mirror of
https://github.com/danbulant/robotparser-rs
synced 2026-05-24 12:35:49 +00:00
Feature gate hyper
This commit is contained in:
parent
7e3a47b7f2
commit
8ede3b4218
3 changed files with 19 additions and 3 deletions
|
|
@ -11,12 +11,17 @@ repository = "https://github.com/messense/robotparser-rs"
|
||||||
version = "0.6.0"
|
version = "0.6.0"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
hyper = "0.9"
|
url = "1.2"
|
||||||
url = "1.1"
|
|
||||||
|
[dependencies.hyper]
|
||||||
|
version = "0.9"
|
||||||
|
optional = true
|
||||||
|
|
||||||
[dependencies.clippy]
|
[dependencies.clippy]
|
||||||
optional = true
|
optional = true
|
||||||
version = "^0.*"
|
version = "^0.*"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
|
default = ["http"]
|
||||||
|
http = ["hyper"]
|
||||||
unstable = []
|
unstable = []
|
||||||
|
|
|
||||||
12
src/lib.rs
12
src/lib.rs
|
|
@ -16,7 +16,7 @@
|
||||||
//!
|
//!
|
||||||
//! # Examples
|
//! # Examples
|
||||||
//!
|
//!
|
||||||
//! ```
|
//! ```ignore
|
||||||
//! extern crate robotparser;
|
//! extern crate robotparser;
|
||||||
//!
|
//!
|
||||||
//! use robotparser::RobotFileParser;
|
//! use robotparser::RobotFileParser;
|
||||||
|
|
@ -33,19 +33,27 @@
|
||||||
#![cfg_attr(feature="clippy", warn(cyclomatic_complexity))]
|
#![cfg_attr(feature="clippy", warn(cyclomatic_complexity))]
|
||||||
|
|
||||||
extern crate url;
|
extern crate url;
|
||||||
|
#[cfg(feature = "http")]
|
||||||
extern crate hyper;
|
extern crate hyper;
|
||||||
|
|
||||||
|
#[cfg(feature = "http")]
|
||||||
use std::io::Read;
|
use std::io::Read;
|
||||||
use std::cell::{Cell, RefCell};
|
use std::cell::{Cell, RefCell};
|
||||||
use std::borrow::Cow;
|
use std::borrow::Cow;
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
use url::Url;
|
use url::Url;
|
||||||
|
|
||||||
|
#[cfg(feature = "http")]
|
||||||
use hyper::Client;
|
use hyper::Client;
|
||||||
|
#[cfg(feature = "http")]
|
||||||
use hyper::header::UserAgent;
|
use hyper::header::UserAgent;
|
||||||
|
#[cfg(feature = "http")]
|
||||||
use hyper::status::StatusCode;
|
use hyper::status::StatusCode;
|
||||||
|
#[cfg(feature = "http")]
|
||||||
use hyper::client::Response;
|
use hyper::client::Response;
|
||||||
|
|
||||||
|
#[cfg(feature = "http")]
|
||||||
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
|
const USER_AGENT: &'static str = "robotparser-rs (https://crates.io/crates/robotparser)";
|
||||||
|
|
||||||
/// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
|
/// A rule line is a single "Allow:" (allowance==True) or "Disallow:"
|
||||||
|
|
@ -243,6 +251,7 @@ impl<'a> RobotFileParser<'a> {
|
||||||
self.last_checked.set(0i64);
|
self.last_checked.set(0i64);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http")]
|
||||||
/// Reads the robots.txt URL and feeds it to the parser.
|
/// Reads the robots.txt URL and feeds it to the parser.
|
||||||
pub fn read(&self) {
|
pub fn read(&self) {
|
||||||
let client = Client::new();
|
let client = Client::new();
|
||||||
|
|
@ -266,6 +275,7 @@ impl<'a> RobotFileParser<'a> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http")]
|
||||||
/// Reads the HTTP response and feeds it to the parser.
|
/// Reads the HTTP response and feeds it to the parser.
|
||||||
pub fn from_response(&self, response: &mut Response) {
|
pub fn from_response(&self, response: &mut Response) {
|
||||||
let mut buf = String::new();
|
let mut buf = String::new();
|
||||||
|
|
|
||||||
|
|
@ -213,6 +213,7 @@ fn test_robots_txt_13() {
|
||||||
robot_test_simple(doc, good, bad);
|
robot_test_simple(doc, good, bad);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http")]
|
||||||
#[test]
|
#[test]
|
||||||
fn test_robots_txt_read() {
|
fn test_robots_txt_read() {
|
||||||
let parser = RobotFileParser::new("http://www.python.org/robots.txt");
|
let parser = RobotFileParser::new("http://www.python.org/robots.txt");
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue