dan/robotparser-rs
1
0
Fork 0
mirror of https://github.com/danbulant/robotparser-rs synced 2026-05-24 12:35:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Better documentation

Browse source
This commit is contained in:
messense 2016-03-06 16:31:44 +08:00
parent 2d9c028dfa
commit 7f9d08b307
2 changed files with 27 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -14,7 +14,7 @@ Add it to your ``Cargo.toml``:
```toml ```toml
[dependencies] [dependencies]
robotparser = "*" robotparser = "0.4"
``` ```
Add ``extern crate robotparser`` to your crate root and your're good to go! Add ``extern crate robotparser`` to your crate root and your're good to go!

26
src/lib.rs
View file

@ -4,6 +4,31 @@
//! The robots.txt Exclusion Protocol is implemented as specified in //! The robots.txt Exclusion Protocol is implemented as specified in
//! http://www.robotstxt.org/norobots-rfc.txt //! http://www.robotstxt.org/norobots-rfc.txt
//! //!
//! # Installation
//!
//! Add it to your ``Cargo.toml``:
//!
//! ```toml
//! [dependencies]
//! robotparser = "0.4"
//! ```
//!
//! Add ``extern crate robotparser`` to your crate root and your're good to go!
//!
//! # Examples
//!
//! ```
//! extern crate robotparser;
//!
//! use robotparser::RobotFileParser;
//!
//! fn main() {
//! let parser = RobotFileParser::new("http://www.python.org/robots.txt");
//! parser.read();
//! assert!(parser.can_fetch("*", "http://www.python.org/robots.txt"));
//! }
//! ```
#![cfg_attr(feature="clippy", feature(plugin))] #![cfg_attr(feature="clippy", feature(plugin))]
#![cfg_attr(feature="clippy", plugin(clippy))] #![cfg_attr(feature="clippy", plugin(clippy))]
#![cfg_attr(feature="clippy", deny(clippy))] #![cfg_attr(feature="clippy", deny(clippy))]
@ -37,6 +62,7 @@ struct Entry {
sitemaps: Vec<Url>, sitemaps: Vec<Url>,
} }
/// robots.txt file parser
#[derive(Debug, Eq, PartialEq, Clone)] #[derive(Debug, Eq, PartialEq, Clone)]
pub struct RobotFileParser { pub struct RobotFileParser {
entries: RefCell<Vec<Entry>>, entries: RefCell<Vec<Entry>>,