diff --git a/README.md b/README.md index f8de130..044a9d4 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Add it to your ``Cargo.toml``: ```toml [dependencies] -robotparser = "*" +robotparser = "0.4" ``` Add ``extern crate robotparser`` to your crate root and your're good to go! diff --git a/src/lib.rs b/src/lib.rs index 56c7528..f6afaad 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,31 @@ //! The robots.txt Exclusion Protocol is implemented as specified in //! http://www.robotstxt.org/norobots-rfc.txt //! +//! # Installation +//! +//! Add it to your ``Cargo.toml``: +//! +//! ```toml +//! [dependencies] +//! robotparser = "0.4" +//! ``` +//! +//! Add ``extern crate robotparser`` to your crate root and your're good to go! +//! +//! # Examples +//! +//! ``` +//! extern crate robotparser; +//! +//! use robotparser::RobotFileParser; +//! +//! fn main() { +//! let parser = RobotFileParser::new("http://www.python.org/robots.txt"); +//! parser.read(); +//! assert!(parser.can_fetch("*", "http://www.python.org/robots.txt")); +//! } +//! ``` + #![cfg_attr(feature="clippy", feature(plugin))] #![cfg_attr(feature="clippy", plugin(clippy))] #![cfg_attr(feature="clippy", deny(clippy))] @@ -37,6 +62,7 @@ struct Entry { sitemaps: Vec, } +/// robots.txt file parser #[derive(Debug, Eq, PartialEq, Clone)] pub struct RobotFileParser { entries: RefCell>,