mirror of
https://github.com/danbulant/robotparser-rs
synced 2026-05-19 04:18:42 +00:00
2d19755779
* Migrated sites into robotsparser file. * Robots.txt refactoring. * Migrated to new version of url and reqwest.
32 lines
714 B
TOML
32 lines
714 B
TOML
[package]
|
|
authors = ["messense <messense@icloud.com>"]
|
|
description = "robots.txt parser for Rust"
|
|
documentation = "https://messense.github.io/robotparser-rs"
|
|
homepage = "https://github.com/messense/robotparser-rs"
|
|
keywords = ["robot", "robots", "parser", "spider", "crawler"]
|
|
license = "MIT"
|
|
name = "robotparser"
|
|
readme = "README.md"
|
|
repository = "https://github.com/messense/robotparser-rs"
|
|
version = "0.11.0"
|
|
edition = "2018"
|
|
|
|
[dependencies]
|
|
url = "2"
|
|
percent-encoding = "2.1"
|
|
|
|
[dependencies.reqwest]
|
|
version = "0.10.1"
|
|
optional = true
|
|
features = ["blocking"]
|
|
|
|
[dependencies.futures]
|
|
version = "0.3"
|
|
optional = true
|
|
|
|
[features]
|
|
default = ["reqwest", "futures"]
|
|
unstable = []
|
|
|
|
[dev-dependencies]
|
|
tokio = "0.2.11"
|