diff --git a/src/parser/robots_txt_parser.rs b/src/parser/robots_txt_parser.rs index 75e660f..7c22bb9 100644 --- a/src/parser/robots_txt_parser.rs +++ b/src/parser/robots_txt_parser.rs @@ -267,8 +267,5 @@ impl Parser { fn ignore_bom(input: &str) -> &str { const BOM: &str = "\u{feff}"; - if input.starts_with(BOM) { - return &input[BOM.len()..]; - } - input + input.trim_start_matches(BOM) }