From 4d3e7efe25135e3e7bb83250cc17d645a1bac178 Mon Sep 17 00:00:00 2001 From: Yehuda Katz Date: Mon, 9 Sep 2019 10:43:10 -0700 Subject: [PATCH] Close a bunch of holes in external command args Previously, there was a single parsing rule for "bare words" that applied to both internal and external commands. This meant that, because `cargo +nightly` needed to work, we needed to add `+` as a valid character in bare words. The number of characters continued to grow, and the situation was becoming untenable. The current strategy would eventually eat up all syntax and make it impossible to add syntax like `@foo` to internal commands. This patch significantly restricts bare words and introduces a new token type (`ExternalWord`). An `ExternalWord` expands to an error in the internal syntax, but expands to a bare word in the external syntax. `ExternalWords` are highlighted in grey in the shell. --- src/errors.rs | 15 +++ src/evaluate/evaluator.rs | 7 +- src/parser/hir.rs | 8 ++ src/parser/hir/baseline_parse.rs | 46 ++++--- src/parser/hir/baseline_parse_tokens.rs | 16 +-- src/parser/parse/call_node.rs | 16 +++ src/parser/parse/parser.rs | 161 +++++++++++++++++++----- src/parser/parse/pipeline.rs | 36 ++++++ src/parser/parse/token_tree.rs | 21 ++-- src/parser/parse/token_tree_builder.rs | 32 ++++- src/parser/parse/tokens.rs | 6 +- src/parser/parse_command.rs | 3 +- src/shell/helper.rs | 6 +- 13 files changed, 300 insertions(+), 73 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index 3eb8e33e..d97435d2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -38,6 +38,7 @@ pub enum ArgumentError { MissingMandatoryFlag(String), MissingMandatoryPositional(String), MissingValueForName(String), + InvalidExternalWord, } #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] @@ -136,6 +137,16 @@ impl ShellError { .start() } + pub(crate) fn invalid_external_word(span: Span) -> ShellError { + ProximateShellError::ArgumentError { + command: "Invalid argument to Nu command (did you mean to call an external command?)" + .into(), + error: ArgumentError::InvalidExternalWord, + span, + } + .start() + } + pub(crate) fn parse_error( error: nom::Err<(nom5_locate::LocatedSpan<&str>, nom::error::ErrorKind)>, ) -> ShellError { @@ -190,6 +201,10 @@ impl ShellError { error, span, } => match error { + ArgumentError::InvalidExternalWord => Diagnostic::new( + Severity::Error, + format!("Invalid bare word for Nu command (did you intend to invoke an external command?)")) + .with_label(Label::new_primary(span)), ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new( Severity::Error, format!( diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index ee241583..6419ab73 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -1,5 +1,5 @@ use crate::data::base::Block; -use crate::errors::Description; +use crate::errors::{ArgumentError, Description}; use crate::parser::{ hir::{self, Expression, RawExpression}, CommandRegistry, Text, @@ -39,6 +39,11 @@ pub(crate) fn evaluate_baseline_expr( ) -> Result, ShellError> { match &expr.item { RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(literal), source)), + RawExpression::ExternalWord => Err(ShellError::argument_error( + "Invalid external word", + ArgumentError::InvalidExternalWord, + expr.span(), + )), RawExpression::FilePath(path) => Ok(Value::path(path.clone()).tagged(expr.span())), RawExpression::Synthetic(hir::Synthetic::String(s)) => Ok(Value::string(s).tagged_unknown()), RawExpression::Variable(var) => evaluate_reference(var, scope, source), diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 3e155cc0..aaf5bb77 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -83,6 +83,7 @@ impl ToDebug for Call { #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub enum RawExpression { Literal(Literal), + ExternalWord, Synthetic(Synthetic), Variable(Variable), Binary(Box), @@ -113,6 +114,7 @@ impl RawExpression { match self { RawExpression::Literal(literal) => literal.type_name(), RawExpression::Synthetic(synthetic) => synthetic.type_name(), + RawExpression::ExternalWord => "externalword", RawExpression::FilePath(..) => "filepath", RawExpression::Variable(..) => "variable", RawExpression::List(..) => "list", @@ -189,6 +191,7 @@ impl ToDebug for Expression { match self.item() { RawExpression::Literal(l) => l.tagged(self.span()).fmt_debug(f, source), RawExpression::FilePath(p) => write!(f, "{}", p.display()), + RawExpression::ExternalWord => write!(f, "{}", self.span().slice(source)), RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{:?}", s), RawExpression::Variable(Variable::It(_)) => write!(f, "$it"), RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)), @@ -225,6 +228,11 @@ impl From> for Expression { } } +/// Literals are expressions that are: +/// +/// 1. Copy +/// 2. Can be evaluated without additional context +/// 3. Evaluation cannot produce an error #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub enum Literal { Number(Number), diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs index d76a88d5..4437a6d3 100644 --- a/src/parser/hir/baseline_parse.rs +++ b/src/parser/hir/baseline_parse.rs @@ -1,10 +1,14 @@ use crate::context::Context; +use crate::errors::ShellError; use crate::parser::{hir, RawToken, Token}; use crate::Text; use std::path::PathBuf; -pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression { - match *token.item() { +pub fn baseline_parse_single_token( + token: &Token, + source: &Text, +) -> Result { + Ok(match *token.item() { RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()), RawToken::Size(int, unit) => { hir::Expression::size(int.to_number(source), unit, token.span()) @@ -14,17 +18,22 @@ pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Express hir::Expression::it_variable(span, token.span()) } RawToken::Variable(span) => hir::Expression::variable(span, token.span()), - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Bare => hir::Expression::bare(token.span()), - } + }) } -pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expression { - match *token.item() { +pub fn baseline_parse_token_as_number( + token: &Token, + source: &Text, +) -> Result { + Ok(match *token.item() { RawToken::Variable(span) if span.slice(source) == "it" => { hir::Expression::it_variable(span, token.span()) } - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Variable(span) => hir::Expression::variable(span, token.span()), RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()), RawToken::Size(number, unit) => { @@ -32,33 +41,38 @@ pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expr } RawToken::Bare => hir::Expression::bare(token.span()), RawToken::String(span) => hir::Expression::string(span, token.span()), - } + }) } -pub fn baseline_parse_token_as_string(token: &Token, source: &Text) -> hir::Expression { - match *token.item() { +pub fn baseline_parse_token_as_string( + token: &Token, + source: &Text, +) -> Result { + Ok(match *token.item() { RawToken::Variable(span) if span.slice(source) == "it" => { hir::Expression::it_variable(span, token.span()) } - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Variable(span) => hir::Expression::variable(span, token.span()), RawToken::Number(_) => hir::Expression::bare(token.span()), RawToken::Size(_, _) => hir::Expression::bare(token.span()), RawToken::Bare => hir::Expression::bare(token.span()), RawToken::String(span) => hir::Expression::string(span, token.span()), - } + }) } pub fn baseline_parse_token_as_path( token: &Token, context: &Context, source: &Text, -) -> hir::Expression { - match *token.item() { +) -> Result { + Ok(match *token.item() { RawToken::Variable(span) if span.slice(source) == "it" => { hir::Expression::it_variable(span, token.span()) } - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Variable(span) => hir::Expression::variable(span, token.span()), RawToken::Number(_) => hir::Expression::bare(token.span()), RawToken::Size(_, _) => hir::Expression::bare(token.span()), @@ -69,7 +83,7 @@ pub fn baseline_parse_token_as_path( RawToken::String(span) => { hir::Expression::file_path(expand_path(span.slice(source), context), token.span()) } - } + }) } pub fn expand_path(string: &str, context: &Context) -> PathBuf { diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index ca9b0bb3..13c7630f 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -33,7 +33,6 @@ pub fn baseline_parse_tokens( Ok(exprs) } - #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SyntaxType { Any, @@ -62,7 +61,7 @@ impl std::fmt::Display for SyntaxType { SyntaxType::Path => write!(f, "Path"), SyntaxType::Binary => write!(f, "Binary"), SyntaxType::Block => write!(f, "Block"), - SyntaxType::Boolean => write!(f, "Boolean") + SyntaxType::Boolean => write!(f, "Boolean"), } } } @@ -81,7 +80,7 @@ pub fn baseline_parse_next_expr( match (syntax_type, next) { (SyntaxType::Path, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_path(token, context, source)) + return baseline_parse_token_as_path(token, context, source) } (SyntaxType::Path, token) => { @@ -92,7 +91,7 @@ pub fn baseline_parse_next_expr( } (SyntaxType::String, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_string(token, source)); + return baseline_parse_token_as_string(token, source); } (SyntaxType::String, token) => { @@ -103,7 +102,7 @@ pub fn baseline_parse_next_expr( } (SyntaxType::Number, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_number(token, source)); + return Ok(baseline_parse_token_as_number(token, source)?); } (SyntaxType::Number, token) => { @@ -115,7 +114,7 @@ pub fn baseline_parse_next_expr( // TODO: More legit member processing (SyntaxType::Member, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_string(token, source)); + return baseline_parse_token_as_string(token, source); } (SyntaxType::Member, token) => { @@ -245,7 +244,7 @@ pub fn baseline_parse_semantic_token( source: &Text, ) -> Result { match token { - TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), + TokenNode::Token(token) => baseline_parse_single_token(token, source), TokenNode::Call(_call) => unimplemented!(), TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, context, source), TokenNode::Pipeline(_pipeline) => unimplemented!(), @@ -315,7 +314,8 @@ pub fn baseline_parse_path( RawToken::Number(_) | RawToken::Size(..) | RawToken::Variable(_) - | RawToken::External(_) => { + | RawToken::ExternalCommand(_) + | RawToken::ExternalWord => { return Err(ShellError::type_error( "String", token.type_name().simple_spanned(part), diff --git a/src/parser/parse/call_node.rs b/src/parser/parse/call_node.rs index 2869abb4..eb715cd3 100644 --- a/src/parser/parse/call_node.rs +++ b/src/parser/parse/call_node.rs @@ -1,5 +1,7 @@ use crate::parser::TokenNode; +use crate::traits::ToDebug; use getset::Getters; +use std::fmt; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)] pub struct CallNode { @@ -24,3 +26,17 @@ impl CallNode { } } } + +impl ToDebug for CallNode { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + write!(f, "{}", self.head.debug(source))?; + + if let Some(children) = &self.children { + for child in children { + write!(f, "{}", child.debug(source))? + } + } + + Ok(()) + } +} diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index f230c36c..a691fb24 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -236,12 +236,34 @@ pub fn bare(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = take_while1(is_start_bare_char)(input)?; let (input, _) = take_while(is_bare_char)(input)?; + + let next_char = &input.fragment.chars().nth(0); + + if let Some(next_char) = next_char { + if is_external_word_char(*next_char) { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::TakeWhile1, + ))); + } + } + let end = input.offset; Ok((input, TokenTreeBuilder::spanned_bare((start, end)))) }) } +pub fn external_word(input: NomSpan) -> IResult { + trace_step(input, "bare", move |input| { + let start = input.offset; + let (input, _) = take_while1(is_external_word_char)(input)?; + let end = input.offset; + + Ok((input, TokenTreeBuilder::spanned_external_word((start, end)))) + }) +} + pub fn var(input: NomSpan) -> IResult { trace_step(input, "var", move |input| { let start = input.offset; @@ -364,8 +386,17 @@ pub fn size(input: NomSpan) -> IResult { pub fn leaf(input: NomSpan) -> IResult { trace_step(input, "leaf", move |input| { - let (input, node) = - alt((size, string, operator, flag, shorthand, var, external, bare))(input)?; + let (input, node) = alt(( + size, + string, + operator, + flag, + shorthand, + var, + external, + bare, + external_word, + ))(input)?; Ok((input, node)) }) @@ -582,26 +613,13 @@ pub fn pipeline(input: NomSpan) -> IResult { } fn make_call_list( - head: Option<( - Option, - Tagged, - Option - )>, - items: Vec<( - NomSpan, - Option, - Tagged, - Option, - )>, + head: Option<(Option, Tagged, Option)>, + items: Vec<(NomSpan, Option, Tagged, Option)>, ) -> Vec { let mut out = vec![]; if let Some(head) = head { - let el = PipelineElement::new( - None, - head.0.map(Span::from), - head.1, - head.2.map(Span::from)); + let el = PipelineElement::new(None, head.0.map(Span::from), head.1, head.2.map(Span::from)); out.push(el); } @@ -611,7 +629,8 @@ fn make_call_list( Some(pipe).map(Span::from), ws1.map(Span::from), call, - ws2.map(Span::from)); + ws2.map(Span::from), + ); out.push(el); } @@ -628,40 +647,39 @@ fn int(frag: &str, neg: Option) -> i64 { } } +fn is_external_word_char(c: char) -> bool { + match c { + ';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' => false, + other if other.is_whitespace() => false, + _ => true, + } +} + fn is_start_bare_char(c: char) -> bool { match c { + '+' => false, _ if c.is_alphabetic() => true, - _ if c.is_numeric() => true, '.' => true, '\\' => true, '/' => true, '_' => true, '-' => true, - '@' => true, - '*' => true, - '?' => true, '~' => true, - '+' => true, _ => false, } } fn is_bare_char(c: char) -> bool { match c { + '+' => false, _ if c.is_alphanumeric() => true, - ':' => true, '.' => true, '\\' => true, '/' => true, '_' => true, '-' => true, - '@' => true, - '*' => true, - '?' => true, '=' => true, '~' => true, - '+' => true, - '%' => true, _ => false, } } @@ -724,6 +742,44 @@ mod tests { } } + macro_rules! equal_tokens { + ($source:tt -> $tokens:expr) => { + let result = apply(pipeline, "pipeline", $source); + let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens); + + if result != expected_tree { + let debug_result = format!("{}", result.debug($source)); + let debug_expected = format!("{}", expected_tree.debug(&expected_source)); + + if debug_result == debug_expected { + assert_eq!( + result, expected_tree, + "NOTE: actual and expected had equivalent debug serializations, source={:?}, debug_expected={:?}", + $source, + debug_expected + ) + } else { + assert_eq!(debug_result, debug_expected) + } + } + + // apply(pipeline, "pipeline", r#"cargo +nightly run"#), + // build_token(b::pipeline(vec![( + // None, + // b::call( + // b::bare("cargo"), + // vec![ + // b::sp(), + // b::external_word("+nightly"), + // b::sp(), + // b::bare("run") + // ] + // ), + // None + // )])) + }; + } + #[test] fn test_integer() { assert_leaf! { @@ -854,7 +910,7 @@ mod tests { fn test_external() { assert_leaf! { parsers [ external ] - "^ls" -> 0..3 { External(span(1, 3)) } + "^ls" -> 0..3 { ExternalCommand(span(1, 3)) } } } @@ -1058,6 +1114,46 @@ mod tests { ); } + #[test] + fn test_external_word() { + let _ = pretty_env_logger::try_init(); + + equal_tokens!( + "cargo +nightly run" -> + b::pipeline(vec![( + None, + b::call( + b::bare("cargo"), + vec![ + b::sp(), + b::external_word("+nightly"), + b::sp(), + b::bare("run") + ] + ), + None + )]) + ); + + equal_tokens!( + "rm foo%bar" -> + b::pipeline(vec![( + None, + b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]), + None + )]) + ); + + equal_tokens!( + "rm foo%bar" -> + b::pipeline(vec![( + None, + b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]), + None + )]) + ); + } + #[test] fn test_smoke_pipeline() { let _ = pretty_env_logger::try_init(); @@ -1178,7 +1274,6 @@ mod tests { } fn build_token(block: CurriedToken) -> TokenNode { - let mut builder = TokenTreeBuilder::new(); - block(&mut builder) + TokenTreeBuilder::build(block).0 } } diff --git a/src/parser/parse/pipeline.rs b/src/parser/parse/pipeline.rs index 75155d14..64a899c1 100644 --- a/src/parser/parse/pipeline.rs +++ b/src/parser/parse/pipeline.rs @@ -1,7 +1,9 @@ use crate::parser::CallNode; +use crate::traits::ToDebug; use crate::{Span, Tagged}; use derive_new::new; use getset::Getters; +use std::fmt; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] pub struct Pipeline { @@ -9,6 +11,20 @@ pub struct Pipeline { pub(crate) post_ws: Option, } +impl ToDebug for Pipeline { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + for part in &self.parts { + write!(f, "{}", part.debug(source))?; + } + + if let Some(post_ws) = self.post_ws { + write!(f, "{}", post_ws.slice(source))? + } + + Ok(()) + } +} + #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] pub struct PipelineElement { pub pipe: Option, @@ -17,3 +33,23 @@ pub struct PipelineElement { call: Tagged, pub post_ws: Option, } + +impl ToDebug for PipelineElement { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + if let Some(pipe) = self.pipe { + write!(f, "{}", pipe.slice(source))?; + } + + if let Some(pre_ws) = self.pre_ws { + write!(f, "{}", pre_ws.slice(source))?; + } + + write!(f, "{}", self.call.debug(source))?; + + if let Some(post_ws) = self.post_ws { + write!(f, "{}", post_ws.slice(source))?; + } + + Ok(()) + } +} diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index df189a1a..f69c176e 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -1,5 +1,6 @@ use crate::errors::ShellError; use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; +use crate::traits::ToDebug; use crate::{Span, Tagged, Text}; use derive_new::new; use enum_utils::FromStr; @@ -22,6 +23,12 @@ pub enum TokenNode { Path(Tagged), } +impl ToDebug for TokenNode { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + write!(f, "{:?}", self.old_debug(&Text::from(source))) + } +} + pub struct DebugTokenNode<'a> { node: &'a TokenNode, source: &'a Text, @@ -34,11 +41,11 @@ impl fmt::Debug for DebugTokenNode<'_> { TokenNode::Call(s) => { write!(f, "(")?; - write!(f, "{:?}", s.head().debug(self.source))?; + write!(f, "{}", s.head().debug(self.source))?; if let Some(children) = s.children() { for child in children { - write!(f, "{:?}", child.debug(self.source))?; + write!(f, "{}", child.debug(self.source))?; } } @@ -57,7 +64,7 @@ impl fmt::Debug for DebugTokenNode<'_> { )?; for child in d.children() { - write!(f, "{:?}", child.debug(self.source))?; + write!(f, "{:?}", child.old_debug(self.source))?; } write!( @@ -70,7 +77,7 @@ impl fmt::Debug for DebugTokenNode<'_> { } ) } - TokenNode::Pipeline(_) => write!(f, ""), + TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)), TokenNode::Error(s) => write!(f, " for {:?}", s.span().slice(self.source)), rest => write!(f, "{}", rest.span().slice(self.source)), } @@ -115,7 +122,7 @@ impl TokenNode { .to_string() } - pub fn debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> { + pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> { DebugTokenNode { node: self, source } } @@ -140,7 +147,7 @@ impl TokenNode { pub fn is_external(&self) -> bool { match self { TokenNode::Token(Tagged { - item: RawToken::External(..), + item: RawToken::ExternalCommand(..), .. }) => true, _ => false, @@ -150,7 +157,7 @@ impl TokenNode { pub fn expect_external(&self) -> Span { match self { TokenNode::Token(Tagged { - item: RawToken::External(span), + item: RawToken::ExternalCommand(span), .. }) => *span, _ => panic!("Only call expect_external if you checked is_external first"), diff --git a/src/parser/parse/token_tree_builder.rs b/src/parser/parse/token_tree_builder.rs index 9dd1ebc1..8034e8b0 100644 --- a/src/parser/parse/token_tree_builder.rs +++ b/src/parser/parse/token_tree_builder.rs @@ -14,15 +14,19 @@ use derive_new::new; pub struct TokenTreeBuilder { #[new(default)] pos: usize, + + #[new(default)] + output: String, } pub type CurriedToken = Box TokenNode + 'static>; pub type CurriedCall = Box Tagged + 'static>; impl TokenTreeBuilder { - pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> TokenNode { + pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) { let mut builder = TokenTreeBuilder::new(); - block(&mut builder) + let node = block(&mut builder); + (node, builder.output) } pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken { @@ -56,7 +60,8 @@ impl TokenTreeBuilder { pipe, pre_span.map(Span::from), call, - post_span.map(Span::from))); + post_span.map(Span::from), + )); loop { match input.next() { @@ -147,9 +152,27 @@ impl TokenTreeBuilder { )) } + pub fn external_word(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&input); + b.pos = end; + + TokenTreeBuilder::spanned_external_word((start, end)) + }) + } + + pub fn spanned_external_word(input: impl Into) -> TokenNode { + TokenNode::Token(Tagged::from_simple_spanned_item( + RawToken::ExternalWord, + input.into(), + )) + } + pub fn spanned_external(input: impl Into, span: impl Into) -> TokenNode { TokenNode::Token(Tagged::from_simple_spanned_item( - RawToken::External(input.into()), + RawToken::ExternalCommand(input.into()), span.into(), )) } @@ -422,6 +445,7 @@ impl TokenTreeBuilder { fn consume(&mut self, input: &str) -> (usize, usize) { let start = self.pos; self.pos += input.len(); + self.output.push_str(input); (start, self.pos) } } diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index ed9c1f72..0bb2e3f1 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -10,7 +10,8 @@ pub enum RawToken { Size(RawNumber, Unit), String(Span), Variable(Span), - External(Span), + ExternalCommand(Span), + ExternalWord, Bare, } @@ -50,7 +51,8 @@ impl RawToken { RawToken::Size(..) => "Size", RawToken::String(_) => "String", RawToken::Variable(_) => "Variable", - RawToken::External(_) => "External", + RawToken::ExternalCommand(_) => "ExternalCommand", + RawToken::ExternalWord => "ExternalWord", RawToken::Bare => "String", } } diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index 33ad25e6..e0fc9d86 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -6,6 +6,7 @@ use crate::parser::{ hir::{self, NamedArguments}, Flag, RawToken, TokenNode, }; +use crate::traits::ToDebug; use crate::{Span, Tag, Tagged, Text}; use log::trace; @@ -248,7 +249,7 @@ pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source itertools::join( tail.debug_remaining() .iter() - .map(|i| format!("%{:?}%", i.debug(source))), + .map(|i| format!("%{}%", i.debug(&source))), " " ) ); diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 8e21c50e..462f3752 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -136,9 +136,13 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String { .. }) => Color::Green.normal().paint(token_node.span().slice(line)), TokenNode::Token(Tagged { - item: RawToken::External(..), + item: RawToken::ExternalCommand(..), .. }) => Color::Cyan.bold().paint(token_node.span().slice(line)), + TokenNode::Token(Tagged { + item: RawToken::ExternalWord, + .. + }) => Color::Black.bold().paint(token_node.span().slice(line)), }; styled.to_string()