diff --git a/src/errors.rs b/src/errors.rs index 3eb8e33e..d97435d2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -38,6 +38,7 @@ pub enum ArgumentError { MissingMandatoryFlag(String), MissingMandatoryPositional(String), MissingValueForName(String), + InvalidExternalWord, } #[derive(Debug, Eq, PartialEq, Clone, Ord, PartialOrd, Serialize, Deserialize)] @@ -136,6 +137,16 @@ impl ShellError { .start() } + pub(crate) fn invalid_external_word(span: Span) -> ShellError { + ProximateShellError::ArgumentError { + command: "Invalid argument to Nu command (did you mean to call an external command?)" + .into(), + error: ArgumentError::InvalidExternalWord, + span, + } + .start() + } + pub(crate) fn parse_error( error: nom::Err<(nom5_locate::LocatedSpan<&str>, nom::error::ErrorKind)>, ) -> ShellError { @@ -190,6 +201,10 @@ impl ShellError { error, span, } => match error { + ArgumentError::InvalidExternalWord => Diagnostic::new( + Severity::Error, + format!("Invalid bare word for Nu command (did you intend to invoke an external command?)")) + .with_label(Label::new_primary(span)), ArgumentError::MissingMandatoryFlag(name) => Diagnostic::new( Severity::Error, format!( diff --git a/src/evaluate/evaluator.rs b/src/evaluate/evaluator.rs index ee241583..6419ab73 100644 --- a/src/evaluate/evaluator.rs +++ b/src/evaluate/evaluator.rs @@ -1,5 +1,5 @@ use crate::data::base::Block; -use crate::errors::Description; +use crate::errors::{ArgumentError, Description}; use crate::parser::{ hir::{self, Expression, RawExpression}, CommandRegistry, Text, @@ -39,6 +39,11 @@ pub(crate) fn evaluate_baseline_expr( ) -> Result, ShellError> { match &expr.item { RawExpression::Literal(literal) => Ok(evaluate_literal(expr.copy_span(literal), source)), + RawExpression::ExternalWord => Err(ShellError::argument_error( + "Invalid external word", + ArgumentError::InvalidExternalWord, + expr.span(), + )), RawExpression::FilePath(path) => Ok(Value::path(path.clone()).tagged(expr.span())), RawExpression::Synthetic(hir::Synthetic::String(s)) => Ok(Value::string(s).tagged_unknown()), RawExpression::Variable(var) => evaluate_reference(var, scope, source), diff --git a/src/parser/hir.rs b/src/parser/hir.rs index 3e155cc0..aaf5bb77 100644 --- a/src/parser/hir.rs +++ b/src/parser/hir.rs @@ -83,6 +83,7 @@ impl ToDebug for Call { #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub enum RawExpression { Literal(Literal), + ExternalWord, Synthetic(Synthetic), Variable(Variable), Binary(Box), @@ -113,6 +114,7 @@ impl RawExpression { match self { RawExpression::Literal(literal) => literal.type_name(), RawExpression::Synthetic(synthetic) => synthetic.type_name(), + RawExpression::ExternalWord => "externalword", RawExpression::FilePath(..) => "filepath", RawExpression::Variable(..) => "variable", RawExpression::List(..) => "list", @@ -189,6 +191,7 @@ impl ToDebug for Expression { match self.item() { RawExpression::Literal(l) => l.tagged(self.span()).fmt_debug(f, source), RawExpression::FilePath(p) => write!(f, "{}", p.display()), + RawExpression::ExternalWord => write!(f, "{}", self.span().slice(source)), RawExpression::Synthetic(Synthetic::String(s)) => write!(f, "{:?}", s), RawExpression::Variable(Variable::It(_)) => write!(f, "$it"), RawExpression::Variable(Variable::Other(s)) => write!(f, "${}", s.slice(source)), @@ -225,6 +228,11 @@ impl From> for Expression { } } +/// Literals are expressions that are: +/// +/// 1. Copy +/// 2. Can be evaluated without additional context +/// 3. Evaluation cannot produce an error #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] pub enum Literal { Number(Number), diff --git a/src/parser/hir/baseline_parse.rs b/src/parser/hir/baseline_parse.rs index d76a88d5..4437a6d3 100644 --- a/src/parser/hir/baseline_parse.rs +++ b/src/parser/hir/baseline_parse.rs @@ -1,10 +1,14 @@ use crate::context::Context; +use crate::errors::ShellError; use crate::parser::{hir, RawToken, Token}; use crate::Text; use std::path::PathBuf; -pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Expression { - match *token.item() { +pub fn baseline_parse_single_token( + token: &Token, + source: &Text, +) -> Result { + Ok(match *token.item() { RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()), RawToken::Size(int, unit) => { hir::Expression::size(int.to_number(source), unit, token.span()) @@ -14,17 +18,22 @@ pub fn baseline_parse_single_token(token: &Token, source: &Text) -> hir::Express hir::Expression::it_variable(span, token.span()) } RawToken::Variable(span) => hir::Expression::variable(span, token.span()), - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Bare => hir::Expression::bare(token.span()), - } + }) } -pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expression { - match *token.item() { +pub fn baseline_parse_token_as_number( + token: &Token, + source: &Text, +) -> Result { + Ok(match *token.item() { RawToken::Variable(span) if span.slice(source) == "it" => { hir::Expression::it_variable(span, token.span()) } - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Variable(span) => hir::Expression::variable(span, token.span()), RawToken::Number(number) => hir::Expression::number(number.to_number(source), token.span()), RawToken::Size(number, unit) => { @@ -32,33 +41,38 @@ pub fn baseline_parse_token_as_number(token: &Token, source: &Text) -> hir::Expr } RawToken::Bare => hir::Expression::bare(token.span()), RawToken::String(span) => hir::Expression::string(span, token.span()), - } + }) } -pub fn baseline_parse_token_as_string(token: &Token, source: &Text) -> hir::Expression { - match *token.item() { +pub fn baseline_parse_token_as_string( + token: &Token, + source: &Text, +) -> Result { + Ok(match *token.item() { RawToken::Variable(span) if span.slice(source) == "it" => { hir::Expression::it_variable(span, token.span()) } - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Variable(span) => hir::Expression::variable(span, token.span()), RawToken::Number(_) => hir::Expression::bare(token.span()), RawToken::Size(_, _) => hir::Expression::bare(token.span()), RawToken::Bare => hir::Expression::bare(token.span()), RawToken::String(span) => hir::Expression::string(span, token.span()), - } + }) } pub fn baseline_parse_token_as_path( token: &Token, context: &Context, source: &Text, -) -> hir::Expression { - match *token.item() { +) -> Result { + Ok(match *token.item() { RawToken::Variable(span) if span.slice(source) == "it" => { hir::Expression::it_variable(span, token.span()) } - RawToken::External(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalCommand(span) => hir::Expression::external_command(span, token.span()), + RawToken::ExternalWord => return Err(ShellError::invalid_external_word(token.span())), RawToken::Variable(span) => hir::Expression::variable(span, token.span()), RawToken::Number(_) => hir::Expression::bare(token.span()), RawToken::Size(_, _) => hir::Expression::bare(token.span()), @@ -69,7 +83,7 @@ pub fn baseline_parse_token_as_path( RawToken::String(span) => { hir::Expression::file_path(expand_path(span.slice(source), context), token.span()) } - } + }) } pub fn expand_path(string: &str, context: &Context) -> PathBuf { diff --git a/src/parser/hir/baseline_parse_tokens.rs b/src/parser/hir/baseline_parse_tokens.rs index ca9b0bb3..13c7630f 100644 --- a/src/parser/hir/baseline_parse_tokens.rs +++ b/src/parser/hir/baseline_parse_tokens.rs @@ -33,7 +33,6 @@ pub fn baseline_parse_tokens( Ok(exprs) } - #[derive(Debug, Copy, Clone, Serialize, Deserialize)] pub enum SyntaxType { Any, @@ -62,7 +61,7 @@ impl std::fmt::Display for SyntaxType { SyntaxType::Path => write!(f, "Path"), SyntaxType::Binary => write!(f, "Binary"), SyntaxType::Block => write!(f, "Block"), - SyntaxType::Boolean => write!(f, "Boolean") + SyntaxType::Boolean => write!(f, "Boolean"), } } } @@ -81,7 +80,7 @@ pub fn baseline_parse_next_expr( match (syntax_type, next) { (SyntaxType::Path, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_path(token, context, source)) + return baseline_parse_token_as_path(token, context, source) } (SyntaxType::Path, token) => { @@ -92,7 +91,7 @@ pub fn baseline_parse_next_expr( } (SyntaxType::String, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_string(token, source)); + return baseline_parse_token_as_string(token, source); } (SyntaxType::String, token) => { @@ -103,7 +102,7 @@ pub fn baseline_parse_next_expr( } (SyntaxType::Number, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_number(token, source)); + return Ok(baseline_parse_token_as_number(token, source)?); } (SyntaxType::Number, token) => { @@ -115,7 +114,7 @@ pub fn baseline_parse_next_expr( // TODO: More legit member processing (SyntaxType::Member, TokenNode::Token(token)) => { - return Ok(baseline_parse_token_as_string(token, source)); + return baseline_parse_token_as_string(token, source); } (SyntaxType::Member, token) => { @@ -245,7 +244,7 @@ pub fn baseline_parse_semantic_token( source: &Text, ) -> Result { match token { - TokenNode::Token(token) => Ok(baseline_parse_single_token(token, source)), + TokenNode::Token(token) => baseline_parse_single_token(token, source), TokenNode::Call(_call) => unimplemented!(), TokenNode::Delimited(delimited) => baseline_parse_delimited(delimited, context, source), TokenNode::Pipeline(_pipeline) => unimplemented!(), @@ -315,7 +314,8 @@ pub fn baseline_parse_path( RawToken::Number(_) | RawToken::Size(..) | RawToken::Variable(_) - | RawToken::External(_) => { + | RawToken::ExternalCommand(_) + | RawToken::ExternalWord => { return Err(ShellError::type_error( "String", token.type_name().simple_spanned(part), diff --git a/src/parser/parse/call_node.rs b/src/parser/parse/call_node.rs index 2869abb4..eb715cd3 100644 --- a/src/parser/parse/call_node.rs +++ b/src/parser/parse/call_node.rs @@ -1,5 +1,7 @@ use crate::parser::TokenNode; +use crate::traits::ToDebug; use getset::Getters; +use std::fmt; #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters)] pub struct CallNode { @@ -24,3 +26,17 @@ impl CallNode { } } } + +impl ToDebug for CallNode { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + write!(f, "{}", self.head.debug(source))?; + + if let Some(children) = &self.children { + for child in children { + write!(f, "{}", child.debug(source))? + } + } + + Ok(()) + } +} diff --git a/src/parser/parse/parser.rs b/src/parser/parse/parser.rs index f230c36c..a691fb24 100644 --- a/src/parser/parse/parser.rs +++ b/src/parser/parse/parser.rs @@ -236,12 +236,34 @@ pub fn bare(input: NomSpan) -> IResult { let start = input.offset; let (input, _) = take_while1(is_start_bare_char)(input)?; let (input, _) = take_while(is_bare_char)(input)?; + + let next_char = &input.fragment.chars().nth(0); + + if let Some(next_char) = next_char { + if is_external_word_char(*next_char) { + return Err(nom::Err::Error(nom::error::make_error( + input, + nom::error::ErrorKind::TakeWhile1, + ))); + } + } + let end = input.offset; Ok((input, TokenTreeBuilder::spanned_bare((start, end)))) }) } +pub fn external_word(input: NomSpan) -> IResult { + trace_step(input, "bare", move |input| { + let start = input.offset; + let (input, _) = take_while1(is_external_word_char)(input)?; + let end = input.offset; + + Ok((input, TokenTreeBuilder::spanned_external_word((start, end)))) + }) +} + pub fn var(input: NomSpan) -> IResult { trace_step(input, "var", move |input| { let start = input.offset; @@ -364,8 +386,17 @@ pub fn size(input: NomSpan) -> IResult { pub fn leaf(input: NomSpan) -> IResult { trace_step(input, "leaf", move |input| { - let (input, node) = - alt((size, string, operator, flag, shorthand, var, external, bare))(input)?; + let (input, node) = alt(( + size, + string, + operator, + flag, + shorthand, + var, + external, + bare, + external_word, + ))(input)?; Ok((input, node)) }) @@ -582,26 +613,13 @@ pub fn pipeline(input: NomSpan) -> IResult { } fn make_call_list( - head: Option<( - Option, - Tagged, - Option - )>, - items: Vec<( - NomSpan, - Option, - Tagged, - Option, - )>, + head: Option<(Option, Tagged, Option)>, + items: Vec<(NomSpan, Option, Tagged, Option)>, ) -> Vec { let mut out = vec![]; if let Some(head) = head { - let el = PipelineElement::new( - None, - head.0.map(Span::from), - head.1, - head.2.map(Span::from)); + let el = PipelineElement::new(None, head.0.map(Span::from), head.1, head.2.map(Span::from)); out.push(el); } @@ -611,7 +629,8 @@ fn make_call_list( Some(pipe).map(Span::from), ws1.map(Span::from), call, - ws2.map(Span::from)); + ws2.map(Span::from), + ); out.push(el); } @@ -628,40 +647,39 @@ fn int(frag: &str, neg: Option) -> i64 { } } +fn is_external_word_char(c: char) -> bool { + match c { + ';' | '|' | '#' | '-' | '"' | '\'' | '$' | '(' | ')' | '[' | ']' | '{' | '}' | '`' => false, + other if other.is_whitespace() => false, + _ => true, + } +} + fn is_start_bare_char(c: char) -> bool { match c { + '+' => false, _ if c.is_alphabetic() => true, - _ if c.is_numeric() => true, '.' => true, '\\' => true, '/' => true, '_' => true, '-' => true, - '@' => true, - '*' => true, - '?' => true, '~' => true, - '+' => true, _ => false, } } fn is_bare_char(c: char) -> bool { match c { + '+' => false, _ if c.is_alphanumeric() => true, - ':' => true, '.' => true, '\\' => true, '/' => true, '_' => true, '-' => true, - '@' => true, - '*' => true, - '?' => true, '=' => true, '~' => true, - '+' => true, - '%' => true, _ => false, } } @@ -724,6 +742,44 @@ mod tests { } } + macro_rules! equal_tokens { + ($source:tt -> $tokens:expr) => { + let result = apply(pipeline, "pipeline", $source); + let (expected_tree, expected_source) = TokenTreeBuilder::build($tokens); + + if result != expected_tree { + let debug_result = format!("{}", result.debug($source)); + let debug_expected = format!("{}", expected_tree.debug(&expected_source)); + + if debug_result == debug_expected { + assert_eq!( + result, expected_tree, + "NOTE: actual and expected had equivalent debug serializations, source={:?}, debug_expected={:?}", + $source, + debug_expected + ) + } else { + assert_eq!(debug_result, debug_expected) + } + } + + // apply(pipeline, "pipeline", r#"cargo +nightly run"#), + // build_token(b::pipeline(vec![( + // None, + // b::call( + // b::bare("cargo"), + // vec![ + // b::sp(), + // b::external_word("+nightly"), + // b::sp(), + // b::bare("run") + // ] + // ), + // None + // )])) + }; + } + #[test] fn test_integer() { assert_leaf! { @@ -854,7 +910,7 @@ mod tests { fn test_external() { assert_leaf! { parsers [ external ] - "^ls" -> 0..3 { External(span(1, 3)) } + "^ls" -> 0..3 { ExternalCommand(span(1, 3)) } } } @@ -1058,6 +1114,46 @@ mod tests { ); } + #[test] + fn test_external_word() { + let _ = pretty_env_logger::try_init(); + + equal_tokens!( + "cargo +nightly run" -> + b::pipeline(vec![( + None, + b::call( + b::bare("cargo"), + vec![ + b::sp(), + b::external_word("+nightly"), + b::sp(), + b::bare("run") + ] + ), + None + )]) + ); + + equal_tokens!( + "rm foo%bar" -> + b::pipeline(vec![( + None, + b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]), + None + )]) + ); + + equal_tokens!( + "rm foo%bar" -> + b::pipeline(vec![( + None, + b::call(b::bare("rm"), vec![b::sp(), b::external_word("foo%bar"),]), + None + )]) + ); + } + #[test] fn test_smoke_pipeline() { let _ = pretty_env_logger::try_init(); @@ -1178,7 +1274,6 @@ mod tests { } fn build_token(block: CurriedToken) -> TokenNode { - let mut builder = TokenTreeBuilder::new(); - block(&mut builder) + TokenTreeBuilder::build(block).0 } } diff --git a/src/parser/parse/pipeline.rs b/src/parser/parse/pipeline.rs index 75155d14..64a899c1 100644 --- a/src/parser/parse/pipeline.rs +++ b/src/parser/parse/pipeline.rs @@ -1,7 +1,9 @@ use crate::parser::CallNode; +use crate::traits::ToDebug; use crate::{Span, Tagged}; use derive_new::new; use getset::Getters; +use std::fmt; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, new)] pub struct Pipeline { @@ -9,6 +11,20 @@ pub struct Pipeline { pub(crate) post_ws: Option, } +impl ToDebug for Pipeline { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + for part in &self.parts { + write!(f, "{}", part.debug(source))?; + } + + if let Some(post_ws) = self.post_ws { + write!(f, "{}", post_ws.slice(source))? + } + + Ok(()) + } +} + #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Getters, new)] pub struct PipelineElement { pub pipe: Option, @@ -17,3 +33,23 @@ pub struct PipelineElement { call: Tagged, pub post_ws: Option, } + +impl ToDebug for PipelineElement { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + if let Some(pipe) = self.pipe { + write!(f, "{}", pipe.slice(source))?; + } + + if let Some(pre_ws) = self.pre_ws { + write!(f, "{}", pre_ws.slice(source))?; + } + + write!(f, "{}", self.call.debug(source))?; + + if let Some(post_ws) = self.post_ws { + write!(f, "{}", post_ws.slice(source))?; + } + + Ok(()) + } +} diff --git a/src/parser/parse/token_tree.rs b/src/parser/parse/token_tree.rs index df189a1a..f69c176e 100644 --- a/src/parser/parse/token_tree.rs +++ b/src/parser/parse/token_tree.rs @@ -1,5 +1,6 @@ use crate::errors::ShellError; use crate::parser::parse::{call_node::*, flag::*, operator::*, pipeline::*, tokens::*}; +use crate::traits::ToDebug; use crate::{Span, Tagged, Text}; use derive_new::new; use enum_utils::FromStr; @@ -22,6 +23,12 @@ pub enum TokenNode { Path(Tagged), } +impl ToDebug for TokenNode { + fn fmt_debug(&self, f: &mut fmt::Formatter, source: &str) -> fmt::Result { + write!(f, "{:?}", self.old_debug(&Text::from(source))) + } +} + pub struct DebugTokenNode<'a> { node: &'a TokenNode, source: &'a Text, @@ -34,11 +41,11 @@ impl fmt::Debug for DebugTokenNode<'_> { TokenNode::Call(s) => { write!(f, "(")?; - write!(f, "{:?}", s.head().debug(self.source))?; + write!(f, "{}", s.head().debug(self.source))?; if let Some(children) = s.children() { for child in children { - write!(f, "{:?}", child.debug(self.source))?; + write!(f, "{}", child.debug(self.source))?; } } @@ -57,7 +64,7 @@ impl fmt::Debug for DebugTokenNode<'_> { )?; for child in d.children() { - write!(f, "{:?}", child.debug(self.source))?; + write!(f, "{:?}", child.old_debug(self.source))?; } write!( @@ -70,7 +77,7 @@ impl fmt::Debug for DebugTokenNode<'_> { } ) } - TokenNode::Pipeline(_) => write!(f, ""), + TokenNode::Pipeline(pipeline) => write!(f, "{}", pipeline.debug(self.source)), TokenNode::Error(s) => write!(f, " for {:?}", s.span().slice(self.source)), rest => write!(f, "{}", rest.span().slice(self.source)), } @@ -115,7 +122,7 @@ impl TokenNode { .to_string() } - pub fn debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> { + pub fn old_debug<'a>(&'a self, source: &'a Text) -> DebugTokenNode<'a> { DebugTokenNode { node: self, source } } @@ -140,7 +147,7 @@ impl TokenNode { pub fn is_external(&self) -> bool { match self { TokenNode::Token(Tagged { - item: RawToken::External(..), + item: RawToken::ExternalCommand(..), .. }) => true, _ => false, @@ -150,7 +157,7 @@ impl TokenNode { pub fn expect_external(&self) -> Span { match self { TokenNode::Token(Tagged { - item: RawToken::External(span), + item: RawToken::ExternalCommand(span), .. }) => *span, _ => panic!("Only call expect_external if you checked is_external first"), diff --git a/src/parser/parse/token_tree_builder.rs b/src/parser/parse/token_tree_builder.rs index 9dd1ebc1..8034e8b0 100644 --- a/src/parser/parse/token_tree_builder.rs +++ b/src/parser/parse/token_tree_builder.rs @@ -14,15 +14,19 @@ use derive_new::new; pub struct TokenTreeBuilder { #[new(default)] pos: usize, + + #[new(default)] + output: String, } pub type CurriedToken = Box TokenNode + 'static>; pub type CurriedCall = Box Tagged + 'static>; impl TokenTreeBuilder { - pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> TokenNode { + pub fn build(block: impl FnOnce(&mut Self) -> TokenNode) -> (TokenNode, String) { let mut builder = TokenTreeBuilder::new(); - block(&mut builder) + let node = block(&mut builder); + (node, builder.output) } pub fn pipeline(input: Vec<(Option<&str>, CurriedCall, Option<&str>)>) -> CurriedToken { @@ -56,7 +60,8 @@ impl TokenTreeBuilder { pipe, pre_span.map(Span::from), call, - post_span.map(Span::from))); + post_span.map(Span::from), + )); loop { match input.next() { @@ -147,9 +152,27 @@ impl TokenTreeBuilder { )) } + pub fn external_word(input: impl Into) -> CurriedToken { + let input = input.into(); + + Box::new(move |b| { + let (start, end) = b.consume(&input); + b.pos = end; + + TokenTreeBuilder::spanned_external_word((start, end)) + }) + } + + pub fn spanned_external_word(input: impl Into) -> TokenNode { + TokenNode::Token(Tagged::from_simple_spanned_item( + RawToken::ExternalWord, + input.into(), + )) + } + pub fn spanned_external(input: impl Into, span: impl Into) -> TokenNode { TokenNode::Token(Tagged::from_simple_spanned_item( - RawToken::External(input.into()), + RawToken::ExternalCommand(input.into()), span.into(), )) } @@ -422,6 +445,7 @@ impl TokenTreeBuilder { fn consume(&mut self, input: &str) -> (usize, usize) { let start = self.pos; self.pos += input.len(); + self.output.push_str(input); (start, self.pos) } } diff --git a/src/parser/parse/tokens.rs b/src/parser/parse/tokens.rs index ed9c1f72..0bb2e3f1 100644 --- a/src/parser/parse/tokens.rs +++ b/src/parser/parse/tokens.rs @@ -10,7 +10,8 @@ pub enum RawToken { Size(RawNumber, Unit), String(Span), Variable(Span), - External(Span), + ExternalCommand(Span), + ExternalWord, Bare, } @@ -50,7 +51,8 @@ impl RawToken { RawToken::Size(..) => "Size", RawToken::String(_) => "String", RawToken::Variable(_) => "Variable", - RawToken::External(_) => "External", + RawToken::ExternalCommand(_) => "ExternalCommand", + RawToken::ExternalWord => "ExternalWord", RawToken::Bare => "String", } } diff --git a/src/parser/parse_command.rs b/src/parser/parse_command.rs index 33ad25e6..e0fc9d86 100644 --- a/src/parser/parse_command.rs +++ b/src/parser/parse_command.rs @@ -6,6 +6,7 @@ use crate::parser::{ hir::{self, NamedArguments}, Flag, RawToken, TokenNode, }; +use crate::traits::ToDebug; use crate::{Span, Tag, Tagged, Text}; use log::trace; @@ -248,7 +249,7 @@ pub fn trace_remaining(desc: &'static str, tail: hir::TokensIterator<'_>, source itertools::join( tail.debug_remaining() .iter() - .map(|i| format!("%{:?}%", i.debug(source))), + .map(|i| format!("%{}%", i.debug(&source))), " " ) ); diff --git a/src/shell/helper.rs b/src/shell/helper.rs index 8e21c50e..462f3752 100644 --- a/src/shell/helper.rs +++ b/src/shell/helper.rs @@ -136,9 +136,13 @@ fn paint_token_node(token_node: &TokenNode, line: &str) -> String { .. }) => Color::Green.normal().paint(token_node.span().slice(line)), TokenNode::Token(Tagged { - item: RawToken::External(..), + item: RawToken::ExternalCommand(..), .. }) => Color::Cyan.bold().paint(token_node.span().slice(line)), + TokenNode::Token(Tagged { + item: RawToken::ExternalWord, + .. + }) => Color::Black.bold().paint(token_node.span().slice(line)), }; styled.to_string()