From 5ff4bcfb7a837faacbd5cab86f2469b273708b58 Mon Sep 17 00:00:00 2001 From: Michael Angerman Date: Sun, 27 Dec 2020 21:34:27 -0800 Subject: [PATCH] Nucli refactor crate stream (#2828) * nu-stream is building on its own, now clean up Cargo.toml * replace the stream crate in nu-cli * cc * since we moved stream out of the nu-cli crate and into its own crate we need to remove pub(crate) and just make it pub * clean up the prelude and hand merge everything together * clean up Cargo.tom * cargo fmt along with Cargo.lock --- Cargo.lock | 11 +++ crates/nu-cli/Cargo.toml | 1 + .../nu-cli/src/commands/classified/block.rs | 2 +- crates/nu-cli/src/evaluation_context.rs | 2 +- crates/nu-cli/src/examples.rs | 2 +- crates/nu-cli/src/lib.rs | 3 +- crates/nu-cli/src/prelude.rs | 6 +- crates/nu-cli/src/shell/shell.rs | 2 +- crates/nu-cli/src/shell/shell_manager.rs | 2 +- crates/nu-stream/Cargo.toml | 18 ++++ .../src/stream => nu-stream/src}/input.rs | 2 +- .../stream => nu-stream/src}/interruptible.rs | 0 .../stream/mod.rs => nu-stream/src/lib.rs} | 2 + .../src/stream => nu-stream/src}/output.rs | 2 +- crates/nu-stream/src/prelude.rs | 93 +++++++++++++++++++ 15 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 crates/nu-stream/Cargo.toml rename crates/{nu-cli/src/stream => nu-stream/src}/input.rs (99%) rename crates/{nu-cli/src/stream => nu-stream/src}/interruptible.rs (100%) rename crates/{nu-cli/src/stream/mod.rs => nu-stream/src/lib.rs} (88%) rename crates/{nu-cli/src/stream => nu-stream/src}/output.rs (98%) create mode 100644 crates/nu-stream/src/prelude.rs diff --git a/Cargo.lock b/Cargo.lock index b536a82b..df38e09d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3181,6 +3181,7 @@ dependencies = [ "nu-plugin", "nu-protocol", "nu-source", + "nu-stream", "nu-table", "nu-test-support", "nu-value-ext", @@ -3358,6 +3359,16 @@ dependencies = [ "termcolor", ] +[[package]] +name = "nu-stream" +version = "0.24.2" +dependencies = [ + "futures 0.3.8", + "nu-errors", + "nu-protocol", + "nu-source", +] + [[package]] name = "nu-table" version = "0.24.2" diff --git a/crates/nu-cli/Cargo.toml b/crates/nu-cli/Cargo.toml index 9652f144..6df2a039 100644 --- a/crates/nu-cli/Cargo.toml +++ b/crates/nu-cli/Cargo.toml @@ -18,6 +18,7 @@ nu-parser = {version = "0.24.2", path = "../nu-parser"} nu-plugin = {version = "0.24.2", path = "../nu-plugin"} nu-protocol = {version = "0.24.2", path = "../nu-protocol"} nu-source = {version = "0.24.2", path = "../nu-source"} +nu-stream = {version = "0.24.2", path = "../nu-stream"} nu-table = {version = "0.24.2", path = "../nu-table"} nu-test-support = {version = "0.24.2", path = "../nu-test-support"} nu-value-ext = {version = "0.24.2", path = "../nu-value-ext"} diff --git a/crates/nu-cli/src/commands/classified/block.rs b/crates/nu-cli/src/commands/classified/block.rs index 0e457c7d..b33ac499 100644 --- a/crates/nu-cli/src/commands/classified/block.rs +++ b/crates/nu-cli/src/commands/classified/block.rs @@ -2,7 +2,6 @@ use crate::commands::classified::expr::run_expression_block; use crate::commands::classified::internal::run_internal_command; use crate::evaluation_context::EvaluationContext; use crate::prelude::*; -use crate::stream::InputStream; use async_recursion::async_recursion; use futures::stream::TryStreamExt; use nu_errors::ShellError; @@ -10,6 +9,7 @@ use nu_protocol::hir::{ Block, Call, ClassifiedCommand, Expression, Pipeline, SpannedExpression, Synthetic, }; use nu_protocol::{ReturnSuccess, UntaggedValue, Value}; +use nu_stream::InputStream; use std::sync::atomic::Ordering; #[async_recursion] diff --git a/crates/nu-cli/src/evaluation_context.rs b/crates/nu-cli/src/evaluation_context.rs index 7e6f5e80..ed840e35 100644 --- a/crates/nu-cli/src/evaluation_context.rs +++ b/crates/nu-cli/src/evaluation_context.rs @@ -2,9 +2,9 @@ use crate::commands::{command::CommandArgs, Command, UnevaluatedCallInfo}; use crate::env::host::Host; use crate::prelude::*; use crate::shell::shell_manager::ShellManager; -use crate::stream::{InputStream, OutputStream}; use nu_protocol::hir; use nu_source::{Tag, Text}; +use nu_stream::{InputStream, OutputStream}; use parking_lot::Mutex; use std::error::Error; use std::sync::atomic::AtomicBool; diff --git a/crates/nu-cli/src/examples.rs b/crates/nu-cli/src/examples.rs index 8c734d70..1f7bba77 100644 --- a/crates/nu-cli/src/examples.rs +++ b/crates/nu-cli/src/examples.rs @@ -17,7 +17,7 @@ use crate::commands::{ StrCollect, WholeStreamCommand, Wrap, }; use crate::evaluation_context::EvaluationContext; -use crate::stream::{InputStream, OutputStream}; +use nu_stream::{InputStream, OutputStream}; use async_trait::async_trait; use futures::executor::block_on; diff --git a/crates/nu-cli/src/lib.rs b/crates/nu-cli/src/lib.rs index 523ab1f2..105d608e 100644 --- a/crates/nu-cli/src/lib.rs +++ b/crates/nu-cli/src/lib.rs @@ -31,7 +31,6 @@ mod keybinding; mod path; mod plugin; mod shell; -mod stream; pub mod types; pub mod utils; @@ -54,11 +53,11 @@ pub use crate::env::environment_syncer::EnvironmentSyncer; pub use crate::env::host::BasicHost; pub use crate::evaluation_context::EvaluationContext; pub use crate::prelude::ToOutputStream; -pub use crate::stream::{InputStream, InterruptibleStream, OutputStream}; pub use nu_data::config; pub use nu_data::dict::TaggedListBuilder; pub use nu_data::primitive; pub use nu_data::value; +pub use nu_stream::{InputStream, InterruptibleStream, OutputStream}; pub use nu_value_ext::ValueExt; pub use num_traits::cast::ToPrimitive; diff --git a/crates/nu-cli/src/prelude.rs b/crates/nu-cli/src/prelude.rs index 8189ccf3..6ec9892b 100644 --- a/crates/nu-cli/src/prelude.rs +++ b/crates/nu-cli/src/prelude.rs @@ -36,7 +36,7 @@ macro_rules! trace_stream { ); }); - $crate::stream::InputStream::from_stream(objects.boxed()) + nu_stream::InputStream::from_stream(objects.boxed()) } else { $expr } @@ -61,7 +61,7 @@ macro_rules! trace_out_stream { ); }); - $crate::stream::OutputStream::new(objects) + nu_stream::OutputStream::new(objects) } else { $expr } @@ -84,7 +84,6 @@ pub(crate) use crate::shell::filesystem_shell::FilesystemShell; pub(crate) use crate::shell::help_shell::HelpShell; pub(crate) use crate::shell::shell_manager::ShellManager; pub(crate) use crate::shell::value_shell::ValueShell; -pub(crate) use crate::stream::{InputStream, InterruptibleStream, OutputStream}; pub(crate) use bigdecimal::BigDecimal; pub(crate) use futures::stream::BoxStream; pub(crate) use futures::{Stream, StreamExt}; @@ -94,6 +93,7 @@ pub(crate) use nu_source::{ b, AnchorLocation, DebugDocBuilder, PrettyDebug, PrettyDebugWithSource, Span, SpannedItem, Tag, TaggedItem, Text, }; +pub(crate) use nu_stream::{InputStream, InterruptibleStream, OutputStream}; pub(crate) use nu_value_ext::ValueExt; pub(crate) use num_bigint::BigInt; pub(crate) use num_traits::cast::ToPrimitive; diff --git a/crates/nu-cli/src/shell/shell.rs b/crates/nu-cli/src/shell/shell.rs index 035be9e5..c32312af 100644 --- a/crates/nu-cli/src/shell/shell.rs +++ b/crates/nu-cli/src/shell/shell.rs @@ -7,7 +7,7 @@ use crate::commands::mkdir::MkdirArgs; use crate::commands::move_::mv::Arguments as MvArgs; use crate::commands::rm::RemoveArgs; use crate::prelude::*; -use crate::stream::OutputStream; +use nu_stream::OutputStream; use encoding_rs::Encoding; use nu_errors::ShellError; diff --git a/crates/nu-cli/src/shell/shell_manager.rs b/crates/nu-cli/src/shell/shell_manager.rs index ec1156d1..b0c829f3 100644 --- a/crates/nu-cli/src/shell/shell_manager.rs +++ b/crates/nu-cli/src/shell/shell_manager.rs @@ -9,7 +9,7 @@ use crate::commands::rm::RemoveArgs; use crate::prelude::*; use crate::shell::filesystem_shell::FilesystemShell; use crate::shell::shell::Shell; -use crate::stream::OutputStream; +use nu_stream::OutputStream; use encoding_rs::Encoding; use nu_errors::ShellError; diff --git a/crates/nu-stream/Cargo.toml b/crates/nu-stream/Cargo.toml new file mode 100644 index 00000000..336736b0 --- /dev/null +++ b/crates/nu-stream/Cargo.toml @@ -0,0 +1,18 @@ +[package] +authors = ["The Nu Project Contributors"] +description = "Nushell stream" +edition = "2018" +license = "MIT" +name = "nu-stream" +version = "0.24.2" + +[dependencies] +nu-errors = {version = "0.24.2", path = "../nu-errors"} +nu-protocol = {version = "0.24.2", path = "../nu-protocol"} +nu-source = {version = "0.24.2", path = "../nu-source"} + +futures = {version = "0.3.5", features = ["compat", "io-compat"]} + +[features] +stable = [] +trace = [] diff --git a/crates/nu-cli/src/stream/input.rs b/crates/nu-stream/src/input.rs similarity index 99% rename from crates/nu-cli/src/stream/input.rs rename to crates/nu-stream/src/input.rs index 5137d633..ce40e63d 100644 --- a/crates/nu-cli/src/stream/input.rs +++ b/crates/nu-stream/src/input.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use futures::stream::{iter, once}; use nu_errors::ShellError; use nu_protocol::{Primitive, Type, UntaggedValue, Value}; -use nu_source::{Tagged, TaggedItem}; +use nu_source::{PrettyDebug, Tag, Tagged, TaggedItem}; pub struct InputStream { values: BoxStream<'static, Value>, diff --git a/crates/nu-cli/src/stream/interruptible.rs b/crates/nu-stream/src/interruptible.rs similarity index 100% rename from crates/nu-cli/src/stream/interruptible.rs rename to crates/nu-stream/src/interruptible.rs diff --git a/crates/nu-cli/src/stream/mod.rs b/crates/nu-stream/src/lib.rs similarity index 88% rename from crates/nu-cli/src/stream/mod.rs rename to crates/nu-stream/src/lib.rs index ee08c663..17d8ba95 100644 --- a/crates/nu-cli/src/stream/mod.rs +++ b/crates/nu-stream/src/lib.rs @@ -1,3 +1,5 @@ +mod prelude; + mod input; mod interruptible; mod output; diff --git a/crates/nu-cli/src/stream/output.rs b/crates/nu-stream/src/output.rs similarity index 98% rename from crates/nu-cli/src/stream/output.rs rename to crates/nu-stream/src/output.rs index 1da51567..e4fd0e39 100644 --- a/crates/nu-cli/src/stream/output.rs +++ b/crates/nu-stream/src/output.rs @@ -4,7 +4,7 @@ use nu_protocol::{ReturnSuccess, ReturnValue, Value}; use std::iter::IntoIterator; pub struct OutputStream { - pub(crate) values: BoxStream<'static, ReturnValue>, + pub values: BoxStream<'static, ReturnValue>, } impl OutputStream { diff --git a/crates/nu-stream/src/prelude.rs b/crates/nu-stream/src/prelude.rs new file mode 100644 index 00000000..6a536ed7 --- /dev/null +++ b/crates/nu-stream/src/prelude.rs @@ -0,0 +1,93 @@ +#[macro_export] +macro_rules! return_err { + ($expr:expr) => { + match $expr { + Err(_) => return, + Ok(expr) => expr, + }; + }; +} + +#[macro_export] +macro_rules! stream { + ($($expr:expr),*) => {{ + let mut v = VecDeque::new(); + + $( + v.push_back($expr); + )* + + v + }} +} + +#[macro_export] +macro_rules! trace_stream { + (target: $target:tt, $desc:tt = $expr:expr) => {{ + if log::log_enabled!(target: $target, log::Level::Trace) { + use futures::stream::StreamExt; + + let objects = $expr.inspect(move |o| { + trace!( + target: $target, + "{} = {}", + $desc, + nu_source::PrettyDebug::plain_string(o, 70) + ); + }); + + $crate::stream::InputStream::from_stream(objects.boxed()) + } else { + $expr + } + }}; +} + +#[macro_export] +macro_rules! trace_out_stream { + (target: $target:tt, $desc:tt = $expr:expr) => {{ + if log::log_enabled!(target: $target, log::Level::Trace) { + use futures::stream::StreamExt; + + let objects = $expr.inspect(move |o| { + trace!( + target: $target, + "{} = {}", + $desc, + match o { + Err(err) => format!("{:?}", err), + Ok(value) => value.display(), + } + ); + }); + + $crate::stream::OutputStream::new(objects) + } else { + $expr + } + }}; +} + +pub(crate) use futures::stream::BoxStream; +pub(crate) use futures::{Stream, StreamExt}; +pub(crate) use std::collections::VecDeque; +pub(crate) use std::future::Future; +pub(crate) use std::sync::Arc; + +pub(crate) use crate::{InputStream, OutputStream}; + +pub trait ToOutputStream { + fn to_output_stream(self) -> OutputStream; +} + +impl ToOutputStream for T +where + T: Stream + Send + 'static, + U: Into, +{ + fn to_output_stream(self) -> OutputStream { + OutputStream { + values: self.map(|item| item.into()).boxed(), + } + } +}