feat(linter): add runner for import-plugin (#858)

Cargo.lock

@@ -1626,6 +1626,7 @@ dependencies = [
 name = "oxc_linter"
 version = "0.0.0"
 dependencies = [
+ "dashmap",
  "insta",
  "itertools 0.11.0",
  "lazy_static",
@@ -1638,6 +1639,7 @@ dependencies = [
  "oxc_formatter",
  "oxc_macros",
  "oxc_parser",
+ "oxc_resolver",
  "oxc_semantic",
  "oxc_span",
  "oxc_syntax",

crates/oxc_cli/src/command.rs

@@ -68,6 +68,10 @@ pub struct LintOptions {
     #[bpaf(external(lint_filter), map(LintFilter::into_tuple), many)]
     pub filter: Vec<(AllowWarnDeny, String)>,
 
+    /// Use the experimental import plugin and detect ESM problems
+    #[bpaf(switch, hide_usage)]
+    pub import_plugin: bool,
+
     #[bpaf(external)]
     pub fix_options: FixOptions,
 

crates/oxc_cli/src/lint/mod.rs

@@ -26,6 +26,7 @@ impl Runner for LintRunner {
         let CliLintOptions {
             paths,
             filter,
+            import_plugin,
             warning_options,
             ignore_options,
             fix_options,
@@ -34,25 +35,27 @@ impl Runner for LintRunner {
 
         let now = std::time::Instant::now();
 
+        let paths = Walk::new(&paths, &ignore_options).paths();
+        let number_of_files = paths.len();
+
+        let cwd = std::env::current_dir().unwrap().into_boxed_path();
         let lint_options = LintOptions::default()
             .with_filter(filter)
             .with_fix(fix_options.fix)
-            .with_timing(misc_options.timing);
-        let lint_service = LintService::new(lint_options);
+            .with_timing(misc_options.timing)
+            .with_import_plugin(import_plugin);
+        let lint_service = LintService::new(cwd, &paths, lint_options);
 
         let diagnostic_service = DiagnosticService::default()
             .with_quiet(warning_options.quiet)
             .with_max_warnings(warning_options.max_warnings);
 
-        let paths = Walk::new(&paths, &ignore_options).paths();
-        let number_of_files = paths.len();
-
         // Spawn linting in another thread so diagnostics can be printed immediately from diagnostic_service.run.
         rayon::spawn({
             let tx_error = diagnostic_service.sender().clone();
             let lint_service = lint_service.clone();
             move || {
-                lint_service.run(paths, &tx_error);
+                lint_service.run(&tx_error);
             }
         });
         diagnostic_service.run();

crates/oxc_cli/src/snapshots/oxc_cli__command__snapshot__default.snap

@@ -44,6 +44,7 @@ Available positional items:
     PATH                      Single file, single path or list of paths
 
 Available options:
+        --import-plugin       Use the experimental import plugin and detect ESM problems
     -h, --help                Prints help information
 
 

crates/oxc_cli/src/snapshots/oxc_cli__command__snapshot__help_help.snap

@@ -57,6 +57,7 @@ Available positional items:
     PATH                      Single file, single path or list of paths
 
 Available options:
+        --import-plugin       Use the experimental import plugin and detect ESM problems
     -h, --help                Prints help information
 
 

crates/oxc_cli/src/walk.rs

@@ -52,6 +52,10 @@ impl ignore::ParallelVisitor for WalkCollector {
 impl Walk {
     /// # Panics
     pub fn new(paths: &[PathBuf], options: &IgnoreOptions) -> Self {
+        let paths = paths
+            .iter()
+            .map(|p| p.canonicalize().unwrap_or_else(|_| p.clone()))
+            .collect::<Vec<_>>();
         let mut inner = ignore::WalkBuilder::new(&paths[0]);
 
         if let Some(paths) = paths.get(1..) {

crates/oxc_linter/Cargo.toml

@@ -22,6 +22,7 @@ oxc_macros      = { workspace = true }
 oxc_semantic    = { workspace = true }
 oxc_syntax      = { workspace = true }
 oxc_formatter   = { workspace = true }
+oxc_resolver    = { workspace = true }
 
 rayon       = { workspace = true }
 lazy_static = { workspace = true }                        # used in oxc_macros
@@ -31,6 +32,7 @@ rustc-hash  = { workspace = true }
 phf         = { workspace = true, features = ["macros"] }
 num-traits  = { workspace = true }
 itertools   = { workspace = true }
+dashmap     = { workspace = true }
 
 rust-lapper = "1.1.0"
 once_cell   = "1.18.0"

crates/oxc_linter/src/options.rs

@@ -9,6 +9,7 @@ pub struct LintOptions {
     pub filter: Vec<(AllowWarnDeny, String)>,
     pub fix: bool,
     pub timing: bool,
+    pub import_plugin: bool,
 }
 
 impl Default for LintOptions {
@@ -17,6 +18,7 @@ impl Default for LintOptions {
             filter: vec![(AllowWarnDeny::Deny, String::from("correctness"))],
             fix: false,
             timing: false,
+            import_plugin: false,
         }
     }
 }
@@ -41,6 +43,12 @@ impl LintOptions {
         self.timing = yes;
         self
     }
+
+    #[must_use]
+    pub fn with_import_plugin(mut self, yes: bool) -> Self {
+        self.import_plugin = yes;
+        self
+    }
 }
 
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]

crates/oxc_linter/src/service.rs

@@ -1,66 +1,178 @@
-use std::{fs, path::Path, rc::Rc, sync::Arc};
+use dashmap::DashMap;
+use std::{
+    collections::HashMap,
+    fs,
+    path::Path,
+    rc::Rc,
+    sync::{Arc, Condvar, Mutex},
+};
 
 use oxc_allocator::Allocator;
 use oxc_diagnostics::{DiagnosticSender, DiagnosticService};
 use oxc_parser::Parser;
-use oxc_semantic::SemanticBuilder;
-use oxc_span::SourceType;
+use oxc_resolver::{ResolveOptions, Resolver};
+use oxc_semantic::{ModuleRecord, SemanticBuilder};
+use oxc_span::{SourceType, VALID_EXTENSIONS};
+use rustc_hash::FxHashSet;
 
 use crate::{Fixer, LintContext, LintOptions, Linter, Message};
-use rayon::prelude::{IntoParallelIterator, ParallelIterator};
+use rayon::{iter::ParallelBridge, prelude::ParallelIterator};
 
 #[derive(Clone)]
 pub struct LintService {
-    linter: Arc<Linter>,
+    runtime: Arc<Runtime>,
 }
 
 impl LintService {
-    pub fn new(options: LintOptions) -> Self {
-        let linter = Arc::new(Linter::from_options(options));
-        Self { linter }
+    pub fn new(cwd: Box<Path>, paths: &[Box<Path>], options: LintOptions) -> Self {
+        let linter = Linter::from_options(options);
+        let runtime = Arc::new(Runtime::new(cwd, paths, linter));
+        Self { runtime }
+    }
+
+    #[cfg(test)]
+    pub(crate) fn from_linter(cwd: Box<Path>, paths: &[Box<Path>], linter: Linter) -> Self {
+        let runtime = Arc::new(Runtime::new(cwd, paths, linter));
+        Self { runtime }
     }
 
     pub fn linter(&self) -> &Linter {
-        &self.linter
+        &self.runtime.linter
+    }
+
+    pub fn number_of_dependencies(&self) -> usize {
+        self.runtime.module_map.len() - self.runtime.paths.len()
     }
 
     /// # Panics
-    pub fn run(&self, paths: Vec<Box<Path>>, tx_error: &DiagnosticSender) {
-        paths.into_par_iter().for_each_with(&self.linter, |linter, path| {
-            Self::run_path(linter, &path, tx_error);
-        });
+    pub fn run(&self, tx_error: &DiagnosticSender) {
+        self.runtime
+            .paths
+            .iter()
+            .par_bridge()
+            .for_each_with(&self.runtime, |runtime, path| runtime.process_path(path, tx_error));
         tx_error.send(None).unwrap();
     }
 
-    /// # Panics
-    fn run_path(linter: &Arc<Linter>, path: &Path, tx_error: &DiagnosticSender) {
-        let tx_error = tx_error.clone();
+    /// For tests
+    #[cfg(test)]
+    pub(crate) fn run_source<'a>(
+        &self,
+        allocator: &'a Allocator,
+        source_text: &'a str,
+        check_syntax_errors: bool,
+        tx_error: &DiagnosticSender,
+    ) -> Vec<Message<'a>> {
+        self.runtime
+            .paths
+            .iter()
+            .flat_map(|path| {
+                let source_type = SourceType::from_path(path).unwrap();
+                self.runtime.init_cache_state(path);
+                self.runtime.process_source(
+                    path,
+                    allocator,
+                    source_text,
+                    source_type,
+                    check_syntax_errors,
+                    tx_error,
+                )
+            })
+            .collect::<Vec<_>>()
+    }
+}
+
+/// `CacheState` and `CacheStateEntry` are used to fix the problem where
+/// there is a brief moment when a concurrent fetch can miss the cache.
+///
+/// Given `ModuleMap` is a `DashMap`, which conceptually is a `RwLock<HashMap>`.
+/// When two requests read the map at the exact same time from different threads,
+/// both will miss the cache so both thread will make a request.
+///
+/// See the "problem section" in <https://medium.com/@polyglot_factotum/rust-concurrency-patterns-condvars-and-locks-e278f18db74f>
+/// and the solution is copied here to fix the issue.
+type CacheState = Mutex<HashMap<Box<Path>, Arc<(Mutex<CacheStateEntry>, Condvar)>>>;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum CacheStateEntry {
+    ReadyToConstruct,
+    PendingStore(usize),
+}
+
+/// Keyed by canonicalized path
+type ModuleMap = DashMap<Box<Path>, Arc<ModuleRecord>>;
+
+pub struct Runtime {
+    cwd: Box<Path>,
+    /// All paths to lint
+    paths: FxHashSet<Box<Path>>,
+    linter: Linter,
+    resolver: Resolver,
+    module_map: ModuleMap,
+    cache_state: CacheState,
+}
+
+impl Runtime {
+    fn new(cwd: Box<Path>, paths: &[Box<Path>], linter: Linter) -> Self {
+        Self {
+            cwd,
+            paths: paths.iter().cloned().collect(),
+            linter,
+            resolver: Self::resolver(),
+            module_map: ModuleMap::default(),
+            cache_state: CacheState::default(),
+        }
+    }
+
+    fn resolver() -> Resolver {
+        Resolver::new(ResolveOptions {
+            condition_names: vec!["node".into(), "import".into()],
+            extension_alias: vec![
+                (".js".into(), vec![".js".into(), ".tsx".into(), "ts".into()]),
+                (".mjs".into(), vec![".mjs".into(), ".mts".into()]),
+            ],
+            extensions: VALID_EXTENSIONS.iter().map(|ext| format!(".{ext}")).collect(),
+            ..ResolveOptions::default()
+        })
+    }
+
+    fn process_path(&self, path: &Path, tx_error: &DiagnosticSender) {
+        let Ok(source_type) = SourceType::from_path(path) else { return };
+
+        if self.init_cache_state(path) {
+            return;
+        }
+
         let allocator = Allocator::default();
         let source_text =
             fs::read_to_string(path).unwrap_or_else(|_| panic!("Failed to read {path:?}"));
 
-        let mut messages = Self::run_source(linter, path, &allocator, &source_text, true);
+        let mut messages =
+            self.process_source(path, &allocator, &source_text, source_type, true, tx_error);
 
-        if linter.options().fix {
+        if self.linter.options().fix {
             let fix_result = Fixer::new(&source_text, messages).fix();
             fs::write(path, fix_result.fixed_code.as_bytes()).unwrap();
             messages = fix_result.messages;
         }
 
-        let errors = messages.into_iter().map(|m| m.error).collect();
-        let diagnostics = DiagnosticService::wrap_diagnostics(path, &source_text, errors);
-        tx_error.send(Some(diagnostics)).unwrap();
+        if !messages.is_empty() {
+            let errors = messages.into_iter().map(|m| m.error).collect();
+            let path = path.strip_prefix(&self.cwd).unwrap();
+            let diagnostics = DiagnosticService::wrap_diagnostics(path, &source_text, errors);
+            tx_error.send(Some(diagnostics)).unwrap();
+        }
     }
 
-    pub(crate) fn run_source<'a>(
-        linter: &Linter,
+    fn process_source<'a>(
+        &self,
         path: &Path,
         allocator: &'a Allocator,
         source_text: &'a str,
+        source_type: SourceType,
         check_syntax_errors: bool,
+        tx_error: &DiagnosticSender,
     ) -> Vec<Message<'a>> {
-        let source_type =
-            SourceType::from_path(path).unwrap_or_else(|_| panic!("Incorrect {path:?}"));
         let ret = Parser::new(allocator, source_text, source_type)
             .allow_return_outside_function(true)
             .parse();
@@ -70,17 +182,104 @@ impl LintService {
         };
 
         let program = allocator.alloc(ret.program);
-        let semantic_ret = SemanticBuilder::new(source_text, source_type)
+
+        // Build the module record to unblock other threads from waiting for too long.
+        // The semantic model is not built at this stage.
+        let semantic_builder = SemanticBuilder::new(source_text, source_type)
             .with_trivias(ret.trivias)
             .with_check_syntax_error(check_syntax_errors)
-            .with_module_record_builder(true)
-            .build(program);
+            .build_module_record(program);
+        let module_record = semantic_builder.module_record();
+
+        if self.linter.options().import_plugin {
+            self.module_map
+                .insert(path.to_path_buf().into_boxed_path(), Arc::clone(&module_record));
+            self.update_cache_state(path);
+
+            // Stop if the current module is not marked for lint.
+            if !self.paths.contains(path) {
+                return vec![];
+            }
+
+            let dir = path.parent().unwrap();
+
+            // Retrieve all dependency modules from this module.
+            module_record
+                .module_requests
+                .keys()
+                .cloned()
+                .par_bridge()
+                .map_with(&self.resolver, |resolver, specifier| {
+                    resolver.resolve(dir, &specifier).ok()
+                })
+                .flatten()
+                .filter(|r| !self.module_map.contains_key(r.path()))
+                .for_each_with(tx_error, |tx_error, resolution| {
+                    self.process_path(resolution.path(), tx_error);
+                });
+        }
+
+        let semantic_ret = semantic_builder.build(program);
 
         if !semantic_ret.errors.is_empty() {
             return semantic_ret.errors.into_iter().map(|err| Message::new(err, None)).collect();
         };
 
         let lint_ctx = LintContext::new(&Rc::new(semantic_ret.semantic));
-        linter.run(lint_ctx)
+        self.linter.run(lint_ctx)
+    }
+
+    fn init_cache_state(&self, path: &Path) -> bool {
+        if !self.linter.options().import_plugin {
+            return false;
+        }
+
+        let (lock, cvar) = {
+            let mut state_map = self.cache_state.lock().unwrap();
+            &*Arc::clone(state_map.entry(path.to_path_buf().into_boxed_path()).or_insert_with(
+                || Arc::new((Mutex::new(CacheStateEntry::ReadyToConstruct), Condvar::new())),
+            ))
+        };
+
+        let mut state = cvar
+            .wait_while(lock.lock().unwrap(), |state| {
+                matches!(*state, CacheStateEntry::PendingStore(_))
+            })
+            .unwrap();
+
+        if self.module_map.get(path).is_some() {
+            return true;
+        }
+
+        let i = if let CacheStateEntry::PendingStore(i) = *state { i } else { 0 };
+        *state = CacheStateEntry::PendingStore(i + 1);
+
+        if *state == CacheStateEntry::ReadyToConstruct {
+            cvar.notify_one();
+        }
+        drop(state);
+        false
+    }
+
+    fn update_cache_state(&self, path: &Path) {
+        let (lock, cvar) = {
+            let mut state_map = self.cache_state.lock().unwrap();
+            &*Arc::clone(
+                state_map
+                    .get_mut(path)
+                    .expect("Entry in http-cache state to have been previously inserted"),
+            )
+        };
+        let mut state = lock.lock().unwrap();
+        if let CacheStateEntry::PendingStore(i) = *state {
+            let new = i - 1;
+            if new == 0 {
+                *state = CacheStateEntry::ReadyToConstruct;
+                // Notify the next thread waiting in line, if there is any.
+                cvar.notify_one();
+            } else {
+                *state = CacheStateEntry::PendingStore(new);
+            }
+        }
     }
 }

crates/oxc_linter/src/tester.rs

@@ -1,7 +1,8 @@
-use std::{path::PathBuf, sync::Arc};
+use std::path::PathBuf;
 
 use oxc_allocator::Allocator;
 use oxc_diagnostics::miette::{GraphicalReportHandler, GraphicalTheme, NamedSource};
+use oxc_diagnostics::DiagnosticService;
 use serde_json::Value;
 
 use crate::{rules::RULES, Fixer, LintOptions, LintService, Linter, RuleEnum};
@@ -99,8 +100,12 @@ impl Tester {
         let allocator = Allocator::default();
         let rule = self.find_rule().read_json(config);
         let options = LintOptions::default().with_fix(is_fix);
-        let linter = Arc::new(Linter::from_options(options).with_rules(vec![rule]));
-        let result = LintService::run_source(&linter, &path, &allocator, source_text, false);
+        let linter = Linter::from_options(options).with_rules(vec![rule]);
+        let cwd = PathBuf::new().into_boxed_path();
+        let lint_service = LintService::from_linter(cwd, &[path.clone().into_boxed_path()], linter);
+        let diagnostic_service = DiagnosticService::default();
+        let tx_error = diagnostic_service.sender();
+        let result = lint_service.run_source(&allocator, source_text, false, tx_error);
 
         if result.is_empty() {
             return TestResult::Passed;

crates/oxc_semantic/src/builder.rs

@@ -63,8 +63,8 @@ pub struct SemanticBuilder<'a> {
     pub scope: ScopeTree,
     pub symbols: SymbolTable,
 
-    with_module_record_builder: bool,
-    pub module_record_builder: ModuleRecordBuilder,
+    pub(crate) module_record: Arc<ModuleRecord>,
+
     unused_labels: UnusedLabels<'a>,
 
     jsdoc: JSDocBuilder<'a>,
@@ -97,8 +97,7 @@ impl<'a> SemanticBuilder<'a> {
             nodes: AstNodes::default(),
             scope,
             symbols: SymbolTable::default(),
-            with_module_record_builder: false,
-            module_record_builder: ModuleRecordBuilder::default(),
+            module_record: Arc::new(ModuleRecord::default()),
             unused_labels: UnusedLabels { scopes: vec![], curr_scope: 0, labels: vec![] },
             jsdoc: JSDocBuilder::new(source_text, &trivias),
             check_syntax_error: false,
@@ -113,34 +112,35 @@ impl<'a> SemanticBuilder<'a> {
         self
     }
 
-    #[must_use]
-    pub fn with_module_record_builder(mut self, yes: bool) -> Self {
-        self.with_module_record_builder = yes;
-        self
-    }
-
     #[must_use]
     pub fn with_check_syntax_error(mut self, yes: bool) -> Self {
         self.check_syntax_error = yes;
         self
     }
 
+    /// Get the built module record from `build_module_record`
+    pub fn module_record(&self) -> Arc<ModuleRecord> {
+        Arc::clone(&self.module_record)
+    }
+
+    /// Build the module record with a shallow AST visit
+    #[must_use]
+    pub fn build_module_record(mut self, program: &'a Program<'a>) -> Self {
+        let mut module_record_builder = ModuleRecordBuilder::default();
+        module_record_builder.visit(program);
+        self.module_record = Arc::new(module_record_builder.build());
+        self
+    }
+
     pub fn build(mut self, program: &'a Program<'a>) -> SemanticBuilderReturn<'a> {
-        // First AST pass
         if !self.source_type.is_typescript_definition() {
             self.visit_program(program);
-        }
 
-        // Second partial AST pass on top level import / export statements
-        let module_record = if self.with_module_record_builder {
-            self.module_record_builder.visit(program);
+            // Checking syntax error on module record requires scope information from the previous AST pass
             if self.check_syntax_error {
                 EarlyErrorJavaScript::check_module_record(&self);
             }
-            self.module_record_builder.build()
-        } else {
-            ModuleRecord::default()
-        };
+        }
 
         let semantic = Semantic {
             source_text: self.source_text,
@@ -149,7 +149,7 @@ impl<'a> SemanticBuilder<'a> {
             nodes: self.nodes,
             scopes: self.scope,
             symbols: self.symbols,
-            module_record: Arc::new(module_record),
+            module_record: Arc::clone(&self.module_record),
             jsdoc: self.jsdoc.build(),
             unused_labels: self.unused_labels.labels,
         };

crates/oxc_semantic/src/checker/javascript.rs

@@ -101,7 +101,7 @@ fn check_module_record(ctx: &SemanticBuilder<'_>) {
         return;
     }
 
-    let module_record = &ctx.module_record_builder.module_record;
+    let module_record = &ctx.module_record;
 
     // It is a Syntax Error if any element of the ExportedBindings of ModuleItemList
     // does not also occur in either the VarDeclaredNames of ModuleItemList, or the LexicallyDeclaredNames of ModuleItemList.

crates/oxc_semantic/src/module_record/mod.rs

@@ -20,7 +20,7 @@ mod module_record_tests {
         let program = allocator.alloc(ret.program);
         let semantic_ret = SemanticBuilder::new(source_text, source_type)
             .with_trivias(ret.trivias)
-            .with_module_record_builder(true)
+            .build_module_record(program)
             .build(program);
         Arc::clone(&semantic_ret.semantic.module_record)
     }

crates/oxc_semantic/tests/modules.rs

@@ -23,8 +23,7 @@ fn test_exports() {
         export { foo };
         export default defaultExport;
         ",
-    )
-    .with_module_record_builder(true);
+    );
 
     test.has_some_symbol("foo").is_exported().test();
 

crates/oxc_semantic/tests/util/mod.rs

@@ -14,8 +14,6 @@ pub struct SemanticTester {
     allocator: Allocator,
     source_type: SourceType,
     source_text: &'static str,
-    /// SemanticBuilder option
-    use_module_record_builder: bool,
 }
 
 impl SemanticTester {
@@ -35,12 +33,7 @@ impl SemanticTester {
     }
 
     pub fn new(source_text: &'static str, source_type: SourceType) -> Self {
-        Self {
-            allocator: Allocator::default(),
-            source_type,
-            source_text,
-            use_module_record_builder: true,
-        }
+        Self { allocator: Allocator::default(), source_type, source_text }
     }
 
     /// Set the [`SourceType`] to TypeScript (or JavaScript, using `false`)
@@ -57,13 +50,6 @@ impl SemanticTester {
         self
     }
 
-    /// Set [`SemanticBuilder`]'s `with_module_record_builder` option
-    #[allow(dead_code)]
-    pub fn with_module_record_builder(mut self, yes: bool) -> Self {
-        self.use_module_record_builder = yes;
-        self
-    }
-
     /// Parse the source text and produce a new [`Semantic`]
     #[allow(unstable_name_collisions)]
     pub fn build(&self) -> Semantic<'_> {
@@ -86,7 +72,7 @@ impl SemanticTester {
         let semantic_ret = SemanticBuilder::new(self.source_text, self.source_type)
             .with_check_syntax_error(true)
             .with_trivias(parse.trivias)
-            .with_module_record_builder(self.use_module_record_builder)
+            .build_module_record(program)
             .build(program);
 
         if !semantic_ret.errors.is_empty() {

tasks/benchmark/benches/semantic.rs

@@ -26,7 +26,7 @@ fn bench_semantic(criterion: &mut Criterion) {
                 let program = allocator.alloc(ret.program);
                 b.iter_with_large_drop(|| {
                     SemanticBuilder::new(source_text, source_type)
-                        .with_module_record_builder(true)
+                        .build_module_record(program)
                         .build(program)
                 });
             },

tasks/coverage/src/suite.rs

@@ -302,8 +302,8 @@ pub trait Case: Sized + Sync + Send + UnwindSafe {
         let program = allocator.alloc(parser_ret.program);
         let semantic_ret = SemanticBuilder::new(source_text, source_type)
             .with_trivias(parser_ret.trivias)
-            .with_module_record_builder(true)
             .with_check_syntax_error(true)
+            .build_module_record(program)
             .build(program);
         if let Some(res) = self.check_semantic(&semantic_ret.semantic) {
             return res;