diff --git a/crates/oxc_resolver/src/cache.rs b/crates/oxc_resolver/src/cache.rs
index 99352defe..472c14aeb 100644
--- a/crates/oxc_resolver/src/cache.rs
+++ b/crates/oxc_resolver/src/cache.rs
@@ -1,22 +1,25 @@
 use std::{
-    hash::BuildHasherDefault,
+    borrow::Borrow,
+    convert::AsRef,
+    hash::{BuildHasherDefault, Hash, Hasher},
+    ops::Deref,
     path::{Path, PathBuf},
     sync::{Arc, OnceLock},
 };
 
-use dashmap::DashMap;
+use dashmap::DashSet;
 use rustc_hash::FxHasher;
 
 use crate::{package_json::PackageJson, FileMetadata, FileSystem, ResolveError};
 
 pub struct Cache<Fs> {
     pub(crate) fs: Fs,
-    cache: DashMap<Box<Path>, Arc<CacheValue>, BuildHasherDefault<FxHasher>>,
+    cache: DashSet<CacheValue, BuildHasherDefault<FxHasher>>,
 }
 
 impl<Fs: FileSystem> Default for Cache<Fs> {
     fn default() -> Self {
-        Self { fs: Fs::default(), cache: DashMap::default() }
+        Self { fs: Fs::default(), cache: DashSet::default() }
     }
 }
 
@@ -28,36 +31,74 @@ impl<Fs: FileSystem> Cache<Fs> {
     /// # Panics
     ///
     /// * Path is file but does not have a parent
-    pub fn dirname(&self, cache_value: &Arc<CacheValue>) -> Arc<CacheValue> {
-        Arc::clone(if cache_value.is_file(&self.fs) {
+    pub fn dirname<'a>(&self, cache_value: &'a CacheValue) -> &'a CacheValue {
+        if cache_value.is_file(&self.fs) {
             cache_value.parent.as_ref().unwrap()
         } else {
             cache_value
-        })
+        }
     }
 
-    pub fn value(&self, path: &Path) -> Arc<CacheValue> {
+    pub fn value(&self, path: &Path) -> CacheValue {
         if let Some(cache_entry) = self.cache.get(path) {
-            return Arc::clone(cache_entry.value());
+            return cache_entry.key().clone();
         }
         let parent = path.parent().map(|p| self.value(p));
-        let data = Arc::new(CacheValue::new(path.to_path_buf().into_boxed_path(), parent));
-        self.cache.insert(path.to_path_buf().into_boxed_path(), Arc::clone(&data));
+        let data =
+            CacheValue(Arc::new(CacheValueImpl::new(path.to_path_buf().into_boxed_path(), parent)));
+        self.cache.insert(data.clone());
         data
     }
 }
 
-#[derive(Debug, Clone)]
-pub struct CacheValue {
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct CacheValue(Arc<CacheValueImpl>);
+
+impl Deref for CacheValue {
+    type Target = CacheValueImpl;
+
+    fn deref(&self) -> &Self::Target {
+        self.0.as_ref()
+    }
+}
+
+impl AsRef<CacheValueImpl> for CacheValue {
+    fn as_ref(&self) -> &CacheValueImpl {
+        self.0.as_ref()
+    }
+}
+
+impl Borrow<Path> for CacheValue {
+    fn borrow(&self) -> &Path {
+        &self.path
+    }
+}
+
+#[derive(Debug)]
+pub struct CacheValueImpl {
     path: Box<Path>,
-    parent: Option<Arc<CacheValue>>,
+    parent: Option<CacheValue>,
     meta: OnceLock<Option<FileMetadata>>,
     symlink: OnceLock<Option<PathBuf>>,
     package_json: OnceLock<Option<Result<Arc<PackageJson>, ResolveError>>>,
 }
 
-impl CacheValue {
-    fn new(path: Box<Path>, parent: Option<Arc<Self>>) -> Self {
+impl Hash for CacheValueImpl {
+    fn hash<H: Hasher>(&self, h: &mut H) {
+        self.path.hash(h);
+    }
+}
+
+impl PartialEq for CacheValueImpl {
+    fn eq(&self, other: &Self) -> bool {
+        self.path == other.path
+    }
+}
+
+impl Eq for CacheValueImpl {}
+
+impl CacheValueImpl {
+    fn new(path: Box<Path>, parent: Option<CacheValue>) -> Self {
         Self {
             path,
             parent,
diff --git a/crates/oxc_resolver/src/lib.rs b/crates/oxc_resolver/src/lib.rs
index cb0afffb6..56d53eee5 100644
--- a/crates/oxc_resolver/src/lib.rs
+++ b/crates/oxc_resolver/src/lib.rs
@@ -21,7 +21,6 @@ use std::{
     borrow::Cow,
     ffi::OsStr,
     path::{Path, PathBuf},
-    sync::Arc,
 };
 
 use crate::{
@@ -38,7 +37,7 @@ pub use crate::{
     resolution::Resolution,
 };
 
-type ResolveState = Result<Option<Arc<CacheValue>>, ResolveError>;
+type ResolveState = Result<Option<CacheValue>, ResolveError>;
 
 /// Resolver with the current operating system as the file system
 pub type Resolver = ResolverGeneric<FileSystemOs>;
@@ -108,9 +107,9 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
     /// Y: path
     fn require(
         &self,
-        cache_value: &Arc<CacheValue>,
+        cache_value: &CacheValue,
         request: &Request,
-    ) -> Result<Arc<CacheValue>, ResolveError> {
+    ) -> Result<CacheValue, ResolveError> {
         match request.path {
             // 1. If X is a core module,
             //    a. return the core module
@@ -139,9 +138,9 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
     // 3. If X begins with './' or '/' or '../'
     fn require_relative(
         &self,
-        cache_value: &Arc<CacheValue>,
+        cache_value: &CacheValue,
         request: &str,
-    ) -> Result<Arc<CacheValue>, ResolveError> {
+    ) -> Result<CacheValue, ResolveError> {
         if let Some(path) = self.load_package_self(cache_value, request)? {
             return Ok(path);
         }
@@ -163,16 +162,16 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
 
     fn require_path(
         &self,
-        cache_value: &Arc<CacheValue>,
+        cache_value: &CacheValue,
         request: &str,
-    ) -> Result<Arc<CacheValue>, ResolveError> {
+    ) -> Result<CacheValue, ResolveError> {
         let dirname = self.cache.dirname(cache_value);
         // 5. LOAD_PACKAGE_SELF(X, dirname(Y))
-        if let Some(path) = self.load_package_self(&dirname, request)? {
+        if let Some(path) = self.load_package_self(dirname, request)? {
             return Ok(path);
         }
         // 6. LOAD_NODE_MODULES(X, dirname(Y))
-        if let Some(path) = self.load_node_modules(&dirname, request)? {
+        if let Some(path) = self.load_node_modules(dirname, request)? {
             return Ok(path);
         }
         let cache_value = self.cache.value(&cache_value.path().join(request));
@@ -183,7 +182,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
         Err(ResolveError::NotFound(cache_value.to_path_buf().into_boxed_path()))
     }
 
-    fn load_as_file(&self, cache_value: &Arc<CacheValue>) -> ResolveState {
+    fn load_as_file(&self, cache_value: &CacheValue) -> ResolveState {
         // enhanced-resolve feature: extension_alias
         if let Some(path) = self.load_extension_alias(cache_value)? {
             return Ok(Some(path));
@@ -202,11 +201,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
         Ok(None)
     }
 
-    fn load_extensions(
-        &self,
-        cache_value: &Arc<CacheValue>,
-        extensions: &[String],
-    ) -> ResolveState {
+    fn load_extensions(&self, cache_value: &CacheValue, extensions: &[String]) -> ResolveState {
         let mut path_with_extension = cache_value.path().to_path_buf();
         for extension in extensions {
             path_with_extension.set_extension(extension);
@@ -218,11 +213,11 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
         Ok(None)
     }
 
-    fn load_symlink(&self, cache_value: &Arc<CacheValue>) -> Option<PathBuf> {
+    fn load_symlink(&self, cache_value: &CacheValue) -> Option<PathBuf> {
         if self.options.symlinks { cache_value.symlink(&self.cache.fs) } else { None }
     }
 
-    fn load_index(&self, cache_value: &Arc<CacheValue>) -> ResolveState {
+    fn load_index(&self, cache_value: &CacheValue) -> ResolveState {
         for main_file in &self.options.main_files {
             let main_path = cache_value.path().join(main_file);
             let cache_value = self.cache.value(&main_path);
@@ -241,7 +236,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
         Ok(None)
     }
 
-    fn load_alias_or_file(&self, cache_value: &Arc<CacheValue>) -> ResolveState {
+    fn load_alias_or_file(&self, cache_value: &CacheValue) -> ResolveState {
         if let Some(package_json) = cache_value.find_package_json(&self.cache.fs)? {
             let path = cache_value.path();
             if let Some(path) = self.load_browser_field(path, None, &package_json)? {
@@ -249,12 +244,12 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
             }
         }
         if cache_value.is_file(&self.cache.fs) {
-            return Ok(Some(Arc::clone(cache_value)));
+            return Ok(Some(cache_value.clone()));
         }
         Ok(None)
     }
 
-    fn load_as_directory(&self, cache_value: &Arc<CacheValue>) -> ResolveState {
+    fn load_as_directory(&self, cache_value: &CacheValue) -> ResolveState {
         // TODO: Only package.json is supported, so warn about having other values
         // Checking for empty files is needed for omitting checks on package.json
         // 1. If X/package.json is a file,
@@ -288,7 +283,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
         self.load_index(cache_value)
     }
 
-    fn load_node_modules(&self, cache_value: &Arc<CacheValue>, request: &str) -> ResolveState {
+    fn load_node_modules(&self, cache_value: &CacheValue, request: &str) -> ResolveState {
         // 1. let DIRS = NODE_MODULES_PATHS(START)
         let dirs = self.node_module_paths(cache_value.path());
         // 2. for each DIR in DIRS:
@@ -338,7 +333,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
     /// # Panics
     ///
     /// * Parent of package.json is None
-    fn load_package_self(&self, cache_value: &Arc<CacheValue>, request: &str) -> ResolveState {
+    fn load_package_self(&self, cache_value: &CacheValue, request: &str) -> ResolveState {
         if let Some(package_json) = cache_value.find_package_json(&self.cache.fs)? {
             if let Some(path) =
                 self.load_browser_field(cache_value.path(), Some(request), &package_json)?
@@ -365,12 +360,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
         Ok(None)
     }
 
-    fn load_alias(
-        &self,
-        cache_value: &Arc<CacheValue>,
-        request: &str,
-        alias: &Alias,
-    ) -> ResolveState {
+    fn load_alias(&self, cache_value: &CacheValue, request: &str, alias: &Alias) -> ResolveState {
         for (alias, requests) in alias {
             let exact_match = alias.strip_prefix(request).is_some_and(|c| c == "$");
             if request.starts_with(alias) || exact_match {
@@ -411,7 +401,7 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
     /// # Errors
     ///
     /// * [ResolveError::ExtensionAlias]: When all of the aliased extensions are not found
-    fn load_extension_alias(&self, cache_value: &Arc<CacheValue>) -> ResolveState {
+    fn load_extension_alias(&self, cache_value: &CacheValue) -> ResolveState {
         let Some(path_extension) = cache_value.path().extension() else { return Ok(None) };
         let Some((_, extensions)) =
             self.options.extension_alias.iter().find(|(ext, _)| OsStr::new(ext) == path_extension)
@@ -426,9 +416,9 @@ impl<Fs: FileSystem> ResolverGeneric<Fs> {
 
     fn load_roots(
         &self,
-        cache_value: &Arc<CacheValue>,
+        cache_value: &CacheValue,
         request: &str,
-    ) -> Result<Arc<CacheValue>, ResolveError> {
+    ) -> Result<CacheValue, ResolveError> {
         debug_assert!(request.starts_with('/'));
         if self.options.roots.is_empty() {
             let cache_value = self.cache.value(Path::new("/"));