perf: read directory in bulk in the background at startup (sxyazi#599)

README.md

@@ -12,7 +12,7 @@ Yazi (means "duck") is a terminal file manager written in Rust, based on non-blo
 - 🧰 Integration with fd, rg, fzf, zoxide
 - 💫 Vim-like input/select component, auto-completion for cd paths
 - 🏷️ Multi-Tab Support, Scrollable Preview (for videos, PDFs, archives, directories, code, etc.)
-- 🔄 Batch Renaming, Visual Mode, File Chooser
+- 🔄 Bulk Renaming, Visual Mode, File Chooser
 - 🎨 Theme System, Custom Layouts, Trash Bin, CSI u
 - ... and more!
 

yazi-core/src/folder/files.rs

@@ -1,7 +1,7 @@
 use std::{collections::{BTreeMap, BTreeSet}, mem, ops::Deref, sync::atomic::Ordering};
 
 use anyhow::Result;
-use tokio::{fs, select, sync::mpsc::{self, UnboundedReceiver}};
+use tokio::{fs::{self, DirEntry}, select, sync::mpsc::{self, UnboundedReceiver}};
 use yazi_config::{manager::SortBy, MANAGER};
 use yazi_shared::fs::{File, Url, FILES_TICKET};
 
@@ -66,6 +66,34 @@ impl Files {
 		});
 		Ok(rx)
 	}
+
+	pub async fn from_dir_bulk(url: &Url) -> Result<Vec<File>> {
+		let mut it = fs::read_dir(url).await?;
+		let mut items = Vec::with_capacity(5000);
+		while let Ok(Some(item)) = it.next_entry().await {
+			items.push(item);
+		}
+
+		let (first, rest) = items.split_at(items.len() / 3);
+		let (second, third) = rest.split_at(items.len() / 3);
+		async fn go(entities: &[DirEntry]) -> Vec<File> {
+			let mut files = Vec::with_capacity(entities.len() / 3 + 1);
+			for entry in entities {
+				if let Ok(meta) = entry.metadata().await {
+					files.push(File::from_meta(Url::from(entry.path()), meta).await);
+				}
+			}
+			files
+		}
+
+		Ok(
+			futures::future::join_all([go(first), go(second), go(third)])
+				.await
+				.into_iter()
+				.flatten()
+				.collect(),
+		)
+	}
 }
 
 impl Files {

yazi-core/src/manager/commands/open.rs

@@ -8,13 +8,16 @@ use yazi_shared::{emit, event::{EventQuit, Exec}, fs::{File, Url}, Layer, MIME_D
 use crate::{manager::Manager, select::Select, tasks::Tasks};
 
 pub struct Opt {
-	targets:     Option<Vec<(Url, Option<String>)>>,
+	targets:     Vec<(Url, Option<String>)>,
 	interactive: bool,
 }
 
 impl From<Exec> for Opt {
 	fn from(mut e: Exec) -> Self {
-		Self { targets: e.take_data(), interactive: e.named.contains_key("interactive") }
+		Self {
+			targets:     e.take_data().unwrap_or_default(),
+			interactive: e.named.contains_key("interactive"),
+		}
 	}
 }
 
@@ -40,7 +43,7 @@ impl Manager {
 
 		let mut opt = opt.into() as Opt;
 		if todo.is_empty() {
-			opt.targets = Some(done);
+			opt.targets = done;
 			return self.open_do(opt, tasks);
 		}
 
@@ -64,11 +67,12 @@ impl Manager {
 
 	pub fn open_do(&mut self, opt: impl Into<Opt>, tasks: &Tasks) {
 		let opt = opt.into() as Opt;
-		let Some(targets) = opt.targets else {
+		if opt.targets.is_empty() {
 			return;
-		};
+		}
 
-		let targets: Vec<_> = targets
+		let targets: Vec<_> = opt
+			.targets
 			.into_iter()
 			.filter_map(|(u, m)| m.or_else(|| self.mimetype.get(&u).cloned()).map(|m| (u, m)))
 			.collect();

yazi-core/src/manager/watcher.rs

@@ -1,5 +1,6 @@
-use std::{collections::{BTreeMap, BTreeSet}, sync::Arc, time::Duration};
+use std::{collections::{BTreeMap, BTreeSet}, sync::Arc, time::{Duration, SystemTime}};
 
+use anyhow::Result;
 use notify::{event::{MetadataKind, ModifyKind}, EventKind, RecommendedWatcher, RecursiveMode, Watcher as _Watcher};
 use parking_lot::RwLock;
 use tokio::{fs, pin, sync::mpsc::{self, UnboundedReceiver}};
@@ -90,26 +91,27 @@ impl Watcher {
 			return;
 		}
 
-		tokio::spawn(async move {
-			for (url, mtime) in todo {
-				let Ok(meta) = fs::metadata(&url).await else {
-					if let Ok(m) = fs::symlink_metadata(&url).await {
-						FilesOp::Full(url, vec![], m.modified().ok()).emit();
-					} else if let Some(p) = url.parent_url() {
-						FilesOp::Deleting(p, vec![url]).emit();
-					}
-					continue;
-				};
-
-				if meta.modified().ok() == mtime {
-					continue;
+		async fn go(url: Url, mtime: Option<SystemTime>) {
+			let Ok(meta) = fs::metadata(&url).await else {
+				if let Ok(m) = fs::symlink_metadata(&url).await {
+					FilesOp::Full(url, vec![], m.modified().ok()).emit();
+				} else if let Some(p) = url.parent_url() {
+					FilesOp::Deleting(p, vec![url]).emit();
 				}
+				return;
+			};
 
-				if let Ok(rx) = Files::from_dir(&url).await {
-					let files: Vec<_> = UnboundedReceiverStream::new(rx).collect().await;
-					FilesOp::Full(url, files, meta.modified().ok()).emit();
-				}
+			if meta.modified().ok() == mtime {
+				return;
+			}
+
+			if let Ok(files) = Files::from_dir_bulk(&url).await {
+				FilesOp::Full(url, files, meta.modified().ok()).emit();
 			}
+		}
+
+		tokio::spawn(async move {
+			futures::future::join_all(todo.into_iter().map(|(url, mtime)| go(url, mtime))).await;
 		});
 	}