perf: <⚡️> improve walker

.github/workflows/ci.yml

@@ -32,4 +32,4 @@ jobs:
       - name: Setup shell env
         run: nix develop
       - name: test
-        run: cargo test
+        run: cargo test --workspace

npmpink-bin/Cargo.toml

@@ -17,3 +17,4 @@ clap = { version = "4.5.8", features = ["derive"] }
 anyhow = "1.0.86"
 inquire = "0.7.5"
 lazycell = "1.3.0"
+regex = "1.10.5"

npmpink-bin/src/cli.rs

@@ -2,6 +2,7 @@
 // https://docs.rs/clap/latest/clap/_derive/index.html#terminology
 use anyhow::{bail, Result};
 use clap::{Parser, Subcommand};
+use npmpink_core::config::HealthCheckError;
 use npmpink_core::ops::packages::{difference_packages, packages_from_source};
 use npmpink_core::package::Package;
 use npmpink_core::source::Source;
@@ -123,14 +124,27 @@ struct InitArgs {
     force: bool,
 }
 fn cmd_handler_init(args: &InitArgs) -> Result<()> {
-    if !args.force {
-        let _ = Config::healthcheck()?;
-        shell()?.info("npmpink is already initialized")?;
-        return Ok(());
+    let result = Config::healthcheck();
+    match result {
+        Err(err) => match err {
+            HealthCheckError::ConfigFileNotExist => {}
+            _ => {
+                return Err(anyhow::Error::msg(err));
+            }
+        },
+        Ok(_) => {
+            if !args.force {
+                shell()?.info("npmpink is already initialized")?;
+
+                return Ok(());
+            }
+        }
     }
 
     // init config
-    Config::create_from_default()
+    Config::create_from_default()?;
+
+    shell()?.info("inited config file")
 }
 
 /// Update packages inside npmpink.lock to node modules
@@ -141,15 +155,16 @@ fn cmd_handler_sync(cli: &Cli) -> Result<()> {
         lockfile.packages_iter().collect::<Vec<Package>>()
     };
     let pkgs_paths = lockfile_pkgs.iter();
+    let mut sh = shell()?;
 
     for pkg in pkgs_paths {
-        println!("> Link package {}: \n", pkg.name);
+        sh.info(format!("> Link package {}: \n", pkg.name))?;
         Command::new("pnpm")
             .args(["link", &pkg.dir])
             .status()
             .map(|_| ())
             .map_err(anyhow::Error::msg)?;
-        println!("\n> -----------------------------\n");
+        sh.info("\n")?;
     }
 
     Ok(())
@@ -159,7 +174,7 @@ fn cmd_handler_check() -> Result<()> {
     let result = Config::healthcheck();
 
     if result.is_ok() {
-        println!("all checks pass");
+        shell()?.info("all checks pass")?;
         return Ok(());
     }
 
@@ -233,7 +248,7 @@ fn cmd_handler_source_list() -> Result<()> {
     let config = appConfig.lock().unwrap();
 
     for source in config.sources.iter() {
-        println!("{}: {}", source.id, source.path.display());
+        shell()?.info(format!("{}: {}", source.id, source.path.display()))?;
     }
 
     Ok(())
@@ -257,11 +272,18 @@ fn cmd_handler_package_sub_cli(cli: &Cli, command: &PackageSubCli) -> Result<()>
 
 fn cmd_handler_package_list_all(_cli: &Cli) -> Result<()> {
     let config = appConfig.lock().unwrap();
-    let pkgs = config.sources.iter().flat_map(packages_from_source);
+    let mut pkgs = config.sources.iter().flat_map(packages_from_source);
+
+    let mut sh = shell()?;
+
+    if pkgs.by_ref().peekable().peek().is_none() {
+        sh.warn("no packages to list")?;
+    }
 
     for pkg in pkgs {
-        eprintln!("{:?}", pkg.name);
+        sh.info(format!("{:?}", pkg.name))?;
     }
+
     Ok(())
 }
 
@@ -290,7 +312,7 @@ fn cmd_handler_package_add(cli: &Cli) -> Result<()> {
     }
     target.flush_lockfile()?;
 
-    println!("{} packages added", picked.len());
+    shell()?.info(format!("{} packages added", picked.len()))?;
     Ok(())
 }
 
@@ -312,6 +334,6 @@ fn cmd_handler_package_remove(cli: &Cli) -> Result<()> {
     }
     target.flush_lockfile()?;
 
-    println!("{} packages removed", picked.len());
+    shell()?.info(format!("{} packages removed", picked.len()))?;
     Ok(())
 }

npmpink-core/Cargo.toml

@@ -6,11 +6,12 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0.86"
 home = "0.5.9"
+ignore = "0.4.22"
 lazy_static = "1.5.0"
 lazycell = "1.3.0"
 package_json_schema = { version = "0.2.1" }
+regex = "1.10.5"
 serde = { version = "1.0.203", features = ["derive"] }
 serde_json = "1.0.119"
 serde_test = "1.0.176"
 thiserror = "1.0.61"
-wax = "0.6.0"

npmpink-core/src/lib.rs

@@ -4,4 +4,5 @@ pub mod ops;
 pub mod package;
 pub mod source;
 pub mod target;
+pub mod walker;
 pub mod workspace;

npmpink-core/src/lockfile.rs

@@ -98,15 +98,12 @@ mod tests {
                 Token::String("foo_package"),
                 Token::Struct {
                     name: "Package",
-                    len: 4,
+                    len: 3,
                 },
                 Token::String("name"),
                 Token::String("foo"),
                 //
-                Token::String("version"),
-                Token::String("bar"),
-                //
-                Token::String("location"),
+                Token::String("dir"),
                 Token::String("foo/bar"),
                 //
                 Token::String("source_id"),

npmpink-core/src/ops/packages.rs

@@ -34,6 +34,7 @@ pub fn packages_from_source(source: &Source) -> Vec<Package> {
                     pkg.private
                         .as_ref()
                         .is_some_and(package_private_is_not_falsy)
+                        || pkg.private.as_ref().is_none()
                 })
                 .map(|pkg| {
                     Package::new(

npmpink-core/src/package.rs

@@ -59,13 +59,13 @@ mod tests {
             &[
                 Token::Struct {
                     name: "Package",
-                    len: 4,
+                    len: 3,
                 },
                 //
                 Token::String("name"),
                 Token::String("foo"),
                 //
-                Token::String("location"),
+                Token::String("dir"),
                 Token::String("bar"),
                 //
                 Token::String("source_id"),

npmpink-core/src/walker/mod.rs

@@ -0,0 +1,12 @@
+pub mod walk;
+
+use std::path::PathBuf;
+pub use walk::{walk, WalkOption};
+
+#[derive(Debug)]
+pub enum ReceiveMode {
+    Buffer,
+    Stream,
+}
+
+pub type ResultItem = PathBuf;

npmpink-core/src/walker/walk.rs

@@ -0,0 +1,115 @@
+use anyhow::Result;
+use ignore::{WalkBuilder, WalkParallel, WalkState};
+use regex::bytes::Regex;
+use std::borrow::Cow;
+use std::path::{Path, PathBuf};
+use std::sync::mpsc::channel;
+
+#[derive(Debug, Clone)]
+pub struct WalkOption {
+    /// How many threads to run.
+    pub threads: usize,
+    /// Math depth to iterate the directories.
+    pub max_depth: usize,
+    pub patterns: Vec<Regex>,
+}
+
+impl WalkOption {
+    pub fn new(patterns: Vec<Regex>) -> WalkOption {
+        WalkOption {
+            patterns,
+            ..Default::default()
+        }
+    }
+    pub fn max_depth(mut self, size: usize) -> Self {
+        self.max_depth = size;
+        self
+    }
+    pub fn threads(mut self, size: usize) -> Self {
+        self.threads = size;
+        self
+    }
+}
+
+impl Default for WalkOption {
+    fn default() -> WalkOption {
+        WalkOption {
+            threads: 5,
+            max_depth: 10,
+            patterns: Vec::new(),
+        }
+    }
+}
+
+/// Create a file tree walker
+pub fn create_concurrent_walker(
+    paths: &[impl AsRef<Path>],
+    option: Option<WalkOption>,
+) -> (WalkParallel, WalkOption) {
+    let option = option.unwrap_or_default();
+
+    let first_path = &paths[0];
+    let mut walker = WalkBuilder::new(first_path);
+    walker
+        .follow_links(false)
+        // .max_filesize(Some(1024 * 1024 * 30))
+        .max_depth(Some(option.max_depth))
+        .threads(option.threads);
+
+    // add paths to search
+    paths.iter().skip(1).for_each(|p| {
+        walker.add(p);
+    });
+
+    (walker.build_parallel(), option)
+}
+
+pub fn walk(paths: &[impl AsRef<Path>], option: Option<WalkOption>) -> Result<Vec<PathBuf>> {
+    let (walker, option) = create_concurrent_walker(paths, option);
+
+    let patterns = &option.patterns;
+    let mut buffers = Vec::<PathBuf>::new();
+
+    let (tx, rx) = channel::<PathBuf>();
+
+    walker.run(|| {
+        Box::new(|result| {
+            if result.is_err() {
+                return WalkState::Skip;
+            }
+            let entry = result.unwrap();
+            // skip root.
+            if entry.depth() == 0 {
+                return WalkState::Continue;
+            }
+
+            let entry_path = entry.path();
+            if !is_match_pattern(entry_path, patterns.as_slice()) {
+                return WalkState::Continue;
+            }
+
+            tx.send(entry_path.to_path_buf()).unwrap();
+
+            WalkState::Continue
+        })
+    });
+
+    drop(tx);
+
+    while let Ok(re) = rx.recv() {
+        buffers.push(re);
+    }
+
+    Ok(buffers)
+}
+
+fn is_match_pattern(path: &Path, pattern: &[Regex]) -> bool {
+    let to_match: Cow<[u8]> = {
+        match path.to_string_lossy() {
+            Cow::Owned(string) => Cow::Owned(string.into_bytes()),
+            Cow::Borrowed(string) => Cow::Borrowed(string.as_bytes()),
+        }
+    };
+
+    pattern.iter().all(|pat| pat.is_match(&to_match))
+}

npmpink-core/src/workspace/package_json_walker.rs

@@ -1,41 +1,22 @@
+use crate::walker;
 use anyhow::Result;
+use regex::bytes::Regex;
 use std::path::{Path, PathBuf};
-use wax::{any, Glob};
 
-pub(super) fn walk_package_jsons_under_path(
-    path: impl AsRef<Path>,
-) -> Result<impl 'static + Iterator<Item = PathBuf>> {
-    let glob = Glob::new("**/package.json").unwrap().into_owned();
-    let walker = glob
-        .walk(path)
-        .not(any([
-            "**/node_modules/**",
-            "**/dist/**",
-            "**/src/**",
-            "**/public/**",
-            "**/.git/**",
-            "**/.direnv/**",
-        ]))
-        .unwrap();
+pub(super) fn walk_package_jsons_under_path(path: impl AsRef<Path>) -> Result<Vec<PathBuf>> {
+    let wo = walker::WalkOption::new(vec![Regex::new(r"package\.json$").unwrap()]);
+    let paths = walker::walk(&[path], Some(wo))?;
 
-    Ok(walker
-        .filter(|entry| entry.is_ok())
-        .map(|entry| entry.unwrap().path().to_path_buf())
-        // clone the path from the iterator results
-        .collect::<Vec<PathBuf>>()
-        .into_iter())
+    Ok(paths)
 }
 
-#[ignore]
 #[test]
 fn test_walk_package_jsons_under_path() {
     let workspace_root = concat!(
         env!("CARGO_WORKSPACE_DIR"),
         "assets_/fixtures_npm_workspaces"
     );
 
-    let paths = walk_package_jsons_under_path(workspace_root)
-        .unwrap()
-        .collect::<Vec<PathBuf>>();
+    let paths = walk_package_jsons_under_path(workspace_root).unwrap();
     assert!(paths.len() == 5);
 }